summaryrefslogtreecommitdiffstats
path: root/metrics-proxy
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@verizonmedia.com>2020-02-20 09:42:19 +0100
committerJon Bratseth <bratseth@verizonmedia.com>2020-02-20 09:42:19 +0100
commit5acf4c47e98674cdf73289a782dfda9da7041ead (patch)
tree9a2720a3326326cc2a0b69d29b6877e4039d5f18 /metrics-proxy
parentd2449a3e66075e7d680263a204302e83b5ba0148 (diff)
parent1cc70ca6f328e7e88e8b4e279cac7544624f055b (diff)
Merge branch 'master' into bratseth/node-metrics
Diffstat (limited to 'metrics-proxy')
-rw-r--r--metrics-proxy/CMakeLists.txt2
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java11
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java3
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ErrorResponse.java32
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/HttpHandlerBase.java78
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/JsonResponse.java30
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java10
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java32
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessor.java39
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java2
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java69
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessor.java24
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java6
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java92
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java2
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java8
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java77
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java20
-rw-r--r--metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/processing/MetricsProcessor.java31
-rw-r--r--metrics-proxy/src/main/resources/configdefinitions/node-info.def5
-rw-r--r--metrics-proxy/src/main/resources/configdefinitions/telegraf.def20
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/ErrorResponseTest.java25
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java6
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java19
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessorTest.java63
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessorTest.java59
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessorTest.java43
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java196
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java173
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java53
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java3
-rw-r--r--metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java15
-rw-r--r--metrics-proxy/src/test/resources/generic-sample.json9
33 files changed, 895 insertions, 362 deletions
diff --git a/metrics-proxy/CMakeLists.txt b/metrics-proxy/CMakeLists.txt
index 41fedb8e8c4..7587159165d 100644
--- a/metrics-proxy/CMakeLists.txt
+++ b/metrics-proxy/CMakeLists.txt
@@ -6,5 +6,7 @@ install_config_definition(src/main/resources/configdefinitions/consumers.def ai.
install_config_definition(src/main/resources/configdefinitions/monitoring.def ai.vespa.metricsproxy.core.monitoring.def)
install_config_definition(src/main/resources/configdefinitions/metrics-nodes.def ai.vespa.metricsproxy.http.application.metrics-nodes.def)
install_config_definition(src/main/resources/configdefinitions/node-dimensions.def ai.vespa.metricsproxy.metric.dimensions.node-dimensions.def)
+install_config_definition(src/main/resources/configdefinitions/node-info.def ai.vespa.metricsproxy.http.metrics.node-info.def)
install_config_definition(src/main/resources/configdefinitions/rpc-connector.def ai.vespa.metricsproxy.rpc.rpc-connector.def)
install_config_definition(src/main/resources/configdefinitions/vespa-services.def ai.vespa.metricsproxy.service.vespa-services.def)
+install_config_definition(src/main/resources/configdefinitions/telegraf.def ai.vespa.metricsproxy.telegraf.telegraf.def)
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java
index 4c4015220bc..53a05ef88f0 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java
@@ -79,6 +79,16 @@ public class MetricsManager {
* @return Metrics for all matching services.
*/
public List<MetricsPacket> getMetrics(List<VespaService> services, Instant startTime) {
+ return getMetricsAsBuilders(services, startTime).stream()
+ .map(MetricsPacket.Builder::build)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Returns the metrics for the given services, in mutable state for further processing.
+ * NOTE: Use {@link #getMetrics(List, Instant)} instead, unless further processing of the metrics is necessary.
+ */
+ public List<MetricsPacket.Builder> getMetricsAsBuilders(List<VespaService> services, Instant startTime) {
if (services.isEmpty()) return Collections.emptyList();
log.log(DEBUG, () -> "Updating services prior to fetching metrics, number of services= " + services.size());
@@ -99,7 +109,6 @@ public class MetricsManager {
.map(builder -> builder.putDimensionsIfAbsent(getGlobalDimensions()))
.map(builder -> builder.putDimensionsIfAbsent(extraDimensions))
.map(builder -> adjustTimestamp(builder, startTime))
- .map(MetricsPacket.Builder::build)
.collect(Collectors.toList());
}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java
index 1b03d3b01f9..c04dca465a1 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java
@@ -24,6 +24,7 @@ import java.util.Set;
import java.util.logging.Logger;
import java.util.stream.Collectors;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_SERVICE_ID;
import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId;
import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
@@ -40,7 +41,7 @@ public class VespaMetrics {
public static final ConsumerId VESPA_CONSUMER_ID = toConsumerId("Vespa");
public static final DimensionId METRIC_TYPE_DIMENSION_ID = toDimensionId("metrictype");
- public static final DimensionId INSTANCE_DIMENSION_ID = toDimensionId("instance");
+ public static final DimensionId INSTANCE_DIMENSION_ID = toDimensionId(INTERNAL_SERVICE_ID);
private final MetricsConsumers metricsConsumers;
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ErrorResponse.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ErrorResponse.java
deleted file mode 100644
index 9bd30d287d4..00000000000
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ErrorResponse.java
+++ /dev/null
@@ -1,32 +0,0 @@
-// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package ai.vespa.metricsproxy.http;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import java.util.Map;
-import java.util.logging.Logger;
-
-import static java.util.logging.Level.WARNING;
-
-/**
- * @author gjoranv
- */
-public class ErrorResponse extends JsonResponse {
- private static Logger log = Logger.getLogger(ErrorResponse.class.getName());
-
- private static ObjectMapper objectMapper = new ObjectMapper();
-
- public ErrorResponse(int code, String message) {
- super(code, asErrorJson(message));
- }
-
- static String asErrorJson(String message) {
- try {
- return objectMapper.writeValueAsString(Map.of("error", message));
- } catch (JsonProcessingException e) {
- log.log(WARNING, "Could not encode error message to json:", e);
- return "Could not encode error message to json, check the log for details.";
- }
- }
-}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/HttpHandlerBase.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/HttpHandlerBase.java
deleted file mode 100644
index aa82a921e1a..00000000000
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/HttpHandlerBase.java
+++ /dev/null
@@ -1,78 +0,0 @@
-// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package ai.vespa.metricsproxy.http;
-
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.container.jdisc.ThreadedHttpRequestHandler;
-import com.yahoo.restapi.Path;
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-
-import java.net.URI;
-import java.util.List;
-import java.util.Optional;
-import java.util.concurrent.Executor;
-
-import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR;
-import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED;
-import static com.yahoo.jdisc.Response.Status.NOT_FOUND;
-import static com.yahoo.jdisc.Response.Status.OK;
-import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
-import static java.util.logging.Level.WARNING;
-
-/**
- * @author gjoranv
- */
-public abstract class HttpHandlerBase extends ThreadedHttpRequestHandler {
-
- protected HttpHandlerBase(Executor executor) {
- super(executor);
- }
-
- protected abstract Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer);
-
- @Override
- public final HttpResponse handle(HttpRequest request) {
- if (request.getMethod() != GET) return new JsonResponse(METHOD_NOT_ALLOWED, "Only GET is supported");
-
- Path path = new Path(request.getUri());
-
- return doHandle(request.getUri(), path, getConsumer(request))
- .orElse(new ErrorResponse(NOT_FOUND, "No content at given path"));
- }
-
- private String getConsumer(HttpRequest request) {
- return request.getProperty("consumer");
- }
-
- protected JsonResponse resourceListResponse(URI requestUri, List<String> resources) {
- try {
- return new JsonResponse(OK, resourceList(requestUri, resources));
- } catch (JSONException e) {
- log.log(WARNING, "Bad JSON construction in generated resource list for " + requestUri.getPath(), e);
- return new ErrorResponse(INTERNAL_SERVER_ERROR,
- "An error occurred when generating the list of api resources.");
- }
- }
-
- // TODO: Use jackson with a "Resources" class instead of JSONObject
- private static String resourceList(URI requestUri, List<String> resources) throws JSONException {
- int port = requestUri.getPort();
- String host = requestUri.getHost();
- StringBuilder base = new StringBuilder("http://");
- base.append(host);
- if (port >= 0) {
- base.append(":").append(port);
- }
- String uriBase = base.toString();
- JSONArray linkList = new JSONArray();
- for (String api : resources) {
- JSONObject resource = new JSONObject();
- resource.put("url", uriBase + api);
- linkList.put(resource);
- }
- return new JSONObject().put("resources", linkList).toString(4);
- }
-
-}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/JsonResponse.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/JsonResponse.java
deleted file mode 100644
index 9de5933bd1f..00000000000
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/JsonResponse.java
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package ai.vespa.metricsproxy.http;
-
-import com.yahoo.container.jdisc.HttpResponse;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.charset.Charset;
-
-/**
- * @author gjoranv
- */
-public class JsonResponse extends HttpResponse {
- private final byte[] data;
-
- public JsonResponse(int code, String data) {
- super(code);
- this.data = data.getBytes(Charset.forName(DEFAULT_CHARACTER_ENCODING));
- }
-
- @Override
- public String getContentType() {
- return "application/json";
- }
-
- @Override
- public void render(OutputStream outputStream) throws IOException {
- outputStream.write(data);
- }
-}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java
index 7386305ad34..51bdae1aab3 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java
@@ -47,6 +47,16 @@ public class ValuesFetcher {
.collect(Collectors.toList());
}
+ public List<MetricsPacket.Builder> fetchMetricsAsBuilders(String requestedConsumer) throws JsonRenderingException {
+ ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers);
+
+ return metricsManager.getMetricsAsBuilders(vespaServices.getVespaServices(), Instant.now())
+ .stream()
+ .filter(builder -> builder.hasConsumer(consumer))
+ .collect(Collectors.toList());
+ }
+
+
public List<MetricsPacket> fetchAllMetrics() throws JsonRenderingException {
return metricsManager.getMetrics(vespaServices.getVespaServices(), Instant.now());
}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java
index ce2e383f0d2..d9303e80dcd 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java
@@ -3,17 +3,17 @@
package ai.vespa.metricsproxy.http.application;
import ai.vespa.metricsproxy.core.MetricsConsumers;
-import ai.vespa.metricsproxy.http.ErrorResponse;
-import ai.vespa.metricsproxy.http.HttpHandlerBase;
-import ai.vespa.metricsproxy.http.JsonResponse;
import ai.vespa.metricsproxy.metric.model.ConsumerId;
import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor;
import com.google.inject.Inject;
+import com.yahoo.container.handler.metrics.ErrorResponse;
+import com.yahoo.container.handler.metrics.HttpHandlerBase;
+import com.yahoo.container.handler.metrics.JsonResponse;
import com.yahoo.container.jdisc.HttpResponse;
import com.yahoo.restapi.Path;
import java.net.URI;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -23,6 +23,7 @@ import java.util.logging.Level;
import static ai.vespa.metricsproxy.http.ValuesFetcher.getConsumerOrDefault;
import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericApplicationModel;
+import static ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor.applyProcessors;
import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR;
import static com.yahoo.jdisc.Response.Status.OK;
import static java.util.stream.Collectors.toList;
@@ -35,7 +36,9 @@ import static java.util.stream.Collectors.toList;
public class ApplicationMetricsHandler extends HttpHandlerBase {
public static final String V1_PATH = "/applicationmetrics/v1";
- static final String VALUES_PATH = V1_PATH + "/values";
+ public static final String VALUES_PATH = V1_PATH + "/values";
+
+ private static final int MAX_DIMENSIONS = 10;
private final ApplicationMetricsRetriever metricsRetriever;
private final MetricsConsumers metricsConsumers;
@@ -60,7 +63,10 @@ public class ApplicationMetricsHandler extends HttpHandlerBase {
try {
ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers);
var buildersByNode = metricsRetriever.getMetrics(consumer);
- var metricsByNode = processAndBuild(buildersByNode);
+ var metricsByNode = processAndBuild(buildersByNode,
+ new ServiceIdDimensionProcessor(),
+ new ClusterIdDimensionProcessor(),
+ new PublicDimensionsProcessor(MAX_DIMENSIONS));
return new JsonResponse(OK, toGenericApplicationModel(metricsByNode).serialize());
} catch (Exception e) {
@@ -69,8 +75,8 @@ public class ApplicationMetricsHandler extends HttpHandlerBase {
}
}
- private Map<Node, List<MetricsPacket>> processAndBuild(Map<Node, List<MetricsPacket.Builder>> buildersByNode,
- MetricsProcessor... processors) {
+ private static Map<Node, List<MetricsPacket>> processAndBuild(Map<Node, List<MetricsPacket.Builder>> buildersByNode,
+ MetricsProcessor... processors) {
var metricsByNode = new HashMap<Node, List<MetricsPacket>>();
buildersByNode.forEach((node, builders) -> {
@@ -84,14 +90,4 @@ public class ApplicationMetricsHandler extends HttpHandlerBase {
return metricsByNode;
}
- private MetricsPacket.Builder applyProcessors(MetricsPacket.Builder builder, MetricsProcessor... processors) {
- Arrays.stream(processors).forEach(processor -> processor.process(builder));
- return builder;
- }
-
- interface MetricsProcessor {
- // Processes the metrics packet builder in-place.
- void process(MetricsPacket.Builder builder);
- }
-
}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessor.java
new file mode 100644
index 00000000000..292c6da3de2
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessor.java
@@ -0,0 +1,39 @@
+package ai.vespa.metricsproxy.http.application;
+
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor;
+
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.CLUSTER_ID;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_CLUSTER_ID;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_CLUSTER_TYPE;
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+
+/**
+ * Replaces the current cluster ID dimension value with "clustertype/clusterid".
+ *
+ * @author gjoranv
+ */
+public class ClusterIdDimensionProcessor implements MetricsProcessor {
+
+ @Override
+ public void process(MetricsPacket.Builder builder) {
+ String clusterType = emptyIfNull(builder.getDimensionValue(toDimensionId(INTERNAL_CLUSTER_TYPE)));
+ String clusterId = emptyIfNull(builder.getDimensionValue(toDimensionId(INTERNAL_CLUSTER_ID)));
+
+ String newClusterId;
+ if (! clusterType.isEmpty() && ! clusterId.isEmpty())
+ newClusterId = clusterType + "/" + clusterId;
+ else if (! clusterType.isEmpty())
+ newClusterId = clusterType;
+ else if (! clusterId.isEmpty())
+ newClusterId = clusterId;
+ else
+ return; // Both type and id were null or empty
+
+ builder.putDimension(toDimensionId(CLUSTER_ID), newClusterId);
+ }
+
+ private String emptyIfNull(String s) {
+ return s == null ? "" : s;
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java
index c8a8e65be5d..c439a037774 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java
@@ -25,7 +25,7 @@ public class Node {
}
public Node(String role, String hostname, int port, String path) {
- Objects.requireNonNull(role, "Null configId is not allowed");
+ Objects.requireNonNull(role, "Null role is not allowed");
Objects.requireNonNull(hostname, "Null hostname is not allowed");
Objects.requireNonNull(path, "Null path is not allowed");
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java
new file mode 100644
index 00000000000..4d1d57644b5
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java
@@ -0,0 +1,69 @@
+package ai.vespa.metricsproxy.http.application;
+
+import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions;
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor;
+
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Ensures that only whitelisted dimensions are retained in the given metrics packet, and that
+ * there are no more dimensions than the given maximum number.
+ *
+ * @author gjoranv
+ */
+public class PublicDimensionsProcessor implements MetricsProcessor {
+
+ private final int maxDimensions;
+ private Set<DimensionId> publicDimensions = getPublicDimensions();
+
+ public PublicDimensionsProcessor(int maxDimensions) {
+ int numCommonDimensions = PublicDimensions.commonDimensions.size();
+ if (numCommonDimensions > maxDimensions) {
+ throw new IllegalArgumentException(String.format(
+ ("The maximum number of dimensions (%d) cannot be smaller than the number of " +
+ "common metrics dimensions (%d)."), maxDimensions, numCommonDimensions));
+ }
+ this.maxDimensions = maxDimensions;
+ }
+
+ @Override
+ public void process(MetricsPacket.Builder builder) {
+ Set<DimensionId> dimensionsToRetain = builder.getDimensionIds();
+ dimensionsToRetain.retainAll(publicDimensions);
+
+ if (dimensionsToRetain.size() > maxDimensions) {
+ for (var metricDim : getMetricDimensions()) {
+ dimensionsToRetain.remove(metricDim);
+ if (dimensionsToRetain.size() <= maxDimensions) break;
+ }
+ }
+
+ builder.retainDimensions(dimensionsToRetain);
+
+ // Extra safeguard, to make sure we don't exceed the limit of some metric systems.
+ if (builder.getDimensionIds().size() > maxDimensions) {
+ throw new IllegalStateException(String.format(
+ "Metrics packet is only allowed to have %d dimensions, but has: %s", maxDimensions, builder.getDimensionIds()));
+ }
+ }
+
+ static Set<DimensionId> getPublicDimensions() {
+ return toDimensionIds(PublicDimensions.publicDimensions);
+ }
+
+ static Set<DimensionId> getMetricDimensions() {
+ return toDimensionIds(PublicDimensions.metricDimensions);
+ }
+
+ static Set<DimensionId> toDimensionIds(Collection<String> dimensionNames) {
+ return dimensionNames.stream()
+ .map(DimensionId::toDimensionId)
+ .collect(Collectors.toCollection(LinkedHashSet::new));
+
+ }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessor.java
new file mode 100644
index 00000000000..45f211a5704
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessor.java
@@ -0,0 +1,24 @@
+package ai.vespa.metricsproxy.http.application;
+
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor;
+
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_SERVICE_ID;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.SERVICE_ID;
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+
+/**
+ * Copies the value of the internally used 'instance' dimension to the more aptly named 'serviceId'.
+ *
+ * @author gjoranv
+ */
+public class ServiceIdDimensionProcessor implements MetricsProcessor {
+
+ @Override
+ public void process(MetricsPacket.Builder builder) {
+ String serviceIdValue = builder.getDimensionValue(toDimensionId(INTERNAL_SERVICE_ID));
+ if (serviceIdValue != null)
+ builder.putDimension(toDimensionId(SERVICE_ID), serviceIdValue);
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java
index 28a24c5dc25..bf4d7b55989 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV1Handler.java
@@ -3,13 +3,13 @@ package ai.vespa.metricsproxy.http.metrics;
import ai.vespa.metricsproxy.core.MetricsConsumers;
import ai.vespa.metricsproxy.core.MetricsManager;
-import ai.vespa.metricsproxy.http.ErrorResponse;
-import ai.vespa.metricsproxy.http.HttpHandlerBase;
-import ai.vespa.metricsproxy.http.JsonResponse;
import ai.vespa.metricsproxy.http.ValuesFetcher;
import ai.vespa.metricsproxy.metric.model.MetricsPacket;
import ai.vespa.metricsproxy.service.VespaServices;
import com.google.inject.Inject;
+import com.yahoo.container.handler.metrics.ErrorResponse;
+import com.yahoo.container.handler.metrics.HttpHandlerBase;
+import com.yahoo.container.handler.metrics.JsonResponse;
import com.yahoo.container.jdisc.HttpResponse;
import com.yahoo.restapi.Path;
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java
new file mode 100644
index 00000000000..71d7857e48a
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java
@@ -0,0 +1,92 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.http.metrics;
+
+import ai.vespa.metricsproxy.core.MetricsConsumers;
+import ai.vespa.metricsproxy.core.MetricsManager;
+import ai.vespa.metricsproxy.http.ValuesFetcher;
+import ai.vespa.metricsproxy.http.application.ClusterIdDimensionProcessor;
+import ai.vespa.metricsproxy.http.application.Node;
+import ai.vespa.metricsproxy.http.application.PublicDimensionsProcessor;
+import ai.vespa.metricsproxy.http.application.ServiceIdDimensionProcessor;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor;
+import ai.vespa.metricsproxy.service.VespaServices;
+import com.google.inject.Inject;
+import com.yahoo.container.handler.metrics.ErrorResponse;
+import com.yahoo.container.handler.metrics.HttpHandlerBase;
+import com.yahoo.container.handler.metrics.JsonResponse;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.restapi.Path;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.Executor;
+import java.util.logging.Level;
+
+import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericApplicationModel;
+import static ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor.applyProcessors;
+import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR;
+import static com.yahoo.jdisc.Response.Status.OK;
+import static java.util.Collections.singletonMap;
+import static java.util.stream.Collectors.toList;
+
+/**
+ * Http handler for the metrics/v2 rest api.
+ *
+ * @author gjoranv
+ */
+public class MetricsV2Handler extends HttpHandlerBase {
+
+ public static final String V2_PATH = "/metrics/v2";
+ public static final String VALUES_PATH = V2_PATH + "/values";
+ private static final int MAX_DIMENSIONS = 10;
+
+ private final ValuesFetcher valuesFetcher;
+ private final NodeInfoConfig nodeInfoConfig;
+
+ @Inject
+ public MetricsV2Handler(Executor executor,
+ MetricsManager metricsManager,
+ VespaServices vespaServices,
+ MetricsConsumers metricsConsumers,
+ NodeInfoConfig nodeInfoConfig) {
+ super(executor);
+ this.nodeInfoConfig = nodeInfoConfig;
+ valuesFetcher = new ValuesFetcher(metricsManager, vespaServices, metricsConsumers);
+ }
+
+ @Override
+ public Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer) {
+ if (apiPath.matches(V2_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH)));
+ if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer));
+ return Optional.empty();
+ }
+
+ private JsonResponse valuesResponse(String consumer) {
+ try {
+ List<MetricsPacket.Builder> builders = valuesFetcher.fetchMetricsAsBuilders(consumer);
+ List<MetricsPacket> metrics = processAndBuild(builders,
+ new ServiceIdDimensionProcessor(),
+ new ClusterIdDimensionProcessor(),
+ new PublicDimensionsProcessor(MAX_DIMENSIONS));
+
+ Node localNode = new Node(nodeInfoConfig.role(), nodeInfoConfig.hostname(), 0, "");
+ Map<Node, List<MetricsPacket>> metricsByNode = singletonMap(localNode, metrics);
+ return new JsonResponse(OK, toGenericApplicationModel(metricsByNode).serialize());
+ } catch (Exception e) {
+ log.log(Level.WARNING, "Got exception when rendering metrics:", e);
+ return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage());
+ }
+ }
+
+ private static List<MetricsPacket> processAndBuild(List<MetricsPacket.Builder> builders,
+ MetricsProcessor... processors) {
+ return builders.stream()
+ .map(builder -> applyProcessors(builder, processors))
+ .map(MetricsPacket.Builder::build)
+ .collect(toList());
+ }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java
index 5f3723df94d..eeabb1e03ac 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java
@@ -3,12 +3,12 @@ package ai.vespa.metricsproxy.http.prometheus;
import ai.vespa.metricsproxy.core.MetricsConsumers;
import ai.vespa.metricsproxy.core.MetricsManager;
-import ai.vespa.metricsproxy.http.HttpHandlerBase;
import ai.vespa.metricsproxy.http.TextResponse;
import ai.vespa.metricsproxy.http.ValuesFetcher;
import ai.vespa.metricsproxy.metric.model.MetricsPacket;
import ai.vespa.metricsproxy.service.VespaServices;
import com.google.inject.Inject;
+import com.yahoo.container.handler.metrics.HttpHandlerBase;
import com.yahoo.container.jdisc.HttpResponse;
import com.yahoo.restapi.Path;
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java
index 38011d089d4..2b22ea101d8 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/yamas/YamasHandler.java
@@ -4,17 +4,17 @@ package ai.vespa.metricsproxy.http.yamas;
import ai.vespa.metricsproxy.core.MetricsConsumers;
import ai.vespa.metricsproxy.core.MetricsManager;
import ai.vespa.metricsproxy.http.ValuesFetcher;
-import ai.vespa.metricsproxy.node.NodeMetricGatherer;
-import ai.vespa.metricsproxy.http.ErrorResponse;
-import ai.vespa.metricsproxy.http.HttpHandlerBase;
-import ai.vespa.metricsproxy.http.JsonResponse;
import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions;
import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions;
import ai.vespa.metricsproxy.metric.model.MetricsPacket;
import ai.vespa.metricsproxy.metric.model.json.JsonRenderingException;
import ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil;
+import ai.vespa.metricsproxy.node.NodeMetricGatherer;
import ai.vespa.metricsproxy.service.VespaServices;
import com.google.inject.Inject;
+import com.yahoo.container.handler.metrics.ErrorResponse;
+import com.yahoo.container.handler.metrics.HttpHandlerBase;
+import com.yahoo.container.handler.metrics.JsonResponse;
import com.yahoo.container.jdisc.HttpResponse;
import com.yahoo.restapi.Path;
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java
new file mode 100644
index 00000000000..8d525310d6f
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java
@@ -0,0 +1,77 @@
+package ai.vespa.metricsproxy.metric.dimensions;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * The names of all dimensions that are publicly available, in addition to some dimensions that
+ * are used in the process of composing these public dimensions.
+ *
+ * 'INTERNAL' in this context means non-public.
+ *
+ * @author gjoranv
+ */
+public final class PublicDimensions {
+ private PublicDimensions() { }
+
+ public static final String APPLICATION_ID = "applicationId"; // <tenant.app.instance>
+ public static final String ZONE = "zone";
+
+ // The public CLUSTER_ID dimension value is composed from the two non-public dimensions.
+ // Node-specific.
+ public static final String INTERNAL_CLUSTER_TYPE = "clustertype";
+ public static final String INTERNAL_CLUSTER_ID = "clusterid";
+ public static final String CLUSTER_ID = "clusterId";
+
+ // Internal name (instance) is confusing, so renamed to 'serviceId' for public use.
+ // This is added by the metrics-proxy.
+ public static final String INTERNAL_SERVICE_ID = "instance";
+ public static final String SERVICE_ID = "serviceId";
+
+ // From host-admin, currently (Jan 2020) only included for 'vespa.node' metrics
+ public static final String HOSTNAME = "host";
+
+
+ /** Metric specific dimensions **/
+ public static final String API = "api"; // feed
+ public static final String CHAIN = "chain"; // query
+ public static final String DOCUMENT_TYPE = "documenttype"; // content
+ public static final String ENDPOINT = "endpoint"; // query
+ public static final String GC_NAME = "gcName"; // container
+ public static final String HTTP_METHOD = "httpMethod"; // container
+ public static final String OPERATION = "operation"; // feed
+ public static final String RANK_PROFILE = "rankProfile"; // content
+ public static final String REASON = "reason"; // query (degraded etc.)
+ public static final String STATUS = "status"; // feed
+
+
+ // Dimensions that are valid (but not necessarily used) for all metrics.
+ public static List<String> commonDimensions =
+ List.of(APPLICATION_ID,
+ CLUSTER_ID,
+ HOSTNAME,
+ SERVICE_ID,
+ ZONE);
+
+ // Dimensions that are only used for a subset of metrics.
+ public static List<String> metricDimensions =
+ List.of(API,
+ CHAIN,
+ DOCUMENT_TYPE,
+ ENDPOINT,
+ GC_NAME,
+ HTTP_METHOD,
+ OPERATION,
+ RANK_PROFILE,
+ REASON,
+ STATUS);
+
+
+ /**
+ * All public dimensions, common dimensions first, then dimensions for individual metrics
+ */
+ public static final List<String> publicDimensions = Stream.concat(commonDimensions.stream(), metricDimensions.stream())
+ .collect(Collectors.toList());
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java
index f9cc17f94ea..8d5a1f50918 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java
@@ -158,11 +158,31 @@ public class MetricsPacket {
return this;
}
+ /**
+ * Returns a modifiable copy of the dimension IDs of this builder, usually for use with {@link #retainDimensions(Collection)}.
+ */
+ public Set<DimensionId> getDimensionIds() {
+ return new LinkedHashSet<>(dimensions.keySet());
+ }
+
+ public String getDimensionValue(DimensionId id) {
+ return dimensions.get(id);
+ }
+
+ public Builder retainDimensions(Collection<DimensionId> idsToRetain) {
+ dimensions.keySet().retainAll(idsToRetain);
+ return this;
+ }
+
public Builder addConsumers(Set<ConsumerId> extraConsumers) {
if (extraConsumers != null) consumers.addAll(extraConsumers);
return this;
}
+ public boolean hasConsumer(ConsumerId id) {
+ return consumers.contains(id);
+ }
+
public MetricsPacket build() {
return new MetricsPacket(statusCode, statusMessage, timestamp, service, metrics, dimensions, consumers);
}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/processing/MetricsProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/processing/MetricsProcessor.java
new file mode 100644
index 00000000000..7860825e075
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/processing/MetricsProcessor.java
@@ -0,0 +1,31 @@
+package ai.vespa.metricsproxy.metric.model.processing;
+
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+
+import java.util.Arrays;
+
+/**
+ * Interface for classes that make amendments to a metrics packet builder.
+ * Includes a utility method to apply a list of processors to a metrics packet.
+ *
+ * @author gjoranv
+ */
+public interface MetricsProcessor {
+
+ /**
+ * Processes the metrics packet builder in-place.
+ */
+ void process(MetricsPacket.Builder builder);
+
+
+ /**
+ * Helper method to apply a list of processors to a metrics packet builder.
+ * Returns the metrics packet builder (which has been processed in-place) for
+ * convenient use in stream processing.
+ */
+ static MetricsPacket.Builder applyProcessors(MetricsPacket.Builder builder, MetricsProcessor... processors) {
+ Arrays.stream(processors).forEach(processor -> processor.process(builder));
+ return builder;
+ }
+
+}
diff --git a/metrics-proxy/src/main/resources/configdefinitions/node-info.def b/metrics-proxy/src/main/resources/configdefinitions/node-info.def
new file mode 100644
index 00000000000..e66433a96d0
--- /dev/null
+++ b/metrics-proxy/src/main/resources/configdefinitions/node-info.def
@@ -0,0 +1,5 @@
+# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package=ai.vespa.metricsproxy.http.metrics
+
+role string
+hostname string
diff --git a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def
new file mode 100644
index 00000000000..f3b5db35d52
--- /dev/null
+++ b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def
@@ -0,0 +1,20 @@
+# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package=ai.vespa.metricsproxy.telegraf
+
+# Metrics pull/push interval
+intervalSeconds int default=60
+
+
+# The consumer to get metrics for
+vespa.consumer string default="default"
+
+
+cloudWatch[].region string default="us-east-1"
+cloudWatch[].namespace string
+
+# Only valid and required for hosted Vespa
+cloudWatch[].accessKeyName string default=""
+cloudWatch[].secretKeyName string default=""
+
+# Only valid and optional for self-hosted Vespa
+cloudWatch[].profile string default=""
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/ErrorResponseTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/ErrorResponseTest.java
deleted file mode 100644
index ccf04560906..00000000000
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/ErrorResponseTest.java
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package ai.vespa.metricsproxy.http;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author gjoranv
- */
-public class ErrorResponseTest {
-
- @Test
- public void error_message_is_wrapped_in_json_object() {
- var json = ErrorResponse.asErrorJson("bad");
- assertEquals("{\"error\":\"bad\"}", json);
- }
-
- @Test
- public void quotes_are_escaped() {
- var json = ErrorResponse.asErrorJson("Message \" with \" embedded quotes.");
- assertEquals("{\"error\":\"Message \\\" with \\\" embedded quotes.\"}", json);
- }
-
-}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java
index 77c3a719cd9..d776368687d 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java
@@ -24,6 +24,7 @@ import java.util.List;
import static ai.vespa.metricsproxy.http.ValuesFetcher.DEFAULT_PUBLIC_CONSUMER_ID;
import static ai.vespa.metricsproxy.metric.ExternalMetrics.VESPA_NODE_SERVICE_ID;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON;
import static ai.vespa.metricsproxy.service.DummyService.METRIC_1;
/**
@@ -61,11 +62,12 @@ public class HttpHandlerTestBase {
}
protected static MetricsConsumers getMetricsConsumers() {
+ // Must use a whitelisted dimension to avoid it being removed for the MetricsV2Handler
var defaultConsumerDimension = new ConsumersConfig.Consumer.Metric.Dimension.Builder()
- .key("consumer-dim").value("default-val");
+ .key(REASON).value("default-val");
var customConsumerDimension = new ConsumersConfig.Consumer.Metric.Dimension.Builder()
- .key("consumer-dim").value("custom-val");
+ .key(REASON).value("custom-val");
return new MetricsConsumers(new ConsumersConfig.Builder()
.consumer(new ConsumersConfig.Consumer.Builder()
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java
index 074b7877430..155bbf094a1 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java
@@ -3,6 +3,7 @@ package ai.vespa.metricsproxy.http.application;
import ai.vespa.metricsproxy.core.ConsumersConfig;
import ai.vespa.metricsproxy.core.MetricsConsumers;
+import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions;
import ai.vespa.metricsproxy.metric.model.json.GenericApplicationModel;
import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel;
import ai.vespa.metricsproxy.metric.model.json.GenericMetrics;
@@ -19,6 +20,7 @@ import org.junit.Test;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Map;
import java.util.concurrent.Executors;
import static ai.vespa.metricsproxy.TestUtil.getFileContents;
@@ -34,6 +36,7 @@ import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options
import static com.yahoo.collections.CollectionUtil.first;
import static java.util.stream.Collectors.toList;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -138,6 +141,22 @@ public class ApplicationMetricsHandlerTest {
}
@Test
+ public void metrics_processors_are_applied() {
+ GenericApplicationModel jsonModel = getResponseAsJsonModel(DEFAULT_PUBLIC_CONSUMER_ID.id);
+
+ GenericService searchnode = jsonModel.nodes.get(0).services.get(0);
+ Map<String, String> dimensions = searchnode.metrics.get(0).dimensions;
+ assertEquals(6, dimensions.size());
+ assertEquals("music.default", dimensions.get(PublicDimensions.APPLICATION_ID));
+ assertEquals("container/default", dimensions.get(PublicDimensions.CLUSTER_ID));
+ assertEquals("us-west", dimensions.get(PublicDimensions.ZONE));
+ assertEquals("search/", dimensions.get(PublicDimensions.API));
+ assertEquals("music", dimensions.get(PublicDimensions.DOCUMENT_TYPE));
+ assertEquals("qrserver0", dimensions.get(PublicDimensions.SERVICE_ID));
+ assertFalse(dimensions.containsKey("non-public"));
+ }
+
+ @Test
public void consumer_is_propagated_in_uri_to_retriever() {
GenericApplicationModel jsonModel = getResponseAsJsonModel(CUSTOM_CONSUMER);
GenericJsonModel nodeModel = jsonModel.nodes.get(0);
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessorTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessorTest.java
new file mode 100644
index 00000000000..0f4c6a885ec
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ClusterIdDimensionProcessorTest.java
@@ -0,0 +1,63 @@
+package ai.vespa.metricsproxy.http.application;
+
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import org.junit.Test;
+
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.CLUSTER_ID;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_CLUSTER_ID;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_CLUSTER_TYPE;
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author gjoranv
+ */
+public class ClusterIdDimensionProcessorTest {
+ private static final DimensionId NEW_ID_DIMENSION = toDimensionId(CLUSTER_ID);
+
+ @Test
+ public void cluster_id_is_replaced_with_type_and_id() {
+ var builder = new MetricsPacket.Builder(toServiceId("foo"));
+ builder.putDimension(toDimensionId(INTERNAL_CLUSTER_TYPE), "type") ;
+ builder.putDimension(toDimensionId(INTERNAL_CLUSTER_ID), "id") ;
+
+ assertEquals("type/id", newClusterId(builder));
+ }
+
+ @Test
+ public void cluster_id_is_type_when_id_is_null() {
+ var builder = new MetricsPacket.Builder(toServiceId("foo"));
+ builder.putDimension(toDimensionId(INTERNAL_CLUSTER_TYPE), "type") ;
+
+ assertEquals(newClusterId(builder), "type");
+ }
+
+ @Test
+ public void cluster_id_is_id_when_type_is_null() {
+ var builder = new MetricsPacket.Builder(toServiceId("foo"));
+ builder.putDimension(toDimensionId(INTERNAL_CLUSTER_ID), "id") ;
+
+ assertEquals(newClusterId(builder), "id");
+ }
+
+ @Test
+ public void cluster_id_is_not_added_when_both_type_and_id_are_null() {
+ var builder = new MetricsPacket.Builder(toServiceId("foo"));
+
+ var processor = new ClusterIdDimensionProcessor();
+ processor.process(builder);
+
+ assertFalse(builder.getDimensionIds().contains(NEW_ID_DIMENSION));
+ }
+
+ private String newClusterId(MetricsPacket.Builder builder) {
+ var processor = new ClusterIdDimensionProcessor();
+ processor.process(builder);
+
+ return builder.getDimensionValue(NEW_ID_DIMENSION);
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessorTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessorTest.java
new file mode 100644
index 00000000000..c6f47d70a1f
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessorTest.java
@@ -0,0 +1,59 @@
+package ai.vespa.metricsproxy.http.application;
+
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import org.junit.Test;
+
+import static ai.vespa.metricsproxy.http.application.PublicDimensionsProcessor.getPublicDimensions;
+import static ai.vespa.metricsproxy.http.application.PublicDimensionsProcessor.toDimensionIds;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.APPLICATION_ID;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.commonDimensions;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.publicDimensions;
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author gjoranv
+ */
+public class PublicDimensionsProcessorTest {
+
+ @Test
+ public void non_public_dimensions_are_removed() {
+ var builder = new MetricsPacket.Builder(toServiceId("foo"))
+ .putDimension(toDimensionId("a"), "");
+
+ var processor = new PublicDimensionsProcessor(10);
+ processor.process(builder);
+ assertTrue(builder.getDimensionIds().isEmpty());
+ }
+
+ @Test
+ public void public_dimensions_are_retained() {
+ var builder = new MetricsPacket.Builder(toServiceId("foo"))
+ .putDimension(toDimensionId(APPLICATION_ID), "app");
+
+ var processor = new PublicDimensionsProcessor(10);
+ processor.process(builder);
+ assertEquals(1, builder.getDimensionIds().size());
+ assertEquals(toDimensionId(APPLICATION_ID), builder.getDimensionIds().iterator().next());
+ }
+
+ @Test
+ public void common_dimensions_have_priority_when_there_are_too_many() {
+ var builder = new MetricsPacket.Builder(toServiceId("foo"));
+ getPublicDimensions()
+ .forEach(dimensionId -> builder.putDimension(dimensionId, dimensionId.id));
+ assertEquals(publicDimensions.size(), builder.getDimensionIds().size());
+
+ var processor = new PublicDimensionsProcessor(commonDimensions.size());
+ processor.process(builder);
+
+ var includedDimensions = builder.getDimensionIds();
+ assertEquals(commonDimensions.size(), includedDimensions.size());
+
+ toDimensionIds(commonDimensions).forEach(commonDimension ->
+ assertTrue(includedDimensions.contains(commonDimension)));
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessorTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessorTest.java
new file mode 100644
index 00000000000..3237abdbfa3
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ServiceIdDimensionProcessorTest.java
@@ -0,0 +1,43 @@
+package ai.vespa.metricsproxy.http.application;
+
+import ai.vespa.metricsproxy.metric.model.DimensionId;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import org.junit.Test;
+
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_SERVICE_ID;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.SERVICE_ID;
+import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId;
+import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author gjoranv
+ */
+public class ServiceIdDimensionProcessorTest {
+ private static final DimensionId NEW_ID_DIMENSION = toDimensionId(SERVICE_ID);
+
+ @Test
+ public void new_service_id_is_added_when_internal_service_id_exists() {
+ var builder = new MetricsPacket.Builder(toServiceId("foo"));
+ builder.putDimension(toDimensionId(INTERNAL_SERVICE_ID), "service");
+
+ var processor = new ServiceIdDimensionProcessor();
+ processor.process(builder);
+
+ assertTrue(builder.getDimensionIds().contains(NEW_ID_DIMENSION));
+ assertEquals("service", builder.getDimensionValue(NEW_ID_DIMENSION));
+ }
+
+ @Test
+ public void new_service_id_is_not_added_when_internal_service_id_is_null() {
+ var builder = new MetricsPacket.Builder(toServiceId("foo"));
+
+ var processor = new ServiceIdDimensionProcessor();
+ processor.process(builder);
+
+ assertFalse(builder.getDimensionIds().contains(NEW_ID_DIMENSION));
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java
new file mode 100644
index 00000000000..1c5ce695155
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java
@@ -0,0 +1,196 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.http.metrics;
+
+import ai.vespa.metricsproxy.http.HttpHandlerTestBase;
+import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel;
+import ai.vespa.metricsproxy.metric.model.json.GenericMetrics;
+import ai.vespa.metricsproxy.metric.model.json.GenericService;
+import ai.vespa.metricsproxy.service.DownService;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_SERVICE_ID;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.SERVICE_ID;
+import static ai.vespa.metricsproxy.metric.model.StatusCode.DOWN;
+import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper;
+import static ai.vespa.metricsproxy.service.DummyService.METRIC_1;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author gjoranv
+ */
+public abstract class MetricsHandlerTestBase<MODEL> extends HttpHandlerTestBase {
+
+ static String rootUri;
+ static String valuesUri;
+
+ Class<MODEL> modelClass;
+
+ abstract GenericJsonModel getGenericJsonModel(MODEL model);
+
+ private MODEL getResponseAsJsonModel(String consumer) {
+ String response = testDriver.sendRequest(valuesUri + "?consumer=" + consumer).readAll();
+ try {
+ return createObjectMapper().readValue(response, modelClass);
+ } catch (IOException e) {
+ fail("Failed to create json model: " + e.getMessage());
+ throw new RuntimeException(e);
+ }
+ }
+
+ private GenericJsonModel getResponseAsGenericJsonModel(String consumer) {
+ return getGenericJsonModel(getResponseAsJsonModel(consumer));
+ }
+
+ @Test
+ public void invalid_path_yields_error_response() throws Exception {
+ String response = testDriver.sendRequest(rootUri + "/invalid").readAll();
+ JSONObject root = new JSONObject(response);
+ assertTrue(root.has("error"));
+ }
+
+ @Test
+ public void root_response_contains_values_uri() throws Exception {
+ String response = testDriver.sendRequest(rootUri).readAll();
+ JSONObject root = new JSONObject(response);
+ assertTrue(root.has("resources"));
+
+ JSONArray resources = root.getJSONArray("resources");
+ assertEquals(1, resources.length());
+
+ JSONObject valuesUrl = resources.getJSONObject(0);
+ assertEquals(valuesUri, valuesUrl.getString("url"));
+ }
+
+ @Ignore
+ @Test
+ public void visually_inspect_values_response() throws Exception {
+ String response = testDriver.sendRequest(valuesUri).readAll();
+ ObjectMapper mapper = createObjectMapper();
+ var jsonModel = mapper.readValue(response, modelClass);
+ System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonModel));
+ }
+
+ @Test
+ public void no_explicit_consumer_gives_the_default_consumer() {
+ String responseDefaultConsumer = testDriver.sendRequest(valuesUri + "?consumer=default").readAll();
+ String responseNoConsumer = testDriver.sendRequest(valuesUri).readAll();
+ assertEqualsExceptTimestamps(responseDefaultConsumer, responseNoConsumer);
+ }
+
+ @Test
+ public void unknown_consumer_gives_the_default_consumer() {
+ String response = testDriver.sendRequest(valuesUri).readAll();
+ String responseUnknownConsumer = testDriver.sendRequest(valuesUri + "?consumer=not_defined").readAll();
+ assertEqualsExceptTimestamps(response, responseUnknownConsumer);
+ }
+
+ private void assertEqualsExceptTimestamps(String s1, String s2) {
+ assertEquals(replaceTimestamps(s1), replaceTimestamps(s2));
+ }
+
+ private String replaceTimestamps(String s) {
+ return s.replaceAll("timestamp\":\\d+,", "timestamp\":1,");
+ }
+
+ @Test
+ public void response_contains_node_metrics() {
+ GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER);
+
+ assertNotNull(jsonModel.node);
+ assertEquals(1, jsonModel.node.metrics.size());
+ assertEquals(12.345, jsonModel.node.metrics.get(0).values.get(CPU_METRIC), 0.0001d);
+ }
+
+ @Test
+ public void response_contains_service_metrics() {
+ GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER);
+
+ assertEquals(2, jsonModel.services.size());
+ GenericService dummyService = jsonModel.services.get(0);
+ assertEquals(2, dummyService.metrics.size());
+
+ GenericMetrics dummy0Metrics = getMetricsForService("dummy0", dummyService);
+ assertEquals(1L, dummy0Metrics.values.get(METRIC_1).longValue());
+ assertEquals("default-val", dummy0Metrics.dimensions.get(REASON));
+
+ GenericMetrics dummy1Metrics = getMetricsForService("dummy1", dummyService);
+ assertEquals(6L, dummy1Metrics.values.get(METRIC_1).longValue());
+ assertEquals("default-val", dummy1Metrics.dimensions.get(REASON));
+ }
+
+ @Test
+ public void custom_consumer_gets_only_its_whitelisted_metrics() {
+ GenericJsonModel jsonModel = getResponseAsGenericJsonModel(CUSTOM_CONSUMER);
+
+ assertNotNull(jsonModel.node);
+ // TODO: see comment in ExternalMetrics.setExtraMetrics
+ // assertEquals(0, jsonModel.node.metrics.size());
+
+ assertEquals(2, jsonModel.services.size());
+ GenericService dummyService = jsonModel.services.get(0);
+ assertEquals(2, dummyService.metrics.size());
+
+ GenericMetrics dummy0Metrics = getMetricsForService("dummy0", dummyService);
+ assertEquals("custom-val", dummy0Metrics.dimensions.get(REASON));
+
+ GenericMetrics dummy1Metrics = getMetricsForService("dummy1", dummyService);
+ assertEquals("custom-val", dummy1Metrics.dimensions.get(REASON));
+ }
+
+ private static GenericMetrics getMetricsForService(String serviceInstance, GenericService service) {
+ for (var metrics : service.metrics) {
+ if (getServiceIdDimension(metrics).equals(serviceInstance))
+ return metrics;
+ }
+ fail("Could not find metrics for service instance " + serviceInstance);
+ throw new RuntimeException();
+ }
+
+ @Test
+ public void all_timestamps_are_equal_and_non_zero() {
+ GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER);
+
+ Long nodeTimestamp = jsonModel.node.timestamp;
+ assertNotEquals(0L, (long) nodeTimestamp);
+ for (var service : jsonModel.services)
+ assertEquals(nodeTimestamp, service.timestamp);
+ }
+
+ @Test
+ public void all_consumers_get_health_from_service_that_is_down() {
+ assertDownServiceHealth(DEFAULT_CONSUMER);
+ assertDownServiceHealth(CUSTOM_CONSUMER);
+ }
+
+ private void assertDownServiceHealth(String consumer) {
+ GenericJsonModel jsonModel = getResponseAsGenericJsonModel(consumer);
+
+ GenericService downService = jsonModel.services.get(1);
+ assertEquals(DOWN.status, downService.status.code);
+ assertEquals("No response", downService.status.description);
+
+ // Service should output metric dimensions, even without metrics, because they contain important info about the service.
+ assertEquals(1, downService.metrics.size());
+ assertEquals(0, downService.metrics.get(0).values.size());
+ assertFalse(downService.metrics.get(0).dimensions.isEmpty());
+ assertEquals(DownService.NAME, getServiceIdDimension(downService.metrics.get(0)));
+ }
+
+ private static String getServiceIdDimension(GenericMetrics metrics) {
+ var instanceDimension = metrics.dimensions.get(INTERNAL_SERVICE_ID);
+ return instanceDimension != null ? instanceDimension : metrics.dimensions.get(SERVICE_ID);
+ }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java
index 22f61114622..fe823466f7b 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java
@@ -1,46 +1,30 @@
// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package ai.vespa.metricsproxy.http.metrics;
-import ai.vespa.metricsproxy.http.HttpHandlerTestBase;
import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel;
-import ai.vespa.metricsproxy.metric.model.json.GenericMetrics;
-import ai.vespa.metricsproxy.metric.model.json.GenericService;
-import ai.vespa.metricsproxy.service.DownService;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.yahoo.container.jdisc.RequestHandlerTestDriver;
-import org.json.JSONArray;
-import org.json.JSONObject;
+import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import java.io.IOException;
import java.util.concurrent.Executors;
-import static ai.vespa.metricsproxy.core.VespaMetrics.INSTANCE_DIMENSION_ID;
import static ai.vespa.metricsproxy.http.metrics.MetricsV1Handler.V1_PATH;
import static ai.vespa.metricsproxy.http.metrics.MetricsV1Handler.VALUES_PATH;
-import static ai.vespa.metricsproxy.metric.model.StatusCode.DOWN;
-import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper;
-import static ai.vespa.metricsproxy.service.DummyService.METRIC_1;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
/**
* @author gjoranv
*/
@SuppressWarnings("UnstableApiUsage")
-public class MetricsV1HandlerTest extends HttpHandlerTestBase {
+public class MetricsV1HandlerTest extends MetricsHandlerTestBase<GenericJsonModel> {
private static final String V1_URI = URI_BASE + V1_PATH;
private static final String VALUES_URI = URI_BASE + VALUES_PATH;
+
@BeforeClass
public static void setup() {
+ rootUri = V1_URI;
+ valuesUri = VALUES_URI;
var handler = new MetricsV1Handler(Executors.newSingleThreadExecutor(),
getMetricsManager(),
vespaServices,
@@ -48,149 +32,14 @@ public class MetricsV1HandlerTest extends HttpHandlerTestBase {
testDriver = new RequestHandlerTestDriver(handler);
}
- private GenericJsonModel getResponseAsJsonModel(String consumer) {
- String response = testDriver.sendRequest(VALUES_URI + "?consumer=" + consumer).readAll();
- try {
- return createObjectMapper().readValue(response, GenericJsonModel.class);
- } catch (IOException e) {
- fail("Failed to create json model: " + e.getMessage());
- throw new RuntimeException(e);
- }
- }
-
- @Test
- public void v1_response_contains_values_uri() throws Exception {
- String response = testDriver.sendRequest(V1_URI).readAll();
- JSONObject root = new JSONObject(response);
- assertTrue(root.has("resources"));
-
- JSONArray resources = root.getJSONArray("resources");
- assertEquals(1, resources.length());
-
- JSONObject valuesUrl = resources.getJSONObject(0);
- assertEquals(VALUES_URI, valuesUrl.getString("url"));
- }
-
- @Ignore
- @Test
- public void visually_inspect_values_response() throws Exception {
- String response = testDriver.sendRequest(VALUES_URI).readAll();
- ObjectMapper mapper = createObjectMapper();
- var jsonModel = mapper.readValue(response, GenericJsonModel.class);
- System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonModel));
- }
-
- @Test
- public void no_explicit_consumer_gives_the_default_consumer() {
- String responseDefaultConsumer = testDriver.sendRequest(VALUES_URI + "?consumer=default").readAll();
- String responseNoConsumer = testDriver.sendRequest(VALUES_URI).readAll();
- assertEqualsExceptTimestamps(responseDefaultConsumer, responseNoConsumer);
- }
-
- @Test
- public void unknown_consumer_gives_the_default_consumer() {
- String response = testDriver.sendRequest(VALUES_URI).readAll();
- String responseUnknownConsumer = testDriver.sendRequest(VALUES_URI + "?consumer=not_defined").readAll();
- assertEqualsExceptTimestamps(response, responseUnknownConsumer);
- }
-
- private void assertEqualsExceptTimestamps(String s1, String s2) {
- assertEquals(replaceTimestamps(s1), replaceTimestamps(s2));
- }
-
- @Test
- public void response_contains_node_metrics() {
- GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER);
-
- assertNotNull(jsonModel.node);
- assertEquals(1, jsonModel.node.metrics.size());
- assertEquals(12.345, jsonModel.node.metrics.get(0).values.get(CPU_METRIC), 0.0001d);
- }
-
- @Test
- public void response_contains_service_metrics() {
- GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER);
-
- assertEquals(2, jsonModel.services.size());
- GenericService dummyService = jsonModel.services.get(0);
- assertEquals(2, dummyService.metrics.size());
-
- GenericMetrics dummy0Metrics = getMetricsForInstance("dummy0", dummyService);
- assertEquals(1L, dummy0Metrics.values.get(METRIC_1).longValue());
- assertEquals("default-val", dummy0Metrics.dimensions.get("consumer-dim"));
-
- GenericMetrics dummy1Metrics = getMetricsForInstance("dummy1", dummyService);
- assertEquals(6L, dummy1Metrics.values.get(METRIC_1).longValue());
- assertEquals("default-val", dummy1Metrics.dimensions.get("consumer-dim"));
- }
-
- @Test
- public void all_consumers_get_health_from_service_that_is_down() {
- assertDownServiceHealth(DEFAULT_CONSUMER);
- assertDownServiceHealth(CUSTOM_CONSUMER);
- }
-
- @Test
- public void all_timestamps_are_equal_and_non_zero() {
- GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER);
-
- Long nodeTimestamp = jsonModel.node.timestamp;
- assertNotEquals(0L, (long) nodeTimestamp);
- for (var service : jsonModel.services)
- assertEquals(nodeTimestamp, service.timestamp);
- }
-
- @Test
- public void custom_consumer_gets_only_its_whitelisted_metrics() {
- GenericJsonModel jsonModel = getResponseAsJsonModel(CUSTOM_CONSUMER);
-
- assertNotNull(jsonModel.node);
- // TODO: see comment in ExternalMetrics.setExtraMetrics
- // assertEquals(0, jsonModel.node.metrics.size());
-
- assertEquals(2, jsonModel.services.size());
- GenericService dummyService = jsonModel.services.get(0);
- assertEquals(2, dummyService.metrics.size());
-
- GenericMetrics dummy0Metrics = getMetricsForInstance("dummy0", dummyService);
- assertEquals("custom-val", dummy0Metrics.dimensions.get("consumer-dim"));
-
- GenericMetrics dummy1Metrics = getMetricsForInstance("dummy1", dummyService);
- assertEquals("custom-val", dummy1Metrics.dimensions.get("consumer-dim"));
- }
-
- @Test
- public void invalid_path_yields_error_response() throws Exception {
- String response = testDriver.sendRequest(V1_URI + "/invalid").readAll();
- JSONObject root = new JSONObject(response);
- assertTrue(root.has("error"));
- }
-
- private void assertDownServiceHealth(String consumer) {
- GenericJsonModel jsonModel = getResponseAsJsonModel(consumer);
-
- GenericService downService = jsonModel.services.get(1);
- assertEquals(DOWN.status, downService.status.code);
- assertEquals("No response", downService.status.description);
-
- // Service should output metric dimensions, even without metrics, because they contain important info about the service.
- assertEquals(1, downService.metrics.size());
- assertEquals(0, downService.metrics.get(0).values.size());
- assertFalse(downService.metrics.get(0).dimensions.isEmpty());
- assertEquals(DownService.NAME, downService.metrics.get(0).dimensions.get(INSTANCE_DIMENSION_ID.id));
- }
-
- private String replaceTimestamps(String s) {
- return s.replaceAll("timestamp\":\\d+,", "timestamp\":1,");
+ @Before
+ public void initModelClass() {
+ modelClass = GenericJsonModel.class;
}
- private static GenericMetrics getMetricsForInstance(String instance, GenericService service) {
- for (var metrics : service.metrics) {
- if (metrics.dimensions.get(INSTANCE_DIMENSION_ID.id).equals(instance))
- return metrics;
- }
- fail("Could not find metrics for service instance " + instance);
- throw new RuntimeException();
+ @Override
+ GenericJsonModel getGenericJsonModel(GenericJsonModel genericJsonModel) {
+ return genericJsonModel;
}
}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java
new file mode 100644
index 00000000000..27ee6be4be3
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java
@@ -0,0 +1,53 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.http.metrics;
+
+import ai.vespa.metricsproxy.metric.model.json.GenericApplicationModel;
+import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel;
+import com.yahoo.container.jdisc.RequestHandlerTestDriver;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.util.concurrent.Executors;
+
+import static ai.vespa.metricsproxy.http.metrics.MetricsV2Handler.V2_PATH;
+import static ai.vespa.metricsproxy.http.metrics.MetricsV2Handler.VALUES_PATH;
+
+/**
+ * @author gjoranv
+ */
+@SuppressWarnings("UnstableApiUsage")
+public class MetricsV2HandlerTest extends MetricsHandlerTestBase<GenericApplicationModel> {
+
+ private static final String V2_URI = URI_BASE + V2_PATH;
+ private static final String VALUES_URI = URI_BASE + VALUES_PATH;
+
+
+ @BeforeClass
+ public static void setup() {
+ rootUri = V2_URI;
+ valuesUri = VALUES_URI;
+ var handler = new MetricsV2Handler(Executors.newSingleThreadExecutor(),
+ getMetricsManager(),
+ vespaServices,
+ getMetricsConsumers(),
+ nodeInfoConfig());
+ testDriver = new RequestHandlerTestDriver(handler);
+ }
+
+ @Before
+ public void initModelClass() {
+ modelClass = GenericApplicationModel.class;
+ }
+
+ @Override
+ GenericJsonModel getGenericJsonModel(GenericApplicationModel genericApplicationModel) {
+ return genericApplicationModel.nodes.get(0);
+ }
+
+ private static NodeInfoConfig nodeInfoConfig() {
+ return new NodeInfoConfig.Builder()
+ .role("my-role")
+ .hostname("my-hostname")
+ .build();
+ }
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java
index a85f0425b4b..a224c4090b3 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java
@@ -12,6 +12,7 @@ import org.junit.Test;
import java.util.concurrent.Executors;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -83,7 +84,7 @@ public class PrometheusHandlerTest extends HttpHandlerTestBase {
@Test
public void service_metrics_have_configured_dimensions() {
String dummy0 = getLine(valuesResponse, DummyService.NAME + "0");
- assertTrue(dummy0.contains("consumer_dim=\"default-val\""));
+ assertTrue(dummy0.contains(REASON + "=\"default-val\""));
}
@Test
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java
index 9b37a805245..78c80689299 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java
@@ -14,6 +14,7 @@ import java.util.Map;
import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId;
import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId;
import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static java.util.Collections.singleton;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -45,13 +46,23 @@ public class MetricsPacketTest {
MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo"))
.statusCode(0)
.statusMessage("")
- .addConsumers(Collections.singleton(DUPLICATE_CONSUMER))
- .addConsumers(Collections.singleton(DUPLICATE_CONSUMER))
+ .addConsumers(singleton(DUPLICATE_CONSUMER))
+ .addConsumers(singleton(DUPLICATE_CONSUMER))
.build();
assertEquals(1, packet.consumers().size());
}
@Test
+ public void builder_allows_inspecting_consumers() {
+ var consumer = toConsumerId("my-consumer");
+ var builder = new MetricsPacket.Builder(toServiceId("foo"))
+ .statusCode(0)
+ .statusMessage("")
+ .addConsumers(singleton(consumer));
+ assertTrue(builder.hasConsumer(consumer));
+ }
+
+ @Test
public void builder_can_retain_subset_of_metrics() {
MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo"))
.putMetrics(ImmutableList.of(
diff --git a/metrics-proxy/src/test/resources/generic-sample.json b/metrics-proxy/src/test/resources/generic-sample.json
index de617895f86..c9b02696e69 100644
--- a/metrics-proxy/src/test/resources/generic-sample.json
+++ b/metrics-proxy/src/test/resources/generic-sample.json
@@ -33,7 +33,14 @@
"queries.count": 4
},
"dimensions": {
- "documentType": "music"
+ "applicationId": "music.default",
+ "clustertype": "container",
+ "clusterid": "default",
+ "instance": "qrserver0",
+ "zone": "us-west",
+ "api": "search/",
+ "documenttype": "music",
+ "non-public": "will-be-removed"
}
}
]