From 8b0272c3104080d1f293e6a709208d2ea149fc03 Mon Sep 17 00:00:00 2001 From: gjoranv Date: Tue, 7 May 2019 15:44:37 +0200 Subject: Revert "Gjoranv/New metrics proxy" --- config-model/pom.xml | 6 - .../com/yahoo/searchdefinition/FeatureNames.java | 5 + .../java/com/yahoo/vespa/model/admin/Admin.java | 2 +- .../metricsproxy/ConsumersConfigGenerator.java | 88 ------ .../admin/metricsproxy/MetricsProxyContainer.java | 97 +------ .../metricsproxy/MetricsProxyContainerCluster.java | 162 +---------- .../metricsproxy/VespaServicesConfigGenerator.java | 46 --- .../com/yahoo/vespa/model/container/Container.java | 13 +- .../vespa/model/container/ContainerCluster.java | 2 +- .../vespa/model/container/xml/BundleMapper.java | 17 -- .../MetricsProxyContainerClusterTest.java | 193 ++----------- .../metricsproxy/MetricsProxyContainerTest.java | 146 ---------- .../metricsproxy/MetricsProxyModelTester.java | 103 ------- metrics-proxy/OWNERS | 1 - metrics-proxy/README | 1 - metrics-proxy/pom.xml | 157 ----------- .../vespa/metricsproxy/core/MetricsConsumers.java | 80 ------ .../ai/vespa/metricsproxy/core/MetricsManager.java | 160 ----------- .../ai/vespa/metricsproxy/core/VespaMetrics.java | 307 --------------------- .../ai/vespa/metricsproxy/core/package-info.java | 8 - .../vespa/metricsproxy/metric/AggregationKey.java | 45 --- .../vespa/metricsproxy/metric/ExternalMetrics.java | 99 ------- .../ai/vespa/metricsproxy/metric/HealthMetric.java | 56 ---- .../java/ai/vespa/metricsproxy/metric/Metric.java | 124 --------- .../java/ai/vespa/metricsproxy/metric/Metrics.java | 111 -------- .../metricsproxy/metric/MetricsFormatter.java | 71 ----- .../metric/dimensions/ApplicationDimensions.java | 30 -- .../metric/dimensions/NodeDimensions.java | 30 -- .../metric/dimensions/package-info.java | 8 - .../metricsproxy/metric/model/ConsumerId.java | 38 --- .../metricsproxy/metric/model/DimensionId.java | 38 --- .../vespa/metricsproxy/metric/model/MetricId.java | 39 --- .../metricsproxy/metric/model/MetricsPacket.java | 182 ------------ .../vespa/metricsproxy/metric/model/ServiceId.java | 39 --- .../metricsproxy/metric/model/json/JsonUtil.java | 130 --------- .../metric/model/json/YamasArrayJsonModel.java | 80 ------ .../metric/model/json/YamasJsonModel.java | 130 --------- .../ai/vespa/metricsproxy/metric/package-info.java | 8 - .../ai/vespa/metricsproxy/rpc/RpcConnector.java | 62 ----- .../java/ai/vespa/metricsproxy/rpc/RpcServer.java | 213 -------------- .../metricsproxy/service/ConfigSentinelClient.java | 176 ------------ .../ai/vespa/metricsproxy/service/CpuJiffies.java | 43 --- .../service/DummyHealthMetricFetcher.java | 34 --- .../metricsproxy/service/DummyMetricsFetcher.java | 30 -- .../metricsproxy/service/HttpMetricFetcher.java | 94 ------- .../service/RemoteHealthMetricFetcher.java | 77 ------ .../metricsproxy/service/RemoteMetricsFetcher.java | 129 --------- .../metricsproxy/service/ServiceListener.java | 14 - .../vespa/metricsproxy/service/SystemPoller.java | 259 ----------------- .../metricsproxy/service/SystemPollerProvider.java | 33 --- .../vespa/metricsproxy/service/VespaService.java | 216 --------------- .../vespa/metricsproxy/service/VespaServices.java | 123 --------- .../vespa/metricsproxy/service/package-info.java | 8 - .../configdefinitions/application-dimensions.def | 5 - .../main/resources/configdefinitions/consumers.def | 10 - .../resources/configdefinitions/monitoring.def | 8 - .../configdefinitions/node-dimensions.def | 5 - .../resources/configdefinitions/rpc-connector.def | 4 - .../resources/configdefinitions/vespa-services.def | 10 - .../test/java/ai/vespa/metricsproxy/TestUtil.java | 27 -- .../metricsproxy/core/MetricsManagerTest.java | 244 ---------------- .../metricsproxy/metric/ExternalMetricsTest.java | 73 ----- .../ai/vespa/metricsproxy/metric/MetricsTest.java | 99 ------- .../metric/model/MetricsPacketTest.java | 111 -------- .../metric/model/json/JsonUtilTest.java | 63 ----- .../metric/model/json/YamasJsonModelTest.java | 94 ------- .../vespa/metricsproxy/rpc/IntegrationTester.java | 144 ---------- .../metricsproxy/rpc/RpcHealthMetricsTest.java | 97 ------- .../ai/vespa/metricsproxy/rpc/RpcMetricsTest.java | 208 -------------- .../service/ConfigSentinelClientTest.java | 104 ------- .../metricsproxy/service/ConfigSentinelDummy.java | 61 ---- .../metricsproxy/service/ContainerServiceTest.java | 67 ----- .../vespa/metricsproxy/service/DummyService.java | 36 --- .../metricsproxy/service/MetricsFetcherTest.java | 90 ------ .../service/MockConfigSentinelClient.java | 50 ---- .../vespa/metricsproxy/service/MockHttpServer.java | 56 ---- .../metricsproxy/service/SystemPollerTest.java | 45 --- .../metricsproxy/service/VespaServiceTest.java | 77 ------ .../metricsproxy/service/VespaServicesTest.java | 42 --- .../resources/health-check-failed.response.json | 7 - .../src/test/resources/health-check.response.json | 6 - .../metrics-container-state-multi-chain.json | 281 ------------------- .../src/test/resources/metrics-state.json | 42 --- .../src/test/resources/metrics-storage-simple.json | 38 --- .../src/test/resources/rpc-json-output-check.json | 1 - .../src/test/resources/yamas-array-no-routing.json | 19 -- metrics-proxy/src/test/resources/yamas-array.json | 26 -- pom.xml | 1 - 88 files changed, 40 insertions(+), 6574 deletions(-) delete mode 100644 config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java delete mode 100644 config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/VespaServicesConfigGenerator.java delete mode 100644 config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java delete mode 100644 config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java delete mode 100644 metrics-proxy/OWNERS delete mode 100644 metrics-proxy/README delete mode 100644 metrics-proxy/pom.xml delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonUtil.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ServiceListener.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java delete mode 100644 metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java delete mode 100644 metrics-proxy/src/main/resources/configdefinitions/application-dimensions.def delete mode 100644 metrics-proxy/src/main/resources/configdefinitions/consumers.def delete mode 100644 metrics-proxy/src/main/resources/configdefinitions/monitoring.def delete mode 100644 metrics-proxy/src/main/resources/configdefinitions/node-dimensions.def delete mode 100644 metrics-proxy/src/main/resources/configdefinitions/rpc-connector.def delete mode 100644 metrics-proxy/src/main/resources/configdefinitions/vespa-services.def delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/JsonUtilTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java delete mode 100644 metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java delete mode 100644 metrics-proxy/src/test/resources/health-check-failed.response.json delete mode 100644 metrics-proxy/src/test/resources/health-check.response.json delete mode 100644 metrics-proxy/src/test/resources/metrics-container-state-multi-chain.json delete mode 100644 metrics-proxy/src/test/resources/metrics-state.json delete mode 100644 metrics-proxy/src/test/resources/metrics-storage-simple.json delete mode 100644 metrics-proxy/src/test/resources/rpc-json-output-check.json delete mode 100644 metrics-proxy/src/test/resources/yamas-array-no-routing.json delete mode 100644 metrics-proxy/src/test/resources/yamas-array.json diff --git a/config-model/pom.xml b/config-model/pom.xml index 2c45e292d01..715718f8bb5 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -94,12 +94,6 @@ ${project.version} compile - - com.yahoo.vespa - metrics-proxy - ${project.version} - compile - com.yahoo.vespa container-disc diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java index 2f41b172ab6..d3da9fe1f03 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java @@ -1,4 +1,9 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* + * // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + * + * + */ package com.yahoo.searchdefinition; import com.yahoo.searchlib.rankingexpression.Reference; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index e0956097c84..dd27dd176e0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -207,7 +207,7 @@ public class Admin extends AbstractConfigProducer implements Serializable { var metricsProxyCluster = new MetricsProxyContainerCluster(this, "metrics", deployState); int index = 0; for (var host : hosts) { - var container = new MetricsProxyContainer(metricsProxyCluster, index++, deployState.isHosted()); + var container = new MetricsProxyContainer(metricsProxyCluster, index++); addAndInitializeService(deployState.getDeployLogger(), host, container); metricsProxyCluster.addContainer(container); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java deleted file mode 100644 index 2df4008214a..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package com.yahoo.vespa.model.admin.metricsproxy; - -import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; -import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; - -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.getDefaultMetricsConsumer; - -/** - * Helper class to generate config for metrics consumers. - * - * @author gjoranv - */ -class ConsumersConfigGenerator { - - /** - * @param userConsumers The consumers set up by the user in services.xml - * @return A list of consumer builders (a mapping from consumer to its metrics) - */ - static List generate(Map userConsumers) { - // Normally, the user given consumers should not contain VESPA_CONSUMER_ID, but it's allowed for some internally used applications. - var allConsumers = new LinkedHashMap<>(userConsumers); - allConsumers.put(VESPA_CONSUMER_ID, combineConsumers(getDefaultMetricsConsumer(), allConsumers.get(VESPA_CONSUMER_ID))); - - return allConsumers.values().stream() - .map(ConsumersConfigGenerator::toConsumerBuilder) - .collect(Collectors.toList()); - } - - /* - * Returns a new consumer that is a combination of the two given consumers - * (ignoring the id of the consumers' metric sets). - * If a metric with the same id exists in both consumers, output name and - * dimensions from the 'overriding' consumer is used, but dimensions from 'original' - * are added if they don't exist in 'overriding'. - */ - private static MetricsConsumer combineConsumers(MetricsConsumer original, MetricsConsumer overriding) { - if (overriding == null) return original; - return addMetrics(original, overriding.getMetrics()); - } - - private static MetricsConsumer addMetrics(MetricsConsumer original, Map metrics) { - Map combinedMetrics = new LinkedHashMap<>(original.getMetrics()); - metrics.forEach((name, newMetric) -> - combinedMetrics.put(name, combineMetrics(original.getMetrics().get(name), newMetric))); - - return new MetricsConsumer(original.getId(), - new MetricSet(original.getMetricSet().getId(), combinedMetrics.values())); - } - - private static Metric combineMetrics(@Nullable Metric original, Metric newMetric) { - return original != null ? newMetric.addDimensionsFrom(original) : newMetric; - } - - private static Consumer.Builder toConsumerBuilder(MetricsConsumer consumer) { - Consumer.Builder builder = new Consumer.Builder().name(consumer.getId()); - consumer.getMetrics().values().forEach(metric -> builder.metric(toConsumerMetricBuilder(metric))); - return builder; - } - - private static Consumer.Metric.Builder toConsumerMetricBuilder(Metric metric) { - Consumer.Metric.Builder builder = new Consumer.Metric.Builder().name(metric.name) - .outputname(metric.outputName) - .description(metric.description); - metric.dimensions.forEach((name, value) -> builder.dimension(toMetricDimensionBuilder(name, value))); - return builder; - } - - private static Consumer.Metric.Dimension.Builder toMetricDimensionBuilder(String name, String value) { - return new Consumer.Metric.Dimension.Builder() - .key(name) - .value(value); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 1a66fa72c93..742196c91d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -1,66 +1,22 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - package com.yahoo.vespa.model.admin.metricsproxy; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; -import ai.vespa.metricsproxy.rpc.RpcConnector; -import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; -import ai.vespa.metricsproxy.service.VespaServices; -import ai.vespa.metricsproxy.service.VespaServicesConfig; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.provision.ClusterMembership; import com.yahoo.vespa.model.container.Container; -import java.util.LinkedHashMap; -import java.util.Map; - import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.CANONICAL_FLAVOR; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.CLUSTER_ID; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.CLUSTER_TYPE; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames.FLAVOR; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_NAME; /** * Container running a metrics proxy. * * @author gjoranv */ -public class MetricsProxyContainer extends Container implements - NodeDimensionsConfig.Producer, - RpcConnectorConfig.Producer, - VespaServicesConfig.Producer -{ - - static final class NodeDimensionNames { - static final String FLAVOR = "flavor"; - static final String CANONICAL_FLAVOR = "canonicalFlavor"; - static final String CLUSTER_TYPE = "clustertype"; - static final String CLUSTER_ID = "clusterid"; - } - - private final boolean isHostedVespa; +public class MetricsProxyContainer extends Container { - public MetricsProxyContainer(AbstractConfigProducer parent, int index, boolean isHostedVespa) { + public MetricsProxyContainer(AbstractConfigProducer parent, int index) { super(parent, "" + index, index); - this.isHostedVespa = isHostedVespa; setProp("clustertype", "admin"); setProp("index", String.valueOf(index)); - addNodeSpecificComponents(); - } - - private void addNodeSpecificComponents() { - addMetricsProxyComponent(NodeDimensions.class); - addMetricsProxyComponent(RpcConnector.class); - addMetricsProxyComponent(VespaServices.class); - } - - int metricsRpcPortOffset() { - return numHttpServerPorts; } @Override @@ -70,7 +26,7 @@ public class MetricsProxyContainer extends Container implements @Override public int getWantedPort() { - return 19092; + return 19092; // TODO: current metrics-proxy uses 19091 as rpc port, will now get 19093. } @Override @@ -78,15 +34,9 @@ public class MetricsProxyContainer extends Container implements return true; } - // Must have predictable ports for both http and rpc. - @Override - public boolean requiresConsecutivePorts() { - return true; - } - @Override public int getPortCount() { - return metricsRpcPortOffset() + 1; + return super.getPortCount() + 1; } @Override @@ -95,43 +45,4 @@ public class MetricsProxyContainer extends Container implements portsMeta.on(numHttpServerPorts).tag("rpc").tag("metrics"); } - @Override - public String[] getPortSuffixes() { - var suffixes = super.getPortSuffixes(); - suffixes[metricsRpcPortOffset()] = "rpc/metrics"; - return suffixes; - } - - @Override - public void getConfig(RpcConnectorConfig.Builder builder) { - builder.port(getRelativePort(metricsRpcPortOffset())); - } - - @Override - public void getConfig(VespaServicesConfig.Builder builder) { - builder.service.addAll(VespaServicesConfigGenerator.generate(getHostResource().getServices())); - } - - @Override - public void getConfig(NodeDimensionsConfig.Builder builder) { - Map dimensions = new LinkedHashMap<>(); - if (isHostedVespa) { - getHostResource().getFlavor().ifPresent(flavor -> { - dimensions.put(FLAVOR, flavor.name()); - dimensions.put(CANONICAL_FLAVOR, flavor.canonicalName()); - }); - - getHostResource().primaryClusterMembership().map(ClusterMembership::cluster).ifPresent(cluster -> { - dimensions.put(CLUSTER_TYPE, cluster.type().name()); - dimensions.put(CLUSTER_ID, cluster.id().value()); - }); - - builder.dimensions(dimensions); - } - } - - private void addMetricsProxyComponent(Class componentClass) { - addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index d9a969ab7c9..437df42d531 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -1,183 +1,23 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - package com.yahoo.vespa.model.admin.metricsproxy; -import ai.vespa.metricsproxy.core.MetricsConsumers; -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.metric.ExternalMetrics; -import ai.vespa.metricsproxy.core.VespaMetrics; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; -import ai.vespa.metricsproxy.rpc.RpcServer; -import ai.vespa.metricsproxy.core.MonitoringConfig; -import ai.vespa.metricsproxy.service.SystemPollerProvider; -import ai.vespa.metricsproxy.service.ConfigSentinelClient; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.model.producer.AbstractConfigProducerRoot; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.Admin; -import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; -import com.yahoo.vespa.model.admin.monitoring.Monitoring; -import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.container.ContainerCluster; -import java.nio.file.Path; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Optional; -import java.util.logging.Logger; - -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.APPLICATION; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.APPLICATION_ID; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.INSTANCE; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.LEGACY_APPLICATION; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.TENANT; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.ZONE; -import static com.yahoo.vespa.model.container.xml.BundleMapper.JarSuffix.JAR_WITH_DEPS; -import static com.yahoo.vespa.model.container.xml.BundleMapper.bundlePathFromName; - /** * Container cluster for metrics proxy containers. * * @author gjoranv */ -public class MetricsProxyContainerCluster extends ContainerCluster implements - ApplicationDimensionsConfig.Producer, - ConsumersConfig.Producer, - MonitoringConfig.Producer -{ - public static final Logger log = Logger.getLogger(MetricsProxyContainerCluster.class.getName()); - - private static final String METRICS_PROXY_NAME = "metrics-proxy"; - private static final Path METRICS_PROXY_BUNDLE_FILE = bundlePathFromName(METRICS_PROXY_NAME, JAR_WITH_DEPS); - static final String METRICS_PROXY_BUNDLE_NAME = "com.yahoo.vespa." + METRICS_PROXY_NAME; - - static final String DEFAULT_NAME_IN_MONITORING_SYSTEM = "vespa"; - static final int DEFAULT_MONITORING_INTERVAL = 1; - - static final class AppDimensionNames { - static final String ZONE = "zone"; - static final String APPLICATION_ID = "applicationId"; // tenant.app.instance - static final String TENANT = "tenantName"; - static final String APPLICATION = "applicationName"; - static final String INSTANCE = "instanceName"; - static final String LEGACY_APPLICATION = "app"; // app.instance - } - - private final AbstractConfigProducer parent; - private final ApplicationId applicationId; +public class MetricsProxyContainerCluster extends ContainerCluster { public MetricsProxyContainerCluster(AbstractConfigProducer parent, String name, DeployState deployState) { super(parent, name, name, deployState); - this.parent = parent; - applicationId = deployState.getProperties().applicationId(); - setRpcServerEnabled(false); addDefaultHandlersExceptStatus(); - - addPlatformBundle(METRICS_PROXY_BUNDLE_FILE); - addClusterComponents(); - } - - private void addClusterComponents() { - addMetricsProxyComponent(ApplicationDimensions.class); - addMetricsProxyComponent(ConfigSentinelClient.class); - addMetricsProxyComponent(ExternalMetrics.class); - addMetricsProxyComponent(MetricsConsumers.class); - addMetricsProxyComponent(MetricsManager.class); - addMetricsProxyComponent(RpcServer.class); - addMetricsProxyComponent(SystemPollerProvider.class); - addMetricsProxyComponent(VespaMetrics.class); } @Override protected void doPrepare(DeployState deployState) { } - @Override - public void getConfig(MonitoringConfig.Builder builder) { - builder.systemName(getSystemName()) - .intervalMinutes(getIntervalMinutes()); - } - - @Override - public void getConfig(ConsumersConfig.Builder builder) { - builder.consumer.addAll(ConsumersConfigGenerator.generate(getUserMetricsConsumers())); - } - - @Override - public void getConfig(ApplicationDimensionsConfig.Builder builder) { - if (isHostedVespa()) { - builder.dimensions(applicationDimensions()); - } - } - - // Returns the metricConsumers from services.xml - private Map getUserMetricsConsumers() { - return getAdmin() - .map(this::consumersInAdmin) - .orElse(Collections.emptyMap()); - } - - private Map consumersInAdmin(Admin admin) { - Metrics metrics = admin.getUserMetrics(); - return metrics.getConsumers(); - } - - private Optional getAdmin() { - if (parent != null) { - AbstractConfigProducerRoot r = parent.getRoot(); - if (r instanceof VespaModel) { - VespaModel model = (VespaModel) r; - return Optional.ofNullable(model.getAdmin()); - } - } - return Optional.empty(); - } - - private String getSystemName() { - Monitoring monitoring = getMonitoringService(); - if (monitoring != null && ! monitoring.getClustername().equals("")) - return monitoring.getClustername(); - return DEFAULT_NAME_IN_MONITORING_SYSTEM; - } - - private int getIntervalMinutes() { - Monitoring monitoring = getMonitoringService(); - if (monitoring != null && monitoring.getInterval() != null) { - return monitoring.getInterval(); - } - return DEFAULT_MONITORING_INTERVAL; - } - - private void addMetricsProxyComponent(Class componentClass) { - addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME); - } - - private Map applicationDimensions() { - Map dimensions = new LinkedHashMap<>(); - dimensions.put(ZONE, zoneString(getZone())); - dimensions.put(APPLICATION_ID, serializeWithDots(applicationId)); - dimensions.put(TENANT, applicationId.tenant().value()); - dimensions.put(APPLICATION, applicationId.application().value()); - dimensions.put(INSTANCE, applicationId.instance().value()); - dimensions.put(LEGACY_APPLICATION, applicationId.application().value() + "." + applicationId.instance().value()); - return dimensions; - } - - // ApplicationId uses ':' as separator. - private static String serializeWithDots(ApplicationId applicationId) { - return applicationId.serializedForm().replace(':', '.'); - } - - static String zoneString(Zone zone) { - return zone.environment().value() + "." + zone.region().value(); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/VespaServicesConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/VespaServicesConfigGenerator.java deleted file mode 100644 index 08a81fad0a3..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/VespaServicesConfigGenerator.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package com.yahoo.vespa.model.admin.metricsproxy; - -import ai.vespa.metricsproxy.service.VespaServicesConfig; -import com.yahoo.vespa.model.Service; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * @author gjoranv - */ -public class VespaServicesConfigGenerator { - - public static List generate(List services) { - return services.stream() - .filter(VespaServicesConfigGenerator::doIncludeServiceMetrics) - .map(VespaServicesConfigGenerator::toServiceBuilder) - .collect(Collectors.toList()); - } - - private static boolean doIncludeServiceMetrics(Service s) { - return s.getStartupCommand() != null || s.getServiceType().equals("configserver") || s.getServiceType().equals("config-sentinel"); - } - - private static VespaServicesConfig.Service.Builder toServiceBuilder(Service service) { - VespaServicesConfig.Service.Builder builder = new VespaServicesConfig.Service.Builder() - .configId(service.getConfigId()) - .name(service.getServiceName()) - .port(service.getHealthPort()) - .healthport(service.getHealthPort()); - - service.getDefaultMetricDimensions().forEach((name, value) -> builder.dimension(toServiceDimensionBuilder(name, value))); - return builder; - } - - private static VespaServicesConfig.Service.Dimension.Builder toServiceDimensionBuilder(String name, String value) { - return new VespaServicesConfig.Service.Dimension.Builder() - .key(name) - .value(value); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index f2bd67cdd81..f449feb5fc1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -10,7 +10,6 @@ import com.yahoo.container.QrConfig; import com.yahoo.container.core.ContainerHttpConfig; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; -import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; @@ -37,10 +36,6 @@ import static com.yahoo.container.QrConfig.Filedistributor; import static com.yahoo.container.QrConfig.Rpc; /** - * Note about components: In general, all components should belong to the cluster and not the container. However, - * components that need node specific config must be added at the container level, along with the node-specific - * parts of the config generation (getConfig). - * * @author gjoranv * @author Einar M R Rosenvinge * @author Tony Vaagenes @@ -112,15 +107,11 @@ public abstract class Container extends AbstractService implements return components; } - public final void addComponent(Component c) { + public void addComponent(Component c) { components.addComponent(c); } - public final void addSimpleComponent(String idSpec, String classSpec, String bundleSpec) { - addComponent(new SimpleComponent(new ComponentModel(idSpec, classSpec, bundleSpec))); - } - - public final void addHandler(Handler h) { + public void addHandler(Handler h) { handlers.addComponent(h); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 1f0c3d6f84e..ad087288799 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -200,7 +200,7 @@ public abstract class ContainerCluster addVipHandler(); } - public final void addDefaultHandlersExceptStatus() { + public void addDefaultHandlersExceptStatus() { addDefaultRootHandler(); addMetricStateHandler(); addApplicationStatusHandler(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java index 4c5c14dad87..11d467a61d5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java @@ -15,20 +15,7 @@ import java.util.Optional; */ public class BundleMapper { - public enum JarSuffix { - JAR_WITH_DEPS("-jar-with-dependencies.jar"), - DEPLOY("-deploy.jar"); - - private final String suffix; - - JarSuffix(String suffix) { - this.suffix = suffix; - } - } - public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars")); - - public static final String searchAndDocprocBundle = "container-search-and-docproc"; private static final Map bundleFromClass; @@ -47,10 +34,6 @@ public class BundleMapper { return LIBRARY_PATH.resolve(fileName); } - public static Path bundlePathFromName(String name, JarSuffix suffix) { - return Paths.get(Defaults.getDefaults().underVespaHome(LIBRARY_PATH + name + suffix.suffix)); - } - /** * TODO: This is a temporary hack to ensure that users can use our internal components without * specifying the bundle in which the components reside. Ideally, this information diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 59a72aedaaf..efd13e82361 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -1,186 +1,45 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - package com.yahoo.vespa.model.admin.metricsproxy; -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; -import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import org.junit.Rule; +import com.yahoo.vespa.model.test.VespaModelTester; import org.junit.Test; -import org.junit.rules.ExpectedException; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.zoneString; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_APPLICATION; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_INSTANCE; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_TENANT; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getApplicationDimensionsConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getConsumersConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getHostedModel; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.assertThat; /** * @author gjoranv */ public class MetricsProxyContainerClusterTest { - private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size(); - private static int numVespaMetrics = vespaMetricSet.getMetrics().size(); - private static int numSystemMetrics = systemMetricSet.getMetrics().size(); - private static int numNetworkMetrics = networkMetricSet.getMetrics().size(); - private static int numMetricsForDefaultConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics; - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - - @Test - public void default_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() { - ConsumersConfig consumersConfig = getConsumersConfig(servicesWithAdminOnly()); - assertEquals(consumersConfig.consumer(0).name(), VESPA_CONSUMER_ID); - assertEquals(numMetricsForDefaultConsumer, consumersConfig.consumer(0).metric().size()); - } - - @Test - public void vespa_is_a_reserved_consumer_id() { - String services = String.join("\n", - "", - " ", - " ", - " ", - " ", - " ", - " ", - "" - ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("'Vespa' is not allowed as metrics consumer id"); - getConsumersConfig(services); - } - - @Test - public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() { - String services = String.join("\n", - "", - " ", - " ", - " ", - " ", - " ", - " ", - " ", - " ", - "" - ); - ConsumersConfig config = getConsumersConfig(services); - assertEquals(1, config.consumer().size()); - - // All default metrics are retained - ConsumersConfig.Consumer vespaConsumer = config.consumer(0); - assertEquals(numMetricsForDefaultConsumer + 1, vespaConsumer.metric().size()); - - Metric customMetric1 = new Metric("custom.metric1"); - assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1)); - } - - @Test - public void consumer_id_is_case_insensitive() { - String services = String.join("\n", - "", - " ", - " ", - " ", - " ", - " ", - " ", - " ", - "" - ); - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("'a' is used as id for two metrics consumers"); - getConsumersConfig(services); - } - - @Test - public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() { - String services = String.join("\n", - "", - " ", - " ", - " ", - " ", - " ", - " ", - " ", - " ", - " ", - "" - ); - ConsumersConfig.Consumer consumer = getCustomConsumer(services); - - assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size()); - - Metric customMetric1 = new Metric("custom.metric1"); - Metric customMetric2 = new Metric("custom.metric2"); - assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1)); - assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2)); - } - @Test - public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() { - String services = String.join("\n", - "", - " ", - " ", - " ", - " ", - " ", - " ", - " ", - " ", - " ", - "" - ); - ConsumersConfig.Consumer consumer = getCustomConsumer(services); - assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size()); + public void one_metrics_proxy_container_is_added_to_every_node() { + var numberOfHosts = 4; + var tester = new VespaModelTester(); + tester.enableMetricsProxyContainer(true); + tester.addHosts(4); - Metric customMetric = new Metric("my.extra.metric"); - assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric)); - } + VespaModel model = tester.createModel(servicesXml(), true); + assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); - @Test - public void hosted_application_propagates_application_dimensions() { - VespaModel hostedModel = getHostedModel(servicesWithAdminOnly()); - ApplicationDimensionsConfig config = getApplicationDimensionsConfig(hostedModel); + for (var host : model.getHostSystem().getHosts()) { + assertThat(host.getService(METRICS_PROXY_CONTAINER.serviceName), notNullValue()); + } - assertEquals(zoneString(Zone.defaultZone()), config.dimensions(AppDimensionNames.ZONE)); - assertEquals(MY_TENANT, config.dimensions(AppDimensionNames.TENANT)); - assertEquals(MY_APPLICATION, config.dimensions(AppDimensionNames.APPLICATION)); - assertEquals(MY_INSTANCE, config.dimensions(AppDimensionNames.INSTANCE)); - assertEquals(MY_TENANT + "." + MY_APPLICATION + "." + MY_INSTANCE, config.dimensions(AppDimensionNames.APPLICATION_ID)); - assertEquals(MY_APPLICATION + "." + MY_INSTANCE, config.dimensions(AppDimensionNames.LEGACY_APPLICATION)); } - - private String servicesWithAdminOnly() { - return String.join("\n", "", - " ", - " ", - " ", - "" - ); + private String servicesXml() { + return String.join("\n", "", + "", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + ""); } - } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java deleted file mode 100644 index ff486c6a437..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ /dev/null @@ -1,146 +0,0 @@ -package com.yahoo.vespa.model.admin.metricsproxy; - -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; -import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; -import ai.vespa.metricsproxy.service.VespaServicesConfig; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer.NodeDimensionNames; -import com.yahoo.vespa.model.test.VespaModelTester; -import org.junit.Test; - -import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CONTAINER_CONFIG_ID; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_FLAVOR; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getHostedModel; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getNodeDimensionsConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getRpcConnectorConfig; -import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getVespaServicesConfig; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author gjoranv - */ -public class MetricsProxyContainerTest { - - @Test - public void one_metrics_proxy_container_is_added_to_every_node() { - var numberOfHosts = 4; - var tester = new VespaModelTester(); - tester.enableMetricsProxyContainer(true); - tester.addHosts(numberOfHosts); - - VespaModel model = tester.createModel(servicesWithManyNodes(), true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); - - for (var host : model.getHostSystem().getHosts()) { - assertThat(host.getService(METRICS_PROXY_CONTAINER.serviceName), notNullValue()); - - long metricsProxies = host.getServices().stream() - .filter(s -> s.getClass().equals(MetricsProxyContainer.class)) - .count(); - assertThat(metricsProxies, is(1L)); - } - } - - @Test - public void http_server_is_running_on_expected_port() { - VespaModel model = getModel(servicesWithContent()); - MetricsProxyContainer container = (MetricsProxyContainer)model.id2producer().get(CONTAINER_CONFIG_ID); - assertEquals(19092, container.getSearchPort()); - assertEquals(19092, container.getHealthPort()); - assertEquals("http", container.getPortSuffixes()[0]); - - assertTrue(container.getPortsMeta().getTagsAt(0).contains("http")); - assertTrue(container.getPortsMeta().getTagsAt(0).contains("state")); - } - - @Test - public void rpc_server_is_running_on_expected_port() { - VespaModel model = getModel(servicesWithContent()); - - MetricsProxyContainer container = (MetricsProxyContainer)model.id2producer().get(CONTAINER_CONFIG_ID); - - int rpcPort = container.metricsRpcPortOffset(); - assertTrue(container.getPortsMeta().getTagsAt(rpcPort).contains("rpc")); - assertTrue(container.getPortsMeta().getTagsAt(rpcPort).contains("metrics")); - - assertEquals("rpc/metrics", container.getPortSuffixes()[rpcPort]); - - RpcConnectorConfig config = getRpcConnectorConfig(model); - assertEquals(19094, config.port()); - } - - @Test - public void hosted_application_propagates_node_dimensions() { - String services = servicesWithContent(); - VespaModel hostedModel = getHostedModel(services); - NodeDimensionsConfig config = getNodeDimensionsConfig(hostedModel); - - assertEquals("content", config.dimensions(NodeDimensionNames.CLUSTER_TYPE)); - assertEquals("my-content", config.dimensions(NodeDimensionNames.CLUSTER_ID)); - assertEquals(MY_FLAVOR, config.dimensions(NodeDimensionNames.FLAVOR)); - assertEquals(MY_FLAVOR, config.dimensions(NodeDimensionNames.CANONICAL_FLAVOR)); - } - - - @Test - public void vespa_services_config_has_all_services() { - VespaServicesConfig vespaServicesConfig = getVespaServicesConfig(servicesWithContent()); - assertEquals(6, vespaServicesConfig.service().size()); - - for (var service : vespaServicesConfig.service()) { - if (service.configId().equals("admin/cluster-controllers/0")) { - assertEquals("Wrong service name", "container-clustercontroller", service.name()); - assertEquals(1, service.dimension().size()); - assertEquals("clustername", service.dimension(0).key()); - assertEquals("cluster-controllers", service.dimension(0).value()); - } - } - } - - @Test - public void vespa_services_config_has_service_dimensions() { - VespaServicesConfig vespaServicesConfig = getVespaServicesConfig(servicesWithContent()); - for (var service : vespaServicesConfig.service()) { - if (service.configId().equals("admin/cluster-controllers/0")) { - assertEquals(1, service.dimension().size()); - assertEquals("clustername", service.dimension(0).key()); - assertEquals("cluster-controllers", service.dimension(0).value()); - } - } - } - - - private String servicesWithManyNodes() { - return String.join("\n", - "", - " ", - " ", - " ", - " ", - " ", - " ", - " ", - ""); - } - - private String servicesWithContent() { - return String.join("\n", - "", - " ", - " ", - " ", - " ", - " ", - " ", - " ", - "" - ); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java deleted file mode 100644 index 3e5c8a6ef0d..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package com.yahoo.vespa.model.admin.metricsproxy; - -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; -import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; -import ai.vespa.metricsproxy.service.VespaServicesConfig; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provisioning.FlavorsConfig; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.test.VespaModelTester; - -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; -import static org.junit.Assert.assertEquals; - -/** - * @author gjoranv - */ -class MetricsProxyModelTester { - - static final String MY_TENANT = "mytenant"; - static final String MY_APPLICATION = "myapp"; - static final String MY_INSTANCE = "myinstance"; - static final String MY_FLAVOR = "myflavor"; - - // Used for all configs that are produced by the container, not the cluster. - static final String CONTAINER_CONFIG_ID = "admin/metrics/0"; - - static VespaModel getModel(String servicesXml) { - var numberOfHosts = 1; - var tester = new VespaModelTester(); - tester.enableMetricsProxyContainer(true); - tester.addHosts(numberOfHosts); - tester.setHosted(false); - return tester.createModel(servicesXml, true); - } - - static VespaModel getHostedModel(String servicesXml) { - var numberOfHosts = 2; - var tester = new VespaModelTester(); - tester.enableMetricsProxyContainer(true); - tester.addHosts(flavorFromString(MY_FLAVOR), numberOfHosts); - tester.setHosted(true); - tester.setApplicationId(MY_TENANT, MY_APPLICATION, MY_INSTANCE); - return tester.createModel(servicesXml, true); - } - - static boolean checkMetric(ConsumersConfig.Consumer consumer, Metric metric) { - for (ConsumersConfig.Consumer.Metric m : consumer.metric()) { - if (metric.name.equals(m.name()) && metric.outputName.equals(m.outputname())) - return true; - } - return false; - } - - static ConsumersConfig.Consumer getCustomConsumer(String servicesXml) { - ConsumersConfig config = getConsumersConfig(servicesXml); - assertEquals(2, config.consumer().size()); - for (ConsumersConfig.Consumer consumer : config.consumer()) { - if (! consumer.name().equals(VESPA_CONSUMER_ID)) - return consumer; - } - throw new RuntimeException("Two consumers with the reserved id - this cannot happen."); - } - - static ConsumersConfig getConsumersConfig(String servicesXml) { - return getConsumersConfig(getModel(servicesXml)); - } - - private static ConsumersConfig getConsumersConfig(VespaModel model) { - String configId = "admin/metrics"; - return new ConsumersConfig((ConsumersConfig.Builder) model.getConfig(new ConsumersConfig.Builder(), configId)); - } - - static ApplicationDimensionsConfig getApplicationDimensionsConfig(VespaModel model) { - String configId = "admin/metrics"; - return new ApplicationDimensionsConfig((ApplicationDimensionsConfig.Builder) model.getConfig(new ApplicationDimensionsConfig.Builder(), configId)); - } - - static NodeDimensionsConfig getNodeDimensionsConfig(VespaModel model) { - return new NodeDimensionsConfig((NodeDimensionsConfig.Builder) model.getConfig(new NodeDimensionsConfig.Builder(), CONTAINER_CONFIG_ID)); - } - - static VespaServicesConfig getVespaServicesConfig(String servicesXml) { - VespaModel model = getModel(servicesXml); - return new VespaServicesConfig((VespaServicesConfig.Builder) model.getConfig(new VespaServicesConfig.Builder(), CONTAINER_CONFIG_ID)); - } - - static RpcConnectorConfig getRpcConnectorConfig(VespaModel model) { - return new RpcConnectorConfig((RpcConnectorConfig.Builder) model.getConfig(new RpcConnectorConfig.Builder(), CONTAINER_CONFIG_ID)); - } - - private static Flavor flavorFromString(String name) { - return new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder(). - name(name))); - } - -} diff --git a/metrics-proxy/OWNERS b/metrics-proxy/OWNERS deleted file mode 100644 index 3b2ba1ede81..00000000000 --- a/metrics-proxy/OWNERS +++ /dev/null @@ -1 +0,0 @@ -gjoranv diff --git a/metrics-proxy/README b/metrics-proxy/README deleted file mode 100644 index 4008cb1aa5d..00000000000 --- a/metrics-proxy/README +++ /dev/null @@ -1 +0,0 @@ -The metrics proxy provides a single point of access for metrics from all Vespa services. diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml deleted file mode 100644 index 88c250f31bd..00000000000 --- a/metrics-proxy/pom.xml +++ /dev/null @@ -1,157 +0,0 @@ - - - - 4.0.0 - - com.yahoo.vespa - parent - 7-SNAPSHOT - ../parent/pom.xml - - metrics-proxy - container-plugin - 7-SNAPSHOT - - - - - - com.fasterxml.jackson.core - jackson-annotations - provided - - - com.fasterxml.jackson.core - jackson-core - provided - - - com.fasterxml.jackson.core - jackson-databind - provided - - - com.google.guava - guava - provided - - - com.google.inject - guice - provided - - - com.yahoo.vespa - annotations - ${project.version} - provided - - - com.yahoo.vespa - component - ${project.version} - provided - - - com.yahoo.vespa - config-lib - ${project.version} - provided - - - com.yahoo.vespa - container-di - ${project.version} - provided - - - com.yahoo.vespa - jrt - ${project.version} - provided - - - com.yahoo.vespa - vespajlib - ${project.version} - provided - - - com.yahoo.vespa - vespalog - ${project.version} - provided - - - com.yahoo.vespa - yolean - ${project.version} - provided - - - org.json - json - provided - - - - - - com.yahoo.vespa - http-utils - ${project.version} - - - org.apache.httpcomponents - httpclient - - - - - - junit - junit - test - - - org.hamcrest - hamcrest-core - test - - - - - - com.yahoo.vespa - config-class-plugin - ${project.version} - - - config-gen - - config-gen - - - - - ai.vespa. - - - - - com.yahoo.vespa - bundle-plugin - true - - ${project.groupId}.${project.artifactId} - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java deleted file mode 100644 index 564de0806ca..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsConsumers.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.core; - -import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer; -import ai.vespa.metricsproxy.metric.model.ConsumerId; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collector; - -import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static com.yahoo.stream.CustomCollectors.toLinkedMap; -import static java.util.Collections.unmodifiableSet; -import static java.util.stream.Collectors.collectingAndThen; - -/** - * Contains metrics consumers and their metrics, and mappings between these. - * All collections are final and immutable. - * - * @author gjoranv - */ -public class MetricsConsumers { - - // All metrics for each consumer. - private final Map> consumerMetrics; - - // All consumers for each metric (more useful than the opposite map). - private final Map> consumersByMetric; - - public MetricsConsumers(ConsumersConfig config) { - consumerMetrics = config.consumer().stream().collect( - toUnmodifiableLinkedMap(consumer -> toConsumerId(consumer.name()), Consumer::metric)); - - consumersByMetric = createConsumersByMetric(consumerMetrics); - } - - /** - * @param consumer The consumer - * @return The metrics for the given consumer. - */ - public List getMetricDefinitions(ConsumerId consumer) { - return consumerMetrics.get(consumer); - } - - public Map> getConsumersByMetric() { - return consumersByMetric; - } - - public Set getAllConsumers() { - return unmodifiableSet(consumerMetrics.keySet()); - } - - /** - * Helper function to create mapping from metric to consumers. - * TODO: consider reversing the mapping in metrics-consumers.def instead: metric{}.consumer[] - */ - private static Map> - createConsumersByMetric(Map> metricsByConsumer) { - Map> consumersByMetric = new LinkedHashMap<>(); - metricsByConsumer.forEach( - (consumer, metrics) -> metrics.forEach( - metric -> consumersByMetric.computeIfAbsent(metric, unused -> new ArrayList<>()) - .add(consumer))); - return Collections.unmodifiableMap(consumersByMetric); - } - - public static Collector> toUnmodifiableLinkedMap(Function keyMapper, - Function valueMapper) { - return collectingAndThen(toLinkedMap(keyMapper, valueMapper), Collections::unmodifiableMap); - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java deleted file mode 100644 index fe823c72127..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.core; - -import ai.vespa.metricsproxy.metric.ExternalMetrics; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.service.VespaService; -import ai.vespa.metricsproxy.service.VespaServices; -import com.yahoo.component.Vtag; - -import java.time.Duration; -import java.time.Instant; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static ai.vespa.metricsproxy.metric.ExternalMetrics.extractConfigserverDimensions; -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static com.yahoo.log.LogLevel.DEBUG; -import static java.util.stream.Collectors.toList; - -/** - * Retrieves metrics and performs necessary conversions and additions of metadata. - * - * @author gjoranv - */ -public class MetricsManager { - private static Logger log = Logger.getLogger(MetricsManager.class.getName()); - - static final DimensionId VESPA_VERSION = toDimensionId("vespaVersion"); - - private final VespaServices vespaServices; - private final VespaMetrics vespaMetrics; - private final ExternalMetrics externalMetrics; - private final ApplicationDimensions applicationDimensions; - private final NodeDimensions nodeDimensions; - - private volatile Map extraDimensions = new HashMap<>(); - - public MetricsManager(VespaServices vespaServices, - VespaMetrics vespaMetrics, - ExternalMetrics externalMetrics, - ApplicationDimensions applicationDimensions, - NodeDimensions nodeDimensions) { - this.vespaServices = vespaServices; - this.vespaMetrics = vespaMetrics; - this.externalMetrics = externalMetrics; - this.applicationDimensions = applicationDimensions; - this.nodeDimensions = nodeDimensions; - } - - /** - * Returns all metrics for the given service that are whitelisted for the given consumer. - */ - public String getMetricNamesForServiceAndConsumer(String service, ConsumerId consumer) { - return vespaMetrics.getMetricNames(vespaServices.getMonitoringServices(service), consumer); - } - - public String getMetricsByConfigId(String configId) { - List services = vespaServices.getInstancesById(configId); - vespaServices.updateServices(services); - - return vespaMetrics.getMetricsAsString(services); - } - - /** - * Returns the metrics for the given services. The empty list is returned if no services are given. - * - * @param services The services to retrieve metrics for. - * @return Metrics for all matching services. - */ - public List getMetrics(List services, Instant startTime) { - if (services.isEmpty()) return Collections.emptyList(); - vespaServices.updateServices(services); - - List result = vespaMetrics.getMetrics(services); - log.log(DEBUG, () -> "Got " + result.size() + " metrics packets for vespa services."); - - List externalPackets = externalMetrics.getMetrics().stream() - .filter(MetricsPacket.Builder::hasMetrics) - .collect(toList()); - log.log(DEBUG, () -> "Got " + externalPackets.size() + " external metrics packets with whitelisted metrics."); - - result.addAll(externalPackets); - - return result.stream() - .map(builder -> builder.putDimensionsIfAbsent(getGlobalDimensions())) - .map(builder -> builder.putDimensionsIfAbsent(extraDimensions)) - .map(builder -> adjustTimestamp(builder, startTime)) - .map(MetricsPacket.Builder::build) - .collect(Collectors.toList()); - } - - /** - * Returns a merged map of all global dimensions. - */ - private Map getGlobalDimensions() { - Map globalDimensions = new LinkedHashMap<>(applicationDimensions.getDimensions()); - globalDimensions.putAll(nodeDimensions.getDimensions()); - globalDimensions.put(VESPA_VERSION, Vtag.currentVersion.toFullString()); - return globalDimensions; - } - - /** - * If the metrics packet is less than one minute newer or older than the given startTime, - * set its timestamp to the given startTime. This is done to ensure that metrics retrieved - * from different sources for this invocation get the same timestamp, and a timestamp as close - * as possible to the invocation from the external metrics retrieving client. The assumption - * is that the client requests metrics periodically every minute. - *

- * However, if the timestamp of the packet is too far off in time, we don't adjust it because - * we would otherwise be masking a real problem with retrieving the metrics. - */ - static MetricsPacket.Builder adjustTimestamp(MetricsPacket.Builder builder, Instant startTime) { - Duration age = Duration.between(startTime, builder.getTimestamp()); - if (age.abs().minusMinutes(1).isNegative()) - builder.timestamp(startTime.getEpochSecond()); - return builder; - } - - /** - * Returns the health metrics for the given services. The empty list is returned if no services are given. - * - * @param services The services to retrieve health metrics for. - * @return Health metrics for all matching services. - */ - public List getHealthMetrics(List services) { - if (services.isEmpty()) return Collections.emptyList(); - vespaServices.updateServices(services); - - // TODO: Add global dimensions to health metrics? - return vespaMetrics.getHealthMetrics(services); - } - - public void setExtraMetrics(List packets) { - extraDimensions = extractConfigserverDimensions(packets); - externalMetrics.setExtraMetrics(packets); - } - - /** - * Returns a space separated list of all distinct service names. - */ - public String getAllVespaServices() { - return vespaServices.getVespaServices().stream() - .map(VespaService::getServiceName) - .distinct() - .collect(Collectors.joining(" ")); - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java deleted file mode 100644 index becfd9a54ce..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java +++ /dev/null @@ -1,307 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.core; - - -import ai.vespa.metricsproxy.metric.AggregationKey; -import ai.vespa.metricsproxy.metric.HealthMetric; -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.Metrics; -import ai.vespa.metricsproxy.metric.MetricsFormatter; -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.service.VespaService; -import ai.vespa.metricsproxy.service.VespaServices; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static com.google.common.base.Strings.isNullOrEmpty; -import static com.yahoo.log.LogLevel.DEBUG; - -/** - * @author Unknown - * @author gjoranv - */ -public class VespaMetrics { - private static final Logger log = Logger.getLogger(VespaMetrics.class.getPackage().getName()); - - // MUST be the same as the constant defined in config-model - public static final ConsumerId VESPA_CONSUMER_ID = toConsumerId("Vespa"); - - public static final DimensionId METRIC_TYPE_DIMENSION_ID = toDimensionId("metrictype"); - public static final DimensionId INSTANCE_DIMENSION_ID = toDimensionId("instance"); - - private static final Set DEFAULT_CONSUMERS = Collections.singleton(VESPA_CONSUMER_ID); - - private final MetricsConsumers metricsConsumers; - - private static final MetricsFormatter formatter = new MetricsFormatter(false, false); - - public VespaMetrics(MetricsConsumers metricsConsumers, VespaServices vespaServices) { - this.metricsConsumers = metricsConsumers; - } - - public List getHealthMetrics(List services) { - List result = new ArrayList<>(); - for (VespaService s : services) { - HealthMetric h = s.getHealth(); - MetricsPacket.Builder builder = new MetricsPacket.Builder(toServiceId(s.getMonitoringName())) - .statusCode(h.isOk() ? 0 : 1) - .statusMessage(h.getMessage()) - .putDimension(METRIC_TYPE_DIMENSION_ID, "health") - .putDimension(INSTANCE_DIMENSION_ID, s.getInstanceName()); - - result.add(builder.build()); - } - - return result; - } - - /** - * @param services The services to get metrics for - * @return A list of metrics packet builders (to allow modification by the caller). - */ - public List getMetrics(List services) { - List metricsPackets = new ArrayList<>(); - - log.log(DEBUG, () -> "Updating services prior to fetching metrics, number of services= " + services.size()); - - Map> consumersByMetric = metricsConsumers.getConsumersByMetric(); - - for (VespaService service : services) { - // One metrics packet for system metrics - Optional systemCheck = getSystemMetrics(service); - systemCheck.ifPresent(metricsPackets::add); - - // One metrics packet per set of metrics that share the same dimensions+consumers - // TODO: Move aggregation into MetricsPacket itself? - Metrics serviceMetrics = getServiceMetrics(service, consumersByMetric); - Map> aggregatedMetrics = - aggregateMetrics(service.getDimensions(), serviceMetrics); - - aggregatedMetrics.forEach((aggregationKey, metrics) -> { - MetricsPacket.Builder builder = new MetricsPacket.Builder(toServiceId(service.getMonitoringName())) - .putMetrics(metrics) - .putDimension(METRIC_TYPE_DIMENSION_ID, "standard") - .putDimension(INSTANCE_DIMENSION_ID, service.getInstanceName()) - .putDimensions(aggregationKey.getDimensions()); - setMetaInfo(builder, serviceMetrics.getTimeStamp()); - builder.addConsumers(aggregationKey.getConsumers()); - metricsPackets.add(builder); - }); - } - - return metricsPackets; - } - - /** - * Returns the metrics to output for the given service, with updated timestamp - * In order to include a metric, it must exist in the given map of metric to consumers. - * Each returned metric will contain a collection of consumers that it should be routed to. - */ - private Metrics getServiceMetrics(VespaService service, Map> consumersByMetric) { - Metrics serviceMetrics = new Metrics(); - Metrics allServiceMetrics = service.getMetrics(); - serviceMetrics.setTimeStamp(getMostRecentTimestamp(allServiceMetrics)); - for (Metric candidate : allServiceMetrics.getMetrics()) { - getConfiguredMetrics(candidate.getName(), consumersByMetric.keySet()).forEach( - configuredMetric -> serviceMetrics.add( - metricWithConfigProperties(candidate, configuredMetric, consumersByMetric))); - } - return serviceMetrics; - } - - private Map extractDimensions(Map dimensions, List configuredDimensions) { - if ( ! configuredDimensions.isEmpty()) { - Map dims = new HashMap<>(dimensions); - configuredDimensions.forEach(d -> dims.put(toDimensionId(d.key()), d.value())); - dimensions = Collections.unmodifiableMap(dims); - } - return dimensions; - } - - private Set extractConsumers(List configuredConsumers) { - Set consumers = Collections.emptySet(); - if (configuredConsumers != null) { - if ( configuredConsumers.size() == 1) { - consumers = Collections.singleton(configuredConsumers.get(0)); - } else if (configuredConsumers.size() > 1){ - consumers = new HashSet<>(); - consumers.addAll(configuredConsumers); - consumers = Collections.unmodifiableSet(consumers); - } - } - return consumers; - } - - private Metric metricWithConfigProperties(Metric candidate, - ConsumersConfig.Consumer.Metric configuredMetric, - Map> consumersByMetric) { - Metric metric = candidate.clone(); - metric.setDimensions(extractDimensions(candidate.getDimensions(), configuredMetric.dimension())); - metric.setConsumers(extractConsumers(consumersByMetric.get(configuredMetric))); - - if (!isNullOrEmpty(configuredMetric.outputname())) - metric.setName(configuredMetric.outputname()); - return metric; - } - - /** - * Returns all configured metrics (for any consumer) that have the given id as 'name'. - */ - private static Set getConfiguredMetrics(String id, - Set configuredMetrics) { - return configuredMetrics.stream() - .filter(m -> m.name().equals(id)) - .collect(Collectors.toSet()); - } - - private Optional getSystemMetrics(VespaService service) { - Metrics systemMetrics = service.getSystemMetrics(); - if (systemMetrics.size() == 0) return Optional.empty(); - - MetricsPacket.Builder builder = new MetricsPacket.Builder(toServiceId(service.getMonitoringName())); - setMetaInfo(builder, systemMetrics.getTimeStamp()); - - builder.putDimension(METRIC_TYPE_DIMENSION_ID, "system") - .putDimension(INSTANCE_DIMENSION_ID, service.getInstanceName()) - .putDimensions(service.getDimensions()) - .putMetrics(systemMetrics.getMetrics()); - - builder.addConsumers(metricsConsumers.getAllConsumers()); - return Optional.of(builder); - } - - private long getMostRecentTimestamp(Metrics metrics) { - long mostRecentTimestamp = 0L; - for (Metric metric : metrics.getMetrics()) { - if (metric.getTimeStamp() > mostRecentTimestamp) { - mostRecentTimestamp = metric.getTimeStamp(); - } - } - return mostRecentTimestamp; - } - - private Map> aggregateMetrics(Map serviceDimensions, - Metrics metrics) { - Map> aggregatedMetrics = new HashMap<>(); - - for (Metric metric : metrics.getMetrics() ) { - Map mergedDimensions = new LinkedHashMap<>(); - mergedDimensions.putAll(metric.getDimensions()); - mergedDimensions.putAll(serviceDimensions); - AggregationKey aggregationKey = new AggregationKey(mergedDimensions, metric.getConsumers()); - - if (aggregatedMetrics.containsKey(aggregationKey)) { - aggregatedMetrics.get(aggregationKey).add(metric); - } else { - List ml = new ArrayList<>(); - ml.add(metric); - aggregatedMetrics.put(aggregationKey, ml); - } - } - return aggregatedMetrics; - } - - private List getMetricDefinitions(ConsumerId consumer) { - if (metricsConsumers == null) return Collections.emptyList(); - - List definitions = metricsConsumers.getMetricDefinitions(consumer); - return definitions == null ? Collections.emptyList() : definitions; - } - - private static void setMetaInfo(MetricsPacket.Builder builder, long timestamp) { - builder.timestamp(timestamp) - .statusCode(0) - .statusMessage("Data collected successfully"); - } - - /** - * Returns a string representation of metrics for the given services; - * a space separated list of key=value. - */ - public String getMetricsAsString(List services) { - StringBuilder b = new StringBuilder(); - for (VespaService s : services) { - for (Metric metric : s.getMetrics().getMetrics()) { - String key = metric.getName(); - String alias = key; - - boolean isForwarded = false; - for (ConsumersConfig.Consumer.Metric metricConsumer : getMetricDefinitions(VESPA_CONSUMER_ID)) { - if (metricConsumer.name().equals(key)) { - alias = metricConsumer.outputname(); - isForwarded = true; - } - } - if (isForwarded) { - b.append(formatter.format(s, alias, metric.getValue())) - .append(" "); - } - } - } - return b.toString(); - } - - /** - * Get all metric names for the given services - * - * @return String representation - */ - public String getMetricNames(List services, ConsumerId consumer) { - StringBuilder bufferOn = new StringBuilder(); - StringBuilder bufferOff = new StringBuilder(); - for (VespaService s : services) { - - for (Metric m : s.getMetrics().getMetrics()) { - String description = m.getDescription(); - String alias = ""; - boolean isForwarded = false; - - for (ConsumersConfig.Consumer.Metric metric : getMetricDefinitions(consumer)) { - if (metric.name().equals(m.getName())) { - alias = metric.outputname(); - isForwarded = true; - if (description.isEmpty()) { - description = metric.description(); - } - } - } - - String message = "OFF"; - StringBuilder buffer = bufferOff; - if (isForwarded) { - buffer = bufferOn; - message = "ON"; - } - buffer.append(m.getName()).append('=').append(message); - if (!description.isEmpty()) { - buffer.append(";description=").append(description); - } - if (!alias.isEmpty()) { - buffer.append(";output-name=").append(alias); - } - buffer.append(','); - } - } - - return bufferOn.toString() + bufferOff.toString(); - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java deleted file mode 100644 index 617cf0a1525..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -@ExportPackage -package ai.vespa.metricsproxy.core; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java deleted file mode 100644 index 9eb1b242535..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/AggregationKey.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.DimensionId; - -import java.util.Collections; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -/** - * @author gjoranv - */ -public final class AggregationKey { - - private Map dimensions; - private Set consumers; - - public AggregationKey(Map dimensions, Set consumers) { - this.dimensions = dimensions; - this.consumers = consumers; - } - - public Map getDimensions() { return Collections.unmodifiableMap(dimensions); } - - public Set getConsumers() { return Collections.unmodifiableSet(consumers); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AggregationKey that = (AggregationKey) o; - return Objects.equals(dimensions, that.dimensions) && - Objects.equals(consumers, that.consumers); - } - - @Override - public int hashCode() { - return Objects.hash(dimensions, consumers); - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java deleted file mode 100644 index 62465909798..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -import ai.vespa.metricsproxy.core.MetricsConsumers; -import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import ai.vespa.metricsproxy.metric.model.MetricId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.metric.model.ServiceId; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Logger; - -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static com.yahoo.log.LogLevel.DEBUG; -import static java.util.stream.Collectors.toCollection; - -/** - * This class is responsible for handling metrics received from external processes. - * - * @author gjoranv - */ -public class ExternalMetrics { - private static final Logger log = Logger.getLogger(ExternalMetrics.class.getName()); - - public static final DimensionId ROLE_DIMENSION = toDimensionId("role"); - public static final DimensionId STATE_DIMENSION = toDimensionId("state"); - public static final DimensionId ORCHESTRATOR_STATE_DIMENSION = toDimensionId("orchestratorState"); - - static final ServiceId VESPA_NODE_SERVICE_ID = toServiceId("vespa.node"); - - private volatile List metrics = new ArrayList<>(); - private final MetricsConsumers consumers; - - public ExternalMetrics(MetricsConsumers consumers) { - this.consumers = consumers; - } - - public List getMetrics() { - return metrics; - } - - public void setExtraMetrics(List externalPackets) { - log.log(DEBUG, () -> "Setting new external metrics with " + externalPackets.size() + " metrics packets."); - externalPackets.forEach(packet -> { - packet.addConsumers(consumers.getAllConsumers()) - .service(VESPA_NODE_SERVICE_ID) - .retainMetrics(metricsToRetain()) - .applyOutputNames(outputNamesById()); - }); - metrics = List.copyOf(externalPackets); - } - - private Set metricsToRetain() { - return consumers.getConsumersByMetric().keySet().stream() - .map(configuredMetric -> toMetricId(configuredMetric.name())) - .collect(toCollection(LinkedHashSet::new)); - } - - /** - * Returns a mapping from metric id to a list of the metric's output names. - * Metrics that only have their id as output name are included in the output. - */ - private Map> outputNamesById() { - Map> outputNamesById = new LinkedHashMap<>(); - for (Consumer.Metric metric : consumers.getConsumersByMetric().keySet()) { - MetricId id = toMetricId(metric.name()); - outputNamesById.computeIfAbsent(id, unused -> new ArrayList<>()) - .add(metric.outputname()); - } - return outputNamesById; - } - - /** - * Extracts the node repository dimensions (role, state etc.) from the given packets. - * If the same dimension exists in multiple packets, this implementation gives no guarantees - * about which value is returned. - */ - public static Map extractConfigserverDimensions(Collection packets) { - Map dimensions = new HashMap<>(); - for (MetricsPacket.Builder packet : packets) { - dimensions.putAll(packet.build().dimensions()); - } - dimensions.keySet().retainAll(Set.of(ROLE_DIMENSION, STATE_DIMENSION, ORCHESTRATOR_STATE_DIMENSION)); - return dimensions; - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java deleted file mode 100644 index 41a8c3d414e..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/HealthMetric.java +++ /dev/null @@ -1,56 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -/** - * @author Jo Kristian Bergum - */ -public class HealthMetric { - private final String message; - private final String status; - private final boolean isAlive; - - private HealthMetric(String status, String message, boolean isAlive) { - this.message = message; - this.status = status; - this.isAlive = isAlive; - } - - public static HealthMetric get(String status, String message) { - if (status == null) { - status = ""; - } - if (message == null) { - message = ""; - } - status = status.toLowerCase(); - - if (status.equals("up") || status.equals("ok")) { - return new HealthMetric(status, message, true); - } else { - return new HealthMetric(status, message, false); - } - } - - public static HealthMetric getFailed(String message) { - return new HealthMetric("down", message, false); - } - - public static HealthMetric getOk(String message) { - return new HealthMetric("up", message, true); - } - - public String getMessage() { - return this.message; - } - - public String getStatus() { - return this.status; - } - - public boolean isOk() { - return this.isAlive; - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java deleted file mode 100644 index 59fbe301a49..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metric.java +++ /dev/null @@ -1,124 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.DimensionId; - -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Set; - -/** - * @author Jo Kristian Bergum - */ -public class Metric { - private final long time; - private final Number value; - private final String description; - private String name; - private Map dimensions; - private Set consumers; - - /** - * Creates a new metric instance - * - * @param name The metric name. E.g 'documents' - * @param value The numeric value - * @param time The timestamp of this metric in seconds - */ - public Metric(String name, Number value, long time, Map dimensions, String description) { - this.time = time; - this.value = value; - this.name = name; - this.dimensions = dimensions; - this.description = description; - } - - public Metric(String name, Number value, long timestamp) { - this(name, value, timestamp, Collections.emptyMap(), ""); - } - - public Metric(String name, Number value) { - this(name, value, System.currentTimeMillis() / 1000); - } - - public void setDimensions(Map dimensions) { - this.dimensions = dimensions; - } - - /** - * @return A map of the dimensions registered for this metric - */ - public Map getDimensions() { return dimensions; } - - public void setConsumers(Set consumers) { this.consumers = consumers; } - - /** - * @return The consumers this metric should be routed to. - */ - public Set getConsumers() { return consumers; } - - /** - * @return The number that this metric name represent - */ - public Number getValue() { - return value; - } - - /** - * Set the name of this metric - * - * @param name The name to use for this metric - */ - public void setName(String name) { - this.name = name; - } - - /** - * @return The name of the metric - */ - public String getName() { - return name; - } - - /** - * @return The UTC timestamp for when this metric was collected - */ - public long getTimeStamp() { - return this.time; - } - - @Override - public String toString() { - return "Metric{" + - "time=" + time + - ", name=" + name + - ", value='" + value + '\'' + - ", dimensions=" + dimensions + - '}'; - } - - @Override - public Metric clone() { - return new Metric(name, value, time, new LinkedHashMap<>(dimensions), getDescription()); - } - - /** - * @return the description of this metric - */ - public String getDescription() { - return this.description; - } - - /** Return an adjusted (rounded up) time if necessary */ - public static long adjustTime(long timestamp, long now) { - if ((now == (timestamp+1)) && ((now % 60) == 0)) { - return now; - } - return timestamp; - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java deleted file mode 100644 index ca611368730..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/Metrics.java +++ /dev/null @@ -1,111 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Once a getter is called, the instance is frozen and no more metrics can be added. - * - * @author Unknown - */ -// TODO: remove timestamp, only used as temporary storage. -// TODO: instances of this class can probably be replaced by a simple freezable map. -public class Metrics { - private final List metrics = new ArrayList<>(); - private long timestamp; - private boolean isFrozen = false; - - public Metrics() { - this(System.currentTimeMillis() / 1000L); - } - - public Metrics(long timestamp) { - this.timestamp = timestamp; - } - - private void ensureNotFrozen() { - if (isFrozen) throw new IllegalStateException("Frozen Metrics cannot be modified!"); - - } - - public long getTimeStamp() { - return this.timestamp; - } - - /** - * Update the timestamp - * - * @param timestamp IN UTC seconds resolution - */ - public void setTimeStamp(long timestamp) { - ensureNotFrozen(); - this.timestamp = timestamp; - } - - public void add(Metric m) { - ensureNotFrozen(); - this.timestamp = m.getTimeStamp(); - this.metrics.add(m); - } - - /** - * Get the size of the metrics covered. Note that this might also contain expired metrics - * - * @return size of metrics - */ - public int size() { - return this.metrics.size(); - } - - /** - * TODO: Remove, might be multiple metrics with same name but different dimensions - * - * @param key metric name - * @return the metric, or null - */ - public Metric getMetric(String key) { - isFrozen = true; - for (Metric m: metrics) { - if (m.getName().equals(key)) { - return m; - } - } - return null; - } - - public List getMetrics() { - isFrozen = true; - return Collections.unmodifiableList(metrics); - } - - - /** - * Get a single metric based on the metric name - * TODO: Remove, might be multiple metrics with same name, but different - * - * @param key metric name - * @return The value or null if metric was not found or expired - */ - public Number get(String key) { - isFrozen = true; - Metric m = getMetric(key); - if (m != null) { - return m.getValue(); - } - return null; - } - - public String toString() { - StringBuilder sb = new StringBuilder(); - for (Metric m : metrics) { - sb.append(m.getName()).append(":").append(m.getValue()).append("\n"); - } - return sb.toString(); - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java deleted file mode 100644 index 8858e21486a..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/MetricsFormatter.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -import ai.vespa.metricsproxy.service.VespaService; - -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; -import java.util.Locale; - -/** - * Format metrics as required by users of the "getMetricsById" rpc method. - * - * @author Unknown - */ -public class MetricsFormatter { - private final boolean includeServiceName; - private final boolean isSystemMetric; - private final DecimalFormat df = new DecimalFormat("0.000", new DecimalFormatSymbols(Locale.ENGLISH)); - - public MetricsFormatter(boolean includeServiceName, boolean isSystemMetric) { - this.includeServiceName = includeServiceName; - this.isSystemMetric = isSystemMetric; - } - - public String format(VespaService service, String name, Number value) { - StringBuilder sb = new StringBuilder(); - - if (includeServiceName) { - sb.append(service.getServiceName()).append("."); - } - - if (isSystemMetric) - sb.append(toSystemServiceId(service.getConfigId())); - else - sb.append(toServiceId(service.getConfigId())); - - sb.append(".") - .append(formatMetricName(name)) - .append("="); - - if (value instanceof Double) { - sb.append(df.format(value.doubleValue())); - } else { - sb.append(value.toString()); - } - - return sb.toString(); - } - - private static String formatMetricName(String name) { - name = name.replaceAll("\"", ""); - name = name.replaceAll("\\.", "_"); - return name; - } - - // E.g. container/qrserver.1 -> 'container.qrserver.1' - private static String toServiceId(String configId) { - return "'" + configId.replace("/", ".") + "'"; - } - - // E.g. container/qrserver.1 -> container.'qrserver.1' - private static String toSystemServiceId(String configId) { - String name = configId.replace("/", "."); - name = name.replaceFirst("\\.", ".'") + "'"; - return name; - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java deleted file mode 100644 index ae40f672a32..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/ApplicationDimensions.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.dimensions; - -import ai.vespa.metricsproxy.metric.model.DimensionId; - -import java.util.Map; - -import static ai.vespa.metricsproxy.core.MetricsConsumers.toUnmodifiableLinkedMap; -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; - -/** - * Application-specific but node-agnostic dimensions. - * - * @author gjoranv - */ -public class ApplicationDimensions { - - private final Map dimensions; - - public ApplicationDimensions(ApplicationDimensionsConfig config) { - dimensions = config.dimensions().entrySet().stream().collect( - toUnmodifiableLinkedMap(e -> toDimensionId(e.getKey()), Map.Entry::getValue)); - } - - public Map getDimensions() { return dimensions; } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java deleted file mode 100644 index d2c1799e148..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/NodeDimensions.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.dimensions; - -import ai.vespa.metricsproxy.metric.model.DimensionId; - -import java.util.Map; - -import static ai.vespa.metricsproxy.core.MetricsConsumers.toUnmodifiableLinkedMap; -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; - -/** - * Node-specific metric dimensions. - * - * @author gjoranv - */ -public class NodeDimensions { - - private final Map dimensions; - - public NodeDimensions(NodeDimensionsConfig config) { - dimensions = config.dimensions().entrySet().stream().collect( - toUnmodifiableLinkedMap(e -> toDimensionId(e.getKey()), Map.Entry::getValue)); - } - - public Map getDimensions() { return dimensions; } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java deleted file mode 100644 index f4e5f74313a..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -@ExportPackage -package ai.vespa.metricsproxy.metric.dimensions; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java deleted file mode 100644 index 0d7acd5f354..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java +++ /dev/null @@ -1,38 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model; - -import java.util.Objects; - -/** - * @author gjoranv - */ -public class ConsumerId { - public final String id; - private ConsumerId(String id) { this.id = id; } - - public static ConsumerId toConsumerId(String id) { return new ConsumerId(id); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ConsumerId that = (ConsumerId) o; - return Objects.equals(id, that.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public String toString() { - return "ConsumerId{" + - "id='" + id + '\'' + - '}'; - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java deleted file mode 100644 index 03f4c2c01ff..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/DimensionId.java +++ /dev/null @@ -1,38 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model; - -import java.util.Objects; - -/** - * @author gjoranv - */ -public class DimensionId { - - public final String id; - private DimensionId(String id) { this.id = id; } - - public static DimensionId toDimensionId(String id) { return new DimensionId(id); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - DimensionId that = (DimensionId) o; - return Objects.equals(id, that.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public String toString() { - return "DimensionId{" + - "id='" + id + '\'' + - '}'; - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java deleted file mode 100644 index c93735c7fca..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricId.java +++ /dev/null @@ -1,39 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model; - -import java.util.Objects; - -/** - * @author gjoranv - */ -public class MetricId { - - public final String id; - private MetricId(String id) { this.id = id; } - - public static MetricId toMetricId(String id) { return new MetricId(id); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - MetricId metricId = (MetricId) o; - return Objects.equals(id, metricId.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public String toString() { - return "MetricId{" + - "id='" + id + '\'' + - '}'; - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java deleted file mode 100644 index fa45c6251f6..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model; - -import ai.vespa.metricsproxy.metric.Metric; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; - -import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; -import static java.util.Collections.unmodifiableList; -import static java.util.Collections.unmodifiableMap; -import static java.util.stream.Collectors.joining; - -/** - * Represents a packet of metrics (with meta information) that belong together because they: - *

    - *
  • share both the same dimensions and consumers, AND
    • - *
  • represent the same source, e.g. a vespa service or the system hardware.
    • - *
- * - * @author gjoranv - */ -public class MetricsPacket { - public final int statusCode; - public final String statusMessage; - public final long timestamp; - public final ServiceId service; - private final Map metrics; - private final Map dimensions; - private final List consumers; - - private MetricsPacket(int statusCode, String statusMessage, long timestamp, ServiceId service, - Map metrics, Map dimensions, Set consumers ) { - this.statusCode = statusCode; - this.statusMessage = statusMessage; - this.timestamp = timestamp; - this.service = service; - this.metrics = metrics; - this.dimensions = dimensions; - this.consumers = new ArrayList<>(consumers); - } - - public Map metrics() { - return unmodifiableMap(metrics); - } - - public Map dimensions() { - return unmodifiableMap(dimensions); - } - - public List consumers() { - return unmodifiableList(consumers); - } - - @Override - public String toString() { - return "MetricsPacket{" + - "statusCode=" + statusCode + - ", statusMessage='" + statusMessage + '\'' + - ", timestamp=" + timestamp + - ", service=" + service.id + - ", metrics=" + idMapToString(metrics, id -> id.id) + - ", dimensions=" + idMapToString(dimensions, id -> id.id) + - ", consumers=" + consumers.stream().map(id -> id.id).collect(joining(",", "[", "]")) + - '}'; - } - - private static String idMapToString(Map map, Function idMapper) { - return map.entrySet().stream() - .map(entry -> idMapper.apply(entry.getKey()) + "=" + entry.getValue()) - .collect(joining(",", "{", "}")); - } - - public static class Builder { - // Set sensible defaults here, and use null guard in all setters. - // Except for 'service' for which we require an explicit non-null value. - private ServiceId service; - private int statusCode = 0; - private String statusMessage = ""; - private long timestamp = 0L; - private Map metrics = new LinkedHashMap<>(); - private final Map dimensions = new LinkedHashMap<>(); - private final Set consumers = new LinkedHashSet<>(); - - public Builder(ServiceId service) { - Objects.requireNonNull(service, "Service cannot be null."); - this.service = service; - } - - public Builder service(ServiceId service) { - if (service == null) throw new IllegalArgumentException("Service cannot be null."); - this.service = service; - return this; - } - - public Builder statusCode(Integer statusCode) { - if (statusCode != null) this.statusCode = statusCode; - return this; - } - - public Builder statusMessage(String statusMessage) { - if (statusMessage != null) this.statusMessage = statusMessage; - return this; - } - - public Builder timestamp(Long timestamp) { - if (timestamp != null) this.timestamp = timestamp; - return this; - } - - public Builder putMetrics(Collection extraMetrics) { - if (extraMetrics != null) - extraMetrics.forEach(metric -> metrics.put(toMetricId(metric.getName()), - metric.getValue().doubleValue())); - return this; - } - - public Builder putMetric(MetricId id, Number value) { - metrics.put(id, value); - return this; - } - - public Builder retainMetrics(Set idsToRetain) { - metrics.keySet().retainAll(idsToRetain); - return this; - } - - public Builder applyOutputNames(Map> outputNamesById) { - Map newMetrics = new LinkedHashMap<>(); - outputNamesById.forEach((id, outputNames) -> { - if (metrics.containsKey(id)) - outputNames.forEach(outputName -> newMetrics.put(toMetricId(outputName), metrics.get(id))); - }); - metrics = newMetrics; - return this; - } - - public Builder putDimension(DimensionId id, String value) { - dimensions.put(id, value); - return this; - } - - public Builder putDimensions(Map extraDimensions) { - if (extraDimensions != null) dimensions.putAll(extraDimensions); - return this; - } - - public Builder putDimensionsIfAbsent(Map extraDimensions) { - if (extraDimensions != null) extraDimensions.forEach(dimensions::putIfAbsent); - return this; - } - - public Builder addConsumers(Set extraConsumers) { - if (extraConsumers != null) consumers.addAll(extraConsumers); - return this; - } - - public MetricsPacket build() { - return new MetricsPacket(statusCode, statusMessage, timestamp, service, metrics, dimensions, consumers); - } - - public boolean hasMetrics() { - return ! metrics.isEmpty(); - } - - public Instant getTimestamp() { return Instant.ofEpochSecond(timestamp); } - - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java deleted file mode 100644 index b61ead75b72..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ServiceId.java +++ /dev/null @@ -1,39 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model; - -import java.util.Objects; - -/** - * @author gjoranv - */ -public class ServiceId { - - public final String id; - private ServiceId(String id) { this.id = id; } - - public static ServiceId toServiceId(String id) { return new ServiceId(id); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ServiceId serviceId = (ServiceId) o; - return Objects.equals(id, serviceId.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public String toString() { - return "ServiceId{" + - "id='" + id + '\'' + - '}'; - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonUtil.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonUtil.java deleted file mode 100644 index f48e5759528..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/JsonUtil.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model.json; - -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableMap; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static com.yahoo.stream.CustomCollectors.toLinkedMap; -import static java.util.Collections.emptyList; -import static java.util.logging.Level.WARNING; - -/** - * @author gjoranv - */ -public class JsonUtil { - private static final Logger log = Logger.getLogger(JsonUtil.class.getName()); - - static final String YAMAS_ROUTING = "yamas"; - - public static MetricsPacket.Builder toMetricsPacketBuilder(YamasJsonModel jsonModel) { - if (jsonModel.application == null) - throw new IllegalArgumentException("Service id cannot be null"); - - return new MetricsPacket.Builder(toServiceId(jsonModel.application)) - .statusCode(jsonModel.status_code) - .statusMessage(jsonModel.status_msg) - .timestamp(jsonModel.timestamp) - .putMetrics(jsonModel.getMetricsList()) - .putDimensions(jsonModel.getDimensionsById()) - .addConsumers(jsonModel.getYamasConsumers()); - } - - public static YamasArrayJsonModel toYamasArray(Collection metricsPackets) { - YamasArrayJsonModel yamasArray = toYamasArray(metricsPackets, false); - - // Add a single status object at the end - yamasArray.metrics.stream().findFirst().map(YamasJsonModel::getYamasConsumers) - .ifPresent(consumers -> yamasArray.add(getStatusYamasModel("Data collected successfully", 0, consumers))); - return yamasArray; - } - - public static YamasArrayJsonModel toYamasArray(Collection metricsPackets, boolean addStatus) { - YamasArrayJsonModel yamasArray = new YamasArrayJsonModel(); - metricsPackets.forEach(packet -> yamasArray.add(toYamasModel(packet, addStatus))); - return yamasArray; - } - - /** - * Converts the given json formatted string to a list of metrics packet builders. - * Note that this method returns an empty list if an IOException occurs, - * and logs a warning as a side effect. - */ - public static List toMetricsPackets(String jsonString) { - List packets = new ArrayList<>(); - try { - JsonParser jp = new JsonFactory().createParser(jsonString); - jp.setCodec(new ObjectMapper()); - while (jp.nextToken() != null) { - YamasJsonModel jsonModel = jp.readValueAs(YamasJsonModel.class); - packets.add(toMetricsPacketBuilder(jsonModel)); - } - return packets; - } catch (IOException e) { - log.log(WARNING, "Could not create metrics packet from string:\n" + jsonString, e); - return emptyList(); - } - } - - private static YamasJsonModel getStatusYamasModel(String statusMessage, int statusCode, Collection consumers) { - YamasJsonModel model = new YamasJsonModel(); - model.status_code = statusCode; - model.status_msg = statusMessage; - model.application = "yms_check_vespa"; - model.routing = ImmutableMap.of(YAMAS_ROUTING, toYamasJsonNamespaces(consumers)); - return model; - } - - private static YamasJsonModel toYamasModel(MetricsPacket packet, boolean addStatus) { - YamasJsonModel model = new YamasJsonModel(); - - if (addStatus) { - model.status_code = packet.statusCode; - model.status_msg = packet.statusMessage; - } - - model.application = packet.service.id; - model.timestamp = (packet.timestamp == 0L) ? null : packet.timestamp; - - if (packet.metrics().isEmpty()) model.metrics = null; - else { - model.metrics = packet.metrics().entrySet().stream().collect( - toLinkedMap(id2metric -> id2metric.getKey().id, - id2metric -> id2metric.getValue().doubleValue())); - } - - if (packet.dimensions().isEmpty()) model.dimensions = null; - else { - model.dimensions = packet.dimensions().entrySet().stream().collect( - toLinkedMap(id2dim -> id2dim.getKey().id, - Map.Entry::getValue)); - } - - if (packet.consumers().isEmpty()) model.routing = null; - else model.routing = ImmutableMap.of(YAMAS_ROUTING, toYamasJsonNamespaces(packet.consumers())); - - return model; - } - - private static YamasJsonModel.YamasJsonNamespace toYamasJsonNamespaces(Collection consumers) { - YamasJsonModel.YamasJsonNamespace namespaces = new YamasJsonModel.YamasJsonNamespace(); - namespaces.namespaces = consumers.stream().map(consumer -> consumer.id).collect(Collectors.toList()); - return namespaces; - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java deleted file mode 100644 index fdac0521256..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasArrayJsonModel.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model.json; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.Version; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.module.SimpleModule; - -import java.io.IOException; -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; - -/** - * Datamodel for the metricsproxy representation of multiple yamas checks. - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class YamasArrayJsonModel { - @JsonProperty("metrics") - public final List metrics = new ArrayList<>(); - - public void add(List results) { - metrics.addAll(results); - } - - public void add(YamasJsonModel result) { - metrics.add(result); - } - - public void add(YamasArrayJsonModel array) { - metrics.addAll(array.metrics); - } - - /** - * Convenience method to serialize. - *

- * Custom floating point serializer to avoid scientifc notation - * - * @return Serialized json - */ - public String serialize() { - ObjectMapper mapper = new ObjectMapper(); - SimpleModule module = new SimpleModule("DoubleSerializer", - new Version(1, 0, 0, "", null, null)); - module.addSerializer(Double.class, new DoubleSerializer()); - mapper.registerModule(module); - - if (metrics.size() > 0) { - try { - return mapper.writeValueAsString(this); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } - } - - return "{}"; // Backwards compatability - } - - public class DoubleSerializer extends JsonSerializer { - @Override - public void serialize(Double value, JsonGenerator jgen, - SerializerProvider provider) throws IOException, JsonProcessingException { - DecimalFormat df = new DecimalFormat("#.####", new DecimalFormatSymbols(Locale.ENGLISH)); - df.setMaximumFractionDigits(13); - jgen.writeNumber(df.format(value)); - } - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java deleted file mode 100644 index 5fdbe9577be..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModel.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model.json; - -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.yahoo.stream.CustomCollectors.toLinkedMap; -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static java.util.Collections.emptySet; - -/** - * Datamodel for Yamas execute output - *

- * Used to read from original yamas checks and annotate with routing information. - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonPropertyOrder({ "status_code", "timestamp", "application", "metrics", "dimensions", "routing", "status_msg"}) -public class YamasJsonModel { - @JsonProperty("status_code") - public Integer status_code; - @JsonProperty("status_msg") - public String status_msg; - @JsonProperty("timestamp") - public Long timestamp; - @JsonProperty("application") - public String application; - @JsonProperty("metrics") - public Map metrics; - @JsonProperty("dimensions") - public Map dimensions; - @JsonProperty("routing") - public Map routing; - - public static class YamasJsonNamespace { - @JsonProperty("namespaces") - public List namespaces; - } - - // NOTE: do not rename to 'setMetrics', as jackson will try to use it. - public void resetMetrics(List newMetrics) { - metrics = new LinkedHashMap<>(); - newMetrics.forEach(metric -> metrics.put(metric.getName(), metric.getValue().doubleValue())); - } - - /** - * Convenience method to add targets to the routing object - * - * @param names Namespaces E.g "Vespa" - */ - public void addRouting(Set names) { - // Setup routing structure if not already existing - if (routing == null) { - routing = new HashMap<>(); - } - - if (! routing.containsKey("yamas")) { - routing.put("yamas", new YamasJsonModel.YamasJsonNamespace()); - } - YamasJsonModel.YamasJsonNamespace namespace = routing.get("yamas"); - - if (namespace.namespaces == null) { - namespace.namespaces = new ArrayList<>(); - } - - namespace.namespaces.addAll(names.stream().map(consumer -> consumer.id).collect(Collectors.toList())); - } - - /** - * Convenience method to add dimensions - */ - public void addDimensions(Map additionalDimensions, boolean replace) { - additionalDimensions.forEach((k,v) -> { - addDimension(k.id, v, replace); - }); - } - - /** - * Convenience method to add dimensions - */ - public void addDimension(String key, String value, boolean replace) { - if (dimensions == null) { - dimensions = new HashMap<>(); - } - if (!dimensions.containsKey(key) || replace) { - dimensions.put(key, value); - } - } - - List getMetricsList() { - if (metrics == null) return emptyList(); - - return metrics.keySet().stream() - .map(name -> new Metric(name, metrics.get(name))) - .collect(Collectors.toList()); - } - - Map getDimensionsById() { - if (dimensions == null) return emptyMap(); - - return dimensions.keySet().stream().collect(toLinkedMap(DimensionId::toDimensionId, - name -> dimensions.get(name))); - } - - Set getYamasConsumers() { - if (routing == null || routing.get("yamas") == null) return emptySet(); - - return routing.get("yamas").namespaces.stream() - .map(ConsumerId::toConsumerId) - .collect(Collectors.toCollection(LinkedHashSet::new)); - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java deleted file mode 100644 index c72f2484f8c..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -@ExportPackage -package ai.vespa.metricsproxy.metric; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java deleted file mode 100644 index e7feab9926d..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.rpc; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.jrt.Acceptor; -import com.yahoo.jrt.ListenFailedException; -import com.yahoo.jrt.Method; -import com.yahoo.jrt.Spec; -import com.yahoo.jrt.Supervisor; -import com.yahoo.jrt.Transport; - -import java.util.logging.Logger; - -import static com.yahoo.log.LogLevel.DEBUG; -import static java.util.logging.Level.INFO; - -/** - * Contains the connector for the rpc server, to prevent it from going down after component reconfiguration. - * This will only be recreated if the rpc port changes, which should never happen under normal circumstances. - * - * @author gjoranv - */ -public class RpcConnector extends AbstractComponent { - private static final Logger log = Logger.getLogger(RpcConnector.class.getName()); - - private final Supervisor supervisor = new Supervisor(new Transport()); - private final Acceptor acceptor; - - public RpcConnector(RpcConnectorConfig config) { - Spec spec = new Spec(config.port()); - try { - acceptor = supervisor.listen(spec); - log.log(DEBUG, "Listening on " + spec.host() + ":" + spec.port()); - } catch (ListenFailedException e) { - stop(); - log.log(INFO, "Failed listening at " + spec.host() + ":" + spec.port()); - throw new RuntimeException("Could not listen at " + spec, e); - } - } - - /** - * Adds a method. If a method with the same name already exists, it will be replaced. - * @param method The method to add. - */ - public void addMethod(Method method) { - supervisor.addMethod(method); - } - - public void stop() { - acceptor.shutdown().join(); - supervisor.transport().shutdown().join(); - } - - @Override - public void deconstruct() { - stop(); - super.deconstruct(); - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java deleted file mode 100644 index e0e0e7a3f87..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.rpc; - -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.service.VespaService; -import ai.vespa.metricsproxy.service.VespaServices; -import com.yahoo.jrt.ErrorCode; -import com.yahoo.jrt.Method; -import com.yahoo.jrt.Request; -import com.yahoo.jrt.Spec; -import com.yahoo.jrt.StringValue; -import com.yahoo.jrt.Supervisor; -import com.yahoo.jrt.Transport; - -import java.time.Instant; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static ai.vespa.metricsproxy.metric.model.json.JsonUtil.toMetricsPackets; -import static ai.vespa.metricsproxy.metric.model.json.JsonUtil.toYamasArray; -import static com.yahoo.collections.CollectionUtil.mkString; -import static com.yahoo.log.LogLevel.DEBUG; -import static java.util.logging.Level.INFO; -import static java.util.logging.Level.WARNING; - -/** - * Rpc server for the metrics proxy. - * - * When a new object is created after reconfiguration, it will claim ownership of the methods - * in the given {@link RpcConnector}. This is ok because at the time this component is created, - * all components it depends on are already created. - * - * @author gjoranv - */ -public class RpcServer { - - private static final Logger log = Logger.getLogger(RpcServer.class.getName()); - - private static int LOG_SPENT_TIME_LIMIT = 10 * 1000; // ms. same as default client RPC timeout used in rpc_invoke - - private final VespaServices vespaServices; - private final MetricsManager metricsManager; - - public RpcServer(RpcConnector connector, VespaServices vespaServices, MetricsManager metricsManager) { - this.vespaServices = vespaServices; - this.metricsManager = metricsManager; - addMethods(connector); - log.log(DEBUG, "RPC server created"); - } - - private void addMethods(RpcConnector connector) { - // Add/replace this method first to increase likelihood of getting extra metrics and global dimensions - connector.addMethod( - new Method("setExtraMetrics", "s", "", this::setExtraMetrics) - .methodDesc("Set extra metrics that will be added to output from getMetricsForYamas.") - .paramDesc(0, "metricsJson", "The metrics in json format")); - - connector.addMethod( - new Method("getMetricsById", "s", "s", this::getMetricsById) - .methodDesc("Get Vespa metrics for the service with the given Id") - .paramDesc(0, "id", "The id of the service") - .returnDesc(0, "ret", "Vespa metrics")); - - connector.addMethod( - new Method("getServices", "", "s", this::getServices) - .methodDesc("Get Vespa services monitored by this metrics proxy") - .returnDesc(0, "ret", "Vespa metrics")); - - connector.addMethod( - new Method("getMetricsForYamas", "s", "s", this::getMetricsForYamas) - .methodDesc("Get JSON formatted Vespa metrics for a given service name or 'all'") - .paramDesc(0, "service", "The vespa service name, or 'all'") - .returnDesc(0, "ret", "Vespa metrics")); - - connector.addMethod( - new Method("getHealthMetricsForYamas", "s", "s", this::getHealthMetricsForYamas) - .methodDesc("Get JSON formatted Health check for a given service name or 'all'") - .paramDesc(0, "service", "The vespa service name") - .returnDesc(0, "ret", "Vespa metrics")); - - connector.addMethod( - new Method("getAllMetricNamesForService", "ss", "s", this::getAllMetricNamesForService) - .methodDesc("Get metric names known for service ") - .paramDesc(0, "service", "The vespa service name'") - .paramDesc(1, "consumer", "The consumer'") - .returnDesc(0, "ret", "Metric names, one metric name per line")); - } - - void getAllMetricNamesForService(Request req) { - String service = req.parameters().get(0).asString(); - ConsumerId consumer = toConsumerId(req.parameters().get(1).asString()); - withExceptionHandling(req, () -> { - String metricNames = metricsManager.getMetricNamesForServiceAndConsumer(service, consumer); - req.returnValues().add(new StringValue(metricNames)); - }); - } - - void getMetricsById(Request req) { - String id = req.parameters().get(0).asString(); - withExceptionHandling(req, () -> { - String metricsString = metricsManager.getMetricsByConfigId(id); - req.returnValues().add(new StringValue(metricsString)); - }); - } - - - void getServices(Request req) { - withExceptionHandling(req, () -> { - String servicesString = metricsManager.getAllVespaServices(); - req.returnValues().add(new StringValue(servicesString)); - }); - } - - void getMetricsForYamas(Request req) { - Instant startTime = Instant.now(); - req.detach(); - String service = req.parameters().get(0).asString(); - log.log(DEBUG, () -> "getMetricsForYamas called at " + startTime + " with argument: " + service); - List services = vespaServices.getMonitoringServices(service); - log.log(DEBUG, () -> "Getting metrics for services: " + mkString(services, "[", ", ", "]")); - if (services.isEmpty()) setNoServiceError(req, service); - else withExceptionHandling(req, () -> { - List packets = metricsManager.getMetrics(services, startTime); - log.log(DEBUG,() -> "Returning metrics packets:\n" + mkString(packets, "\n")); - req.returnValues().add(new StringValue(toYamasArray(packets).serialize())); - }); - req.returnRequest(); - } - - void getHealthMetricsForYamas(Request req) { - req.detach(); - String service = req.parameters().get(0).asString(); - List services = vespaServices.getMonitoringServices(service); - if (services.isEmpty()) setNoServiceError(req, service); - else withExceptionHandling(req, () -> { - List packets = metricsManager.getHealthMetrics(services); - req.returnValues().add(new StringValue(toYamasArray(packets, true).serialize())); - }); - req.returnRequest(); - } - - void setExtraMetrics(Request req) { - String metricsJson = req.parameters().get(0).asString(); - log.log(DEBUG, "setExtraMetrics called with argument: " + metricsJson); - withExceptionHandling(req, () -> metricsManager.setExtraMetrics(toMetricsPackets(metricsJson))); - } - - private static void withExceptionHandling(Request req, ThrowingRunnable runnable) { - try { - TimeTracker timeTracker = new TimeTracker(req); - runnable.run(); - timeTracker.logSpentTime(); - } catch (Exception e) { - log.log(WARNING, "Got exception when running RPC command " + req.methodName(), e); - setMethodFailedError(req, e); - } catch (Error e) { - log.log(WARNING, "Got error when running RPC command " + req.methodName(), e); - setMethodFailedError(req, e); - } catch (Throwable t) { - log.log(WARNING, "Got throwable (non-error, non-exception) when running RPC command " + req.methodName(), t); - setMethodFailedError(req, t); - } - } - - private static void setMethodFailedError(Request req, Throwable t) { - String msg = "Request failed due to internal error: " + t.getClass().getName() + ": " + t.getMessage(); - req.setError(ErrorCode.METHOD_FAILED, msg); - req.returnValues().add(new StringValue("")); - } - - private static void setNoServiceError(Request req, String serviceName) { - String msg = "No service with name '" + serviceName + "'"; - req.setError(ErrorCode.BAD_REQUEST, msg); - req.returnValues().add(new StringValue("")); - } - - - private static class TimeTracker { - private final long startTime = System.currentTimeMillis(); - private final Request request; - - private TimeTracker(Request request) { - this.request = request; - } - - long spentTime() { - return System.currentTimeMillis() - startTime; - } - - private void logSpentTime() { - Level logLevel = DEBUG; - if (spentTime() > LOG_SPENT_TIME_LIMIT) { - logLevel = INFO; - } - if (log.isLoggable(logLevel)) { - log.log(logLevel, "RPC request '" + request.methodName() + "' with parameters '" + - request.parameters() + "' took " + spentTime() + " ms"); - } - } - } - - private interface ThrowingRunnable { - void run() throws Exception; - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java deleted file mode 100644 index 875b6190763..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import com.google.inject.Inject; -import com.yahoo.log.LogLevel; - -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Logger; - -/** - * Connects to the config sentinel and gets information like pid for the services on the node - */ -public class ConfigSentinelClient { - private final static Logger log = Logger.getLogger(ConfigSentinelClient.class.getName()); - - private final CmdClient client; - - @Inject - public ConfigSentinelClient() { - this.client = new CmdClient(); - } - - /** - * Update all services reading from config sentinel - * - * @param services The list of services - */ - synchronized void updateServiceStatuses(List services) { - try { - setStatus(services); - } catch (Exception e) { - log.log(LogLevel.ERROR, "Unable to update service pids from sentinel", e); - } - } - - - /** - * Update status - * - * @param s The service to update the status for - */ - public synchronized void ping(VespaService s) { - List services = new ArrayList<>(); - services.add(s); - log.log(LogLevel.DEBUG, "Ping for service " + s); - try { - setStatus(services); - } catch (Exception e) { - log.log(LogLevel.ERROR, "Unable to update service pids from sentinel", e); - } - } - - /** - * Update the status (pid check etc) - * - * @param services list of services - * @throws Exception if something went wrong - */ - protected synchronized void setStatus(List services) throws Exception { - InputStream in; - PrintStream out; - client.connect(); - - in = client.getInputStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(in)); - String line; - List updatedServices = new ArrayList<>(); - while ((line = reader.readLine()) != null) { - if (line.equals("")) { - break; - } - - VespaService s = parseServiceString(line, services); - if (s != null) { - updatedServices.add(s); - } - } - - //Check if there are services that were not found in output - //from the sentinel - for (VespaService s : services) { - if ((!s.getServiceName().equals("configserver")) && !updatedServices.contains(s)) { - log.log(LogLevel.DEBUG,"Service " + s + " is no longer found with sentinel - setting alive = false"); - s.setAlive(false); - } - } - - //Close streams - reader.close(); - client.disconnect(); - } - - protected static VespaService parseServiceString(String line, List services) { - String[] parts = line.split(" "); - if (parts.length < 3) - return null; - - String name = parts[0]; - int pid = -1; - String state = null; - VespaService service = null; - - for (VespaService s : services) { - if (s.getInstanceName().compareToIgnoreCase(name) == 0) { - service = s; - break; - } - } - - //Could not find this service - //nothing wrong with that as the check is invoked per line from sentinel - if (service == null) { - return service; - } - - for (int i = 1; i < parts.length; i++) { - String keyValue[] = parts[i].split("="); - - String key = keyValue[0]; - String value = keyValue[1]; - - if (key.equals("state")) { - state = value; - } else if (key.equals("pid")) { - pid = Integer.parseInt(value); - } - } - - if (state != null) { - service.setState(state); - if (pid >= 0 && "RUNNING".equals(state)) { - service.setAlive(true); - service.setPid(pid); - } else { - service.setAlive(false); - - } - } else { - service.setAlive(false); - } - return service; - } - - static class CmdClient { - Process proc; - // NOTE: hostname/port not used yet - void connect() { - String[] args = new String[]{"vespa-sentinel-cmd", "list"}; - try { - proc = Runtime.getRuntime().exec(args); - } catch (Exception e) { - log.log(LogLevel.WARNING, "could not run vespa-sentinel-cmd: "+e); - proc = null; - } - } - void disconnect() { - if (proc.isAlive()) { - proc.destroy(); - } - proc = null; - } - InputStream getInputStream() { - return (proc != null) - ? proc.getInputStream() - : new java.io.ByteArrayInputStream(new byte[0]); - } - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java deleted file mode 100644 index 312e7d5c0c1..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -class CpuJiffies { - private int cpuId; - private long jiffies; - - CpuJiffies(String line) { - parseLine(line); - } - - private void parseLine(String line) { - String elems[]; - String cpuId; - long jiffies; - - elems = line.split("\\s+"); - cpuId = elems[0].substring(3); - if (cpuId.length() == 0) { - this.cpuId = -1; - } else { - this.cpuId = Integer.parseInt(cpuId); - } - - jiffies = 0; - for (int i = 1; i < elems.length; i++) { - jiffies += Long.parseLong(elems[i].replaceAll("[\\n\\r]+", "")); - } - - this.jiffies = jiffies; - } - - public int getCpuId() { - return cpuId; - } - - public long getTotalJiffies() { - return jiffies; - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java deleted file mode 100644 index f87171a42dc..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyHealthMetricFetcher.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.metric.HealthMetric; - -/** - * Dummy class used for getting health status for a vespa service that has no HTTP service - * for getting health status - * - * @author hmusum - */ -public class DummyHealthMetricFetcher extends RemoteHealthMetricFetcher { - - /** - * @param service The service to fetch metrics from - */ - DummyHealthMetricFetcher(VespaService service) { - super(service, 0); - } - - /** - * Connect to remote service over http and fetch metrics - */ - public HealthMetric getHealth(int fetchCount) { - if (service.isAlive()) { - return HealthMetric.getOk("Service is running - pid check only"); - } else { - return HealthMetric.getFailed("Service is not running - pid check only"); - } - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java deleted file mode 100644 index f21d125e279..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/DummyMetricsFetcher.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.metric.Metrics; - -/** - * Dummy class used for getting health status for a vespa service that has no HTTP service - * for getting metrics - * - * @author hmusum - */ -public class DummyMetricsFetcher extends RemoteMetricsFetcher { - - /** - * @param service The service to fetch metrics from - */ - DummyMetricsFetcher(VespaService service) { - super(service, 0); - } - - /** - * Connect to remote service over http and fetch metrics - */ - public Metrics getMetrics(int fetchCount) { - return new Metrics(); - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java deleted file mode 100644 index 9094ef22c20..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ /dev/null @@ -1,94 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.util.http.VespaHttpClientBuilder; -import com.yahoo.log.LogLevel; -import com.yahoo.yolean.Exceptions; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.BasicResponseHandler; -import org.apache.http.impl.client.CloseableHttpClient; - -import java.io.IOException; -import java.net.URI; -import java.util.logging.Logger; - -/** - * HTTP client to get metrics or health data from a service - * - * @author hmusum - * @author bjorncs - */ -public abstract class HttpMetricFetcher { - private final static Logger log = Logger.getLogger(HttpMetricFetcher.class.getPackage().getName()); - public final static String STATE_PATH = "/state/v1/"; - final static String METRICS_PATH = STATE_PATH + "metrics"; - final static String HEALTH_PATH = STATE_PATH + "health"; - // The call to apache will do 3 retries. As long as we check the services in series, we can't have this too high. - public static int CONNECTION_TIMEOUT = 5000; - private final static int SOCKET_TIMEOUT = 60000; - private final URI url; - protected final VespaService service; - private static final CloseableHttpClient httpClient = createHttpClient(); - - - /** - * @param service The service to fetch metrics from - * @param port The port to use - */ - HttpMetricFetcher(VespaService service, int port, String path) { - this.service = service; - - String u = "http://localhost:" + port + path; - this.url = URI.create(u); - log.log(LogLevel.DEBUG, "Fetching metrics from " + u + " with timeout " + CONNECTION_TIMEOUT); - } - - String getJson() throws IOException { - log.log(LogLevel.DEBUG, "Connecting to url " + url + " for service '" + service + "'"); - return httpClient.execute(new HttpGet(url), new BasicResponseHandler()); - } - - public String toString() { - return this.getClass().getSimpleName() + " using " + url; - } - - String errMsgNoResponse(IOException e) { - return "Unable to get response from service '" + service + "': " + - Exceptions.toMessageString(e); - } - - void handleException(Exception e, String data, int timesFetched) { - logMessage("Unable to parse json '" + data + "' for service '" + service + "': " + - Exceptions.toMessageString(e), timesFetched); - } - - private void logMessage(String message, int timesFetched) { - if (service.isAlive() && timesFetched > 5) { - log.log(LogLevel.INFO, message); - } else { - log.log(LogLevel.DEBUG, message); - } - } - - void logMessageNoResponse(String message, int timesFetched) { - if (timesFetched > 5) { - log.log(LogLevel.WARNING, message); - } else { - log.log(LogLevel.INFO, message); - } - } - - private static CloseableHttpClient createHttpClient() { - return VespaHttpClientBuilder.create() - .setUserAgent("metrics-proxy-http-client") - .setDefaultRequestConfig(RequestConfig.custom() - .setConnectTimeout(CONNECTION_TIMEOUT) - .setSocketTimeout(SOCKET_TIMEOUT) - .build()) - .build(); - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java deleted file mode 100644 index 503f582a827..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java +++ /dev/null @@ -1,77 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.metric.HealthMetric; -import com.yahoo.log.LogLevel; -import org.json.JSONException; -import org.json.JSONObject; - -import java.io.IOException; -import java.util.logging.Logger; - -/** - * Fetch health status for a given vespa service - * - * @author Jo Kristian Bergum - */ -public class RemoteHealthMetricFetcher extends HttpMetricFetcher { - - private final static Logger log = Logger.getLogger(RemoteHealthMetricFetcher.class.getPackage().getName()); - - /** - * @param service The service to fetch metrics from - * @param port The port to use - */ - public RemoteHealthMetricFetcher(VespaService service, int port) { - super(service, port, HEALTH_PATH); - } - - /** - * Connect to remote service over http and fetch metrics - */ - public HealthMetric getHealth(int fetchCount) { - String data = "{}"; - try { - data = getJson(); - } catch (IOException e) { - logMessageNoResponse(errMsgNoResponse(e), fetchCount); - } - return createHealthMetrics(data, fetchCount); - } - - /** - * Connect to remote service over http and fetch metrics - */ - HealthMetric createHealthMetrics(String data, int fetchCount) { - HealthMetric healthMetric = HealthMetric.getFailed("Failed fetching status page for service"); - try { - healthMetric = parse(data); - } catch (Exception e) { - handleException(e, data, fetchCount); - } - return healthMetric; - } - - private HealthMetric parse(String data) { - if (data == null || data.isEmpty()) { - return HealthMetric.getFailed("Empty response from status page"); - } - try { - JSONObject o = new JSONObject(data); - JSONObject status = o.getJSONObject("status"); - String code = status.getString("code"); - String message = ""; - if (status.has("message")) { - message = status.getString("message"); - } - return HealthMetric.get(code, message); - - } catch (JSONException e) { - log.log(LogLevel.DEBUG, "Failed to parse json response from metrics page:" + e + ":" + data); - return HealthMetric.getFailed("Not able to parse json from status page"); - } - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java deleted file mode 100644 index a606ec7d8cd..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java +++ /dev/null @@ -1,129 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.Metrics; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; - -/** - * Fetch metrics for a given vespa service - * - * @author Jo Kristian Bergum - */ -public class RemoteMetricsFetcher extends HttpMetricFetcher { - /** - * @param service The service to fetch metrics from - * @param port The port to use - */ - RemoteMetricsFetcher(VespaService service, int port) { - super(service, port, METRICS_PATH); - } - - /** - * Connect to remote service over http and fetch metrics - */ - public Metrics getMetrics(int fetchCount) { - String data = "{}"; - try { - data = getJson(); - } catch (IOException e) { - logMessageNoResponse(errMsgNoResponse(e), fetchCount); - } - - return createMetrics(data, fetchCount); - } - - /** - * Connect to remote service over http and fetch metrics - */ - public Metrics createMetrics(String data, int fetchCount) { - Metrics remoteMetrics = new Metrics(); - try { - remoteMetrics = parse(data); - } catch (Exception e) { - handleException(e, data, fetchCount); - } - - return remoteMetrics; - } - - Metrics parse(String data) throws JSONException { - JSONObject o = new JSONObject(data); - if (!(o.has("metrics"))) { - return new Metrics(); //empty - } - - JSONObject metrics = o.getJSONObject("metrics"); - JSONArray values; - long timestamp; - - try { - JSONObject snapshot = metrics.getJSONObject("snapshot"); - timestamp = (long) snapshot.getDouble("to"); - values = metrics.getJSONArray("values"); - } catch (JSONException e) { - // snapshot might not have been produced. Do not throw exception into log - return new Metrics(); - } - long now = System.currentTimeMillis() / 1000; - timestamp = Metric.adjustTime(timestamp, now); - Metrics m = new Metrics(timestamp); - - Map noDims = Collections.emptyMap(); - Map> uniqueDimensions = new HashMap<>(); - for (int i = 0; i < values.length(); i++) { - JSONObject metric = values.getJSONObject(i); - String name = metric.getString("name"); - String description = ""; - - if (metric.has("description")) { - description = metric.getString("description"); - } - - Map dim = noDims; - if (metric.has("dimensions")) { - JSONObject dimensions = metric.getJSONObject("dimensions"); - StringBuilder sb = new StringBuilder(); - for (Iterator it = dimensions.keys(); it.hasNext(); ) { - String k = (String) it.next(); - String v = dimensions.getString(k); - sb.append(toDimensionId(k)).append(v); - } - if ( ! uniqueDimensions.containsKey(sb.toString())) { - dim = new HashMap<>(); - for (Iterator it = dimensions.keys(); it.hasNext(); ) { - String k = (String) it.next(); - String v = dimensions.getString(k); - dim.put(toDimensionId(k), v); - } - uniqueDimensions.put(sb.toString(), Collections.unmodifiableMap(dim)); - } - dim = uniqueDimensions.get(sb.toString()); - } - - JSONObject aggregates = metric.getJSONObject("values"); - for (Iterator it = aggregates.keys(); it.hasNext(); ) { - String aggregator = (String) it.next(); - Number value = (Number) aggregates.get(aggregator); - StringBuilder metricName = (new StringBuilder()).append(name).append(".").append(aggregator); - m.add(new Metric(metricName.toString(), value, timestamp, dim, description)); - } - } - - return m; - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ServiceListener.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ServiceListener.java deleted file mode 100644 index 810eb5fb908..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ServiceListener.java +++ /dev/null @@ -1,14 +0,0 @@ -/* -* Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import java.util.List; - -/** - * @author Unknown - */ -public interface ServiceListener { - void setServices(List services); -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java deleted file mode 100644 index 9f6614668a5..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.Metrics; -import com.yahoo.log.LogLevel; - -import java.io.BufferedReader; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.ConcurrentHashMap; -import java.util.logging.Logger; - -/** - * Class to get data from the system and update the services at given intervals. - * TODO: rewrite to use ScheduledExecutorService or just call poll() directly. - * - * @author Eirik Nygaard - */ -public class SystemPoller implements ServiceListener { - final private static Logger log = Logger.getLogger(SystemPoller.class.getPackage().getName()); - - private final int pollingIntervalSecs; - private volatile List services; - - private final int memoryTypeVirtual = 0; - private final int memoryTypeResident = 1; - private final Map lastCpuJiffiesMetrics = new ConcurrentHashMap<>(); - private final Timer systemPollTimer; - - private long lastTotalCpuJiffies = -1; - - public SystemPoller(List services, int pollingIntervalSecs) { - this.services = services; - this.pollingIntervalSecs = pollingIntervalSecs; - systemPollTimer = new Timer("systemPollTimer", true); - } - - @Override - public void setServices(List services) { - log.log(LogLevel.DEBUG, "Setting services in SystemPoller to: " + services); - this.services = services; - } - - void stop() { - systemPollTimer.cancel(); - } - - /** - * Return memory usage for a given process, both resident and virtual is - * returned. - * - * @param service The instance to get memory usage for - * @return array[0] = memoryResident, array[1] = memoryVirtual (kB units) - */ - long[] getMemoryUsage(VespaService service) { - long size[] = new long[2]; - BufferedReader br; - int pid = service.getPid(); - - size[0] = 0; - size[1] = 0; - try { - br = new BufferedReader(new FileReader("/proc/" + pid + "/smaps")); - } catch (FileNotFoundException ex) { - markDead(service); - return size; - } - String line; - try { - while ((line = br.readLine()) != null) { - String[] elems = line.split("\\s+"); - /* Memory size is given in kB - convert to bytes by multiply with 1024*/ - if (line.startsWith("Rss:")) { - size[memoryTypeResident] += Long.parseLong(elems[1]) * 1024; - } else if (line.startsWith("Size:")) { - size[memoryTypeVirtual] += Long.parseLong(elems[1]) * 1024; - } - } - - br.close(); - } catch (IOException ex) { - log.log(LogLevel.DEBUG, "Unable to read line from smaps file", ex); - return size; - } - - return size; - } - - /** - * Mark a service as dead. - * - * @param service The service to mark as dead. - */ - private static void markDead(VespaService service) { - service.setAlive(false); - } - - /** - * Poll services for system metrics - */ - void poll() { - long startTime = System.currentTimeMillis(); - boolean someAlive = false; - - /* Don't do any work if there are no known services */ - if (services.isEmpty()) { - schedule(); - return; - } - - log.log(LogLevel.DEBUG, "Monitoring system metrics for " + services.size() + " services"); - - long sysJiffies = getNormalizedSystemJiffies(); - for (VespaService s : services) { - - - if(s.isAlive()) { - someAlive = true; - } - - Metrics metrics = new Metrics(); - log.log(LogLevel.DEBUG, "Current size of system metrics for service " + s + " is " + metrics.size()); - - long[] size = getMemoryUsage(s); - log.log(LogLevel.DEBUG, "Updating memory metric for service " + s); - - metrics.add(new Metric("memory_virt", size[memoryTypeVirtual], startTime / 1000)); - metrics.add(new Metric("memory_rss", size[memoryTypeResident], startTime / 1000)); - - long procJiffies = getPidJiffies(s); - if (lastTotalCpuJiffies >= 0 && lastCpuJiffiesMetrics.containsKey(s)) { - long last = lastCpuJiffiesMetrics.get(s); - long diff = procJiffies - last; - - if (diff >= 0) { - metrics.add(new Metric("cpu", 100 * ((double) diff) / (sysJiffies - lastTotalCpuJiffies), startTime / 1000)); - } - } - lastCpuJiffiesMetrics.put(s, procJiffies); - s.setSystemMetrics(metrics); - } - - lastTotalCpuJiffies = sysJiffies; - - // If none of the services were alive, reschedule in a short time - if (!someAlive) { - reschedule(System.currentTimeMillis() - startTime); - } else { - schedule(); - } - } - - long getPidJiffies(VespaService service) { - BufferedReader in; - String line; - String[] elems; - int pid = service.getPid(); - - try { - in = new BufferedReader(new FileReader("/proc/" + pid + "/stat")); - } catch (FileNotFoundException ex) { - log.log(LogLevel.DEBUG, "Unable to find pid in proc directory " + pid); - service.setAlive(false); - return 0; - } - - try { - line = in.readLine(); - in.close(); - } catch (IOException ex) { - log.log(LogLevel.DEBUG, "Unable to read line from process stat file", ex); - return 0; - } - - elems = line.split(" "); - - /* Add user mode and kernel mode jiffies for the given process */ - return Long.parseLong(elems[13]) + Long.parseLong(elems[14]); - } - - long getNormalizedSystemJiffies() { - BufferedReader in; - String line; - ArrayList jiffies = new ArrayList<>(); - CpuJiffies total = null; - - try { - in = new BufferedReader(new FileReader("/proc/stat")); - } catch (FileNotFoundException ex) { - log.log(LogLevel.ERROR, "Unable to open stat file", ex); - return 0; - } - try { - while ((line = in.readLine()) != null) { - if (line.startsWith("cpu ")) { - total = new CpuJiffies(line); - } else if (line.startsWith("cpu")) { - jiffies.add(new CpuJiffies(line)); - } - } - - in.close(); - } catch (IOException ex) { - log.log(LogLevel.ERROR, "Unable to read line from stat file", ex); - return 0; - } - - /* Normalize so that a process that uses an entire CPU core will get 100% util */ - if (total != null) { - return total.getTotalJiffies() / jiffies.size(); - } else { - return 0; - } - } - - private void schedule(long time) { - try { - systemPollTimer.schedule(new PollTask(this), time); - } catch(IllegalStateException e){ - log.info("Tried to schedule task, but timer was already shut down."); - } - } - - public void schedule() { - schedule(pollingIntervalSecs * 1000); - } - - private void reschedule(long skew) { - long sleep = (pollingIntervalSecs * 1000) - skew; - - // Don't sleep less than 1 min - sleep = Math.max(60 * 1000, sleep); - schedule(sleep); - } - - - private static class PollTask extends TimerTask { - private final SystemPoller poller; - - PollTask(SystemPoller poller) { - this.poller = poller; - } - - @Override - public void run() { - poller.poll(); - } - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java deleted file mode 100644 index dea7b2b4809..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPollerProvider.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.core.MonitoringConfig; -import com.yahoo.container.di.componentgraph.Provider; - -/** - * @author gjoranv - */ -public class SystemPollerProvider implements Provider { - - private final SystemPoller poller; - - /** - * @param services The list of VespaService instances to monitor for System metrics - * @param monitoringConfig The interval in seconds between each polling. - */ - public SystemPollerProvider (VespaServices services, MonitoringConfig monitoringConfig) { - poller = new SystemPoller(services.getVespaServices(), 60 * monitoringConfig.intervalMinutes()); - poller.poll(); - } - - public void deconstruct() { - poller.stop(); - } - - public SystemPoller get() { - return poller; - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java deleted file mode 100644 index d3f674176b3..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaService.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.metric.HealthMetric; -import ai.vespa.metricsproxy.metric.Metrics; -import ai.vespa.metricsproxy.metric.model.DimensionId; - -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; - - -/** - * Represents a Vespa service - * - * @author jobergum - */ -public class VespaService implements Comparable { - - private static final Map EMPTY_DIMENSIONS = Collections.emptyMap(); - private static final String DEFAULT_MONITORING_PREFIX = "vespa"; - public static final String SEPARATOR = "."; - - private final String instanceName; - private final String configId; - private final String serviceName; - private final String monitoringPrefix; - private final Map dimensions; - - - private volatile int pid = -1; - private volatile String state = "UNKNOWN"; - - // Used to keep the last polled system metrics for service - private Metrics systemMetrics; - - private final int statePort; - - private final RemoteHealthMetricFetcher remoteHealthMetricFetcher; - private final RemoteMetricsFetcher remoteMetricsFetcher; - - private boolean isAlive; - - // Used to keep track of log level when health or metrics requests fail - private AtomicInteger metricsFetchCount = new AtomicInteger(0); - private AtomicInteger healthFetchCount = new AtomicInteger(0); - - - public static VespaService create(String name, String id, int statePort) { - return create(name,id, statePort, DEFAULT_MONITORING_PREFIX, EMPTY_DIMENSIONS); - } - - public static VespaService create(String name, String id, int statePort, String monitoringName, Map dimensions) { - String serviceName = name.replaceAll("\\d*$", ""); - return new VespaService(serviceName, name, id, statePort, monitoringName, dimensions); - } - - VespaService(String serviceName, String configId) { - this(serviceName, serviceName, configId); - } - - VespaService(String serviceName, String instanceName, String configId) { - this(serviceName, instanceName, configId, -1, DEFAULT_MONITORING_PREFIX, EMPTY_DIMENSIONS); - } - - private VespaService(String serviceName, String instanceName, String configId, - int statePort, String monitoringPrefix, - Map dimensions) { - this.serviceName = serviceName; - this.instanceName = instanceName; - this.monitoringPrefix = monitoringPrefix; - this.configId = configId; - this.statePort = statePort; - this.dimensions = dimensions; - this.systemMetrics = new Metrics(); - this.isAlive = false; - this.remoteMetricsFetcher = (this.statePort> 0) ? new RemoteMetricsFetcher(this, this.statePort) : new DummyMetricsFetcher(this); - this.remoteHealthMetricFetcher = (this.statePort > 0) ? new RemoteHealthMetricFetcher(this, this.statePort) : new DummyHealthMetricFetcher(this); - } - - /** - * The name used for this service in the monitoring system: - * monitoring-system-name.serviceName - */ - public String getMonitoringName() { - return monitoringPrefix + SEPARATOR + serviceName; - } - - @Override - public int compareTo(VespaService other) { - return this.getInstanceName().compareTo(other.getInstanceName()); - } - - /** - * Get the service name/type. E.g 'searchnode', but not 'searchnode2' - * - * @return the service name - */ - public String getServiceName() { - return this.serviceName; - } - - /** - * Get the instance name. E.g searchnode2 - * - * @return the instance service name - */ - public String getInstanceName() { - return this.instanceName; - } - - public Map getDimensions() { - return dimensions; - } - - /** - * @return The health of this service - */ - public HealthMetric getHealth() { - HealthMetric healthMetric = remoteHealthMetricFetcher.getHealth(healthFetchCount.get()); - healthFetchCount.getAndIncrement(); - return healthMetric; - } - - /** - * Gets the system metrics for this service - * - * @return System metrics - */ - public synchronized Metrics getSystemMetrics() { - return this.systemMetrics; - } - - /** - * Get the Metrics registered for this service. Metrics are fetched over HTTP - * if a metric http port has been defined, otherwise from log file - * - * @return the non-system metrics - */ - public Metrics getMetrics() { - Metrics remoteMetrics = remoteMetricsFetcher.getMetrics(metricsFetchCount.get()); - metricsFetchCount.getAndIncrement(); - return remoteMetrics; - } - - /** - * Gets the config id of this service - * - * @return the config id - */ - public String getConfigId() { - return configId; - } - - /** - * The current pid of this service - * - * @return The pid - */ - public int getPid() { - return this.pid; - } - - /** - * update the pid of this service - * - * @param pid The pid that this service runs as - */ - public void setPid(int pid) { - this.pid = pid; - } - - /** - * Get the string representation of the state of this service - * - * @return string representing the state of this service - obtained from config-sentinel - */ - public String getState() { - return state; - } - - /** - * Update the state of this service - * - * @param state the new state - */ - public void setState(String state) { - this.state = state; - } - - /** - * Check if this pid/service is running - * - * @return true if the service is alive (e.g the pid is running) - */ - public boolean isAlive() { - return (isAlive && (pid >= 0)); - } - - @Override - public String toString() { - return instanceName + ":" + pid + ":" + state + ":" + configId; - } - - public void setAlive(boolean alive) { - this.isAlive = alive; - } - - public synchronized void setSystemMetrics(Metrics systemMetrics) { - this.systemMetrics = systemMetrics; - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java deleted file mode 100644 index 2668c158ed6..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.core.MonitoringConfig; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import ai.vespa.metricsproxy.service.VespaServicesConfig.Service; -import com.google.common.annotations.VisibleForTesting; -import com.google.inject.Inject; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; - -import static ai.vespa.metricsproxy.core.MetricsConsumers.toUnmodifiableLinkedMap; -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static com.yahoo.log.LogLevel.DEBUG; - -/** - * Creates representations for the Vespa services running on the node, - * and provides methods for updating and getting them. - * - * @author gjoranv - */ -public class VespaServices { - private static final Logger log = Logger.getLogger(VespaServices.class.getName()); - - public static final String ALL_SERVICES = "all"; - - private final ConfigSentinelClient sentinel; - private final List services; - - @Inject - public VespaServices(VespaServicesConfig config, MonitoringConfig monitoringConfig, ConfigSentinelClient sentinel) { - this.services = createServices(config, monitoringConfig.systemName()); - this.sentinel = sentinel; - } - - @VisibleForTesting - public VespaServices(List services) { - this.services = services; - sentinel = null; - } - - private List createServices(VespaServicesConfig servicesConfig, String monitoringSystemName) { - List services = new ArrayList<>(); - for (Service s : servicesConfig.service()) { - log.log(DEBUG, "Re-configuring service " + s.name()); - VespaService vespaService = VespaService.create(s.name(), s.configId(), s.healthport(), monitoringSystemName, - createServiceDimensions(s)); - services.add(vespaService); - } - log.log(DEBUG, "Created new services: " + services.size()); - updateServices(services); - return services; - } - - /** - * Sets 'alive=false' for services that are no longer running. - * Note that the status is updated in-place for the given services. - */ - public void updateServices(List services) { - if (sentinel != null) { - log.log(DEBUG, "Updating services "); - sentinel.updateServiceStatuses(services); - } - } - - /** - * Get all known vespa services - * - * @return A list of VespaService objects - */ - public List getVespaServices() { - return Collections.unmodifiableList(services); - } - - /** - * @param id The configid - * @return A list with size 1 as there should only be one service with the given configid - */ - public List getInstancesById(String id) { - List myServices = new ArrayList<>(); - for (VespaService s : services) { - if (s.getConfigId().equals(id)) { - myServices.add(s); - } - } - - return myServices; - } - - /** - * Get services matching pattern for the name used in the monitoring system. - * - * @param service name in monitoring system + service name, without index, e.g: vespa.container - * @return A list of VespaServices - */ - public List getMonitoringServices(String service) { - if (service.equalsIgnoreCase(ALL_SERVICES)) - return services; - - List myServices = new ArrayList<>(); - for (VespaService s : services) { - log.log(DEBUG, () -> "getMonitoringServices. service=" + service + ", checking against " + s + ", which has monitoring name " + s.getMonitoringName()); - if (s.getMonitoringName().equalsIgnoreCase(service)) { - myServices.add(s); - } - } - - return myServices; - } - - private static Map createServiceDimensions(Service service) { - return service.dimension().stream().collect( - toUnmodifiableLinkedMap(dim -> toDimensionId(dim.key()), Service.Dimension::value)); - } - -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java deleted file mode 100644 index b478cdf8b5b..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -@ExportPackage -package ai.vespa.metricsproxy.service; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/metrics-proxy/src/main/resources/configdefinitions/application-dimensions.def b/metrics-proxy/src/main/resources/configdefinitions/application-dimensions.def deleted file mode 100644 index 41b384193e5..00000000000 --- a/metrics-proxy/src/main/resources/configdefinitions/application-dimensions.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package=ai.vespa.metricsproxy.metric.dimensions - -# Dimensions based on application properties -dimensions{} string diff --git a/metrics-proxy/src/main/resources/configdefinitions/consumers.def b/metrics-proxy/src/main/resources/configdefinitions/consumers.def deleted file mode 100644 index a1828b1320c..00000000000 --- a/metrics-proxy/src/main/resources/configdefinitions/consumers.def +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package=ai.vespa.metricsproxy.core - -# Consumers with metric definitions -consumer[].name string default="" -consumer[].metric[].name string -consumer[].metric[].description string default="" -consumer[].metric[].outputname string -consumer[].metric[].dimension[].key string -consumer[].metric[].dimension[].value string diff --git a/metrics-proxy/src/main/resources/configdefinitions/monitoring.def b/metrics-proxy/src/main/resources/configdefinitions/monitoring.def deleted file mode 100644 index 6fe24c86394..00000000000 --- a/metrics-proxy/src/main/resources/configdefinitions/monitoring.def +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package=ai.vespa.metricsproxy.core - -# The rate at which metrics are passed to the monitoring system. Currently (Apr 2019) only used by SystemPoller. -intervalMinutes int default=5 - -# The name used for this application in the monitoring system -systemName string default=vespa diff --git a/metrics-proxy/src/main/resources/configdefinitions/node-dimensions.def b/metrics-proxy/src/main/resources/configdefinitions/node-dimensions.def deleted file mode 100644 index 917cd165a9f..00000000000 --- a/metrics-proxy/src/main/resources/configdefinitions/node-dimensions.def +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package=ai.vespa.metricsproxy.metric.dimensions - -# Dimensions based on node properties -dimensions{} string diff --git a/metrics-proxy/src/main/resources/configdefinitions/rpc-connector.def b/metrics-proxy/src/main/resources/configdefinitions/rpc-connector.def deleted file mode 100644 index 7a8450780de..00000000000 --- a/metrics-proxy/src/main/resources/configdefinitions/rpc-connector.def +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package=ai.vespa.metricsproxy.rpc - -port int diff --git a/metrics-proxy/src/main/resources/configdefinitions/vespa-services.def b/metrics-proxy/src/main/resources/configdefinitions/vespa-services.def deleted file mode 100644 index 6fdacfffd42..00000000000 --- a/metrics-proxy/src/main/resources/configdefinitions/vespa-services.def +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package=ai.vespa.metricsproxy.service - -# Services with service id and the http port for the metrics page -service[].configId string -service[].name string default="" -service[].port int default=-1 -service[].healthport int default=-1 -service[].dimension[].key string -service[].dimension[].value string diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java deleted file mode 100644 index 5997f85a46f..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.stream.Collectors; - -/** - * @author gjoranv - */ -public class TestUtil { - - public static String getContents(String filename) { - InputStream in = TestUtil.class.getClassLoader().getResourceAsStream(filename); - if (in == null) { - throw new RuntimeException("File not found: " + filename); - } - return new BufferedReader(new InputStreamReader(in)).lines().collect(Collectors.joining("\n")); - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java deleted file mode 100644 index 81634323269..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/core/MetricsManagerTest.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.core; - -import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer; -import ai.vespa.metricsproxy.metric.ExternalMetrics; -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.Metrics; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; -import ai.vespa.metricsproxy.metric.model.DimensionId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.metricsproxy.service.DummyService; -import ai.vespa.metricsproxy.service.VespaService; -import ai.vespa.metricsproxy.service.VespaServices; -import com.google.common.collect.ImmutableList; -import org.junit.Before; -import org.junit.Test; - -import java.time.Instant; -import java.util.List; -import java.util.Map; - -import static ai.vespa.metricsproxy.core.MetricsManager.VESPA_VERSION; -import static ai.vespa.metricsproxy.core.VespaMetrics.METRIC_TYPE_DIMENSION_ID; -import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID; -import static ai.vespa.metricsproxy.metric.ExternalMetrics.ROLE_DIMENSION; -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author gjoranv - */ -public class MetricsManagerTest { - - private MetricsManager metricsManager; - - private static final String SERVICE_0_ID = "dummy/id/0"; - private static final String SERVICE_1_ID = "dummy/id/1"; - - private static final List testServices = ImmutableList.of( - new DummyService(0, SERVICE_0_ID), - new DummyService(1, SERVICE_1_ID)); - - private static final String WHITELISTED_METRIC_ID = "whitelisted"; - - @Before - public void setupMetricsManager() { - metricsManager = getMetricsManager(); - } - - @Test - public void each_service_gets_separate_metrics_packets() { - List packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.size(), is(2)); - - assertThat(packets.get(0).dimensions().get(toDimensionId("instance")), is("dummy0")); - assertThat(packets.get(0).metrics().get(toMetricId("c.test")), is(1.0)); - assertThat(packets.get(0).metrics().get(toMetricId("val")), is(1.05)); - - assertThat(packets.get(1).dimensions().get(toDimensionId("instance")), is("dummy1")); - assertThat(packets.get(1).metrics().get(toMetricId("c.test")), is(6.0)); - assertThat(packets.get(1).metrics().get(toMetricId("val")), is(2.35)); - } - - @Test - public void verify_expected_output_from_getMetricsById() { - String dummy0Metrics = metricsManager.getMetricsByConfigId("dummy/id/0"); - assertThat(dummy0Metrics, containsString("'dummy.id.0'.val=1.050")); - assertThat(dummy0Metrics, containsString("'dummy.id.0'.c_test=1")); - - String dummy1Metrics = metricsManager.getMetricsByConfigId("dummy/id/1"); - assertThat(dummy1Metrics, containsString("'dummy.id.1'.val=2.350")); - assertThat(dummy1Metrics, containsString("'dummy.id.1'.c_test=6")); - } - - @Test - public void getServices_returns_service_types() { - assertThat(metricsManager.getAllVespaServices(), is("dummy")); - } - - @Test - public void global_dimensions_are_added_but_do_not_override_metric_dimensions() { - List packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertEquals(2, packets.size()); - assertGlobalDimensions(packets.get(0).dimensions()); - assertGlobalDimensions(packets.get(1).dimensions()); - } - - private void assertGlobalDimensions(Map dimensions) { - assertTrue(dimensions.containsKey(VESPA_VERSION)); - assertEquals("value", dimensions.get(toDimensionId("global"))); - assertEquals("metric-dim", dimensions.get(toDimensionId("dim0"))); - } - - - @Test - public void system_metrics_are_added() { - VespaService service0 = testServices.get(0); - Metrics oldSystemMetrics = service0.getSystemMetrics(); - - service0.getSystemMetrics().add(new Metric("cpu", 1)); - - List packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertEquals(3, packets.size()); - - MetricsPacket systemPacket = packets.get(0); // system metrics are added before other metrics - assertThat(systemPacket.metrics().get(toMetricId("cpu")), is(1.0)); - assertThat(systemPacket.dimensions().get(toDimensionId("metrictype")), is("system")); - - service0.setSystemMetrics(oldSystemMetrics); - } - - @Test - public void extra_metrics_packets_containing_whitelisted_metrics_are_added() { - metricsManager.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("foo")) - .putMetrics(ImmutableList.of(new Metric(WHITELISTED_METRIC_ID, 0))))); - - List packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.size(), is(3)); - } - - @Test - public void extra_metrics_packets_without_whitelisted_metrics_are_not_added() { - metricsManager.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("foo")) - .putMetrics(ImmutableList.of(new Metric("not-whitelisted", 0))))); - - List packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.size(), is(2)); - } - - @Test - public void extra_dimensions_are_added_to_metrics_packets_that_do_not_have_those_dimensions() { - metricsManager.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("foo")) - .putMetrics(ImmutableList.of(new Metric(WHITELISTED_METRIC_ID, 0))) - .putDimension(ROLE_DIMENSION, "role from extraMetrics"))); - - List packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - for (MetricsPacket packet : packets) { - assertThat(packet.dimensions().get(ROLE_DIMENSION), is("role from extraMetrics")); - } - } - - @Test - public void extra_dimensions_do_not_overwrite_existing_dimension_values() { - metricsManager.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("foo")) - .putMetrics(ImmutableList.of(new Metric(WHITELISTED_METRIC_ID, 0))) - .putDimension(METRIC_TYPE_DIMENSION_ID, "from extraMetrics"))); - - List packets = metricsManager.getMetrics(testServices, Instant.EPOCH); - assertThat(packets.get(0).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("standard")); - assertThat(packets.get(1).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("standard")); - assertThat(packets.get(2).dimensions().get(METRIC_TYPE_DIMENSION_ID), is("from extraMetrics")); - } - - @Test - public void timestamp_is_adjusted_when_metric_is_less_than_one_minute_younger_than_start_time() { - Instant START_TIME = Instant.ofEpochSecond(0); - Instant METRIC_TIME = Instant.ofEpochSecond(59); - assertEquals(START_TIME, getAdjustedTimestamp(START_TIME, METRIC_TIME)); - } - - @Test - public void timestamp_is_adjusted_when_metric_is_less_than_one_minute_older_than_start_time() { - Instant START_TIME = Instant.ofEpochSecond(59); - Instant METRIC_TIME = Instant.ofEpochSecond(0); - assertEquals(START_TIME, getAdjustedTimestamp(START_TIME, METRIC_TIME)); - } - - @Test - public void timestamp_is_not_adjusted_when_metric_is_at_least_one_minute_younger_than_start_time() { - Instant START_TIME = Instant.ofEpochSecond(0); - Instant METRIC_TIME = Instant.ofEpochSecond(60); - assertEquals(METRIC_TIME, getAdjustedTimestamp(START_TIME, METRIC_TIME)); - } - - @Test - public void timestamp_is_not_adjusted_when_metric_is_at_least_one_minute_older_than_start_time() { - Instant START_TIME = Instant.ofEpochSecond(60); - Instant METRIC_TIME = Instant.ofEpochSecond(0); - assertEquals(METRIC_TIME, getAdjustedTimestamp(START_TIME, METRIC_TIME)); - } - - private Instant getAdjustedTimestamp(Instant startTime, Instant metricTime) { - MetricsPacket.Builder builder = new MetricsPacket.Builder(toServiceId("foo")) - .timestamp(metricTime.getEpochSecond()); - return MetricsManager.adjustTimestamp(builder, startTime).getTimestamp(); - } - - private MetricsManager getMetricsManager() { - VespaServices vespaServices = new VespaServices(testServices); - MetricsConsumers consumers = getMetricsConsumers(); - VespaMetrics metrics = new VespaMetrics(consumers, vespaServices); - - return new MetricsManager(vespaServices, metrics, new ExternalMetrics(consumers), - getApplicationDimensions(),getNodeDimensions()); - } - - private static MetricsConsumers getMetricsConsumers() { - Consumer.Metric.Dimension.Builder metricDimension = new Consumer.Metric.Dimension.Builder() - .key("dim0").value("metric-dim"); - - return new MetricsConsumers(new ConsumersConfig.Builder() - .consumer(new Consumer.Builder() - .name(VESPA_CONSUMER_ID.id) - .metric(new Consumer.Metric.Builder() - .name(WHITELISTED_METRIC_ID) - .outputname(WHITELISTED_METRIC_ID)) - .metric(new Consumer.Metric.Builder() - .name(DummyService.METRIC_1) - .outputname(DummyService.METRIC_1) - .dimension(metricDimension)) - .metric(new Consumer.Metric.Builder() - .name(DummyService.METRIC_2) - .outputname(DummyService.METRIC_2) - .dimension(metricDimension))) - .build()); - } - - private ApplicationDimensions getApplicationDimensions() { - return new ApplicationDimensions(new ApplicationDimensionsConfig.Builder() - .dimensions("global", "value").build()); - } - - private NodeDimensions getNodeDimensions() { - return new NodeDimensions(new NodeDimensionsConfig.Builder() - .dimensions("dim0", "should not override metric dim").build()); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java deleted file mode 100644 index 11c271d46e4..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/ExternalMetricsTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.core.MetricsConsumers; -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import com.google.common.collect.ImmutableList; -import org.junit.Test; - -import java.util.List; - -import static ai.vespa.metricsproxy.metric.ExternalMetrics.VESPA_NODE_SERVICE_ID; -import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static org.junit.Assert.assertEquals; - -/** - * @author gjoranv - */ -public class ExternalMetricsTest { - private static final ConsumerId CUSTOM_CONSUMER_1 = toConsumerId("consumer-1"); - private static final ConsumerId CUSTOM_CONSUMER_2 = toConsumerId("consumer-2"); - - @Test - public void extra_metrics_are_added() { - MetricsConsumers noConsumers = new MetricsConsumers(new ConsumersConfig.Builder().build()); - ExternalMetrics externalMetrics = new ExternalMetrics(noConsumers); - - externalMetrics.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("foo")))); - - List packets = externalMetrics.getMetrics(); - assertEquals(1, packets.size()); - } - - @Test - public void service_id_is_set_to_vespa_node_id() { - MetricsConsumers noConsumers = new MetricsConsumers(new ConsumersConfig.Builder().build()); - ExternalMetrics externalMetrics = new ExternalMetrics(noConsumers); - externalMetrics.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("replace_with_vespa_node_id")))); - - List packets = externalMetrics.getMetrics(); - assertEquals(1, packets.size()); - assertEquals(VESPA_NODE_SERVICE_ID, packets.get(0).build().service); - } - - @Test - public void custom_consumers_are_added() { - ConsumersConfig consumersConfig = new ConsumersConfig.Builder() - .consumer(new ConsumersConfig.Consumer.Builder().name(CUSTOM_CONSUMER_1.id)) - .consumer(new ConsumersConfig.Consumer.Builder().name(CUSTOM_CONSUMER_2.id)) - .build(); - MetricsConsumers consumers = new MetricsConsumers(consumersConfig); - ExternalMetrics externalMetrics = new ExternalMetrics(consumers); - - externalMetrics.setExtraMetrics(ImmutableList.of( - new MetricsPacket.Builder(toServiceId("foo")))); - - List packets = externalMetrics.getMetrics(); - assertEquals(1, packets.size()); - - List consumerIds = packets.get(0).build().consumers(); - assertEquals(2, consumerIds.size()); - assertEquals(CUSTOM_CONSUMER_1, consumerIds.get(0)); - assertEquals(CUSTOM_CONSUMER_2, consumerIds.get(1)); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java deleted file mode 100644 index b9e6377c27b..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/MetricsTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric; - -import ai.vespa.metricsproxy.service.DummyService; -import ai.vespa.metricsproxy.service.VespaService; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author Unknowm - */ -public class MetricsTest { - - @Test - public void testIterator() { - Metrics m = new Metrics(); - long now = System.currentTimeMillis() / 1000; - m.add(new Metric("a", 1, now)); - m.add(new Metric("b", 2.5, now)); - - //should expire after 0 seconds - m.add(new Metric("c", 2, now)); - - Map map = new HashMap<>(); - - for (Metric metric: m.getMetrics()) { - String k = metric.getName(); - - assertThat(map.containsKey(k), is(false)); - map.put(k, metric.getValue()); - } - - assertThat(map.get("a").intValue(), is(1)); - assertThat(map.get("b").doubleValue(), is(2.5)); - } - - @Test - public void testBasicMetric() { - Metrics m = new Metrics(); - m.add(new Metric("count", 1, System.currentTimeMillis() / 1000)); - assertThat(m.get("count").intValue(), is(1)); - } - - @Test - public void testHealthMetric() { - HealthMetric m = HealthMetric.get(null, null); - assertThat(m.isOk(), is(false)); - m = HealthMetric.get("up", "test message"); - assertThat(m.isOk(), is(true)); - assertThat(m.getMessage(), is("test message")); - m = HealthMetric.get("ok", "test message"); - assertThat(m.isOk(), is(true)); - assertThat(m.getMessage(), is("test message")); - - m = HealthMetric.get("bad", "test message"); - assertThat(m.isOk(), is(false)); - assertThat(m.getStatus(), is("bad")); - } - - @Test - public void testMetricFormatter() { - MetricsFormatter formatter = new MetricsFormatter(false, false); - VespaService service = new DummyService(0, "config.id"); - String data = formatter.format(service, "key", 1); - assertThat(data, is("'config.id'.key=1")); - - formatter = new MetricsFormatter(true, false); - data = formatter.format(service, "key", 1); - assertThat(data, is("dummy.'config.id'.key=1")); - - - formatter = new MetricsFormatter(true, true); - data = formatter.format(service, "key", 1); - assertThat(data, is("dummy.config.'id'.key=1")); - - formatter = new MetricsFormatter(false, true); - data = formatter.format(service, "key", 1); - assertThat(data, is("config.'id'.key=1")); - } - - @Test - public void testTimeAdjustment() { - assertThat(Metric.adjustTime(0L, 0L), is(0L)); - assertThat(Metric.adjustTime(59L, 59L), is(59L)); - assertThat(Metric.adjustTime(60L, 60L), is(60L)); - assertThat(Metric.adjustTime(59L, 60L), is(60L)); - assertThat(Metric.adjustTime(60L, 59L), is(60L)); - assertThat(Metric.adjustTime(59L, 61L), is(59L)); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java deleted file mode 100644 index d522a56a9ac..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model; - -import ai.vespa.metricsproxy.metric.Metric; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import org.junit.Test; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * @author gjoranv - */ -public class MetricsPacketTest { - - @Test - public void service_cannot_be_null() { - try { - MetricsPacket packet = new MetricsPacket.Builder(null) - .statusCode(0) - .statusMessage("") - .timestamp(0L) - .build(); - fail("Expected exception due to null service."); - } catch (Exception e) { - assertEquals("Service cannot be null.", e.getMessage()); - } - } - - @Test - public void consumers_are_always_distinct() { - ConsumerId DUPLICATE_CONSUMER = toConsumerId("duplicateConsumer"); - - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .statusCode(0) - .statusMessage("") - .addConsumers(Collections.singleton(DUPLICATE_CONSUMER)) - .addConsumers(Collections.singleton(DUPLICATE_CONSUMER)) - .build(); - assertEquals(1, packet.consumers().size()); - } - - @Test - public void builder_can_retain_subset_of_metrics() { - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .putMetrics(ImmutableList.of( - new Metric("remove", 1), - new Metric("keep", 2))) - .retainMetrics(ImmutableSet.of(toMetricId("keep"), toMetricId("non-existent"))) - .build(); - - assertFalse("should not contain 'remove'", packet.metrics().containsKey(toMetricId("remove"))); - assertTrue("should contain 'keep'", packet.metrics().containsKey(toMetricId("keep"))); - assertFalse("should not contain 'non-existent'", packet.metrics().containsKey(toMetricId("non-existent"))); - } - - @Test - public void builder_applies_output_names() { - String ONE = "one"; - String TWO = "two"; - String THREE = "three"; - String NON_EXISTENT = "non-existent"; - MetricId ONE_ID = toMetricId(ONE); - MetricId TWO_ID = toMetricId(TWO); - MetricId THREE_ID = toMetricId(THREE); - MetricId NON_EXISTENT_ID = toMetricId(NON_EXISTENT); - - Map> outputNamesById = ImmutableMap.of( - toMetricId(ONE), ImmutableList.of(ONE), - toMetricId(TWO), ImmutableList.of(TWO, "dos"), - toMetricId(THREE), ImmutableList.of("3"), - toMetricId(NON_EXISTENT), ImmutableList.of(NON_EXISTENT)); - - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .putMetrics(ImmutableList.of( - new Metric(ONE, 1), - new Metric(TWO, 2), - new Metric(THREE, 3))) - .applyOutputNames(outputNamesById) - .build(); - - // Only original name - assertTrue(packet.metrics().containsKey(ONE_ID)); - - // Both names - assertTrue(packet.metrics().containsKey(TWO_ID)); - assertTrue(packet.metrics().containsKey(toMetricId("dos"))); - - // Only new name - assertFalse(packet.metrics().containsKey(THREE_ID)); - assertTrue(packet.metrics().containsKey(toMetricId("3"))); - - // Non-existent metric not added - assertFalse(packet.metrics().containsKey(NON_EXISTENT_ID)); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/JsonUtilTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/JsonUtilTest.java deleted file mode 100644 index 28912293fdb..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/JsonUtilTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model.json; - -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import org.junit.Test; - -import java.util.List; - -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static ai.vespa.metricsproxy.metric.model.json.JsonUtil.toMetricsPackets; -import static java.util.Collections.singleton; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -/** - * @author gjoranv - */ -public class JsonUtilTest { - @Test - public void json_model_gets_null_status_by_default() { - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .build(); - YamasJsonModel jsonModel = JsonUtil.toYamasArray(singleton(packet)).metrics.get(0); - assertNull(jsonModel.status_code); - assertNull(jsonModel.status_msg); - } - - @Test - public void status_is_included_in_json_model_when_explicitly_asked_for() { - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .build(); - YamasJsonModel jsonModel = JsonUtil.toYamasArray(singleton(packet), true).metrics.get(0); - assertNotNull(jsonModel.status_code); - assertNotNull(jsonModel.status_msg); - } - - @Test - public void timestamp_0_in_packet_is_translated_to_null_in_json_model() { - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .timestamp(0L) - .build(); - YamasJsonModel jsonModel = JsonUtil.toYamasArray(singleton(packet)).metrics.get(0); - assertNull(jsonModel.timestamp); - } - - @Test - public void empty_consumers_is_translated_to_null_routing_in_json_model() { - MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo")) - .build(); - YamasJsonModel jsonModel = JsonUtil.toYamasArray(singleton(packet)).metrics.get(0); - assertNull(jsonModel.routing); - } - - @Test - public void empty_json_string_yields_empty_packet_list() { - List builders = toMetricsPackets(""); - assertTrue(builders.isEmpty()); - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java deleted file mode 100644 index e91ff32e3b4..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/json/YamasJsonModelTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.metric.model.json; - -import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.Test; - -import java.io.BufferedReader; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Collections; - -import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -/** - * Tests for YamasJsonModel and YamasArrayJsonModel - * - * @author smorgrav - * @author gjoranv - */ -public class YamasJsonModelTest { - - private static final String EXPECTED_JSON = "{\"metrics\":[{\"status_code\":0,\"timestamp\":1400047900,\"application\":\"vespa.searchnode\",\"metrics\":{\"cpu\":55.5555555555555,\"memory_virt\":22222222222,\"memory_rss\":5555555555},\"dimensions\":{\"applicationName\":\"app\",\"tenantName\":\"tenant\",\"metrictype\":\"system\",\"instance\":\"searchnode\",\"applicationInstance\":\"default\",\"clustername\":\"cluster\"},\"routing\":{\"yamas\":{\"namespaces\":[\"Vespa\"]}},\"status_msg\":\"Data collected successfully\"}]}"; - - @Test - public void array_definition_creates_correct_json() throws IOException { - YamasJsonModel jsonModel = getYamasJsonModel("yamas-array.json"); - - YamasArrayJsonModel yamasData = new YamasArrayJsonModel(); - yamasData.add(jsonModel); - - assertEquals(EXPECTED_JSON, yamasData.serialize()); - } - - @Test - public void deserialize_serialize_roundtrip() throws IOException { - YamasJsonModel jsonModel = getYamasJsonModel("yamas-array.json"); - - // Do some sanity checking - assertEquals("vespa.searchnode", jsonModel.application); - assertEquals("Vespa", jsonModel.routing.get("yamas").namespaces.get(0)); - assertEquals(5.555555555E9, jsonModel.metrics.get("memory_rss"), 0.1d); //Not using custom double renderer - - // Serialize and verify - YamasArrayJsonModel yamasArray = new YamasArrayJsonModel(); - yamasArray.add(jsonModel); - String string = yamasArray.serialize(); - assertEquals(EXPECTED_JSON, string); - } - - @Test - public void deserialize_serialize_roundtrip_with_metrics_packet() throws IOException { - YamasJsonModel jsonModel = getYamasJsonModel("yamas-array.json"); - MetricsPacket metricsPacket = JsonUtil.toMetricsPacketBuilder(jsonModel).build(); - - // Do some sanity checking - assertEquals(toServiceId("vespa.searchnode"), metricsPacket.service); - assertEquals(toConsumerId("Vespa"), metricsPacket.consumers().get(0)); - assertEquals(5.555555555E9, metricsPacket.metrics().get(toMetricId("memory_rss")).doubleValue(), 0.1d); //Not using custom double rendrer - - // Serialize and verify - YamasArrayJsonModel yamasArray = JsonUtil.toYamasArray(Collections.singleton(metricsPacket), true); - String string = yamasArray.serialize(); - assertEquals(EXPECTED_JSON, string); - } - - @Test - public void missing_routing_object_makes_it_null() throws IOException { - // Read file that was taken from production (real -life example that is) - String filename = getClass().getClassLoader().getResource("yamas-array-no-routing.json").getFile(); - BufferedReader reader = Files.newBufferedReader(Paths.get(filename)); - ObjectMapper mapper = new ObjectMapper(); - YamasJsonModel jsonModel = mapper.readValue(reader, YamasJsonModel.class); - - // Do some sanity checking - assertNull(jsonModel.routing); - } - - private YamasJsonModel getYamasJsonModel(String testFile) throws IOException { - String filename = getClass().getClassLoader().getResource(testFile).getFile(); - BufferedReader reader = Files.newBufferedReader(Paths.get(filename)); - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(reader, YamasJsonModel.class); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java deleted file mode 100644 index 6bbf4ae1ef5..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.rpc; - -import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer; -import ai.vespa.metricsproxy.core.MetricsConsumers; -import ai.vespa.metricsproxy.core.MetricsManager; -import ai.vespa.metricsproxy.core.MonitoringConfig; -import ai.vespa.metricsproxy.core.VespaMetrics; -import ai.vespa.metricsproxy.metric.ExternalMetrics; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; -import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; -import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig; -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.metric.model.ServiceId; -import ai.vespa.metricsproxy.service.HttpMetricFetcher; -import ai.vespa.metricsproxy.service.MockHttpServer; -import ai.vespa.metricsproxy.service.VespaServices; -import ai.vespa.metricsproxy.service.VespaServicesConfig; -import ai.vespa.metricsproxy.service.VespaServicesConfig.Service; - -import java.io.IOException; - -import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID; -import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; -import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static ai.vespa.metricsproxy.service.HttpMetricFetcher.STATE_PATH; - - -/** - * Setup and shutdown of config and servers for integration-style unit tests. - * - * @author hmusum - * @author gjoranv - */ -public class IntegrationTester implements AutoCloseable { - - static final String MONITORING_SYSTEM = "test-system"; - static final ConsumerId CUSTOM_CONSUMER_ID = toConsumerId("custom-consumer"); - static final String SERVICE_1_CONFIG_ID = "container/qrserver.0"; - static final String SERVICE_2_CONFIG_ID = "storage/cluster.storage/storage/0"; - - private final int httpPort; - private final int rpcPort; - private final RpcConnector connector; - private final MockHttpServer mockHttpServer; - private final VespaServices vespaServices; - - static { - HttpMetricFetcher.CONNECTION_TIMEOUT = 60000; // 60 secs in unit tests - } - - IntegrationTester(int httpPort, int rpcPort) { - if (httpPort == 0 || rpcPort == 0) { - throw new IllegalArgumentException("http port and rpc port must be defined"); - } - this.httpPort = httpPort; - this.rpcPort = rpcPort; - try { - mockHttpServer = new MockHttpServer(httpPort, null, STATE_PATH); - } catch (IOException e) { - throw new RuntimeException("Unable to start web server on port:" + httpPort); - } - - vespaServices = new VespaServices(servicesConfig(), monitoringConfig(), null); - MetricsConsumers consumers = new MetricsConsumers(consumersConfig()); - VespaMetrics vespaMetrics = new VespaMetrics(consumers, vespaServices); - ExternalMetrics externalMetrics = new ExternalMetrics(consumers); - ApplicationDimensions appDimensions = new ApplicationDimensions(applicationDimensionsConfig()); - NodeDimensions nodeDimensions = new NodeDimensions(nodeDimensionsConfig()); - - connector = new RpcConnector(rpcConnectorConfig()); - RpcServer server = new RpcServer(connector, vespaServices, new MetricsManager(vespaServices, vespaMetrics, externalMetrics, appDimensions, nodeDimensions)); - } - - MockHttpServer httpServer() { - return mockHttpServer; - } - - VespaServices vespaServices() { return vespaServices; } - - @Override - public void close() { - mockHttpServer.close(); - this.connector.stop(); - } - - private RpcConnectorConfig rpcConnectorConfig() { - return new RpcConnectorConfig.Builder() - .port(rpcPort) - .build(); - } - - private ConsumersConfig consumersConfig() { - return new ConsumersConfig.Builder() - .consumer(createConsumer(VESPA_CONSUMER_ID, "foo.count", "foo_count")) - .consumer(createConsumer(CUSTOM_CONSUMER_ID, "foo.count", "foo.count")) - .build(); - } - - private static Consumer.Builder createConsumer(ConsumerId consumerId, String metricName, String outputName) { - return new Consumer.Builder() - .name(consumerId.id) - .metric(new Consumer.Metric.Builder() - .dimension(new Consumer.Metric.Dimension.Builder().key("somekey").value("somevalue")) - .name(metricName) - .outputname(outputName)); - } - - private VespaServicesConfig servicesConfig() { - return new VespaServicesConfig.Builder() - .service(createService(toServiceId("qrserver"), SERVICE_1_CONFIG_ID, httpPort)) - .service(createService(toServiceId("storagenode"), SERVICE_2_CONFIG_ID, httpPort)) - .build(); - } - - private static Service.Builder createService(ServiceId serviceId, String configId, int port) { - return new Service.Builder() - .name(serviceId.id) - .configId(configId) - .port(port) - .healthport(port) - .dimension(new Service.Dimension.Builder().key("serviceDim").value("serviceDimValue")); - } - - private MonitoringConfig monitoringConfig() { - return new MonitoringConfig.Builder() - .systemName(MONITORING_SYSTEM) - .build(); - } - - private ApplicationDimensionsConfig applicationDimensionsConfig() { - return new ApplicationDimensionsConfig.Builder().build(); - } - - private NodeDimensionsConfig nodeDimensionsConfig() { - return new NodeDimensionsConfig.Builder().build(); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java deleted file mode 100644 index 20fb69e410e..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcHealthMetricsTest.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.rpc; - -import ai.vespa.metricsproxy.TestUtil; -import ai.vespa.metricsproxy.metric.HealthMetric; -import ai.vespa.metricsproxy.service.MockHttpServer; -import ai.vespa.metricsproxy.service.VespaService; -import com.yahoo.jrt.Request; -import com.yahoo.jrt.Spec; -import com.yahoo.jrt.StringValue; -import com.yahoo.jrt.Supervisor; -import com.yahoo.jrt.Target; -import com.yahoo.jrt.Transport; -import org.junit.Test; - -import java.util.List; - -import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_1_CONFIG_ID; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; - -/** - * @author jobergum - * @author gjoranv - */ -public class RpcHealthMetricsTest { - - private static final String HEALTH_OK_RESPONSE = - TestUtil.getContents("health-check.response.json"); - private static final String HEALTH_FAILED_RESPONSE = - TestUtil.getContents("health-check-failed.response.json"); - private static final String WANTED_RPC_RESPONSE = - TestUtil.getContents("rpc-json-output-check.json").trim(); - - // see factory/doc/port-ranges.txt - private static final int httpPort = 18635; - private static final int rpcPort = 18636; - - @Test - public void expected_response_is_returned() { - try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) { - - MockHttpServer mockHttpServer = tester.httpServer(); - mockHttpServer.setResponse(HEALTH_OK_RESPONSE); - List services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); - - assertThat(services.size(), is(1)); - VespaService qrserver = services.get(0); - HealthMetric h = qrserver.getHealth(); - assertNotNull("Health metric should never be null", h); - assertThat("Status failed, reason = " + h.getMessage(), h.isOk(), is(true)); - assertThat(h.getMessage(), is("WORKING")); - - mockHttpServer.setResponse(HEALTH_FAILED_RESPONSE); - h = qrserver.getHealth(); - assertNotNull("Health metric should never be null", h); - assertThat("Status should be failed" + h.getMessage(), h.isOk(), is(false)); - assertThat(h.getMessage(), is("SOMETHING FAILED")); - - String jsonRPCMessage = getHealthMetrics(qrserver.getMonitoringName()); - assertThat(jsonRPCMessage, is(WANTED_RPC_RESPONSE)); - } - } - - @Test - public void non_existent_service_name_returns_an_error_message() { - try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) { - String jsonRPCMessage = getHealthMetrics("non-existing service"); - assertThat(jsonRPCMessage, is("105: No service with name 'non-existing service'")); - } - } - - private String getHealthMetrics(String service) { - Supervisor supervisor = new Supervisor(new Transport()); - Target target = supervisor.connect(new Spec("localhost", rpcPort)); - Request req = new Request("getHealthMetricsForYamas"); - req.parameters().add(new StringValue(service)); - String returnValue; - - target.invokeSync(req, 20.0); - if (req.checkReturnTypes("s")) { - returnValue = req.returnValues().get(0).asString(); - } else { - System.out.println("RpcServer invocation failed " + req.errorCode() + ": " + req.errorMessage()); - returnValue = req.errorCode() + ": " + req.errorMessage(); - } - target.close(); - supervisor.transport().shutdown().join(); - - return returnValue; - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java deleted file mode 100644 index f264fd13ddc..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.rpc; - -import ai.vespa.metricsproxy.TestUtil; -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.Metrics; -import ai.vespa.metricsproxy.metric.model.ConsumerId; -import ai.vespa.metricsproxy.service.VespaService; -import com.yahoo.jrt.Request; -import com.yahoo.jrt.Spec; -import com.yahoo.jrt.StringValue; -import com.yahoo.jrt.Supervisor; -import com.yahoo.jrt.Target; -import com.yahoo.jrt.Transport; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.junit.Test; - -import java.util.List; - -import static ai.vespa.metricsproxy.core.VespaMetrics.VESPA_CONSUMER_ID; -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static ai.vespa.metricsproxy.rpc.IntegrationTester.CUSTOM_CONSUMER_ID; -import static ai.vespa.metricsproxy.rpc.IntegrationTester.MONITORING_SYSTEM; -import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_1_CONFIG_ID; -import static ai.vespa.metricsproxy.rpc.IntegrationTester.SERVICE_2_CONFIG_ID; -import static ai.vespa.metricsproxy.service.VespaServices.ALL_SERVICES; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; - -/** - * @author jobergum - * @author gjoranv - */ -public class RpcMetricsTest { - - private static final String METRICS_RESPONSE_CCL = - TestUtil.getContents("metrics-storage-simple.json").trim(); - - // see factory/doc/port-ranges.txt - private static final int httpPort = 18633; - private static final int rpcPort = 18634; - - @Test - public void testGetMetrics() throws Exception { - try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) { - tester.httpServer().setResponse(METRICS_RESPONSE_CCL); - List services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); - - assertThat("#Services should be 1 for config id " + SERVICE_1_CONFIG_ID, services.size(), is(1)); - - VespaService qrserver = services.get(0); - assertThat(qrserver.getMonitoringName(), is(MONITORING_SYSTEM + VespaService.SEPARATOR + "qrserver")); - - Metrics metrics = qrserver.getMetrics(); - assertThat("Fetched number of metrics is not correct", metrics.size(), is(2)); - Metric m = metrics.getMetric("foo.count"); - assertNotNull("Did not find expected metric with name 'foo.count'", m); - Metric m2 = metrics.getMetric("bar.count"); - assertNotNull("Did not find expected metric with name 'bar.count'", m2); - - // Setup RPC client - Supervisor supervisor = new Supervisor(new Transport()); - Target target = supervisor.connect(new Spec("localhost", rpcPort)); - - verifyMetricsFromRpcRequest(qrserver, target); - - services = tester.vespaServices().getInstancesById(SERVICE_2_CONFIG_ID); - assertThat("#Services should be 1 for config id " + SERVICE_2_CONFIG_ID, services.size(), is(1)); - - VespaService storageService = services.get(0); - verfiyMetricsFromServiceObject(storageService); - - String metricsById = getMetricsById(storageService.getConfigId(), target); - assertThat(metricsById, is("'storage.cluster.storage.storage.0'.foo_count=1 ")); - - String jsonResponse = getMetricsForYamas("non-existing", target).trim(); - assertThat(jsonResponse, is("105: No service with name 'non-existing'")); - - verifyMetricsFromRpcRequestForAllServices(target); - - // Shutdown RPC - target.close(); - supervisor.transport().shutdown().join(); - } - } - - private static void verifyMetricsFromRpcRequest(VespaService service, Target target) throws JSONException { - String jsonResponse = getMetricsForYamas(service.getMonitoringName(), target).trim(); - JSONArray metrics = new JSONObject(jsonResponse).getJSONArray("metrics"); - assertThat("Expected 3 metric messages", metrics.length(), is(3)); - for (int i = 0; i < metrics.length() - 1; i++) { // The last "metric message" contains only status code/message - JSONObject jsonObject = metrics.getJSONObject(i); - assertFalse(jsonObject.has("status_code")); - assertFalse(jsonObject.has("status_msg")); - assertThat(jsonObject.getJSONObject("dimensions").getString("foo"), is("bar")); - assertThat(jsonObject.getJSONObject("dimensions").getString("bar"), is("foo")); - assertThat(jsonObject.getJSONObject("dimensions").getString("serviceDim"), is("serviceDimValue")); - assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").length(), is(1)); - if (jsonObject.getJSONObject("metrics").has("foo_count")) { - assertThat(jsonObject.getJSONObject("metrics").getInt("foo_count"), is(1)); - assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").get(0), is(VESPA_CONSUMER_ID.id)); - } else { - assertThat(jsonObject.getJSONObject("metrics").getInt("foo.count"), is(1)); - assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").get(0), is(CUSTOM_CONSUMER_ID.id)); - } - } - - verifyStatusMessage(metrics.getJSONObject(metrics.length() - 1)); - } - - private void verfiyMetricsFromServiceObject(VespaService service) { - Metrics storageMetrics = service.getMetrics(); - assertThat(storageMetrics.size(), is(2)); - Metric foo = storageMetrics.getMetric("foo.count"); - assertNotNull("Did not find expected metric with name 'foo.count'", foo); - assertThat("Expected 2 dimensions for metric foo", foo.getDimensions().size(), is(2)); - assertThat("Metric foo did not contain correct dimension mapping for key = foo.count", foo.getDimensions().containsKey(toDimensionId("foo")), is(true)); - assertThat("Metric foo did not contain correct dimension", foo.getDimensions().get(toDimensionId("foo")), is("bar")); - assertThat("Metric foo did not contain correct dimension", foo.getDimensions().containsKey(toDimensionId("bar")), is(true)); - assertThat("Metric foo did not contain correct dimension for key = bar", foo.getDimensions().get(toDimensionId("bar")), is("foo")); - } - - private void verifyMetricsFromRpcRequestForAllServices(Target target) throws JSONException { - // Verify that metrics for all services can be retrieved in one request. - String allServicesResponse = getMetricsForYamas(ALL_SERVICES, target).trim(); - JSONArray allServicesMetrics = new JSONObject(allServicesResponse).getJSONArray("metrics"); - assertThat(allServicesMetrics.length(), is(5)); - } - - @Test - public void testGetAllMetricNames() { - try (IntegrationTester tester = new IntegrationTester(httpPort, rpcPort)) { - - tester.httpServer().setResponse(METRICS_RESPONSE_CCL); - List services = tester.vespaServices().getInstancesById(SERVICE_1_CONFIG_ID); - - assertThat(services.size(), is(1)); - Metrics metrics = services.get(0).getMetrics(); - assertThat("Fetched number of metrics is not correct", metrics.size(), is(2)); - Metric m = metrics.getMetric("foo.count"); - assertNotNull("Did not find expected metric with name 'foo.count'", m); - - - Metric m2 = metrics.getMetric("bar.count"); - assertNotNull("Did not find expected metric with name 'bar'", m2); - - // Setup RPC - Supervisor supervisor = new Supervisor(new Transport()); - Target target = supervisor.connect(new Spec("localhost", rpcPort)); - - String response = getAllMetricNamesForService(services.get(0).getMonitoringName(), VESPA_CONSUMER_ID, target); - assertThat(response, is("foo.count=ON;output-name=foo_count,bar.count=OFF,")); - - // Shutdown RPC - target.close(); - supervisor.transport().shutdown().join(); - } - } - - private static String getMetricsForYamas(String service, Target target) { - Request req = new Request("getMetricsForYamas"); - req.parameters().add(new StringValue(service)); - return invoke(req, target); - } - - private String getMetricsById(String service, Target target) { - Request req = new Request("getMetricsById"); - req.parameters().add(new StringValue(service)); - return invoke(req, target); - } - - private String getAllMetricNamesForService(String service, ConsumerId consumer, Target target) { - Request req = new Request("getAllMetricNamesForService"); - req.parameters().add(new StringValue(service)); - req.parameters().add(new StringValue(consumer.id)); - return invoke(req, target); - } - - private static String invoke(Request req, Target target) { - String returnValue; - target.invokeSync(req, 20.0); - if (req.checkReturnTypes("s")) { - returnValue = req.returnValues().get(0).asString(); - } else { - System.out.println(req.methodName() + " from rpcserver - Invocation failed " - + req.errorCode() + ": " + req.errorMessage()); - returnValue = req.errorCode() + ": " + req.errorMessage(); - } - return returnValue; - } - - private static void verifyStatusMessage(JSONObject jsonObject) throws JSONException { - assertThat(jsonObject.getInt("status_code"), is(0)); - assertThat(jsonObject.getString("status_msg"), notNullValue()); - assertThat(jsonObject.getString("application"), notNullValue()); - assertThat(jsonObject.getString("routing"), notNullValue()); - assertThat(jsonObject.length(), is(4)); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java deleted file mode 100644 index bd61b8443aa..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author Unknown - */ -public class ConfigSentinelClientTest { - - @Test - public void testConfigSentinelClient() { - ConfigSentinelDummy configsentinel = new ConfigSentinelDummy(); - List services = new ArrayList<>(); - VespaService docproc = new VespaService("docprocservice", "docproc/cluster.x.indexing/0"); - VespaService searchnode4 = new VespaService("searchnode4", "search/cluster.x/g0/c1/r1"); - VespaService qrserver = new VespaService("qrserver", "container/qrserver.0"); - - services.add(searchnode4); - services.add(qrserver); - services.add(docproc); - - MockConfigSentinelClient client = new MockConfigSentinelClient(configsentinel); - client.updateServiceStatuses(services); - - assertThat(qrserver.getPid(), is(6520)); - assertThat(qrserver.getState(), is("RUNNING")); - assertThat(qrserver.isAlive(), is(true)); - assertThat(searchnode4.getPid(), is(6534)); - assertThat(searchnode4.getState(), is("RUNNING")); - assertThat(searchnode4.isAlive(), is(true)); - - assertThat(docproc.getPid(), is(-1)); - assertThat(docproc.getState(), is("FINISHED")); - assertThat(docproc.isAlive(), is(false)); - - - configsentinel.reConfigure(); - - client.ping(docproc); - assertThat(docproc.getPid(), is(100)); - assertThat(docproc.getState(), is("RUNNING")); - assertThat(docproc.isAlive(), is(true)); - - //qrserver has yet not been checked - assertThat(qrserver.isAlive(), is(true)); - - client.updateServiceStatuses(services); - - assertThat(docproc.getPid(), is(100)); - assertThat(docproc.getState(), is("RUNNING")); - assertThat(docproc.isAlive(), is(true)); - //qrserver is no longer running on this node - so should be false - assertThat(qrserver.isAlive(), is(false)); - } - - @Test - public void testElastic() throws Exception { - String response = "container state=RUNNING mode=AUTO pid=14338 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"get/container.0\"\n" + - "container-clustercontroller state=RUNNING mode=AUTO pid=25020 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/cluster-controllers/0\"\n" + - "distributor state=RUNNING mode=AUTO pid=25024 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/distributor/0\"\n" + - "docprocservice state=RUNNING mode=AUTO pid=11973 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"docproc/cluster.search.indexing/0\"\n" + - "logd state=RUNNING mode=AUTO pid=25016 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/vespa19.dev.gq1.yahoo.com/logd\"\n" + - "logserver state=RUNNING mode=AUTO pid=25018 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/logserver\"\n" + - "metricsproxy state=RUNNING mode=AUTO pid=13107 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/vespa19.dev.gq1.yahoo.com/metricsproxy\"\n" + - "searchnode state=RUNNING mode=AUTO pid=25023 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/search/cluster.search/0\"\n" + - "slobrok state=RUNNING mode=AUTO pid=25019 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n" + - "topleveldispatch state=RUNNING mode=AUTO pid=25026 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/search/cluster.search/tlds/tld.0\"\n" + - "\n"; - - ConfigSentinelDummy configsentinel = new ConfigSentinelDummy(response); - List services = new ArrayList<>(); - - VespaService container = VespaService.create("container", "get/container.0", -1); - - VespaService containerClusterController = - VespaService.create("container-clustercontroller", "get/container.0", -1); - - VespaService notPresent = VespaService.create("dummy","fake", -1); - - services.add(container); - services.add(containerClusterController); - services.add(notPresent); - - MockConfigSentinelClient client = new MockConfigSentinelClient(configsentinel); - client.updateServiceStatuses(services); - assertThat(container.isAlive(),is(true)); - assertThat(container.getPid(),is(14338)); - assertThat(container.getState(),is("RUNNING")); - - assertThat(containerClusterController.isAlive(),is(true)); - assertThat(containerClusterController.getPid(),is(25020)); - assertThat(containerClusterController.getState(),is("RUNNING")); - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java deleted file mode 100644 index 108f5c18e1d..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelDummy.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ -package ai.vespa.metricsproxy.service; - -/** - * @author Eirik Nygaard - */ -public class ConfigSentinelDummy { - private String serviceList = - "docprocservice state=FINISHED mode=MANUAL pid=6555 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"docproc/cluster.x.indexing/0\"\n" - + "distributor state=RUNNING mode=AUTO pid=6548 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/distributor/0\"\n" - + "fleetcontroller state=RUNNING mode=AUTO pid=6543 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/fleetcontroller/0\"\n" - + "storagenode state=RUNNING mode=AUTO pid=6539 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/storage/0\"\n" - + "searchnode4 state=RUNNING mode=AUTO pid=6534 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c1/r1\"\n" - + "qrserver2 state=RUNNING mode=AUTO pid=6521 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"container/qrserver.1\"\n" - + "logserver state=RUNNING mode=AUTO pid=6518 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/logserver\"\n" - + "logd state=RUNNING mode=AUTO pid=6517 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/dell-bl5s7.trondheim.corp.yahoo.com/logd\"\n" - + "searchnode2 state=RUNNING mode=AUTO pid=6527 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r1\"\n" - + "topleveldispatch2 state=RUNNING mode=AUTO pid=6525 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.1\"\n" - + "topleveldispatch state=RUNNING mode=AUTO pid=6524 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.0\"\n" - + "clustercontroller2 state=RUNNING mode=AUTO pid=6523 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/1\"\n" - + "clustercontroller state=RUNNING mode=AUTO pid=6522 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/0\"\n" - + "slobrok state=RUNNING mode=AUTO pid=6519 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n" - + "searchnode3 state=RUNNING mode=AUTO pid=6529 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c1/r0\"\n" - + "searchnode state=RUNNING mode=AUTO pid=6526 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r0\"\n" - + "qrserver state=RUNNING mode=AUTO pid=6520 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"container/qrserver.0\"\n" - + "\n"; - - - public ConfigSentinelDummy() { - } - - public ConfigSentinelDummy(String response) { - serviceList = response; - } - - public void reConfigure() { - this.serviceList = "docprocservice state=RUNNING mode=AUTO pid=100 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"docproc/cluster.x.indexing/0\"\n" - + "distributor state=RUNNING mode=AUTO pid=6548 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/distributor/0\"\n" - + "fleetcontroller state=RUNNING mode=AUTO pid=6543 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/fleetcontroller/0\"\n" - + "storagenode state=RUNNING mode=AUTO pid=6539 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"storage/cluster.storage/storage/0\"\n" - + "searchnode4 state=RUNNING mode=AUTO pid=6534 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c1/r1\"\n" - + "qrserver2 state=RUNNING mode=AUTO pid=6521 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"container/qrserver.1\"\n" - + "logserver state=RUNNING mode=AUTO pid=6518 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/logserver\"\n" - + "logd state=RUNNING mode=AUTO pid=6517 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"hosts/dell-bl5s7.trondheim.corp.yahoo.com/logd\"\n" - + "searchnode2 state=RUNNING mode=AUTO pid=6527 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r1\"\n" - + "topleveldispatch2 state=RUNNING mode=AUTO pid=6525 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.1\"\n" - + "topleveldispatch state=RUNNING mode=AUTO pid=6524 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/tlds/tld.0\"\n" - + "clustercontroller2 state=RUNNING mode=AUTO pid=6523 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/1\"\n" - + "clustercontroller state=RUNNING mode=AUTO pid=6522 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/rtx/0\"\n" - + "slobrok state=RUNNING mode=AUTO pid=6519 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"admin/slobrok.0\"\n" - + "searchnode3 state=RUNNING mode=AUTO pid=6529 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c1/r0\"\n" - + "searchnode state=RUNNING mode=AUTO pid=6526 exitstatus=0 autostart=TRUE autorestart=TRUE id=\"search/cluster.x/g0/c0/r0\"\n" - + "\n"; - } - - public String getServiceList() { - return serviceList; - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java deleted file mode 100644 index 4174b18f3a7..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.TestUtil; -import ai.vespa.metricsproxy.metric.Metric; -import org.json.JSONException; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author Unknown - */ -public class ContainerServiceTest { - - private MockHttpServer service; - private int csPort; - - @BeforeClass - public static void init() { - HttpMetricFetcher.CONNECTION_TIMEOUT = 60000; // 60 secs in unit tests - } - - @Before - public void setupHTTPServer() { - csPort = 18637; // see factory/doc/port-ranges.txt - try { - String response = TestUtil.getContents("metrics-container-state-multi-chain.json"); - service = new MockHttpServer(csPort, response, HttpMetricFetcher.METRICS_PATH); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @Test - public void testMultipleQueryDimensions() throws JSONException { - int count = 0; - VespaService service = VespaService.create("service1", "id", csPort); - for (Metric m : service.getMetrics().getMetrics()) { - if (m.getName().equals("queries.rate")) { - count++; - System.out.println("Name: " + m.getName() + " value: " + m.getValue()); - if (m.getDimensions().get(toDimensionId("chain")).equals("asvBlendingResult")) { - assertThat((Double)m.getValue(), is(26.4)); - } else if (m.getDimensions().get(toDimensionId("chain")).equals("blendingResult")) { - assertThat((Double)m.getValue(), is(0.36666666666666664)); - } else { - assertThat("Unknown unknown chain", false, is(true)); - } - } - } - assertThat(count, is(2)); - } - - @After - public void shutdown() { - this.service.close(); - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java deleted file mode 100644 index 380a992aead..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/DummyService.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.metric.Metric; -import ai.vespa.metricsproxy.metric.Metrics; - -/** - * @author Unknown - */ -public class DummyService extends VespaService { - static final String NAME = "dummy"; - public static final String METRIC_1 = "c.test"; - public static final String METRIC_2 = "val"; - - private final int num; - - public DummyService(int num, String configid) { - super(NAME, NAME + num, configid); - this.num = num; - } - - @Override - public Metrics getMetrics() { - Metrics m = new Metrics(); - - long timestamp = System.currentTimeMillis() / 1000; - m.add(new Metric(METRIC_1, 5 * num + 1, timestamp)); - m.add(new Metric(METRIC_2, 1.3 * num + 1.05, timestamp)); - - return m; - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java deleted file mode 100644 index 27e1bb97943..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.TestUtil; -import ai.vespa.metricsproxy.metric.Metrics; -import org.junit.Test; - -import java.io.File; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author Unknowm - */ -public class MetricsFetcherTest { - private static int port = 9; //port number is not used in this test - - @Test - public void testStateFormatMetricsParse() { - String jsonData = TestUtil.getContents("metrics-state.json"); - RemoteMetricsFetcher fetcher = new RemoteMetricsFetcher(new DummyService(0, "dummy/id/0"), port); - Metrics metrics = fetcher.createMetrics(jsonData, 0); - assertThat("Wrong number of metrics", metrics.size(), is(10)); - assertThat("Wrong value for metric", metrics.get("query_hits.count").intValue(), is(28)); - assertThat("Wrong value for metric ", metrics.get("queries.rate").doubleValue(), is(0.4667)); - assertThat("Wrong timestamp", metrics.getTimeStamp(), is(1334134700L)); - } - - @Test - public void testEmptyJson() { - String jsonData = "{}"; - RemoteMetricsFetcher fetcher = new RemoteMetricsFetcher(new DummyService(0, "dummy/id/0"), port); - Metrics metrics = fetcher.createMetrics(jsonData, 0); - assertThat("Wrong number of metrics", metrics.size(), is(0)); - } - - @Test - public void testErrors() { - String jsonData; - Metrics metrics; - - RemoteMetricsFetcher fetcher = new RemoteMetricsFetcher(new DummyService(0, "dummy/id/0"), port); - - jsonData = ""; - metrics = fetcher.createMetrics(jsonData, 0); - assertThat("Wrong number of metrics", metrics.size(), is(0)); - - jsonData = "{\n" + - "\"status\" : {\n" + - " \"code\" : \"up\",\n" + - " \"message\" : \"Everything ok here\"\n" + - "}\n" + - "}"; - metrics = fetcher.createMetrics(jsonData, 0); - assertThat("Wrong number of metrics", metrics.size(), is(0)); - - jsonData = "{\n" + - "\"status\" : {\n" + - " \"code\" : \"up\",\n" + - " \"message\" : \"Everything ok here\"\n" + - "},\n" + - "\"metrics\" : {\n" + - " \"snapshot\" : {\n" + - " \"from\" : 1334134640.089,\n" + - " \"to\" : 1334134700.088\n" + - " },\n" + - " \"values\" : [\n" + - " {\n" + - " \"name\" : \"queries\",\n" + - " \"description\" : \"Number of queries executed during snapshot interval\",\n" + - " \"values\" : {\n" + - " \"count\" : null,\n" + - " \"rate\" : 0.4667\n" + - " },\n" + - " \"dimensions\" : {\n" + - " \"searcherid\" : \"x\"\n" + - " }\n" + - " }\n" + "" + - " ]\n" + - "}\n" + - "}"; - - metrics = fetcher.createMetrics(jsonData, 0); - assertThat("Wrong number of metrics", metrics.size(), is(0)); - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java deleted file mode 100644 index 917c529e63e..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ -package ai.vespa.metricsproxy.service; - -import com.yahoo.log.LogLevel; - -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Logger; - -/** - * Mock config sentinel - * - * @author hmusum - */ -public class MockConfigSentinelClient extends ConfigSentinelClient { - private final ConfigSentinelDummy configSentinel; - private final static Logger log = Logger.getLogger(MockConfigSentinelClient.class.getPackage().getName()); - - public MockConfigSentinelClient(ConfigSentinelDummy configSentinel) { - super(); - this.configSentinel = configSentinel; - } - - @Override - protected synchronized void setStatus(List services) throws Exception { - List updatedServices = new ArrayList<>(); - String[] lines = configSentinel.getServiceList().split("\n"); - for (String line : lines) { - if (line.equals("")) { - break; - } - - VespaService s = parseServiceString(line, services); - if (s != null) { - updatedServices.add(s); - } - } - - //Check if there are services that were not found in - //from the sentinel - for (VespaService s : services) { - if (!updatedServices.contains(s)) { - log.log(LogLevel.DEBUG, "Service " + s + " is no longer found with sentinel - setting alive = false"); - s.setAlive(false); - } - } - } -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java deleted file mode 100644 index fdf2fae3081..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockHttpServer.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpHandler; -import com.sun.net.httpserver.HttpServer; - -import java.io.IOException; -import java.io.OutputStream; -import java.net.InetSocketAddress; - -/** - * @author jobergum - */ -public class MockHttpServer { - - private String response; - private HttpServer server; - - /** - * Mock http server that will return response as body - * - * @param port the port to listen to - * @param response the response to return along with 200 OK - * @param path the file path that the server will accept requests for. E.g /state/v1/metrics - */ - public MockHttpServer(int port, String response, String path) throws IOException { - this.response = response; - this.server = HttpServer.create(new InetSocketAddress(port), 10); - this.server.createContext(path, new MyHandler()); - this.server.setExecutor(null); // creates a default executor - this.server.start(); - System.out.println("Started web server on port " + port); - } - - public synchronized void setResponse(String r) { - this.response = r; - } - - public void close() { - this.server.stop(0); - } - - private class MyHandler implements HttpHandler { - public void handle(HttpExchange t) throws IOException { - t.sendResponseHeaders(200, response.length()); - OutputStream os = t.getResponseBody(); - os.write(response.getBytes()); - os.close(); - } - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java deleted file mode 100644 index a42d52b7ea6..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/SystemPollerTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author Unknown - */ -public class SystemPollerTest { - - @Test - public void testSystemPoller() { - DummyService s = new DummyService(0, "id"); - List services = new ArrayList<>(); - services.add(s); - - SystemPoller poller = new SystemPoller(services, 60*5); - assertThat(s.isAlive(), is(false)); - services.remove(0); - poller.setServices(services); - long n = poller.getPidJiffies(s); - assertThat(n, is(0L)); - long[] memusage = poller.getMemoryUsage(s); - assertThat(memusage[0], is(0L)); - assertThat(memusage[1], is(0L)); - } - - @Test - public void testCPUJiffies() { - String line = "cpu1 102180864 789 56766899 12800020140 1654757 0 0"; - CpuJiffies n = new CpuJiffies(line); - assertThat(n.getCpuId(), is(1)); - assertThat(n.getTotalJiffies(), is(12960623449L)); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java deleted file mode 100644 index 13be98db23a..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServiceTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import ai.vespa.metricsproxy.TestUtil; -import ai.vespa.metricsproxy.metric.Metrics; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.File; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author Unknown - */ -public class VespaServiceTest { - private MockHttpServer service; - private int csPort; - private static final String response; - - static { - response = TestUtil.getContents("metrics-state.json"); - HttpMetricFetcher.CONNECTION_TIMEOUT = 60000; // 60 secs in unit tests - } - - @Before - public void setupHTTPServer() { - csPort = 18632; // see factory/doc/port-ranges.txt - try { - service = new MockHttpServer(csPort, response, HttpMetricFetcher.METRICS_PATH); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @Test - public void testService() { - VespaService service = new VespaService("qrserver", "container/qrserver.0"); - assertThat(service.getServiceName(), is("qrserver")); - assertThat(service.getInstanceName(), is("qrserver")); - assertThat(service.getPid(), is(-1)); - assertThat(service.getConfigId(), is("container/qrserver.0")); - - - service = VespaService.create("qrserver2", "container/qrserver.0", -1); - assertThat(service.getServiceName(), is("qrserver")); - assertThat(service.getInstanceName(), is("qrserver2")); - assertThat(service.getPid(), is(-1)); - assertThat(service.getConfigId(), is("container/qrserver.0")); - } - - @Test - // TODO: Make it possible to test this without running a HTTP server to create the response - public void testMetricsFetching() { - VespaService service = VespaService.create("service1", "id", csPort); - Metrics metrics = service.getMetrics(); - assertThat(metrics.getMetric("queries.count").getValue().intValue(), is(28)); - - // Shutdown server and check that no metrics are returned (should use empty metrics - // when unable to fetch new metrics) - shutdown(); - - metrics = service.getMetrics(); - assertThat(metrics.size(), is(0)); - } - - @After - public void shutdown() { - this.service.close(); - } - -} diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java deleted file mode 100644 index bd0b670ca35..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/VespaServicesTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - */ - -package ai.vespa.metricsproxy.service; - -import com.google.common.collect.ImmutableList; -import org.junit.Test; - -import java.util.List; - -import static ai.vespa.metricsproxy.service.VespaServices.ALL_SERVICES; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * TODO: add more tests - * - * @author gjoranv - */ -public class VespaServicesTest { - - @Test - public void services_can_be_retrieved_from_monitoring_name() { - List dummyServices = ImmutableList.of( - new DummyService(0, "dummy/id/0"), - new DummyService(1, "dummy/id/1")); - VespaServices services = new VespaServices(dummyServices); - - assertThat(services.getMonitoringServices("vespa.dummy").size(), is(2)); - } - - @Test - public void all_services_can_be_retrieved_by_using_special_name() { - List dummyServices = ImmutableList.of( - new DummyService(0, "dummy/id/0")); - VespaServices services = new VespaServices(dummyServices); - - assertThat(services.getMonitoringServices(ALL_SERVICES).size(), is(1)); - } - -} diff --git a/metrics-proxy/src/test/resources/health-check-failed.response.json b/metrics-proxy/src/test/resources/health-check-failed.response.json deleted file mode 100644 index e118f10ec5e..00000000000 --- a/metrics-proxy/src/test/resources/health-check-failed.response.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "status": { - "code": "down", - "message":"SOMETHING FAILED" - }, - "metrics": [] - } diff --git a/metrics-proxy/src/test/resources/health-check.response.json b/metrics-proxy/src/test/resources/health-check.response.json deleted file mode 100644 index 8e3858ec5d8..00000000000 --- a/metrics-proxy/src/test/resources/health-check.response.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "status": { - "code": "OK", - "message":"WORKING" - } - } \ No newline at end of file diff --git a/metrics-proxy/src/test/resources/metrics-container-state-multi-chain.json b/metrics-proxy/src/test/resources/metrics-container-state-multi-chain.json deleted file mode 100644 index 76d0be50cca..00000000000 --- a/metrics-proxy/src/test/resources/metrics-container-state-multi-chain.json +++ /dev/null @@ -1,281 +0,0 @@ -{ - "metrics": { - "snapshot": { - "from": 1.383132197389E9, - "to": 1.383132257389E9 - }, - "values": [ - { - "name": "search_connections", - "values": { - "average": 3.459204315576534, - "count": 1, - "last": 3.459204315576534, - "max": 3.459204315576534, - "min": 3.459204315576534, - "rate": 0.016666666666666666 - } - }, - { - "name": "active_queries", - "values": { - "average": 0, - "count": 1, - "last": 0, - "max": 0, - "min": 0, - "rate": 0.016666666666666666 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverNumFailedResponseWrites", - "values": { - "count": 85, - "rate": 1.4166666666666667 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverNumFailedResponses", - "values": { - "count": 8, - "rate": 0.13333333333333333 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverNumConnections", - "values": { - "count": 1630, - "rate": 27.166666666666668 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverNumSuccessfulResponses", - "values": { - "count": 1621, - "rate": 27.016666666666666 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverNetworkLatency", - "values": { - "average": 0.11715958713775308, - "count": 20152, - "last": 0, - "max": 55, - "min": 0, - "rate": 335.8666666666667 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverNumSuccessfulResponseWrites", - "values": { - "count": 20152, - "rate": 335.8666666666667 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverTotalSuccessfulResponseLatency", - "values": { - "average": 90.88401253918495, - "count": 1595, - "last": 80, - "max": 233, - "min": 0, - "rate": 26.583333333333332 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverNumRequests", - "values": { - "count": 1633, - "rate": 27.216666666666665 - } - }, - { - "dimensions": { - "serverName": "qrs-server", - "serverPort": "4080" - }, - "name": "serverTotalFailedResponseLatency", - "values": { - "average": 0.75, - "count": 8, - "last": 1, - "max": 1, - "min": 0, - "rate": 0.13333333333333333 - } - }, - { - "dimensions": {"chain": "asvBlendingResult"}, - "name": "query_latency", - "values": { - "average": 83.35949367088608, - "count": 1580, - "last": 61, - "max": 224, - "min": 12, - "rate": 26.333333333333332 - } - }, - { - "dimensions": {"chain": "asvBlendingResult"}, - "name": "max_query_latency", - "values": { - "average": 83.35949367088608, - "count": 1580, - "last": 61, - "max": 224, - "min": 12, - "rate": 26.333333333333332 - } - }, - { - "dimensions": {"chain": "asvBlendingResult"}, - "name": "peak_qps", - "values": { - "average": 25.87656434951563, - "count": 6, - "last": 23.681592039800993, - "max": 29.7659845295212, - "min": 23.681592039800993, - "rate": 0.1 - } - }, - { - "dimensions": {"chain": "asvBlendingResult"}, - "name": "queries", - "values": { - "count": 1584, - "rate": 26.4 - } - }, - { - "dimensions": {"chain": "asvBlendingResult"}, - "name": "mean_query_latency", - "values": { - "average": 83.35949367088608, - "count": 1580, - "last": 61, - "max": 224, - "min": 12, - "rate": 26.333333333333332 - } - }, - { - "dimensions": {"chain": "asvBlendingResult"}, - "name": "hits_per_query", - "values": { - "average": 173.70126582278482, - "count": 1580, - "last": 175, - "max": 175, - "min": 5, - "rate": 26.333333333333332 - } - }, - { - "dimensions": {"chain": "blendingResult"}, - "name": "query_latency", - "values": { - "average": 39.40909090909091, - "count": 22, - "last": 26, - "max": 174, - "min": 13, - "rate": 0.36666666666666664 - } - }, - { - "dimensions": {"chain": "blendingResult"}, - "name": "max_query_latency", - "values": { - "average": 39.40909090909091, - "count": 22, - "last": 26, - "max": 174, - "min": 13, - "rate": 0.36666666666666664 - } - }, - { - "dimensions": {"chain": "blendingResult"}, - "name": "peak_qps", - "values": { - "average": 0.5890415170417276, - "count": 3, - "last": 0.40488561981240295, - "max": 0.864528399757932, - "min": 0.40488561981240295, - "rate": 0.05 - } - }, - { - "dimensions": {"chain": "blendingResult"}, - "name": "queries", - "values": { - "count": 22, - "rate": 0.36666666666666664 - } - }, - { - "dimensions": {"chain": "blendingResult"}, - "name": "mean_query_latency", - "values": { - "average": 39.40909090909091, - "count": 22, - "last": 26, - "max": 174, - "min": 13, - "rate": 0.36666666666666664 - } - }, - { - "dimensions": {"chain": "blendingResult"}, - "name": "hits_per_query", - "values": { - "average": 47.5, - "count": 22, - "last": 28, - "max": 176, - "min": 5, - "rate": 0.36666666666666664 - } - } - ] - }, - "status": {"code": "up"}, - "time": 1383132269767 -} \ No newline at end of file diff --git a/metrics-proxy/src/test/resources/metrics-state.json b/metrics-proxy/src/test/resources/metrics-state.json deleted file mode 100644 index b7773e5fb8b..00000000000 --- a/metrics-proxy/src/test/resources/metrics-state.json +++ /dev/null @@ -1,42 +0,0 @@ -{ -"status" : { - "code" : "up", - "message" : "Everything ok here" -}, -"metrics" : { - "snapshot" : { - "from" : 1334134640.089, - "to" : 1334134700.088 - }, - "values" : [ - { - "name" : "queries", - "description" : "Number of queries executed during snapshot interval", - "values" : { - "count" : 28, - "rate" : 0.4667 - }, - "dimensions" : { - "searcherid" : "x" - } - }, - { - "name" : "query_hits", - "description" : "Number of documents matched per query during snapshot interval", - "values" : { - "count" : 28, - "rate" : 0.4667, - "average" : 128.3, - "min" : 0, - "max" : 10000, - "sum" : 3584, - "median" : 124.0, - "std_deviation": 5.43 - }, - "dimensions" : { - "searcherid" : "x" - } - } - ] -} -} diff --git a/metrics-proxy/src/test/resources/metrics-storage-simple.json b/metrics-proxy/src/test/resources/metrics-storage-simple.json deleted file mode 100644 index 00715b52046..00000000000 --- a/metrics-proxy/src/test/resources/metrics-storage-simple.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "status" : { - "code" : "up", - "message": "All good" - }, - - "metrics" : { - "snapshot" : { - "from": 1335523285, - "to": 1335525685 - }, - "values": [ - - { - "name" : "foo", - "values" : { - "count" : 1 - }, - "dimensions" : { - "foo": "bar", - "bar" : "foo" - } - }, - - { - "name" : "bar", - "values" : { - "count" : 2 - }, - "dimensions" : { - "d0": "d0val", - "d1" : "d1val" - } - } - ] - - } -} diff --git a/metrics-proxy/src/test/resources/rpc-json-output-check.json b/metrics-proxy/src/test/resources/rpc-json-output-check.json deleted file mode 100644 index 701a06d82b2..00000000000 --- a/metrics-proxy/src/test/resources/rpc-json-output-check.json +++ /dev/null @@ -1 +0,0 @@ -{"metrics":[{"status_code":1,"application":"test-system.qrserver","dimensions":{"metrictype":"health","instance":"qrserver"},"status_msg":"SOMETHING FAILED"}]} \ No newline at end of file diff --git a/metrics-proxy/src/test/resources/yamas-array-no-routing.json b/metrics-proxy/src/test/resources/yamas-array-no-routing.json deleted file mode 100644 index 8f21e8253b9..00000000000 --- a/metrics-proxy/src/test/resources/yamas-array-no-routing.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "status_code" : 0, - "timestamp" : 1400047900, - "application" : "vespa.searchnode", - "metrics" : { - "cpu" : 55.5555555555555, - "memory_virt" : 22222222222, - "memory_rss" : 5555555555 - }, - "dimensions" : { - "applicationName" : "app", - "tenantName" : "tenant", - "metrictype" : "system", - "instance" : "searchnode", - "applicationInstance" : "default", - "clustername" : "cluster" - }, - "status_msg" : "Data collected successfully" -} diff --git a/metrics-proxy/src/test/resources/yamas-array.json b/metrics-proxy/src/test/resources/yamas-array.json deleted file mode 100644 index c9293623b25..00000000000 --- a/metrics-proxy/src/test/resources/yamas-array.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "status_code" : 0, - "timestamp" : 1400047900, - "application" : "vespa.searchnode", - "metrics" : { - "cpu" : 55.5555555555555, - "memory_virt" : 22222222222, - "memory_rss" : 5555555555 - }, - "dimensions" : { - "applicationName" : "app", - "tenantName" : "tenant", - "metrictype" : "system", - "instance" : "searchnode", - "applicationInstance" : "default", - "clustername" : "cluster" - }, - "routing" : { - "yamas" : { - "namespaces" : [ - "Vespa" - ] - } - }, - "status_msg" : "Data collected successfully" -} diff --git a/pom.xml b/pom.xml index 8fc9cb21db8..c17b7d4a101 100644 --- a/pom.xml +++ b/pom.xml @@ -96,7 +96,6 @@ messagebus-disc messagebus metrics - metrics-proxy model-evaluation model-integration node-repository -- cgit v1.2.3