diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2019-06-17 13:33:09 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-06-17 13:33:09 +0200 |
commit | 0186420fe998eb7c8d741102ee268dcadda7d91b (patch) | |
tree | 15cc166a6a03ab2dec3c715230373abbaeae262c | |
parent | c73d733980a78a469f7b1bf34ccb98188e779baa (diff) | |
parent | 756579381f60bc881d7633cb6f14deb9c03b72d6 (diff) |
Merge pull request #9822 from vespa-engine/freva/simplify-metrics
Node-Admin: Simplify metrics
23 files changed, 446 insertions, 513 deletions
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index 8cdb0bee7c2..43313392cdb 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -26,9 +26,8 @@ import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.dockerapi.exception.ContainerNotFoundException; import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; import com.yahoo.vespa.hosted.dockerapi.exception.DockerExecTimeoutException; -import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper; -import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Counter; +import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import java.io.ByteArrayOutputStream; import java.time.Duration; @@ -56,19 +55,18 @@ public class DockerImpl implements Docker { private final DockerClient dockerClient; private final DockerImageGarbageCollector dockerImageGC; - private final CounterWrapper numberOfDockerDaemonFails; + private final Counter numberOfDockerApiFails; @Inject - public DockerImpl(MetricReceiverWrapper metricReceiverWrapper) { - this(createDockerClient(), metricReceiverWrapper); + public DockerImpl(Metrics metrics) { + this(createDockerClient(), metrics); } - DockerImpl(DockerClient dockerClient, MetricReceiverWrapper metricReceiver) { + DockerImpl(DockerClient dockerClient, Metrics metrics) { this.dockerClient = dockerClient; this.dockerImageGC = new DockerImageGarbageCollector(this); - Dimensions dimensions = new Dimensions.Builder().add("role", "docker").build(); - numberOfDockerDaemonFails = metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "daemon.api_fails"); + numberOfDockerApiFails = metrics.declareCounter("docker.api_fails"); } @Override @@ -86,7 +84,7 @@ public class DockerImpl implements Docker { return true; } } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to pull image '" + image.asString() + "'", e); } } @@ -110,7 +108,7 @@ public class DockerImpl implements Docker { } catch (NotFoundException e) { return Optional.empty(); } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to inspect image '" + dockerImage.asString() + "'", e); } } @@ -146,7 +144,7 @@ public class DockerImpl implements Docker { return new ProcessResult(state.getExitCode(), new String(output.toByteArray()), new String(errors.toByteArray())); } catch (RuntimeException | InterruptedException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Container '" + containerName.asString() + "' failed to execute " + Arrays.toString(command), e); } @@ -171,7 +169,7 @@ public class DockerImpl implements Docker { } catch (NotFoundException ignored) { return Optional.empty(); } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to get info for container '" + container + "'", e); } } @@ -186,7 +184,7 @@ public class DockerImpl implements Docker { } catch (NotFoundException ignored) { return Optional.empty(); } catch (RuntimeException | InterruptedException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to get stats for container '" + containerName.asString() + "'", e); } } @@ -200,7 +198,7 @@ public class DockerImpl implements Docker { } catch (NotModifiedException ignored) { // If is already started, ignore } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to start container '" + containerName.asString() + "'", e); } } @@ -214,7 +212,7 @@ public class DockerImpl implements Docker { } catch (NotModifiedException ignored) { // If is already stopped, ignore } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to stop container '" + containerName.asString() + "'", e); } } @@ -226,7 +224,7 @@ public class DockerImpl implements Docker { } catch (NotFoundException e) { throw new ContainerNotFoundException(containerName); } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to delete container '" + containerName.asString() + "'", e); } } @@ -253,7 +251,7 @@ public class DockerImpl implements Docker { } catch (NotFoundException e) { throw new ContainerNotFoundException(containerName); } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to update container '" + containerName.asString() + "' to " + resources, e); } } @@ -307,7 +305,7 @@ public class DockerImpl implements Docker { try { return dockerClient.listContainersCmd().withShowAll(true).exec(); } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to list all containers", e); } } @@ -316,7 +314,7 @@ public class DockerImpl implements Docker { try { return dockerClient.listImagesCmd().withShowAll(true).exec(); } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to list all images", e); } } @@ -327,7 +325,7 @@ public class DockerImpl implements Docker { } catch (NotFoundException ignored) { // Image was already deleted, ignore } catch (RuntimeException e) { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerException("Failed to delete docker image " + dockerImage.asString(), e); } } @@ -357,7 +355,7 @@ public class DockerImpl implements Docker { logger.log(LogLevel.INFO, "Download completed: " + dockerImage.asString()); removeScheduledPoll(dockerImage); } else { - numberOfDockerDaemonFails.add(); + numberOfDockerApiFails.increment(); throw new DockerClientException("Could not download image: " + dockerImage); } } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Counter.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Counter.java new file mode 100644 index 00000000000..3a0b820c846 --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Counter.java @@ -0,0 +1,28 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi.metrics; + +/** + * @author freva + */ +public class Counter implements MetricValue { + private final Object lock = new Object(); + + private long value = 0; + + public void increment() { + add(1L); + } + + public void add(long n) { + synchronized (lock) { + value += n; + } + } + + @Override + public Number getValue() { + synchronized (lock) { + return value; + } + } +} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/CounterWrapper.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/CounterWrapper.java deleted file mode 100644 index 55c42271674..00000000000 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/CounterWrapper.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.dockerapi.metrics; - -import com.yahoo.metrics.simple.Counter; - -/** - * Forwards sample to {@link com.yahoo.metrics.simple.Counter} to be displayed in /state/v1/metrics, - * while also saving the value so it can be accessed programatically later. - * - * @author valerijf - */ -public class CounterWrapper implements MetricValue { - private final Object lock = new Object(); - - private final Counter counter; - private long value = 0; - - CounterWrapper(Counter counter) { - this.counter = counter; - } - - public void add() { - add(1L); - } - - public void add(long n) { - synchronized (lock) { - counter.add(n); - value += n; - } - } - - @Override - public Number getValue() { - synchronized (lock) { - return value; - } - } -} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java index 770ff5e2216..ef59c4b17d6 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/DimensionMetrics.java @@ -4,47 +4,67 @@ package com.yahoo.vespa.hosted.dockerapi.metrics; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; /** * @author freva */ public class DimensionMetrics { - private final static ObjectMapper objectMapper = new ObjectMapper(); + private static final ObjectMapper objectMapper = new ObjectMapper(); + private static final Map<String, Object> routing = Map.of("yamas", Map.of("namespaces", List.of("Vespa"))); private final String application; private final Dimensions dimensions; private final Map<String, Number> metrics; DimensionMetrics(String application, Dimensions dimensions, Map<String, Number> metrics) { - this.application = application; - this.dimensions = dimensions; - this.metrics = metrics; + this.application = Objects.requireNonNull(application); + this.dimensions = Objects.requireNonNull(dimensions); + this.metrics = Objects.requireNonNull(metrics); } - Map<String, Object> getMetrics() { - final Map<String, Object> routing = new HashMap<>(); - final Map<String, Object> routingMonitoring = new HashMap<>(); - routing.put("yamas", routingMonitoring); - routingMonitoring.put("namespaces", Collections.singletonList("Vespa")); - - Map<String, Object> report = new HashMap<>(); + public String toSecretAgentReport() throws JsonProcessingException { + Map<String, Object> report = new TreeMap<>(); report.put("application", application); - report.put("dimensions", dimensions.dimensionsMap); - report.put("metrics", metrics); + report.put("dimensions", new TreeMap<>(dimensions.asMap())); + report.put("metrics", new TreeMap<>(metrics)); report.put("routing", routing); - return report; - } - - public String toSecretAgentReport() throws JsonProcessingException { - Map<String, Object> report = getMetrics(); report.put("timestamp", System.currentTimeMillis() / 1000); return objectMapper.writeValueAsString(report); } + public String getApplication() { + return application; + } + + public Dimensions getDimensions() { + return dimensions; + } + + public Map<String, Number> getMetrics() { + return metrics; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DimensionMetrics that = (DimensionMetrics) o; + return application.equals(that.application) && + dimensions.equals(that.dimensions) && + metrics.equals(that.metrics); + } + + @Override + public int hashCode() { + return Objects.hash(application, dimensions, metrics); + } + public static class Builder { private final String application; private final Dimensions dimensions; diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java index 586622100fb..63b92e06505 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Dimensions.java @@ -1,20 +1,24 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.dockerapi.metrics; -import java.util.Collections; import java.util.HashMap; import java.util.Map; /** - * Each metric reported to secret agent has dimensions. - * - * @author valerijf + * @author freva */ public class Dimensions { - final Map<String, Object> dimensionsMap; - private Dimensions(Map<String, Object> dimensionsMap) { - this.dimensionsMap = dimensionsMap; + public static final Dimensions NONE = new Dimensions(Map.of()); + + private final Map<String, String> dimensionsMap; + + public Dimensions(Map<String, String> dimensionsMap) { + this.dimensionsMap = Map.copyOf(dimensionsMap); + } + + public Map<String, String> asMap() { + return dimensionsMap; } @Override @@ -45,7 +49,7 @@ public class Dimensions { } public Dimensions build() { - return new Dimensions(Collections.unmodifiableMap(new HashMap<>(dimensionsMap))); + return new Dimensions(dimensionsMap); } } } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Gauge.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Gauge.java new file mode 100644 index 00000000000..b413475fc2b --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Gauge.java @@ -0,0 +1,24 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi.metrics; + +/** + * @author freva + */ +public class Gauge implements MetricValue { + private final Object lock = new Object(); + + private double value; + + public void sample(double x) { + synchronized (lock) { + this.value = x; + } + } + + @Override + public Number getValue() { + synchronized (lock) { + return value; + } + } +} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/GaugeWrapper.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/GaugeWrapper.java deleted file mode 100644 index 02e1f15a94f..00000000000 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/GaugeWrapper.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.dockerapi.metrics; - -import com.yahoo.metrics.simple.Gauge; - -/** - * Forwards sample to {@link com.yahoo.metrics.simple.Gauge} to be displayed in /state/v1/metrics, - * while also saving the value so it can be accessed programatically later. - * - * @author valerijf - */ -public class GaugeWrapper implements MetricValue { - private final Object lock = new Object(); - - private final Gauge gauge; - private double value; - - GaugeWrapper(Gauge gauge) { - this.gauge = gauge; - } - - public void sample(double x) { - synchronized (lock) { - gauge.sample(x); - this.value = x; - } - } - - @Override - public Number getValue() { - synchronized (lock) { - return value; - } - } -} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java deleted file mode 100644 index 58126a59cbb..00000000000 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.dockerapi.metrics; - -import com.google.inject.Inject; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.metrics.simple.Point; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * Export metrics to both /state/v1/metrics and makes them available programmatically. - * Each metric belongs to a monitoring application - * - * @author freva - */ -public class MetricReceiverWrapper { - // Application names used - public static final String APPLICATION_DOCKER = "docker"; - public static final String APPLICATION_HOST = "vespa.host"; - public static final String APPLICATION_NODE = "vespa.node"; - - private final Object monitor = new Object(); - private final Map<DimensionType, Map<String, ApplicationMetrics>> metrics = new HashMap<>(); - private final MetricReceiver metricReceiver; - - @Inject - public MetricReceiverWrapper(MetricReceiver metricReceiver) { - this.metricReceiver = metricReceiver; - } - - /** - * Declaring the same dimensions and name results in the same CounterWrapper instance (idempotent). - */ - public CounterWrapper declareCounter(String application, Dimensions dimensions, String name) { - return declareCounter(application, dimensions, name, DimensionType.DEFAULT); - } - - public CounterWrapper declareCounter(String application, Dimensions dimensions, String name, DimensionType type) { - synchronized (monitor) { - Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = getOrCreateApplicationMetrics(application, type); - if (!metricsByDimensions.containsKey(dimensions)) metricsByDimensions.put(dimensions, new HashMap<>()); - if (!metricsByDimensions.get(dimensions).containsKey(name)) { - CounterWrapper counter = new CounterWrapper(metricReceiver.declareCounter(name, new Point(dimensions.dimensionsMap))); - metricsByDimensions.get(dimensions).put(name, counter); - } - - return (CounterWrapper) metricsByDimensions.get(dimensions).get(name); - } - } - - /** - * Declaring the same dimensions and name results in the same GaugeWrapper instance (idempotent). - */ - public GaugeWrapper declareGauge(String application, Dimensions dimensions, String name) { - return declareGauge(application, dimensions, name, DimensionType.DEFAULT); - } - - public GaugeWrapper declareGauge(String application, Dimensions dimensions, String name, DimensionType type) { - synchronized (monitor) { - Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = getOrCreateApplicationMetrics(application, type); - if (!metricsByDimensions.containsKey(dimensions)) - metricsByDimensions.put(dimensions, new HashMap<>()); - if (!metricsByDimensions.get(dimensions).containsKey(name)) { - GaugeWrapper gauge = new GaugeWrapper(metricReceiver.declareGauge(name, new Point(dimensions.dimensionsMap))); - metricsByDimensions.get(dimensions).put(name, gauge); - } - - return (GaugeWrapper) metricsByDimensions.get(dimensions).get(name); - } - } - - public List<DimensionMetrics> getDefaultMetrics() { - return getMetricsByType(DimensionType.DEFAULT); - } - - // For testing, returns same as getDefaultMetrics(), but without "timestamp" - public Set<Map<String, Object>> getDefaultMetricsRaw() { - synchronized (monitor) { - Set<Map<String, Object>> dimensionMetrics = new HashSet<>(); - metrics.getOrDefault(DimensionType.DEFAULT, new HashMap<>()) - .forEach((application, applicationMetrics) -> applicationMetrics.metricsByDimensions().entrySet().stream() - .map(entry -> new DimensionMetrics(application, entry.getKey(), - entry.getValue().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, value -> value.getValue().getValue())))) - .map(DimensionMetrics::getMetrics) - .forEach(dimensionMetrics::add)); - return dimensionMetrics; - } - } - - public List<DimensionMetrics> getMetricsByType(DimensionType type) { - synchronized (monitor) { - List<DimensionMetrics> dimensionMetrics = new ArrayList<>(); - metrics.getOrDefault(type, new HashMap<>()) - .forEach((application, applicationMetrics) -> applicationMetrics.metricsByDimensions().entrySet().stream() - .map(entry -> new DimensionMetrics(application, entry.getKey(), - entry.getValue().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, value -> value.getValue().getValue())))) - .forEach(dimensionMetrics::add)); - return dimensionMetrics; - } - } - - public void deleteMetricByDimension(String name, Dimensions dimensionsToRemove, DimensionType type) { - synchronized (monitor) { - Optional.ofNullable(metrics.get(type)) - .map(m -> m.get(name)) - .map(ApplicationMetrics::metricsByDimensions) - .ifPresent(m -> m.remove(dimensionsToRemove)); - } - } - - // For testing - Map<String, Number> getMetricsForDimension(String application, Dimensions dimensions) { - synchronized (monitor) { - Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = getOrCreateApplicationMetrics(application, DimensionType.DEFAULT); - return metricsByDimensions.getOrDefault(dimensions, Collections.emptyMap()) - .entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getValue())); - } - } - - private Map<Dimensions, Map<String, MetricValue>> getOrCreateApplicationMetrics(String application, DimensionType type) { - Map<String, ApplicationMetrics> applicationMetrics = metrics.computeIfAbsent(type, m -> new HashMap<>()); - if (! applicationMetrics.containsKey(application)) { - ApplicationMetrics metrics = new ApplicationMetrics(); - applicationMetrics.put(application, metrics); - } - return applicationMetrics.get(application).metricsByDimensions(); - } - - // "Application" is the monitoring application, not Vespa application - private static class ApplicationMetrics { - private final Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = new LinkedHashMap<>(); - - Map<Dimensions, Map<String, MetricValue>> metricsByDimensions() { - return metricsByDimensions; - } - } - - // Used to distinguish whether metrics have been populated with all tag vaules - public enum DimensionType {DEFAULT, PRETAGGED} -} diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricValue.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricValue.java index 7bd4968747f..b20aa1b11ff 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricValue.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricValue.java @@ -1,8 +1,8 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.dockerapi.metrics; /** - * @author valerijf + * @author freva */ public interface MetricValue { Number getValue(); diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Metrics.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Metrics.java new file mode 100644 index 00000000000..f9b169f0a93 --- /dev/null +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/Metrics.java @@ -0,0 +1,128 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi.metrics; + +import com.google.inject.Inject; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Stores the latest metric for the given application, name, dimension triplet in memory + * + * @author freva + */ +public class Metrics { + // Application names used + public static final String APPLICATION_HOST = "vespa.host"; + public static final String APPLICATION_NODE = "vespa.node"; + + private final Object monitor = new Object(); + private final Map<DimensionType, Map<String, ApplicationMetrics>> metrics = new HashMap<>(); + + @Inject + public Metrics() { } + + /** + * Creates a counter metric under vespa.host application, with no dimensions and default dimension type + * See {@link #declareCounter(String, String, Dimensions, DimensionType)} + */ + public Counter declareCounter(String name) { + return declareCounter(name, Dimensions.NONE); + } + + /** + * Creates a counter metric under vespa.host application, with the given dimensions and default dimension type + * See {@link #declareCounter(String, String, Dimensions, DimensionType)} + */ + public Counter declareCounter(String name, Dimensions dimensions) { + return declareCounter(APPLICATION_HOST, name, dimensions, DimensionType.DEFAULT); + } + + /** Creates a counter metric. This method is idempotent. */ + public Counter declareCounter(String application, String name, Dimensions dimensions, DimensionType type) { + synchronized (monitor) { + return (Counter) getOrCreateApplicationMetrics(application, type) + .computeIfAbsent(dimensions, d -> new HashMap<>()) + .computeIfAbsent(name, n -> new Counter()); + } + } + + /** + * Creates a gauge metric under vespa.host application, with no dimensions and default dimension type + * See {@link #declareGauge(String, String, Dimensions, DimensionType)} + */ + public Gauge declareGauge(String name) { + return declareGauge(name, Dimensions.NONE); + } + + /** + * Creates a gauge metric under vespa.host application, with the given dimensions and default dimension type + * See {@link #declareGauge(String, String, Dimensions, DimensionType)} + */ + public Gauge declareGauge(String name, Dimensions dimensions) { + return declareGauge(APPLICATION_HOST, name, dimensions, DimensionType.DEFAULT); + } + + /** Creates a gauge metric. This method is idempotent */ + public Gauge declareGauge(String application, String name, Dimensions dimensions, DimensionType type) { + synchronized (monitor) { + return (Gauge) getOrCreateApplicationMetrics(application, type) + .computeIfAbsent(dimensions, d -> new HashMap<>()) + .computeIfAbsent(name, n -> new Gauge()); + } + } + + public List<DimensionMetrics> getDefaultMetrics() { + return getMetricsByType(DimensionType.DEFAULT); + } + + public List<DimensionMetrics> getMetricsByType(DimensionType type) { + synchronized (monitor) { + List<DimensionMetrics> dimensionMetrics = new ArrayList<>(); + metrics.getOrDefault(type, Map.of()) + .forEach((application, applicationMetrics) -> applicationMetrics.metricsByDimensions().entrySet().stream() + .map(entry -> new DimensionMetrics(application, entry.getKey(), + entry.getValue().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, value -> value.getValue().getValue())))) + .forEach(dimensionMetrics::add)); + return dimensionMetrics; + } + } + + public void deleteMetricByDimension(String name, Dimensions dimensionsToRemove, DimensionType type) { + synchronized (monitor) { + Optional.ofNullable(metrics.get(type)) + .map(m -> m.get(name)) + .map(ApplicationMetrics::metricsByDimensions) + .ifPresent(m -> m.remove(dimensionsToRemove)); + } + } + + Map<Dimensions, Map<String, MetricValue>> getOrCreateApplicationMetrics(String application, DimensionType type) { + return metrics.computeIfAbsent(type, m -> new HashMap<>()) + .computeIfAbsent(application, app -> new ApplicationMetrics()) + .metricsByDimensions(); + } + + // "Application" is the monitoring application, not Vespa application + private static class ApplicationMetrics { + private final Map<Dimensions, Map<String, MetricValue>> metricsByDimensions = new LinkedHashMap<>(); + + Map<Dimensions, Map<String, MetricValue>> metricsByDimensions() { + return metricsByDimensions; + } + } + + // Used to distinguish whether metrics have been populated with all tag vaules + public enum DimensionType { + /** Default metrics get added default dimensions set in check config */ + DEFAULT, + + /** Pretagged metrics will only get the dimensions explicitly set when creating the counter/gauge */ + PRETAGGED + } +} diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java index df221302575..4843d8f9685 100644 --- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerImplTest.java @@ -14,8 +14,7 @@ import com.github.dockerjava.api.command.PullImageCmd; import com.github.dockerjava.api.exception.NotFoundException; import com.github.dockerjava.core.command.ExecStartResultCallback; import com.yahoo.config.provision.DockerImage; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Matchers; @@ -37,8 +36,8 @@ import static org.mockito.Mockito.when; public class DockerImplTest { private final DockerClient dockerClient = mock(DockerClient.class); - private final MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - private final DockerImpl docker = new DockerImpl(dockerClient, metricReceiver); + private final Metrics metrics = new Metrics(); + private final DockerImpl docker = new DockerImpl(dockerClient, metrics); @Test public void testExecuteCompletes() { diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapperTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapperTest.java deleted file mode 100644 index c20e64d906e..00000000000 --- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapperTest.java +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.dockerapi.metrics; - -import com.yahoo.metrics.simple.MetricReceiver; -import org.junit.Test; - -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * @author freva - */ -public class MetricReceiverWrapperTest { - private static final Dimensions hostDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build(); - private static final String applicationDocker = MetricReceiverWrapper.APPLICATION_DOCKER; - - @Test - public void testDefaultValue() { - MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - - metricReceiver.declareCounter(applicationDocker, hostDimension, "some.name"); - - assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, hostDimension).get("some.name"), 0L); - } - - @Test - public void testSimpleIncrementMetric() { - MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - CounterWrapper counter = metricReceiver.declareCounter(applicationDocker, hostDimension, "a_counter.value"); - - counter.add(5); - counter.add(8); - - Map<String, Number> latestMetrics = metricReceiver.getMetricsForDimension(applicationDocker, hostDimension); - assertEquals("Expected only 1 metric value to be set", 1, latestMetrics.size()); - assertEquals(latestMetrics.get("a_counter.value"), 13L); // 5 + 8 - } - - @Test - public void testSimpleGauge() { - MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - GaugeWrapper gauge = metricReceiver.declareGauge(applicationDocker, hostDimension, "test.gauge"); - - gauge.sample(42); - gauge.sample(-342.23); - - Map<String, Number> latestMetrics = metricReceiver.getMetricsForDimension(applicationDocker, hostDimension); - assertEquals("Expected only 1 metric value to be set", 1, latestMetrics.size()); - assertEquals(latestMetrics.get("test.gauge"), -342.23); - } - - @Test - public void testRedeclaringSameGauge() { - MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - GaugeWrapper gauge = metricReceiver.declareGauge(applicationDocker, hostDimension, "test.gauge"); - gauge.sample(42); - - // Same as hostDimension, but new instance. - Dimensions newDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build(); - GaugeWrapper newGauge = metricReceiver.declareGauge(applicationDocker, newDimension, "test.gauge"); - newGauge.sample(56); - - assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, hostDimension).get("test.gauge"), 56.); - } - - @Test - public void testSameMetricNameButDifferentDimensions() { - MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - GaugeWrapper gauge = metricReceiver.declareGauge(applicationDocker, hostDimension, "test.gauge"); - gauge.sample(42); - - // Not the same as hostDimension. - Dimensions newDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build(); - GaugeWrapper newGauge = metricReceiver.declareGauge(applicationDocker, newDimension, "test.gauge"); - newGauge.sample(56); - - assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, hostDimension).get("test.gauge"), 42.); - assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, newDimension).get("test.gauge"), 56.); - } - - @Test - public void testDeletingMetric() { - MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); - metricReceiver.declareGauge(applicationDocker, hostDimension, "test.gauge"); - - Dimensions differentDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build(); - metricReceiver.declareGauge(applicationDocker, differentDimension, "test.gauge"); - - assertEquals(2, metricReceiver.getDefaultMetricsRaw().size()); - metricReceiver.deleteMetricByDimension(applicationDocker, differentDimension, MetricReceiverWrapper.DimensionType.DEFAULT); - assertEquals(1, metricReceiver.getDefaultMetricsRaw().size()); - assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, hostDimension).size(), 1); - assertEquals(metricReceiver.getMetricsForDimension(applicationDocker, differentDimension).size(), 0); - } -} diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricsTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricsTest.java new file mode 100644 index 00000000000..fc153ee0562 --- /dev/null +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricsTest.java @@ -0,0 +1,99 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.dockerapi.metrics; + +import org.junit.Test; + +import java.util.Map; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.dockerapi.metrics.Metrics.APPLICATION_HOST; +import static com.yahoo.vespa.hosted.dockerapi.metrics.Metrics.DimensionType.DEFAULT; +import static org.junit.Assert.assertEquals; + +/** + * @author freva + */ +public class MetricsTest { + private static final Dimensions hostDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build(); + private final Metrics metrics = new Metrics(); + + @Test + public void testDefaultValue() { + metrics.declareCounter("some.name", hostDimension); + + assertEquals(getMetricsForDimension(hostDimension).get("some.name"), 0L); + } + + @Test + public void testSimpleIncrementMetric() { + Counter counter = metrics.declareCounter("a_counter.value", hostDimension); + + counter.add(5); + counter.add(8); + + Map<String, Number> latestMetrics = getMetricsForDimension(hostDimension); + assertEquals("Expected only 1 metric value to be set", 1, latestMetrics.size()); + assertEquals(latestMetrics.get("a_counter.value"), 13L); // 5 + 8 + } + + @Test + public void testSimpleGauge() { + Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); + + gauge.sample(42); + gauge.sample(-342.23); + + Map<String, Number> latestMetrics = getMetricsForDimension(hostDimension); + assertEquals("Expected only 1 metric value to be set", 1, latestMetrics.size()); + assertEquals(latestMetrics.get("test.gauge"), -342.23); + } + + @Test + public void testRedeclaringSameGauge() { + Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); + gauge.sample(42); + + // Same as hostDimension, but new instance. + Dimensions newDimension = new Dimensions.Builder().add("host", "abc.yahoo.com").build(); + Gauge newGauge = metrics.declareGauge("test.gauge", newDimension); + newGauge.sample(56); + + assertEquals(getMetricsForDimension(hostDimension).get("test.gauge"), 56.); + } + + @Test + public void testSameMetricNameButDifferentDimensions() { + Gauge gauge = metrics.declareGauge("test.gauge", hostDimension); + gauge.sample(42); + + // Not the same as hostDimension. + Dimensions newDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build(); + Gauge newGauge = metrics.declareGauge("test.gauge", newDimension); + newGauge.sample(56); + + assertEquals(getMetricsForDimension(hostDimension).get("test.gauge"), 42.); + assertEquals(getMetricsForDimension(newDimension).get("test.gauge"), 56.); + } + + @Test + public void testDeletingMetric() { + metrics.declareGauge("test.gauge", hostDimension); + + Dimensions differentDimension = new Dimensions.Builder().add("host", "abcd.yahoo.com").build(); + metrics.declareGauge("test.gauge", differentDimension); + + assertEquals(2, metrics.getMetricsByType(DEFAULT).size()); + metrics.deleteMetricByDimension(APPLICATION_HOST, differentDimension, DEFAULT); + assertEquals(1, metrics.getMetricsByType(DEFAULT).size()); + assertEquals(getMetricsForDimension(hostDimension).size(), 1); + assertEquals(getMetricsForDimension(differentDimension).size(), 0); + } + + private Map<String, Number> getMetricsForDimension(Dimensions dimensions) { + return metrics.getOrCreateApplicationMetrics(APPLICATION_HOST, DEFAULT) + .getOrDefault(dimensions, Map.of()) + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getValue())); + } +} diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml index db00c686c99..d5a4dce7c5a 100644 --- a/node-admin/src/main/application/services.xml +++ b/node-admin/src/main/application/services.xml @@ -5,7 +5,7 @@ <!-- Please update container test when changing this file --> <accesslog type="vespa" fileNamePattern="logs/vespa/node-admin/access.log.%Y%m%d%H%M%S" symlinkName="access.log" /> <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/> - <component id="metrics-wrapper" class="com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper" bundle="docker-api"/> + <component id="metrics" class="com.yahoo.vespa.hosted.dockerapi.metrics.Metrics" bundle="docker-api"/> <preprocess:include file="variant.xml" required="false"/> </container> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java index 00ec985ba0c..7de2aae77c8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java @@ -16,11 +16,8 @@ public interface NodeAdmin { /** Start/stop NodeAgents and schedule next NodeAgent ticks with the given NodeAgentContexts */ void refreshContainersToRun(Set<NodeAgentContext> nodeAgentContexts); - /** Gather node agent and its docker container metrics and forward them to the {@code MetricReceiverWrapper} */ - void updateNodeAgentMetrics(); - - /** Gather node admin metrics and forward them to the {@code MetricReceiverWrapper} */ - void updateNodeAdminMetrics(); + /** Update node admin metrics */ + void updateMetrics(); /** * Attempts to freeze/unfreeze all NodeAgents and itself. To freeze a NodeAgent means that diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index 0d520241ac8..cb10eac9e6c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeadmin; -import com.yahoo.vespa.hosted.dockerapi.metrics.CounterWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Counter; import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; -import com.yahoo.vespa.hosted.dockerapi.metrics.GaugeWrapper; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Gauge; +import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgent; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextManager; @@ -39,28 +39,26 @@ public class NodeAdminImpl implements NodeAdmin { private boolean previousWantFrozen; private boolean isFrozen; private Instant startOfFreezeConvergence; - private final Map<String, NodeAgentWithScheduler> nodeAgentWithSchedulerByHostname = new ConcurrentHashMap<>(); - private final GaugeWrapper numberOfContainersInLoadImageState; - private final GaugeWrapper jvmHeapUsed; - private final GaugeWrapper jvmHeapFree; - private final GaugeWrapper jvmHeapTotal; - private final CounterWrapper numberOfUnhandledExceptionsInNodeAgent; + private final Gauge jvmHeapUsed; + private final Gauge jvmHeapFree; + private final Gauge jvmHeapTotal; + private final Counter numberOfUnhandledExceptions; - public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, MetricReceiverWrapper metricReceiver, Clock clock) { + public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, Metrics metrics, Clock clock) { this((NodeAgentWithSchedulerFactory) nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext), - metricReceiver, clock, NODE_AGENT_FREEZE_TIMEOUT, NODE_AGENT_SPREAD); + metrics, clock, NODE_AGENT_FREEZE_TIMEOUT, NODE_AGENT_SPREAD); } - public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, MetricReceiverWrapper metricReceiver, + public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, Metrics metrics, Clock clock, Duration freezeTimeout, Duration spread) { this((NodeAgentWithSchedulerFactory) nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext), - metricReceiver, clock, freezeTimeout, spread); + metrics, clock, freezeTimeout, spread); } NodeAdminImpl(NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory, - MetricReceiverWrapper metricReceiver, Clock clock, Duration freezeTimeout, Duration spread) { + Metrics metrics, Clock clock, Duration freezeTimeout, Duration spread) { this.nodeAgentWithSchedulerFactory = nodeAgentWithSchedulerFactory; this.clock = clock; @@ -70,13 +68,12 @@ public class NodeAdminImpl implements NodeAdmin { this.isFrozen = true; this.startOfFreezeConvergence = clock.instant(); - Dimensions dimensions = new Dimensions.Builder().add("role", "docker").build(); - this.numberOfContainersInLoadImageState = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.image.loading"); - this.numberOfUnhandledExceptionsInNodeAgent = metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.unhandled_exceptions"); + this.numberOfUnhandledExceptions = metrics.declareCounter("unhandled_exceptions", + new Dimensions(Map.of("src", "node-agents"))); - this.jvmHeapUsed = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_HOST, new Dimensions.Builder().build(), "mem.heap.used"); - this.jvmHeapFree = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_HOST, new Dimensions.Builder().build(), "mem.heap.free"); - this.jvmHeapTotal = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_HOST, new Dimensions.Builder().build(), "mem.heap.total"); + this.jvmHeapUsed = metrics.declareGauge("mem.heap.used"); + this.jvmHeapFree = metrics.declareGauge("mem.heap.free"); + this.jvmHeapTotal = metrics.declareGauge("mem.heap.total"); } @Override @@ -105,22 +102,12 @@ public class NodeAdminImpl implements NodeAdmin { } @Override - public void updateNodeAgentMetrics() { - int numberContainersWaitingImage = 0; - int numberOfNewUnhandledExceptions = 0; - + public void updateMetrics() { for (NodeAgentWithScheduler nodeAgentWithScheduler : nodeAgentWithSchedulerByHostname.values()) { - if (nodeAgentWithScheduler.isDownloadingImage()) numberContainersWaitingImage++; - numberOfNewUnhandledExceptions += nodeAgentWithScheduler.getAndResetNumberOfUnhandledExceptions(); + numberOfUnhandledExceptions.add(nodeAgentWithScheduler.getAndResetNumberOfUnhandledExceptions()); nodeAgentWithScheduler.updateContainerNodeMetrics(); } - numberOfContainersInLoadImageState.sample(numberContainersWaitingImage); - numberOfUnhandledExceptionsInNodeAgent.add(numberOfNewUnhandledExceptions); - } - - @Override - public void updateNodeAdminMetrics() { Runtime runtime = Runtime.getRuntime(); long freeMemory = runtime.freeMemory(); long totalMemory = runtime.totalMemory(); @@ -208,7 +195,6 @@ public class NodeAdminImpl implements NodeAdmin { @Override public void stopForHostSuspension() { nodeAgent.stopForHostSuspension(); } @Override public void stopForRemoval() { nodeAgent.stopForRemoval(); } @Override public void updateContainerNodeMetrics() { nodeAgent.updateContainerNodeMetrics(); } - @Override public boolean isDownloadingImage() { return nodeAgent.isDownloadingImage(); } @Override public int getAndResetNumberOfUnhandledExceptions() { return nodeAgent.getAndResetNumberOfUnhandledExceptions(); } @Override public void scheduleTickWith(NodeAgentContext context, Instant at) { nodeAgentScheduler.scheduleTickWith(context, at); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index 2cd15a3ebe4..f3302bd2359 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -80,8 +80,7 @@ public class NodeAdminStateUpdater { metricsScheduler.scheduleAtFixedRate(() -> { try { if (suspendedStates.contains(currentState)) return; - nodeAdmin.updateNodeAgentMetrics(); - nodeAdmin.updateNodeAdminMetrics(); + nodeAdmin.updateMetrics(); } catch (Throwable e) { log.log(Level.WARNING, "Metric fetcher scheduler failed", e); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java index d62cb8e45d9..de5ee1b69a4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java @@ -33,11 +33,6 @@ public interface NodeAgent { void updateContainerNodeMetrics(); /** - * Returns true if NodeAgent is waiting for an image download to finish - */ - boolean isDownloadingImage(); - - /** * Returns and resets number of unhandled exceptions */ int getAndResetNumberOfUnhandledExceptions(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index cfdcefec1e3..da6c5224f74 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -18,7 +18,7 @@ import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; import com.yahoo.vespa.hosted.dockerapi.exception.DockerExecTimeoutException; import com.yahoo.vespa.hosted.dockerapi.metrics.DimensionMetrics; import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; @@ -73,8 +73,6 @@ public class NodeAgentImpl implements NodeAgent { private final DoubleFlag containerCpuCap; private int numberOfUnhandledException = 0; - private DockerImage imageBeingDownloaded = null; - private long currentRebootGeneration = 0; private Optional<Long> currentRestartGeneration = Optional.empty(); @@ -378,14 +376,10 @@ public class NodeAgentImpl implements NodeAgent { || (zone.getSystemName().isCd() && zone.getEnvironment() != Environment.prod); } - private void scheduleDownLoadIfNeeded(NodeSpec node, Optional<Container> container) { - if (node.getWantedDockerImage().equals(container.map(c -> c.image))) return; + private boolean downloadImageIfNeeded(NodeSpec node, Optional<Container> container) { + if (node.getWantedDockerImage().equals(container.map(c -> c.image))) return false; - if (dockerOperations.pullImageAsyncIfNeeded(node.getWantedDockerImage().get())) { - imageBeingDownloaded = node.getWantedDockerImage().get(); - } else if (imageBeingDownloaded != null) { // Image was downloading, but now it's ready - imageBeingDownloaded = null; - } + return node.getWantedDockerImage().map(dockerOperations::pullImageAsyncIfNeeded).orElse(false); } public void converge(NodeAgentContext context) { @@ -448,9 +442,8 @@ public class NodeAgentImpl implements NodeAgent { .filter(diskUtil -> diskUtil >= 0.8) .ifPresent(diskUtil -> storageMaintainer.removeOldFilesFromNode(context)); - scheduleDownLoadIfNeeded(node, container); - if (isDownloadingImage()) { - context.log(logger, "Waiting for image to download " + imageBeingDownloaded.asString()); + if (downloadImageIfNeeded(node, container)) { + context.log(logger, "Waiting for image to download " + context.node().getWantedDockerImage().get().asString()); return; } container = removeContainerIfNeededUpdateContainerState(context, container); @@ -540,7 +533,7 @@ public class NodeAgentImpl implements NodeAgent { Dimensions dimensions = dimensionsBuilder.build(); ContainerStats stats = containerStats.get(); - final String APP = MetricReceiverWrapper.APPLICATION_NODE; + final String APP = Metrics.APPLICATION_NODE; final int totalNumCpuCores = stats.getCpuStats().getOnlineCpus(); final long cpuContainerKernelTime = stats.getCpuStats().getUsageInKernelMode(); final long cpuContainerTotalTime = stats.getCpuStats().getTotalUsage(); @@ -634,11 +627,6 @@ public class NodeAgentImpl implements NodeAgent { } @Override - public boolean isDownloadingImage() { - return imageBeingDownloaded != null; - } - - @Override public int getAndResetNumberOfUnhandledExceptions() { int temp = numberOfUnhandledException; numberOfUnhandledException = 0; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index f9524d32c81..7f0f3fd37f6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -4,11 +4,10 @@ package com.yahoo.vespa.hosted.node.admin.integrationTests; import com.yahoo.collections.Pair; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; -import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.system.ProcessExecuter; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.dockerapi.Docker; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; @@ -92,11 +91,11 @@ public class DockerTester implements AutoCloseable { FileSystem fileSystem = TestFileSystem.create(); DockerOperations dockerOperations = new DockerOperationsImpl(docker, processExecuter, ipAddresses); - MetricReceiverWrapper mr = new MetricReceiverWrapper(MetricReceiver.nullImplementation); + Metrics metrics = new Metrics(); NodeAgentFactory nodeAgentFactory = contextSupplier -> new NodeAgentImpl( contextSupplier, nodeRepository, orchestrator, dockerOperations, storageMaintainer, flagSource, Optional.empty(), Optional.empty(), Optional.empty()); - nodeAdmin = new NodeAdminImpl(nodeAgentFactory, mr, Clock.systemUTC(), Duration.ofMillis(10), Duration.ZERO); + nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, Clock.systemUTC(), Duration.ofMillis(10), Duration.ZERO); NodeAgentContextFactory nodeAgentContextFactory = (nodeSpec, acl) -> new NodeAgentContextImpl.Builder(nodeSpec).acl(acl).fileSystem(fileSystem).build(); nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeAgentContextFactory, nodeRepository, orchestrator, diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index ce2e9f6d7a2..6e645e6c70f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -2,13 +2,11 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.config.provision.NodeType; -import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextFactory; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import org.junit.Test; import org.mockito.InOrder; @@ -40,11 +38,10 @@ import static org.mockito.Mockito.when; public class NodeAdminImplTest { private final NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory = mock(NodeAgentWithSchedulerFactory.class); - private final NodeAgentContextFactory nodeAgentContextFactory = mock(NodeAgentContextFactory.class); private final ManualClock clock = new ManualClock(); private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentWithSchedulerFactory, - new MetricReceiverWrapper(MetricReceiver.nullImplementation), clock, Duration.ZERO, Duration.ZERO); + new Metrics(), clock, Duration.ZERO, Duration.ZERO); @Test public void nodeAgentsAreProperlyLifeCycleManaged() { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index c43750684f6..3a92e345801 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -5,7 +5,6 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeType; import com.yahoo.io.IOUtils; -import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.dockerapi.Container; @@ -13,7 +12,7 @@ import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.dockerapi.ContainerResources; import com.yahoo.vespa.hosted.dockerapi.ContainerStats; import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; +import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembership; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeOwner; @@ -34,10 +33,8 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; -import java.util.Map; +import java.util.List; import java.util.Optional; -import java.util.Set; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -79,7 +76,7 @@ public class NodeAgentImplTest { private final NodeRepository nodeRepository = mock(NodeRepository.class); private final Orchestrator orchestrator = mock(Orchestrator.class); private final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); - private final MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); + private final Metrics metrics = new Metrics(); private final AclMaintainer aclMaintainer = mock(AclMaintainer.class); private final HealthChecker healthChecker = mock(HealthChecker.class); private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class); @@ -713,8 +710,7 @@ public class NodeAgentImplTest { nodeAgent.updateContainerNodeMetrics(); - Set<Map<String, Object>> actualMetrics = metricReceiver.getDefaultMetricsRaw(); - assertEquals(Collections.emptySet(), actualMetrics); + assertEquals(List.of(), metrics.getDefaultMetrics()); } @Test diff --git a/node-admin/src/test/resources/expected.container.system.metrics.txt b/node-admin/src/test/resources/expected.container.system.metrics.txt index c44d72b395e..ec750798c98 100644 --- a/node-admin/src/test/resources/expected.container.system.metrics.txt +++ b/node-admin/src/test/resources/expected.container.system.metrics.txt @@ -1,83 +1,80 @@ s: { - "routing": { - "yamas": { - "namespaces": - ["Vespa"] - } - }, "application": "vespa.node", + "dimensions": { + "host": "host1.test.yahoo.com", + "orchestratorState":"ALLOWED_TO_BE_DOWN", + "parentHostname": "parent.host.name.yahoo.com", + "role": "tenants", + "state": "active" + }, "metrics": { - "mem.limit": 4294967296, - "mem.used": 1073741824, - "mem_total.util": 40.808, - "mem_total.used": 1752707072, - "disk.used": 39625000000, "cpu.sys.util": 3.402, - "disk.util": 15.85, - "cpu.vcpus": 2.0, "cpu.util": 5.4, + "cpu.vcpus": 2.0, + "disk.limit": 250000000000, + "disk.used": 39625000000, + "disk.util": 15.85, + "mem.limit": 4294967296, + "mem.used": 1073741824, "mem.util": 25.0, - "disk.limit": 250000000000 + "mem_total.used": 1752707072, + "mem_total.util": 40.808 }, - "dimensions": { - "host": "host1.test.yahoo.com", - "orchestratorState":"ALLOWED_TO_BE_DOWN", - "role": "tenants", - "state": "active", - "parentHostname": "parent.host.name.yahoo.com" + "routing": { + "yamas": { + "namespaces": ["Vespa"] + } }, "timestamp": 0 } { - "routing": { - "yamas": { - "namespaces": - ["Vespa"] - } - }, "application": "vespa.node", + "dimensions": { + "host": "host1.test.yahoo.com", + "interface": "eth0", + "orchestratorState":"ALLOWED_TO_BE_DOWN", + "parentHostname": "parent.host.name.yahoo.com", + "role": "tenants", + "state": "active" + }, "metrics": { - "net.out.bytes": 20303455, + "net.in.bytes": 19499270, "net.in.dropped": 4, + "net.in.errors": 55, + "net.out.bytes": 20303455, "net.out.dropped": 13, - "net.in.bytes": 19499270, - "net.out.errors": 3, - "net.in.errors": 55 + "net.out.errors": 3 }, - "dimensions": { - "role": "tenants", - "host": "host1.test.yahoo.com", - "orchestratorState":"ALLOWED_TO_BE_DOWN", - "state": "active", - "interface": "eth0", - "parentHostname": "parent.host.name.yahoo.com" + "routing": { + "yamas": { + "namespaces": ["Vespa"] + } }, "timestamp": 0 } { - "routing": { - "yamas": { - "namespaces": - ["Vespa"] - } - }, "application": "vespa.node", + "dimensions": { + "host": "host1.test.yahoo.com", + "interface": "eth1", + "orchestratorState":"ALLOWED_TO_BE_DOWN", + "parentHostname": "parent.host.name.yahoo.com", + "role": "tenants", + "state": "active" + }, "metrics": { - "net.out.bytes": 54246745, + "net.in.bytes": 3245766, "net.in.dropped": 0, + "net.in.errors": 0, + "net.out.bytes": 54246745, "net.out.dropped": 0, - "net.in.bytes": 3245766, - "net.out.errors": 0, - "net.in.errors": 0 + "net.out.errors": 0 }, - "dimensions": { - "role": "tenants", - "host": "host1.test.yahoo.com", - "orchestratorState":"ALLOWED_TO_BE_DOWN", - "state": "active", - "interface": "eth1", - "parentHostname": "parent.host.name.yahoo.com" + "routing": { + "yamas": { + "namespaces": ["Vespa"] + } }, "timestamp": 0 }
\ No newline at end of file |