diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /simplemetrics/src |
Publish
Diffstat (limited to 'simplemetrics/src')
30 files changed, 2749 insertions, 0 deletions
diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java new file mode 100644 index 00000000000..78e217e9658 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java @@ -0,0 +1,202 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; + +import com.yahoo.collections.LazyMap; +import com.yahoo.collections.LazySet; +import com.yahoo.log.LogLevel; + +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * An aggregation of data which is only written to from a single thread. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class Bucket { + boolean gotTimeStamps; + long fromMillis; + long toMillis; + + public Bucket() { + this.gotTimeStamps = false; + this.fromMillis = 0; + this.toMillis = 0; + } + + public Bucket(long fromMillis, long toMillis) { + this.gotTimeStamps = true; + this.fromMillis = fromMillis; + this.toMillis = toMillis; + } + + private static final Logger log = Logger.getLogger(Bucket.class.getName()); + private final Map<Identifier, UntypedMetric> values = LazyMap.newHashMap(); + + public Set<Map.Entry<Identifier, UntypedMetric>> entrySet() { + return values.entrySet(); + } + + void put(Sample x) { + UntypedMetric value = get(x); + Measurement m = x.getMeasurement(); + switch (x.getMetricType()) { + case GAUGE: + value.put(m.getMagnitude()); + break; + case COUNTER: + value.add(m.getMagnitude()); + break; + default: + throw new IllegalArgumentException("Unsupported metric type: " + x.getMetricType()); + } + } + + void put(Identifier id, UntypedMetric value) { + values.put(id, value); + } + + boolean hasIdentifier(Identifier id) { + return values.containsKey(id); + } + + void merge(Bucket other, boolean otherIsNewer) { + final LazySet<String> malformedMetrics = LazySet.newHashSet(); + for (Map.Entry<Identifier, UntypedMetric> entry : other.values.entrySet()) { + String metricName = entry.getKey().getName(); + try { + if (!malformedMetrics.contains(metricName)) { + get(entry.getKey(), entry.getValue()).merge(entry.getValue(), otherIsNewer); + } + } catch (IllegalArgumentException e) { + log.log(LogLevel.WARNING, "Problems merging metric " + metricName + ", possibly ignoring data."); + // avoid spamming the log if there are a lot of mismatching + // threads + malformedMetrics.add(metricName); + } + } + } + + void merge(Bucket other) { + boolean otherIsNewer = resolveTimeStamps(other); + merge(other, otherIsNewer); + } + + private boolean resolveTimeStamps(Bucket other) { + boolean otherIsNewer = other.fromMillis > this.fromMillis; + if (! gotTimeStamps) { + fromMillis = other.fromMillis; + toMillis = other.toMillis; + gotTimeStamps = other.gotTimeStamps; + } else if (other.gotTimeStamps) { + fromMillis = Math.min(fromMillis, other.fromMillis); + toMillis = Math.max(toMillis, other.toMillis); + } + return otherIsNewer; + } + + private UntypedMetric get(Sample sample) { + Identifier dim = sample.getIdentifier(); + UntypedMetric v = values.get(dim); + + if (v == null) { + // please keep inside guard, as sample.getHistogramDefinition(String) touches a volatile + v = new UntypedMetric(sample.getHistogramDefinition(dim.getName())); + values.put(dim, v); + } + return v; + } + + private UntypedMetric get(Identifier dim, UntypedMetric other) { + UntypedMetric v = values.get(dim); + + if (v == null) { + v = new UntypedMetric(other.getMetricDefinition()); + values.put(dim, v); + } + return v; + } + + public Collection<String> getAllMetricNames() { + Set<String> names = new HashSet<>(); + for (Identifier id : values.keySet()) { + names.add(id.getName()); + } + return names; + } + + public Collection<Map.Entry<Point, UntypedMetric>> getValuesForMetric(@NonNull String metricName) { + List<Map.Entry<Point, UntypedMetric>> singleMetric = new ArrayList<>(); + for (Map.Entry<Identifier, UntypedMetric> entry : values.entrySet()) { + if (metricName.equals(entry.getKey().getName())) { + singleMetric.add(locationValuePair(entry)); + } + } + return singleMetric; + } + + public Map<String, List<Map.Entry<Point, UntypedMetric>>> getValuesByMetricName() { + Map<String, List<Map.Entry<Point, UntypedMetric>>> result = new HashMap<>(); + for (Map.Entry<Identifier, UntypedMetric> entry : values.entrySet()) { + List<Map.Entry<Point, UntypedMetric>> singleMetric; + if (result.containsKey(entry.getKey().getName())) { + singleMetric = result.get(entry.getKey().getName()); + } else { + singleMetric = new ArrayList<>(); + result.put(entry.getKey().getName(), singleMetric); + } + singleMetric.add(locationValuePair(entry)); + } + return result; + } + + private SimpleImmutableEntry<Point, UntypedMetric> locationValuePair(Map.Entry<Identifier, UntypedMetric> entry) { + return new SimpleImmutableEntry<>(entry.getKey().getLocation(), entry.getValue()); + } + + @Override + public String toString() { + final int maxLen = 3; + StringBuilder builder = new StringBuilder(); + builder.append("Bucket [values=").append(values != null ? toString(values.entrySet(), maxLen) : null).append("]"); + return builder.toString(); + } + + private String toString(Collection<?> collection, int maxLen) { + StringBuilder builder = new StringBuilder(); + builder.append("["); + int i = 0; + for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) { + if (i > 0) { + builder.append(", "); + } + builder.append(iterator.next()); + } + builder.append("]"); + return builder.toString(); + } + + /** + * This bucket contains data newer than approximately this point in time. + */ + public long getFromMillis() { + return fromMillis; + } + + /** + * This bucket contains data older than approximately this point in time. + */ + public long getToMillis() { + return toMillis; + } +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/Counter.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Counter.java new file mode 100644 index 00000000000..a35be17e75c --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Counter.java @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import com.google.common.annotations.Beta; +import com.yahoo.metrics.simple.UntypedMetric.AssumedType; + +/** + * A counter metric. Create a counter by declaring it with + * {@link MetricReceiver#declareCounter(String)} or + * {@link MetricReceiver#declareCounter(String, Point)}. + * + * @author steinar + */ +@Beta +public class Counter { + private final Point defaultPosition; + private final String name; + private final MetricReceiver metricReceiver; + + Counter(String name, Point defaultPosition, MetricReceiver receiver) { + this.name = name; + this.defaultPosition = defaultPosition; + this.metricReceiver = receiver; + } + + /** + * Increase the dimension-less/zero-point value of this counter by 1. + */ + public void add() { + add(1L, defaultPosition); + } + + /** + * Add to the dimension-less/zero-point value of this counter. + * + * @param n the amount by which to increase this counter + */ + public void add(long n) { + add(n, defaultPosition); + } + + /** + * Increase this metric at the given point by 1. + * + * @param p the point in the metric space at which to increase this metric by 1 + */ + public void add(Point p) { + add(1L, p); + } + + /** + * Add to this metric at the given point. + * + * @param n + * the amount by which to increase this counter + * @param p + * the point in the metric space at which to add to the metric + */ + public void add(long n, Point p) { + metricReceiver.update(new Sample(new Measurement(Long.valueOf(n)), new Identifier(name, p), AssumedType.COUNTER)); + } + + /** + * Create a PointBuilder with default dimension values as given when this + * counter was declared. + * + * @return a PointBuilder reflecting the default dimension values of this + * counter + */ + public PointBuilder builder() { + return new PointBuilder(defaultPosition); + } +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/DimensionCache.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/DimensionCache.java new file mode 100644 index 00000000000..c3537947d66 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/DimensionCache.java @@ -0,0 +1,94 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +/** + * Basically the persistence layer for metrics. Both CPU and memory hungry, but + * it runs in its own little world. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +class DimensionCache { + private final Map<String, LinkedHashMap<Point, UntypedMetric>> persistentData = new HashMap<>(); + private final int pointsToKeep; + + public DimensionCache(int pointsToKeep) { + this.pointsToKeep = pointsToKeep; + } + + void updateDimensionPersistence(Bucket toDelete, Bucket toPresent) { + updatePersistentData(toDelete); + padPresentation(toPresent); + } + + private void padPresentation(Bucket toPresent) { + Map<String, List<Entry<Point, UntypedMetric>>> currentMetricNames = toPresent.getValuesByMetricName(); + + for (Map.Entry<String, List<Entry<Point, UntypedMetric>>> metric : currentMetricNames.entrySet()) { + final int currentDataPoints = metric.getValue().size(); + if (currentDataPoints < pointsToKeep) { + padMetric(metric.getKey(), toPresent, currentDataPoints); + } + } + Set<String> keysMissingFromPresentation = new HashSet<>(persistentData.keySet()); + keysMissingFromPresentation.removeAll(currentMetricNames.keySet()); + for (String cachedMetric : keysMissingFromPresentation) { + padMetric(cachedMetric, toPresent, 0); + } + } + + private void updatePersistentData(Bucket toDelete) { + if (toDelete == null) { + return; + } + for (Map.Entry<String, List<Entry<Point, UntypedMetric>>> metric : toDelete.getValuesByMetricName().entrySet()) { + LinkedHashMap<Point, UntypedMetric> cachedPoints = getCachedMetric(metric.getKey()); + + for (Entry<Point, UntypedMetric> newestInterval : metric.getValue()) { + // overwriting an existing entry does not update the order + // in the map + cachedPoints.remove(newestInterval.getKey()); + cachedPoints.put(newestInterval.getKey(), newestInterval.getValue()); + } + } + } + + private void padMetric(String metric, + Bucket toPresent, + int currentDataPoints) { + final LinkedHashMap<Point, UntypedMetric> cachedPoints = getCachedMetric(metric); + int toAdd = pointsToKeep - currentDataPoints; + @SuppressWarnings({"unchecked","rawtypes"}) + Entry<Point, UntypedMetric>[] cachedEntries = cachedPoints.entrySet().toArray(new Entry[0]); + for (int i = cachedEntries.length - 1; i >= 0 && toAdd > 0; --i) { + Entry<Point, UntypedMetric> leastOld = cachedEntries[i]; + final Identifier id = new Identifier(metric, leastOld.getKey()); + if (!toPresent.hasIdentifier(id)) { + toPresent.put(id, leastOld.getValue().pruneData()); + --toAdd; + } + } + } + + @SuppressWarnings("serial") + private LinkedHashMap<Point, UntypedMetric> getCachedMetric(String metricName) { + LinkedHashMap<Point, UntypedMetric> points = persistentData.get(metricName); + if (points == null) { + points = new LinkedHashMap<Point, UntypedMetric>(16, 0.75f, false) { + protected boolean removeEldestEntry(Map.Entry<Point, UntypedMetric> eldest) { + return size() > pointsToKeep; + } + }; + persistentData.put(metricName, points); + } + return points; + } + +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/Gauge.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Gauge.java new file mode 100644 index 00000000000..87ea7ffdaf0 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Gauge.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import com.google.common.annotations.Beta; +import com.yahoo.metrics.simple.UntypedMetric.AssumedType; + +import edu.umd.cs.findbugs.annotations.Nullable; + +/** + * A gauge metric, i.e. a bucket of arbitrary sample values. Create a gauge + * metric by declaring it with {@link MetricReceiver#declareGauge(String)} or + * {@link MetricReceiver#declareGauge(String, Point)}. + * + * @author steinar + */ +@Beta +public final class Gauge { + @Nullable + private final Point defaultPosition; + private final String name; + private final MetricReceiver receiver; + + Gauge(String name, Point defaultPosition, MetricReceiver receiver) { + this.name = name; + this.defaultPosition = defaultPosition; + this.receiver = receiver; + } + + /** + * Record a sample with default or no position. + * + * @param x + * sample value + */ + public void sample(double x) { + sample(x, defaultPosition); + } + + /** + * Record a sample at the given position. + * + * @param x + * sample value + * @param p + * position/dimension values for the sample + */ + public void sample(double x, Point p) { + receiver.update(new Sample(new Measurement(Double.valueOf(x)), new Identifier(name, p), AssumedType.GAUGE)); + } + + /** + * Create a PointBuilder with the default dimension values reflecting those + * given when this gauge was declared. + * + * @return a builder initialized with defaults from this metric instance + */ + public PointBuilder builder() { + return new PointBuilder(defaultPosition); + } +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/Identifier.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Identifier.java new file mode 100644 index 00000000000..40c9c3e5d45 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Identifier.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +/** + * The name of the metric and its n-dimensional position. Basically a pair of a + * Point and a metric name. Written to be robust against null input as the API + * gives very little guidance. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class Identifier { + private final String name; + private final Point location; + + public Identifier(String name, Point location) { + this.name = name; + this.location = location; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((location == null) ? 0 : location.hashCode()); + result = prime * result + ((name == null) ? 0 : name.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Identifier other = (Identifier) obj; + if (location == null) { + if (other.location != null) { + return false; + } + } else if (!location.equals(other.location)) { + return false; + } + if (name == null) { + if (other.name != null) { + return false; + } + } else if (!name.equals(other.name)) { + return false; + } + return true; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Identifier [name=").append(name).append(", location=").append(location).append("]"); + return builder.toString(); + } + + public String getName() { + return name; + } + + public Point getLocation() { + return location; + } +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/Measurement.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Measurement.java new file mode 100644 index 00000000000..71ed9dd96dc --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Measurement.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +/** + * Wrapper class for the actually measured value. Candidate for removal, but I + * wanted a type instead of some opaque instance of Number. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class Measurement { + private final Number magnitude; + + public Measurement(Number magnitude) { + this.magnitude = magnitude; + } + + Number getMagnitude() { + return magnitude; + } + +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricAggregator.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricAggregator.java new file mode 100644 index 00000000000..a388b1a1cc4 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricAggregator.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import com.yahoo.concurrent.ThreadLocalDirectory; +import com.yahoo.metrics.ManagerConfig; + +/** + * Worker thread to collect the data stored in worker threads and build + * snapshots for external consumption. Using the correct executor gives the + * necessary guarantuess for this being invoked from only a single thread. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +class MetricAggregator implements Runnable { + private final ThreadLocalDirectory<Bucket, Sample> metricsCollection; + private final AtomicReference<Bucket> currentSnapshot; + private int generation = 0; + private final Bucket[] buffer; + private long fromMillis; + private final DimensionCache dimensions; + + MetricAggregator(ThreadLocalDirectory<Bucket, Sample> metricsCollection, AtomicReference<Bucket> currentSnapshot, + ManagerConfig settings) { + if (settings.reportPeriodSeconds() < 10) { + throw new IllegalArgumentException( + "Do not use this metrics implementation" + + " if report periods of less than 10 seconds is desired."); + } + buffer = new Bucket[settings.reportPeriodSeconds()]; + dimensions = new DimensionCache(settings.pointsToKeepPerMetric()); + fromMillis = System.currentTimeMillis(); + this.metricsCollection = metricsCollection; + this.currentSnapshot = currentSnapshot; + } + + @Override + public void run() { + Bucket toDelete = updateBuffer(); + createSnapshot(toDelete); + } + + private void createSnapshot(Bucket toDelete) { + final Bucket toPresent = new Bucket(); + for (Bucket b : buffer) { + if (b == null) { + continue; + } + toPresent.merge(b); + } + dimensions.updateDimensionPersistence(toDelete, toPresent); + currentSnapshot.set(toPresent); + } + + private Bucket updateBuffer() { + List<Bucket> buckets = metricsCollection.fetch(); + final long toMillis = System.currentTimeMillis(); + final int bucketIndex = generation++ % buffer.length; + Bucket bucketToDelete = buffer[bucketIndex]; + Bucket latest = new Bucket(fromMillis, toMillis); + for (Bucket b : buckets) { + latest.merge(b, true); + } + buffer[bucketIndex] = latest; + this.fromMillis = toMillis; + return bucketToDelete; + } + +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java new file mode 100644 index 00000000000..aef7067dc65 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Logger; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.concurrent.ThreadLocalDirectory; +import com.yahoo.concurrent.ThreadLocalDirectory.Updater; +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.metrics.ManagerConfig; +import com.yahoo.log.LogLevel; + +/** + * This is the coordinating class owning the executor and the top level objects + * for measured metrics. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class MetricManager extends AbstractComponent implements Provider<MetricReceiver> { + private static Logger log = Logger.getLogger(MetricManager.class.getName()); + + private final ScheduledThreadPoolExecutor executor; + private final MetricReceiver receiver; + private ThreadLocalDirectory<Bucket, Sample> metricsCollection; + + public MetricManager(ManagerConfig settings) { + this(settings, new MetricUpdater()); + } + + private MetricManager(ManagerConfig settings, Updater<Bucket, Sample> updater) { + log.log(LogLevel.CONFIG, "setting up simple metrics gathering." + + " reportPeriodSeconds=" + settings.reportPeriodSeconds() + + ", pointsToKeepPerMetric=" + settings.pointsToKeepPerMetric()); + metricsCollection = new ThreadLocalDirectory<>(updater); + final AtomicReference<Bucket> currentSnapshot = new AtomicReference<>(null); + executor = new ScheduledThreadPoolExecutor(1); + // Fixed rate, not fixed delay, is it is not too important that each + // bucket has data for exactly one second, but one should strive for + // this.buffer to contain data for as close a period to the report + // interval as possible + executor.scheduleAtFixedRate(new MetricAggregator(metricsCollection, currentSnapshot, settings), 1, 1, TimeUnit.SECONDS); + receiver = new MetricReceiver(metricsCollection, currentSnapshot); + } + + static MetricManager constructWithCustomUpdater(ManagerConfig settings, Updater<Bucket, Sample> updater) { + return new MetricManager(settings, updater); + } + + + @Override + public void deconstruct() { + executor.shutdown(); + } + + @Override + public MetricReceiver get() { + return receiver; + } + +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java new file mode 100644 index 00000000000..a0b94f1e571 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricReceiver.java @@ -0,0 +1,279 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; + +import com.google.common.annotations.Beta; +import com.google.common.collect.ImmutableMap; +import com.yahoo.concurrent.ThreadLocalDirectory; + +/** + * The reception point for measurements. This is the class users should inject + * in constructors for declaring instances of {@link Counter} and {@link Gauge} + * for the actual measurement of metrics. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +@Beta +public class MetricReceiver { + + public static final MetricReceiver nullImplementation = new NullReceiver(); + private final ThreadLocalDirectory<Bucket, Sample> metricsCollection; + private final AtomicReference<Bucket> currentSnapshot; + + // metricSettings is volatile for reading, the lock is for updates + private final Object histogramDefinitionsLock = new Object(); + private volatile Map<String, MetricSettings> metricSettings; + + private static final class NullCounter extends Counter { + NullCounter() { + super(null, null, null); + } + + @Override + public void add() { + } + + @Override + public void add(long n) { + } + + @Override + public void add(Point p) { + } + + @Override + public void add(long n, Point p) { + } + + @Override + public PointBuilder builder() { + return super.builder(); + } + } + + private static final class NullReceiver extends MetricReceiver { + NullReceiver() { + super(null, null); + } + + @Override + public void update(Sample s) { + } + + @Override + public Counter declareCounter(String name) { + return new NullCounter(); + } + + @Override + public Counter declareCounter(String name, Point boundDimensions) { + return new NullCounter(); + } + + @Override + public Gauge declareGauge(String name) { + return null; + } + + @Override + public Gauge declareGauge(String name, Point boundDimensions) { + return null; + } + + @Override + public Gauge declareGauge(String name, Optional<Point> boundDimensions, MetricSettings customSettings) { + return null; + } + + @Override + public PointBuilder pointBuilder() { + return null; + } + + @Override + public Bucket getSnapshot() { + return null; + } + + @Override + void addMetricDefinition(String metricName, MetricSettings definition) { + } + + @Override + MetricSettings getMetricDefinition(String metricName) { + return null; + } + } + + public MetricReceiver(ThreadLocalDirectory<Bucket, Sample> metricsCollection, AtomicReference<Bucket> currentSnapshot) { + this.metricsCollection = metricsCollection; + this.currentSnapshot = currentSnapshot; + metricSettings = new ImmutableMap.Builder<String, MetricSettings>().build(); + } + + /** + * Update a metric. This API is not intended for clients for the + * simplemetrics API, declare a Counter or a Gauge using + * {@link #declareCounter(String)}, {@link #declareCounter(String, Point)}, + * {@link #declareGauge(String)}, or {@link #declareGauge(String, Point)} + * instead. + * + * @param s + * a single simple containing all meta data necessary to update a + * metric + */ + public void update(Sample s) { + // pass around the receiver instead of histogram settings to avoid reading any volatile if unnecessary + s.setReceiver(this); + metricsCollection.update(s); + } + + /** + * Declare a counter metric without setting any default position. + * + * @param name + * the name of the metric + * @return a thread-safe counter + */ + public Counter declareCounter(String name) { + return declareCounter(name, null); + } + + /** + * Declare a counter metric, with default dimension values as given. Create + * the point argument by using a builder from {@link #pointBuilder()}. + * + * @param name + * the name of the metric + * @param boundDimensions + * dimensions which have a fixed value in the life cycle of the + * metric object or null + * @return a thread-safe counter with given default values + */ + public Counter declareCounter(String name, Point boundDimensions) { + return new Counter(name, boundDimensions, this); + } + + /** + * Declare a gauge metric with any default position. + * + * @param name + * the name of the metric + * @return a thread-safe gauge instance + */ + public Gauge declareGauge(String name) { + return declareGauge(name, null); + } + + /** + * Declare a gauge metric, with default dimension values as given. Create + * the point argument by using a builder from {@link #pointBuilder()}. + * + * @param name + * the name of the metric + * @param boundDimensions + * dimensions which have a fixed value in the life cycle of the + * metric object or null + * @return a thread-safe gauge metric + */ + public Gauge declareGauge(String name, Point boundDimensions) { + Optional<Point> optionalOfBoundDimensions; + if (boundDimensions == null) { + optionalOfBoundDimensions = Optional.empty(); + } else { + optionalOfBoundDimensions = Optional.of(boundDimensions); + } + return declareGauge(name, optionalOfBoundDimensions, null); + } + + /** + * Declare a gauge metric, with default dimension values as given. Create + * the point argument by using a builder from {@link #pointBuilder()}. + * MetricSettings instances are built using + * {@link MetricSettings.Builder}. + * + * @param name + * the name of the metric + * @param boundDimensions + * an optional of dimensions which have a fixed value in the life + * cycle of the metric object + * @param customSettings + * any optional settings + * @return a thread-safe gauge metric + */ + public Gauge declareGauge(String name, Optional<Point> boundDimensions, MetricSettings customSettings) { + if (customSettings != null) { + addMetricDefinition(name, customSettings); + } + Point defaultDimensions = null; + if (boundDimensions.isPresent()) { + defaultDimensions = boundDimensions.get(); + } + return new Gauge(name, defaultDimensions, this); + } + + /** + * Create a PointBuilder instance with no default settings. PointBuilder + * instances are not thread-safe. + * + * @return an "empty" point builder instance + */ + public PointBuilder pointBuilder() { + return new PointBuilder(); + } + + /** + * Fetch the latest metric values, aggregated over all threads for the + * configured sample history (by default five minutes). The values will be + * less than 1 second old, and this method has only a memory barrier as side + * effect. + * + * @return the latest five minutes of metrics + */ + public Bucket getSnapshot() { + return currentSnapshot.get(); + } + + /** + * Add how to build a histogram for a given metric. + * + * <p> + * Do note, this is not part of the public API. + * </p> + * + * @param metricName + * the metric where samples should be put in a histogram + * @param definition + * settings for a histogram + */ + void addMetricDefinition(String metricName, MetricSettings definition) { + synchronized (histogramDefinitionsLock) { + // read the volatile _after_ acquiring the lock + Map<String, MetricSettings> oldMetricDefinitions = metricSettings; + Map<String, MetricSettings> builderMap = new HashMap<>(oldMetricDefinitions.size() + 1); + builderMap.putAll(oldMetricDefinitions); + builderMap.put(metricName, definition); + metricSettings = ImmutableMap.copyOf(builderMap); + } + } + + /** + * Get how to build a histogram for a given metric, or null if no histogram + * should be created. + * + * <p> + * Do note, this is not part of the public API. + * </p> + * + * @param metricName + * the name of an arbitrary metric + * @return the corresponding histogram definition or null + */ + MetricSettings getMetricDefinition(String metricName) { + return metricSettings.get(metricName); + } +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricSettings.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricSettings.java new file mode 100644 index 00000000000..f47a796948f --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricSettings.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import com.google.common.annotations.Beta; + +/** + * All information needed for creating any extra data structures associated with + * a single metric, outside of its basic type. + * + * @author steinar + */ +@Beta +public final class MetricSettings { + + /** + * A builder for the immutable MetricSettings instances. + */ + @Beta + public static final class Builder { + private boolean histogram = false; + + /** + * Create a new builder for a MetricSettings instance with default + * settings. + */ + public Builder() { + } + + /** + * Set whether a resulting metric should have a histogram. Default is + * false. + * + * @param histogram + * whether to generate a histogram + * @return this, to facilitate chaining + */ + public Builder histogram(boolean histogram) { + this.histogram = histogram; + return this; + } + + /** + * Build a fresh MetricSettings instance. + * + * @return a MetricSettings instance containing the values set in this + * builder + */ + public MetricSettings build() { + return new MetricSettings(histogram); + } + } + + private final int significantDigits; // could have been static, but would + // just introduce bugs when we must + // expose this setting + private final boolean histogram; + + private MetricSettings(boolean histogram) { + this.histogram = histogram; + this.significantDigits = 2; + } + + int getSignificantdigits() { + return significantDigits; + } + + boolean isHistogram() { + return histogram; + } +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricUpdater.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricUpdater.java new file mode 100644 index 00000000000..e38ca0a41a8 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricUpdater.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import com.yahoo.concurrent.ThreadLocalDirectory.Updater; + +/** + * The link between each single thread and the central data store. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +class MetricUpdater implements Updater<Bucket, Sample> { + + @Override + public Bucket createGenerationInstance(Bucket previous) { + return new Bucket(); + } + + @Override + public Bucket update(Bucket current, Sample x) { + current.put(x); + return current; + } + +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/Point.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Point.java new file mode 100644 index 00000000000..61415f042bc --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Point.java @@ -0,0 +1,126 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import com.google.common.annotations.Beta; +import com.google.common.collect.ImmutableList; +import com.yahoo.collections.Tuple2; +import com.yahoo.jdisc.Metric.Context; + +/** + * An efficiently comparable point in a sparse vector space. + * + * @author steinar + */ +@Beta +public final class Point implements Context { + private final Value[] location; + private final String[] dimensions; + + public Point(Map<String, ?> properties) { + this(buildParameters(properties)); + } + + private Point(Tuple2<String[], Value[]> dimensionsAndLocation) { + this(dimensionsAndLocation.first, dimensionsAndLocation.second); + } + + /** + * Only to be used by simplemetrics itself. + * + * @param dimensions dimension name, Point takes ownership of the array + * @param location dimension values, Point takes ownership of the array + */ + Point(String[] dimensions, Value[] location) { + this.dimensions = dimensions; + this.location = location; + } + + private static Tuple2<String[], Value[]> buildParameters(Map<String, ?> properties) { + String[] dimensions = properties.keySet().toArray(new String[0]); + Arrays.sort(dimensions); + Value[] location = new Value[dimensions.length]; + for (int i = 0; i < dimensions.length; ++i) { + location[i] = Value.of(String.valueOf(properties.get(dimensions[i]))); + } + return new Tuple2<>(dimensions, location); + + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Point other = (Point) obj; + if (!Arrays.equals(dimensions, other.dimensions)) { + return false; + } + if (!Arrays.equals(location, other.location)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(dimensions); + result = prime * result + Arrays.hashCode(location); + return result; + } + + @Override + public String toString() { + final int maxLen = 3; + StringBuilder builder = new StringBuilder(); + builder.append("Point [location=") + .append(Arrays.asList(location).subList(0, Math.min(location.length, maxLen))) + .append(", dimensions=") + .append(Arrays.asList(dimensions).subList(0, Math.min(dimensions.length, maxLen))) + .append("]"); + return builder.toString(); + } + + /** + * Get an immutable list view of the values for each dimension. + */ + public List<Value> location() { + return ImmutableList.copyOf(location); + } + + /** + * Get an immutable list view of the names of each dimension. + */ + public List<String> dimensions() { + return ImmutableList.copyOf(dimensions); + } + + /** + * Get the number of dimensions defined for this Point, i.e. the size of the + * collection returned by {@link #dimensions()}. + */ + public int dimensionality() { + return dimensions.length; + } + + /** package private accessor only for simplemetrics itself */ + String[] getDimensions() { + return dimensions; + } + + /** package private accessor only for simplemetrics itself */ + Value[] getLocation() { + return location; + } +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/PointBuilder.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/PointBuilder.java new file mode 100644 index 00000000000..7f6b797c601 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/PointBuilder.java @@ -0,0 +1,126 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.ArrayList; +import java.util.Collections; + +import com.google.common.annotations.Beta; + +/** + * Single-use builder for the immutable Point instances used to set dimensions + * for a metric. Get a fresh instance either from a corresponding Gauge or Counter, + * or through the MetricReceiver API. + * + * @author steinar + */ +@Beta +public final class PointBuilder { + private ArrayList<String> dimensions; + private ArrayList<Value> location; + private static final String[] dimensionsTypeArgument = new String[0]; + private static final Value[] locationTypeArgument = new Value[0]; + + public enum Discriminator { + LONG, DOUBLE, STRING; + } + + PointBuilder() { + this(null); + } + + PointBuilder(Point p) { + dimensions = new ArrayList<>(); + location = new ArrayList<>(); + if (p != null) { + int size = p.dimensionality(); + dimensions = new ArrayList<>(size); + location = new ArrayList<>(size); + for (String dimensionName : p.getDimensions()) { + dimensions.add(dimensionName); + } + for (Value dimensionValue : p.getLocation()) { + location.add(dimensionValue); + } + } else { + dimensions = new ArrayList<>(4); + location = new ArrayList<>(4); + } + } + + /** + * Set a named dimension to an integer value. + * + * @param dimensionName the name of the dimension to set + * @param dimensionValue to value for the given dimension + * @return this, to facilitate chaining + */ + public PointBuilder set(String dimensionName, long dimensionValue) { + return set(dimensionName, Value.of(dimensionValue)); + } + + /** + * Set a named dimension to a floating point value. + * + * @param dimensionName the name of the dimension to set + * @param dimensionValue to value for the given dimension + * @return this, to facilitate chaining + */ + public PointBuilder set(String dimensionName, double dimensionValue) { + return set(dimensionName, Value.of(dimensionValue)); + } + + /** + * Set a named dimension to a string value. + * + * @param dimensionName the name of the dimension to set + * @param dimensionValue to value for the given dimension + * @return this, to facilitate chaining + */ + public PointBuilder set(String dimensionName, String dimensionValue) { + return set(dimensionName, Value.of(dimensionValue)); + } + + private PointBuilder set(String axisName, Value w) { + // handle setting same axis multiple times nicely + int i = Collections.binarySearch(dimensions, axisName); + if (i < 0) { + dimensions.add(~i, axisName); + location.add(~i, w); + } else { + // only set location, dim obviously exists + location.set(i, w); + } + return this; + } + + /** + * Create a new Point instance using the settings stored in this + * PointBuilder. PointBuilder instances cannot be re-used after build() has + * been invoked. + * + * @return a Point instance reflecting this builder + */ + public Point build() { + Point p; + if (dimensions.size() == 0) { + p = null; + } else { + p = new Point(dimensions.toArray(dimensionsTypeArgument), location.toArray(locationTypeArgument)); + } + // deny builder re-use + dimensions = null; + location = null; + return p; + } + + @Override + public String toString() { + final int maxLen = 3; + StringBuilder builder = new StringBuilder(); + builder.append("PointBuilder [dimensions=") + .append(dimensions != null ? dimensions.subList(0, Math.min(dimensions.size(), maxLen)) : null) + .append(", location=").append(location != null ? location.subList(0, Math.min(location.size(), maxLen)) : null) + .append("]"); + return builder.toString(); + } +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/Sample.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Sample.java new file mode 100644 index 00000000000..b23b886c973 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Sample.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import com.yahoo.metrics.simple.UntypedMetric.AssumedType; + +/** + * A single metric measurement and all the meta data needed to route it + * correctly. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class Sample { + private final Identifier identifier; + private final Measurement measurement; + private final AssumedType metricType; + private MetricReceiver metricReceiver = null; + + public Sample(Measurement measurement, Identifier id, AssumedType t) { + this.identifier = id; + this.measurement = measurement; + this.metricType = t; + } + + Identifier getIdentifier() { + return identifier; + } + + Measurement getMeasurement() { + return measurement; + } + + AssumedType getMetricType() { + return metricType; + } + + void setReceiver(MetricReceiver metricReceiver) { + this.metricReceiver = metricReceiver; + } + + /** + * Get histogram definition for an arbitrary metric. Caveat emptor: This + * involves reading a volatile. + * + * @param metricName + * name of the metric to get histogram definition for + * @return how to define a new histogram or null + */ + MetricSettings getHistogramDefinition(String metricName) { + if (metricReceiver == null) { + return null; + } else { + return metricReceiver.getMetricDefinition(metricName); + } + } + +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/UnitTestSetup.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/UnitTestSetup.java new file mode 100644 index 00000000000..c800eae4f5d --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/UnitTestSetup.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import com.yahoo.metrics.ManagerConfig; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * Common code for running unit tests of simplemetrics + * + * @author eirik + */ +public class UnitTestSetup { + MetricManager metricManager; + MetricReceiver receiver; + ObservableUpdater updater; + + static class ObservableUpdater extends MetricUpdater { + CountDownLatch gotData = new CountDownLatch(1); + private volatile boolean hasBeenAccessed = false; + + @Override + public Bucket createGenerationInstance(Bucket previous) { + if (hasBeenAccessed) { + gotData.countDown(); + } + return super.createGenerationInstance(previous); + } + + @Override + public Bucket update(Bucket current, Sample x) { + hasBeenAccessed = true; + return super.update(current, x); + } + } + + void init() { + updater = new ObservableUpdater(); + metricManager = MetricManager.constructWithCustomUpdater(new ManagerConfig(new ManagerConfig.Builder()), updater); + receiver = metricManager.get(); + } + + void fini() { + receiver = null; + metricManager.deconstruct(); + metricManager = null; + updater = null; + } + + public Bucket getUpdatedSnapshot() throws InterruptedException { + updater.gotData.await(10, TimeUnit.SECONDS); + Bucket s = receiver.getSnapshot(); + long startedWaitingForSnapshot = System.currentTimeMillis(); + // just waiting for the correct snapshot being constructed (yes, this is + // necessary) + while (s == null || s.entrySet().size() == 0) { + if (System.currentTimeMillis() - startedWaitingForSnapshot > (10L * 1000L)) { + throw new RuntimeException("Test timed out."); + } + Thread.sleep(10); + s = receiver.getSnapshot(); + } + return s; + } + + public MetricReceiver getReceiver() { + return receiver; + } +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java new file mode 100644 index 00000000000..75bf9c013ae --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import java.util.logging.Logger; + +import org.HdrHistogram.DoubleHistogram; + +import com.yahoo.log.LogLevel; + +/** + * A gauge or a counter or... who knows? The class for storing a metric when the + * metric has not been declared. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class UntypedMetric { + private static final Logger log = Logger.getLogger(UntypedMetric.class.getName()); + + private long count = 0L; + private double current = 0.0d; + private double max; + private double min; + private double sum; + private AssumedType outputFormat = AssumedType.NONE; + private final DoubleHistogram histogram; + private final MetricSettings metricSettings; + + public enum AssumedType { NONE, GAUGE, COUNTER }; + + UntypedMetric(MetricSettings metricSettings) { + this.metricSettings = metricSettings; + if (metricSettings == null || !metricSettings.isHistogram()) { + histogram = null; + } else { + histogram = new DoubleHistogram(metricSettings.getSignificantdigits()); + } + } + + void add(Number x) { + outputFormat = AssumedType.COUNTER; + count += x.longValue(); + } + + void put(Number x) { + outputFormat = AssumedType.GAUGE; + current = x.doubleValue(); + if (histogram != null) { + histogram.recordValue(current); + } + if (count > 0) { + max = Math.max(current, max); + min = Math.min(current, min); + sum += current; + } else { + max = current; + min = current; + sum = current; + } + ++count; + } + + UntypedMetric pruneData() { + UntypedMetric pruned = new UntypedMetric(null); + pruned.outputFormat = this.outputFormat; + pruned.current = this.current; + return pruned; + } + + void merge(UntypedMetric other, boolean otherIsNewer) throws IllegalArgumentException { + if (outputFormat == AssumedType.NONE) { + outputFormat = other.outputFormat; + } + if (outputFormat != other.outputFormat) { + throw new IllegalArgumentException("Mismatching output formats: " + outputFormat + " and " + other.outputFormat + "."); + } + count += other.count; + if (otherIsNewer) { + current = other.current; + } + max = Math.max(other.max, max); + min = Math.min(other.min, min); + sum += other.sum; + if (histogram != null) { + // some config scenarios may lead to differing histogram settings, + // so doing this defensively + if (other.histogram != null) { + try { + histogram.add(other.histogram); + } catch (ArrayIndexOutOfBoundsException e) { + log.log(LogLevel.WARNING, "Had trouble merging histograms: " + e.getMessage()); + } + } + } + } + + public boolean isCounter() { return outputFormat == AssumedType.COUNTER; } + + public long getCount() { return count; } + public double getLast() { return current; } + public double getMax() { return max; } + public double getMin() { return min; } + public double getSum() { return sum; } + + MetricSettings getMetricDefinition() { + return metricSettings; + } + + public DoubleHistogram getHistogram() { + return histogram; + } +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/Value.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Value.java new file mode 100644 index 00000000000..43b59b3519d --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Value.java @@ -0,0 +1,246 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +/** + * Wrapper for dimension values. + * + * @author steinar + */ +public abstract class Value { + private static final String UNSUPPORTED_VALUE_TYPE = "Unsupported value type."; + + /** + * Marker for the type of the contained value of a Value instance. + */ + public enum Discriminator { + LONG, DOUBLE, STRING; + } + + /** + * Get the long wrapped by a Value if one exists. + * + * @throws UnsupportedOperationException if LONG is not returned by {{@link #getType()}. + */ + public long longValue() throws UnsupportedOperationException { + throw new UnsupportedOperationException(UNSUPPORTED_VALUE_TYPE); + } + + /** + * Get the double wrapped by a Value if one exists. + * + * @throws UnsupportedOperationException if DOUBLE is not returned by {{@link #getType()}. + */ + public double doubleValue() throws UnsupportedOperationException { + throw new UnsupportedOperationException(UNSUPPORTED_VALUE_TYPE); + } + + /** + * Get the string wrapped by a Value if one exists. + * + * @throws UnsupportedOperationException if STRING is not returned by {{@link #getType()}. + */ + public String stringValue() throws UnsupportedOperationException { + throw new UnsupportedOperationException(UNSUPPORTED_VALUE_TYPE); + } + + /** + * Show the (single) supported standard type representation of a Value instance. + */ + public abstract Discriminator getType(); + + private static class LongValue extends Value { + private final long value; + + LongValue(long value) { + this.value = value; + } + + @Override + public long longValue() { + return value; + } + + @Override + public Discriminator getType() { + return Discriminator.LONG; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (value ^ (value >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + LongValue other = (LongValue) obj; + if (value != other.value) { + return false; + } + return true; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("LongValue [value=").append(value).append("]"); + return builder.toString(); + } + } + + private static class DoubleValue extends Value { + private final double value; + + DoubleValue(double value) { + this.value = value; + } + + @Override + public double doubleValue() { + return value; + } + + @Override + public Discriminator getType() { + return Discriminator.DOUBLE; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + long temp; + temp = Double.doubleToLongBits(value); + result = prime * result + (int) (temp ^ (temp >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + DoubleValue other = (DoubleValue) obj; + if (Double.doubleToLongBits(value) != Double.doubleToLongBits(other.value)) { + return false; + } + return true; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("DoubleValue [value=").append(value).append("]"); + return builder.toString(); + } + } + + private static class StringValue extends Value { + private final String value; + + StringValue(String value) { + this.value = value; + } + + @Override + public String stringValue() { + return value; + } + + @Override + public Discriminator getType() { + return Discriminator.STRING; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((value == null) ? 0 : value.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + StringValue other = (StringValue) obj; + if (value == null) { + if (other.value != null) { + return false; + } + } else if (!value.equals(other.value)) { + return false; + } + return true; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("StringValue [value=").append(value).append("]"); + return builder.toString(); + } + } + + /** + * Helper method to wrap a long as a Value. The instance returned may or may + * not be unique. + * + * @param value + * the value to wrap + * @return an immutable wrapper + */ + public static Value of(long value) { + return new LongValue(value); + } + + /** + * Helper method to wrap a double as a Value. The instance returned may or + * may not be unique. + * + * @param value + * the value to wrap + * @return an immutable wrapper + * */ + public static Value of(double value) { + return new DoubleValue(value); + } + + /** + * Helper method to wrap a string as a Value. The instance returned may or + * may not be unique. + * + * @param value + * the value to wrap + * @return an immutable wrapper + */ + public static Value of(String value) { + return new StringValue(value); + } + +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/jdisc/JdiscMetricsFactory.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/jdisc/JdiscMetricsFactory.java new file mode 100644 index 00000000000..f5208b2226c --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/jdisc/JdiscMetricsFactory.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple.jdisc; + +import java.io.PrintStream; +import java.util.logging.Logger; + +import com.yahoo.container.jdisc.MetricConsumerFactory; +import com.yahoo.container.jdisc.state.MetricSnapshot; +import com.yahoo.container.jdisc.state.SnapshotProvider; +import com.yahoo.jdisc.application.MetricConsumer; +import com.yahoo.metrics.simple.Bucket; +import com.yahoo.metrics.simple.MetricReceiver; + +/** + * A factory for all the JDisc API classes. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class JdiscMetricsFactory implements MetricConsumerFactory, SnapshotProvider { + private static final Logger log = Logger.getLogger(JdiscMetricsFactory.class.getName()); + private final SimpleMetricConsumer metricInstance; + private final MetricReceiver metricReceiver; + + public JdiscMetricsFactory(MetricReceiver receiver) { + this.metricReceiver = receiver; + this.metricInstance = new SimpleMetricConsumer(receiver); + } + + @Override + public MetricConsumer newInstance() { + // the underlying implementation is thread safe anyway to allow for stand-alone use + return metricInstance; + } + + + @Override + public MetricSnapshot latestSnapshot() { + Bucket curr = metricReceiver.getSnapshot(); + if (curr == null) { + log.warning("no snapshot from instance of " + metricInstance.getClass()); + return null; + } else { + SnapshotConverter converter = new SnapshotConverter(curr); + return converter.convert(); + } + } + + @Override + public void histogram(PrintStream output) { + Bucket curr = metricReceiver.getSnapshot(); + if (curr == null) { + log.warning("no snapshot from instance of " + metricInstance.getClass()); + return; + } else { + SnapshotConverter converter = new SnapshotConverter(curr); + converter.outputHistograms(output); + return; + } + } + +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/jdisc/SimpleMetricConsumer.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/jdisc/SimpleMetricConsumer.java new file mode 100644 index 00000000000..7f5571418fe --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/jdisc/SimpleMetricConsumer.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple.jdisc; + +import java.util.Map; + +import com.yahoo.jdisc.Metric.Context; +import com.yahoo.jdisc.application.MetricConsumer; +import com.yahoo.metrics.simple.Identifier; +import com.yahoo.metrics.simple.Measurement; +import com.yahoo.metrics.simple.Point; +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.metrics.simple.Sample; +import com.yahoo.metrics.simple.UntypedMetric.AssumedType; + +/** + * The single user facing part of the JDisc interfaces of simple metrics. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class SimpleMetricConsumer implements MetricConsumer { + + private final MetricReceiver receiver; + + public SimpleMetricConsumer(MetricReceiver receiver) { + this.receiver = receiver; + } + + @Override + public void set(String key, Number val, Context ctx) { + receiver.update(new Sample(new Measurement(val), new Identifier(key, getSimpleCoordinate(ctx)), AssumedType.GAUGE)); + } + + @Override + public void add(String key, Number val, Context ctx) { + receiver.update(new Sample(new Measurement(val), new Identifier(key, getSimpleCoordinate(ctx)), AssumedType.COUNTER)); + } + + private Point getSimpleCoordinate(Context ctx) { + if (ctx instanceof Point) { + return (Point) ctx; + } else { + return null; + } + } + + @Override + public Context createContext(Map<String, ?> properties) { + if (properties == null) { + return null; + } else { + return new Point(properties); + } + } + +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/jdisc/SnapshotConverter.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/jdisc/SnapshotConverter.java new file mode 100644 index 00000000000..6d3b6ad6243 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/jdisc/SnapshotConverter.java @@ -0,0 +1,212 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple.jdisc; + +import java.io.PrintStream; +import java.util.*; +import java.util.concurrent.TimeUnit; + +import org.HdrHistogram.DoubleHistogram; + +import com.yahoo.collections.Tuple2; +import com.yahoo.container.jdisc.state.*; +import com.yahoo.metrics.simple.Bucket; +import com.yahoo.metrics.simple.Identifier; +import com.yahoo.metrics.simple.Point; +import com.yahoo.metrics.simple.UntypedMetric; +import com.yahoo.metrics.simple.Value; +import com.yahoo.text.JSON; + +/** + * Convert simple metrics snapshots into jdisc state snapshots. + * + * @author arnej27959 + */ +class SnapshotConverter { + final Bucket snapshot; + final Map<Point, Map<String, MetricValue>> perPointData = new HashMap<>(); + private static final char[] DIGITS = new char[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + + private Map<String, MetricValue> getMap(Point point) { + if (! perPointData.containsKey(point)) { + perPointData.put(point, new HashMap<String, MetricValue>()); + } + return perPointData.get(point); + } + + public SnapshotConverter(Bucket snapshot) { + this.snapshot = snapshot; + } + + static MetricDimensions convert(Point p) { + if (p == null) { + return StateMetricContext.newInstance(null); + } + List<String> dimensions = p.dimensions(); + List<Value> location = p.location(); + Map<String, Object> pointWrapper = new HashMap<>(dimensions.size()); + for (int i = 0; i < dimensions.size(); ++i) { + pointWrapper.put(dimensions.get(i), valueAsString(location.get(i))); + } + return StateMetricContext.newInstance(pointWrapper); + } + + // TODO: just a compatibility wrapper, should be removed ASAP + private static Object valueAsString(Value value) { + switch (value.getType()) { + case STRING: + return value.stringValue(); + case LONG: + return Long.valueOf(value.longValue()); + case DOUBLE: + return Double.valueOf(value.doubleValue()); + default: + throw new IllegalStateException("simplemetrics impl is out of sync with itself, please file a ticket."); + } + } + + + static MetricValue convert(UntypedMetric val) { + if (val.isCounter()) { + return CountMetric.newInstance(val.getCount()); + } else { + if (val.getHistogram() == null) { + return GaugeMetric.newInstance(val.getLast(), val.getMax(), val.getMin(), val.getSum(), val.getCount()); + } else { + return GaugeMetric.newInstance(val.getLast(), val.getMax(), val.getMin(), val.getSum(), val.getCount(), + Optional.of(buildPercentileList(val.getHistogram()))); + } + } + } + + private static List<Tuple2<String, Double>> buildPercentileList(DoubleHistogram histogram) { + final List<Tuple2<String, Double>> prefixAndValues = new ArrayList<>(2); + prefixAndValues.add(new Tuple2<>("95", histogram.getValueAtPercentile(95.0d))); + prefixAndValues.add(new Tuple2<>("99", histogram.getValueAtPercentile(99.0d))); + return prefixAndValues; + } + + MetricSnapshot convert() { + for (Map.Entry<Identifier, UntypedMetric> entry : snapshot.entrySet()) { + Identifier ident = entry.getKey(); + getMap(ident.getLocation()).put(ident.getName(), convert(entry.getValue())); + } + Map<MetricDimensions, MetricSet> data = new HashMap<>(); + for (Map.Entry<Point, Map<String, MetricValue>> entry : perPointData.entrySet()) { + data.put(convert(entry.getKey()), new MetricSet(entry.getValue())); + } + return new MetricSnapshot(snapshot.getFromMillis(), + snapshot.getToMillis(), + TimeUnit.MILLISECONDS, + data); + } + + void outputHistograms(PrintStream output) { + boolean gotHistogram = false; + for (Map.Entry<Identifier, UntypedMetric> entry : snapshot.entrySet()) { + if (entry.getValue().getHistogram() == null) { + continue; + } + gotHistogram = true; + DoubleHistogram histogram = entry.getValue().getHistogram(); + Identifier id = entry.getKey(); + String metricIdentifier = getIdentifierString(id); + output.println("# start of metric " + metricIdentifier); + histogram.outputPercentileDistribution(output, 4, 1.0d, true); + output.println("# end of metric " + metricIdentifier); + } + if (!gotHistogram) { + output.println("# No histograms currently available."); + } + } + + private String getIdentifierString(Identifier id) { + StringBuilder buffer = new StringBuilder(); + Point location = id.getLocation(); + buffer.append(id.getName()); + if (location != null) { + buffer.append(", dimensions: { "); + Iterator<String> dimensions = location.dimensions().iterator(); + Iterator<Value> values = location.location().iterator(); + boolean firstDimension = true; + while (dimensions.hasNext() && values.hasNext()) { + + if (firstDimension) { + firstDimension = false; + } else { + buffer.append(", "); + } + serializeSingleDimension(buffer, dimensions.next(), values.next()); + } + buffer.append(" }"); + } + return buffer.toString(); + + } + + private void serializeSingleDimension(StringBuilder buffer, final String dimensionName, Value dimensionValue) { + buffer.append('"'); + escape(dimensionName, buffer); + buffer.append("\": "); + switch (dimensionValue.getType()) { + case LONG: + buffer.append(Long.toString(dimensionValue.longValue())); + break; + case DOUBLE: + buffer.append(Double.toString(dimensionValue.doubleValue())); + break; + case STRING: + buffer.append('"'); + escape(dimensionValue.stringValue(), buffer); + buffer.append('"'); + break; + default: + buffer.append("\"Unknown type for this dimension, this is a bug.\""); + break; + } + } + + private void escape(final String in, final StringBuilder target) { + for (final char c : in.toCharArray()) { + switch (c) { + case ('"'): + target.append("\\\""); + break; + case ('\\'): + target.append("\\\\"); + break; + case ('\b'): + target.append("\\b"); + break; + case ('\f'): + target.append("\\f"); + break; + case ('\n'): + target.append("\\n"); + break; + case ('\r'): + target.append("\\r"); + break; + case ('\t'): + target.append("\\t"); + break; + default: + if (c < 32) { + target.append("\\u").append(fourDigitHexString(c)); + } else { + target.append(c); + } + break; + } + } + } + + private static char[] fourDigitHexString(final char c) { + final char[] hex = new char[4]; + int in = ((c) & 0xFFFF); + for (int i = 3; i >= 0; --i) { + hex[i] = DIGITS[in & 0xF]; + in >>>= 4; + } + return hex; + } +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/jdisc/package-info.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/jdisc/package-info.java new file mode 100644 index 00000000000..4d34244bec0 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/jdisc/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * JDisc metrics API for simple metrics implementation. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +@ExportPackage +package com.yahoo.metrics.simple.jdisc; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/package-info.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/package-info.java new file mode 100644 index 00000000000..6529b804a43 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/package-info.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * A metrics API with declarable metric, and also an implementation of the + * JDisc Metrics API where the newest state is made continously available. + * + * <p> + * Users should have an instance of {@link com.yahoo.metrics.simple.MetricReceiver} + * injected in the constructor where needed, then declare metrics as instances + * of {@link com.yahoo.metrics.simple.Counter} and + * {@link com.yahoo.metrics.simple.Gauge} using + * {@link com.yahoo.metrics.simple.MetricReceiver#declareCounter(String)}, + * {@link com.yahoo.metrics.simple.MetricReceiver#declareCounter(String, Point)}, + * {@link com.yahoo.metrics.simple.MetricReceiver#declareGauge(String)}, + * {@link com.yahoo.metrics.simple.MetricReceiver#declareGauge(String, Point)}, or + * {@link com.yahoo.metrics.simple.MetricReceiver#declareGauge(String, java.util.Optional, MetricSettings)}. + * </p> + * + * <p> + * Clients input data through the API in {@link com.yahoo.metrics.simple.MetricReceiver} (or + * using the JDisc Metric API it will be received in + * {@link com.yahoo.metrics.simple.jdisc.SimpleMetricConsumer}), while the internal work is + * done by {@link com.yahoo.metrics.simple.MetricAggregator}. Initialization + * is done top-down from {@link com.yahoo.metrics.simple.MetricManager}. The link + * between calls to MetricReceiver and MetricAggregator is the role of + * {@link com.yahoo.metrics.simple.MetricUpdater}. + * </p> + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +@PublicApi +@ExportPackage +package com.yahoo.metrics.simple; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/runtime/MetricProperties.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/runtime/MetricProperties.java new file mode 100644 index 00000000000..2e9082dbe44 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/runtime/MetricProperties.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple.runtime; + +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Constants used by Vespa to make the simple metrics implementation available + * to other components. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class MetricProperties { + private MetricProperties() { + } + + public static final String BUNDLE_SYMBOLIC_NAME = "simplemetrics"; +} diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/runtime/package-info.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/runtime/package-info.java new file mode 100644 index 00000000000..0dc746bb8c2 --- /dev/null +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/runtime/package-info.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Settings and properties used for setting up the simple metrics library in a + * container. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +@ExportPackage +package com.yahoo.metrics.simple.runtime; + +import com.yahoo.osgi.annotation.ExportPackage; + diff --git a/simplemetrics/src/main/resources/configdefinitions/manager.def b/simplemetrics/src/main/resources/configdefinitions/manager.def new file mode 100644 index 00000000000..6f6bef75fd7 --- /dev/null +++ b/simplemetrics/src/main/resources/configdefinitions/manager.def @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 +namespace=metrics + +reportPeriodSeconds int default=300 +pointsToKeepPerMetric int default=100 diff --git a/simplemetrics/src/test/java/com/yahoo/metrics/simple/BucketTest.java b/simplemetrics/src/test/java/com/yahoo/metrics/simple/BucketTest.java new file mode 100644 index 00000000000..00829ddf5f0 --- /dev/null +++ b/simplemetrics/src/test/java/com/yahoo/metrics/simple/BucketTest.java @@ -0,0 +1,213 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import static org.junit.Assert.*; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.logging.Handler; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.Set; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.metrics.simple.UntypedMetric.AssumedType; + +/** + * Functional tests for the value buckets, as implemented in the class Bucket, + * and by extension the value store itself, UntypedValue. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class BucketTest { + private Bucket bucket; + + @Before + public void setUp() throws Exception { + bucket = new Bucket(); + } + + @After + public void tearDown() throws Exception { + bucket = null; + } + + @Test + public final void testEntrySet() { + assertEquals(0, bucket.entrySet().size()); + for (int i = 0; i < 4; ++i) { + bucket.put(new Sample(new Measurement(i), new Identifier("nalle_" + i, null), AssumedType.GAUGE)); + } + assertEquals(4, bucket.entrySet().size()); + for (int i = 0; i < 4; ++i) { + bucket.put(new Sample(new Measurement(i), new Identifier("nalle", + new Point(new ImmutableMap.Builder<String, Integer>().put("dim", Integer.valueOf(i)).build())), + AssumedType.GAUGE)); + } + assertEquals(8, bucket.entrySet().size()); + int nalle = 0, nalle0 = 0, nalle1 = 0, nalle2 = 0, nalle3 = 0; + for (Entry<Identifier, UntypedMetric> x : bucket.entrySet()) { + String metricName = x.getKey().getName(); + switch (metricName) { + case "nalle": + ++nalle; + break; + case "nalle_0": + ++nalle0; + break; + case "nalle_1": + ++nalle1; + break; + case "nalle_2": + ++nalle2; + break; + case "nalle_3": + ++nalle3; + break; + default: + throw new IllegalStateException(); + } + } + assertEquals(4, nalle); + assertEquals(1, nalle0); + assertEquals(1, nalle1); + assertEquals(1, nalle2); + assertEquals(1, nalle3); + } + + @Test + public final void testPutSampleWithUnsupportedType() { + boolean caughtIt = false; + try { + bucket.put(new Sample(new Measurement(1), new Identifier("nalle", null), AssumedType.NONE)); + } catch (Exception e) { + caughtIt = true; + } + assertTrue(caughtIt); + } + + @Test + public final void testPutIdentifierUntypedValue() { + UntypedMetric v = new UntypedMetric(null); + v.add(2); + bucket.put(new Sample(new Measurement(3), new Identifier("nalle", null), AssumedType.GAUGE)); + bucket.put(new Identifier("nalle", null), v); + assertEquals(1, bucket.entrySet().size()); + // check raw overwriting + Entry<Identifier, UntypedMetric> stored = bucket.entrySet().iterator().next(); + assertEquals(new Identifier("nalle", null), stored.getKey()); + assertTrue(stored.getValue().isCounter()); + } + + @Test + public final void testHasIdentifier() { + for (int i = 0; i < 4; ++i) { + bucket.put(new Sample(new Measurement(i), new Identifier("nalle_" + i, new Point( + new ImmutableMap.Builder<String, Integer>().put(String.valueOf(i), Integer.valueOf(i)).build())), + AssumedType.GAUGE)); + } + for (int i = 0; i < 4; ++i) { + assertTrue(bucket.hasIdentifier(new Identifier("nalle_" + i, new Point(new ImmutableMap.Builder<String, Integer>().put( + String.valueOf(i), Integer.valueOf(i)).build())))); + } + } + + @Test + public final void testOkMerge() { + bucket.put(new Sample(new Measurement(2), new Identifier("nalle", null), AssumedType.GAUGE)); + Bucket otherNew = new Bucket(); + otherNew.put(new Sample(new Measurement(3), new Identifier("nalle", null), AssumedType.GAUGE)); + Bucket otherOld = new Bucket(); + otherOld.put(new Sample(new Measurement(5), new Identifier("nalle", null), AssumedType.GAUGE)); + bucket.merge(otherNew, true); + bucket.merge(otherOld, false); + Set<Entry<Identifier, UntypedMetric>> entries = bucket.entrySet(); + assertEquals(1, entries.size()); + Entry<Identifier, UntypedMetric> entry = entries.iterator().next(); + assertEquals(10, entry.getValue().getSum(), 0.0); + assertEquals(3, entry.getValue().getLast(), 0.0); + assertEquals(2, entry.getValue().getMin(), 0.0); + assertEquals(5, entry.getValue().getMax(), 0.0); + assertEquals(3, entry.getValue().getCount()); + } + + private static class CheckThatItWasLogged extends Handler { + final boolean[] loggingMarker; + + public CheckThatItWasLogged(boolean[] loggingMarker) { + this.loggingMarker = loggingMarker; + } + + @Override + public void publish(LogRecord record) { + loggingMarker[0] = true; + } + + @Override + public void flush() { + } + + @Override + public void close() throws SecurityException { + } + } + + @Test + public final void testMismatchedMerge() { + Logger log = Logger.getLogger(Bucket.class.getName()); + boolean[] loggingMarker = new boolean[1]; + loggingMarker[0] = false; + log.setUseParentHandlers(false); + Handler logHandler = new CheckThatItWasLogged(loggingMarker); + log.addHandler(logHandler); + Bucket other = new Bucket(); + bucket.put(new Sample(new Measurement(2), new Identifier("nalle", null), AssumedType.GAUGE)); + other.put(new Sample(new Measurement(3), new Identifier("nalle", null), AssumedType.COUNTER)); + bucket.merge(other, true); + assertTrue(loggingMarker[0]); + log.removeHandler(logHandler); + log.setUseParentHandlers(true); + } + + @Test + public final void testGetAllMetricNames() { + twoMetricsUniqueDimensions(); + Collection<String> names = bucket.getAllMetricNames(); + assertEquals(2, names.size()); + assertTrue(names.contains("nalle")); + assertTrue(names.contains("nalle2")); + } + + @Test + public final void testGetValuesForMetric() { + twoMetricsUniqueDimensions(); + Collection<Entry<Point, UntypedMetric>> values = bucket.getValuesForMetric("nalle"); + assertEquals(4, values.size()); + } + + private void twoMetricsUniqueDimensions() { + for (int i = 0; i < 4; ++i) { + bucket.put(new Sample(new Measurement(i), new Identifier("nalle", new Point(new ImmutableMap.Builder<String, Integer>() + .put(String.valueOf(i), Integer.valueOf(i)).build())), AssumedType.GAUGE)); + bucket.put(new Sample(new Measurement(i), new Identifier("nalle2", new Point( + new ImmutableMap.Builder<String, Integer>().put(String.valueOf(i), Integer.valueOf(i)).build())), + AssumedType.GAUGE)); + } + } + + @Test + public final void testGetValuesByMetricName() { + twoMetricsUniqueDimensions(); + Map<String, List<Entry<Point, UntypedMetric>>> values = bucket.getValuesByMetricName(); + assertEquals(2, values.size()); + assertEquals(4, values.get("nalle").size()); + assertEquals(4, values.get("nalle2").size()); + } + +} diff --git a/simplemetrics/src/test/java/com/yahoo/metrics/simple/CounterTest.java b/simplemetrics/src/test/java/com/yahoo/metrics/simple/CounterTest.java new file mode 100644 index 00000000000..e7dc93eddcb --- /dev/null +++ b/simplemetrics/src/test/java/com/yahoo/metrics/simple/CounterTest.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import static org.junit.Assert.*; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.metrics.ManagerConfig; + +/** + * Functional tests for counters. + * + * @author steinar + */ +public class CounterTest extends UnitTestSetup { + + @Before + public void setUp() throws Exception { + super.init(); + } + + @After + public void tearDown() throws Exception { + super.fini(); + } + + @Test + public final void testAdd() throws InterruptedException { + final String metricName = "unitTestCounter"; + Counter c = receiver.declareCounter(metricName); + c.add(); + Bucket b = getUpdatedSnapshot(); + final Map<String, List<Entry<Point, UntypedMetric>>> valuesByMetricName = b.getValuesByMetricName(); + assertEquals(1, valuesByMetricName.size()); + List<Entry<Point, UntypedMetric>> x = valuesByMetricName.get(metricName); + assertEquals(1, x.size()); + assertNull(x.get(0).getKey()); + assertEquals(1L, x.get(0).getValue().getCount()); + } + + @Test + public final void testAddLong() throws InterruptedException { + final String metricName = "unitTestCounter"; + Counter c = receiver.declareCounter(metricName); + final long twoToThePowerOfFourtyeight = 65536L * 65536L * 65536L; + c.add(twoToThePowerOfFourtyeight); + Bucket b = getUpdatedSnapshot(); + final Map<String, List<Entry<Point, UntypedMetric>>> valuesByMetricName = b.getValuesByMetricName(); + assertEquals(1, valuesByMetricName.size()); + List<Entry<Point, UntypedMetric>> x = valuesByMetricName.get(metricName); + assertEquals(1, x.size()); + assertNull(x.get(0).getKey()); + assertEquals(twoToThePowerOfFourtyeight, x.get(0).getValue().getCount()); + } + + @Test + public final void testAddPoint() throws InterruptedException { + final String metricName = "unitTestCounter"; + Point p = receiver.pointBuilder().set("x", 2L).set("y", 3.0d).set("z", "5").build(); + Counter c = receiver.declareCounter(metricName, p); + c.add(); + Bucket b = getUpdatedSnapshot(); + final Map<String, List<Entry<Point, UntypedMetric>>> valuesByMetricName = b.getValuesByMetricName(); + assertEquals(1, valuesByMetricName.size()); + List<Entry<Point, UntypedMetric>> x = valuesByMetricName.get(metricName); + assertEquals(1, x.size()); + assertEquals(p, x.get(0).getKey()); + assertEquals(1, x.get(0).getValue().getCount()); + } + + @Test + public final void testAddLongPoint() throws InterruptedException { + final String metricName = "unitTestCounter"; + Point p = receiver.pointBuilder().set("x", 2L).set("y", 3.0d).set("z", "5").build(); + Counter c = receiver.declareCounter(metricName, p); + final long twoToThePowerOfFourtyeight = 65536L * 65536L * 65536L; + c.add(twoToThePowerOfFourtyeight, c.builder().set("x", 7).set("_y", 11.0d).set("Z", "13").build()); + Bucket b = getUpdatedSnapshot(); + final Map<String, List<Entry<Point, UntypedMetric>>> valuesByMetricName = b.getValuesByMetricName(); + assertEquals(1, valuesByMetricName.size()); + List<Entry<Point, UntypedMetric>> x = valuesByMetricName.get(metricName); + assertEquals(1, x.size()); + Point actual = x.get(0).getKey(); + assertEquals(5, actual.dimensionality()); + List<String> dimensions = actual.dimensions(); + List<Value> location = actual.location(); + assertEquals(dimensions.size(), location.size()); + Iterator<String> i0 = dimensions.iterator(); + Iterator<Value> i1 = location.iterator(); + Map<String, Value> asMap = new HashMap<>(); + while (i0.hasNext() && i1.hasNext()) { + asMap.put(i0.next(), i1.next()); + } + assertEquals(Value.of(7), asMap.get("x")); + assertEquals(Value.of(3.0d), asMap.get("y")); + assertEquals(Value.of("5"), asMap.get("z")); + assertEquals(Value.of(11.0d), asMap.get("_y")); + assertEquals(Value.of("13"), asMap.get("Z")); + assertEquals(twoToThePowerOfFourtyeight, x.get(0).getValue().getCount()); + } + +} diff --git a/simplemetrics/src/test/java/com/yahoo/metrics/simple/DimensionsCacheTest.java b/simplemetrics/src/test/java/com/yahoo/metrics/simple/DimensionsCacheTest.java new file mode 100644 index 00000000000..8c24baae560 --- /dev/null +++ b/simplemetrics/src/test/java/com/yahoo/metrics/simple/DimensionsCacheTest.java @@ -0,0 +1,110 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import static org.junit.Assert.*; + +import java.util.Collection; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.metrics.simple.UntypedMetric.AssumedType; + +/** + * Functional test for point persistence layer. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class DimensionsCacheTest { + private static final int POINTS_TO_KEEP = 3; + DimensionCache cache; + + @Before + public void setUp() throws Exception { + cache = new DimensionCache(POINTS_TO_KEEP); + } + + @After + public void tearDown() throws Exception { + cache = null; + } + + @Test + public final void smokeTest() { + String metricName = "testMetric"; + Bucket first = new Bucket(); + for (int i = 0; i < 4; ++i) { + populateSingleValue(metricName, first, i); + } + cache.updateDimensionPersistence(null, first); + Bucket second = new Bucket(); + final int newest = 42; + populateSingleValue(metricName, second, newest); + cache.updateDimensionPersistence(first, second); + assertEquals(POINTS_TO_KEEP, second.getValuesForMetric(metricName).size()); + boolean newestFound = false; + for (Entry<Point, UntypedMetric> x : second.getValuesForMetric(metricName)) { + if (x.getValue().getLast() == newest) { + newestFound = true; + } + } + assertTrue("Did not keep newest measurement when padding points.", newestFound); + } + + @Test + public final void testNoBoomWithEmptyBuckets() { + Bucket check = new Bucket(); + cache.updateDimensionPersistence(null, new Bucket()); + cache.updateDimensionPersistence(null, new Bucket()); + cache.updateDimensionPersistence(new Bucket(), check); + assertEquals(0, check.entrySet().size()); + } + + @Test + public final void testUpdateWithNullThenDataThenData() { + Bucket first = new Bucket(); + populateDimensionLessValue("one", first, 2); + cache.updateDimensionPersistence(null, first); + Bucket second = new Bucket(); + populateDimensionLessValue("other", second, 3); + cache.updateDimensionPersistence(first, second); + Collection<String> names = second.getAllMetricNames(); + assertEquals(2, names.size()); + assertTrue(names.contains("one")); + assertTrue(names.contains("other")); + } + + @Test + public final void testUpdateWithNullThenDataThenNoDataThenData() { + Bucket first = new Bucket(); + Bucket second = new Bucket(); + populateDimensionLessValue("first", first, 1.0d); + populateDimensionLessValue("second", second, 2.0d); + cache.updateDimensionPersistence(null, first); + cache.updateDimensionPersistence(first, new Bucket()); + cache.updateDimensionPersistence(new Bucket(), second); + assertEquals(2, second.entrySet().size()); + assertTrue(second.getAllMetricNames().contains("first")); + assertTrue(second.getAllMetricNames().contains("second")); + } + + private void populateDimensionLessValue(String metricName, Bucket bucket, double x) { + Identifier id = new Identifier(metricName, null); + Sample wrappedX = new Sample(new Measurement(Double.valueOf(x)), id, AssumedType.GAUGE); + bucket.put(wrappedX); + } + + private void populateSingleValue(String metricName, Bucket bucket, int i) { + Map<String, Integer> m = new TreeMap<>(); + m.put(String.valueOf(i), Integer.valueOf(i)); + Point p = new Point(m); + Identifier id = new Identifier(metricName, p); + Sample x = new Sample(new Measurement(Double.valueOf(i)), id, AssumedType.GAUGE); + bucket.put(x); + } + +} diff --git a/simplemetrics/src/test/java/com/yahoo/metrics/simple/GaugeTest.java b/simplemetrics/src/test/java/com/yahoo/metrics/simple/GaugeTest.java new file mode 100644 index 00000000000..47cea45ee49 --- /dev/null +++ b/simplemetrics/src/test/java/com/yahoo/metrics/simple/GaugeTest.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import static org.junit.Assert.*; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.metrics.ManagerConfig; + +/** + * Functional tests for gauges. + * + * @author steinar + */ +public class GaugeTest extends UnitTestSetup { + + @Before + public void setUp() throws Exception { + super.init(); + } + + @After + public void tearDown() throws Exception { + super.fini(); + } + + @Test + public final void testSampleDouble() throws InterruptedException { + final String metricName = "unitTestGauge"; + Gauge g = receiver.declareGauge(metricName); + g.sample(1.0d); + Bucket b = getUpdatedSnapshot(); + final Map<String, List<Entry<Point, UntypedMetric>>> valuesByMetricName = b.getValuesByMetricName(); + assertEquals(1, valuesByMetricName.size()); + List<Entry<Point, UntypedMetric>> x = valuesByMetricName.get(metricName); + assertEquals(1, x.size()); + assertNull(x.get(0).getKey()); + assertEquals(1L, x.get(0).getValue().getCount()); + assertEquals(1.0d, x.get(0).getValue().getLast(), 0.0d); + } + + @Test + public final void testSampleDoublePoint() throws InterruptedException { + final String metricName = "unitTestGauge"; + Point p = receiver.pointBuilder().set("x", 2L).set("y", 3.0d).set("z", "5").build(); + Gauge g = receiver.declareGauge(metricName, p); + g.sample(Math.E, g.builder().set("x", 7).set("_y", 11.0d).set("Z", "13").build()); + Bucket b = getUpdatedSnapshot(); + final Map<String, List<Entry<Point, UntypedMetric>>> valuesByMetricName = b.getValuesByMetricName(); + assertEquals(1, valuesByMetricName.size()); + List<Entry<Point, UntypedMetric>> x = valuesByMetricName.get(metricName); + assertEquals(1, x.size()); + Point actual = x.get(0).getKey(); + assertEquals(5, actual.dimensionality()); + List<String> dimensions = actual.dimensions(); + List<Value> location = actual.location(); + assertEquals(dimensions.size(), location.size()); + Iterator<String> i0 = dimensions.iterator(); + Iterator<Value> i1 = location.iterator(); + Map<String, Value> asMap = new HashMap<>(); + while (i0.hasNext() && i1.hasNext()) { + asMap.put(i0.next(), i1.next()); + } + assertEquals(Value.of(7), asMap.get("x")); + assertEquals(Value.of(3.0d), asMap.get("y")); + assertEquals(Value.of("5"), asMap.get("z")); + assertEquals(Value.of(11.0d), asMap.get("_y")); + assertEquals(Value.of("13"), asMap.get("Z")); + assertEquals(Math.E, x.get(0).getValue().getLast(), 1e-15); + } + +} diff --git a/simplemetrics/src/test/java/com/yahoo/metrics/simple/MetricsTest.java b/simplemetrics/src/test/java/com/yahoo/metrics/simple/MetricsTest.java new file mode 100644 index 00000000000..d33e25cab01 --- /dev/null +++ b/simplemetrics/src/test/java/com/yahoo/metrics/simple/MetricsTest.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.simple; + +import static org.junit.Assert.*; + +import java.util.Collection; +import java.util.Map.Entry; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.metrics.ManagerConfig; +import com.yahoo.metrics.simple.jdisc.JdiscMetricsFactory; +import com.yahoo.metrics.simple.jdisc.SimpleMetricConsumer; + +/** + * Functional test for simple metric implementation. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class MetricsTest extends UnitTestSetup { + SimpleMetricConsumer metricApi; + + @Before + public void setUp() throws Exception { + super.init(); + metricApi = (SimpleMetricConsumer) new JdiscMetricsFactory(metricManager.get()).newInstance(); + } + + @After + public void tearDown() throws Exception { + super.fini(); + } + + @Test + public final void smokeTest() throws InterruptedException { + final String metricName = "testMetric"; + metricApi.set(metricName, Double.valueOf(1.0d), null); + updater.gotData.await(10, TimeUnit.SECONDS); + Bucket s = getUpdatedSnapshot(); + Collection<Entry<Point, UntypedMetric>> values = s.getValuesForMetric(metricName); + assertEquals(1, values.size()); + Entry<Point, UntypedMetric> value = values.iterator().next(); + assertNull(value.getKey()); + assertEquals(1.0d, value.getValue().getLast(), 0.0d); // using number exactly expressible as doubles + assertEquals(1L, value.getValue().getCount()); + } + + @Test + public final void testRedefinition() { + MetricReceiver r = metricManager.get(); + final String metricName = "gah"; + r.addMetricDefinition(metricName, new MetricSettings.Builder().build()); + r.addMetricDefinition(metricName, new MetricSettings.Builder().histogram(true).build()); + assertTrue(r.getMetricDefinition(metricName).isHistogram()); + } + +} |