diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /metrics/src/main |
Publish
Diffstat (limited to 'metrics/src/main')
24 files changed, 2873 insertions, 0 deletions
diff --git a/metrics/src/main/java/com/yahoo/metrics/ConsumerSpec.java b/metrics/src/main/java/com/yahoo/metrics/ConsumerSpec.java new file mode 100644 index 00000000000..ec6963bef30 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/ConsumerSpec.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import java.util.HashSet; +import java.util.Set; +import java.util.StringTokenizer; + +/** + * Spec saved from config. If metricSetChildren has content, metric pointed + * to is a metric set. + */ +class ConsumerSpec { + Set<String> includedMetrics = new HashSet<String>(); + + public boolean contains(Metric m) { + return includedMetrics.contains(m.getPath()); + } + + public void register(String path) { + StringTokenizer tokenizer = new StringTokenizer(path, "."); + + String total = ""; + + while (tokenizer.hasMoreTokens()) { + if (!total.isEmpty()) { + total += "."; + } + total += tokenizer.nextToken(); + includedMetrics.add(total); + } + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/CountMetric.java b/metrics/src/main/java/com/yahoo/metrics/CountMetric.java new file mode 100644 index 00000000000..409866e4040 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/CountMetric.java @@ -0,0 +1,222 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.metrics.util.MetricValueSet; +import com.yahoo.metrics.util.ValueType; +import com.yahoo.metrics.util.HasCopy; +import com.yahoo.text.Utf8String; +import com.yahoo.text.XMLWriter; + +import java.util.Collection; +import java.util.Locale; +import java.util.logging.Logger; + +/** + * A metric that counts something. The value should always be positive. + */ +@SuppressWarnings("unchecked") +public class CountMetric extends Metric { + + public static final int LOG_IF_UNSET = 2; + + private static final Utf8String AVERAGE_CHANGE_PER_SECOND = new Utf8String("average_change_per_second"); + private static final Utf8String COUNT = new Utf8String("count"); + private static final Logger log = Logger.getLogger(CountMetric.class.getName()); + private final MetricValueSet<CountValue> values; + private int flags; + + public CountMetric(String name, String tags, String description, MetricSet owner) { + super(name, tags, description, owner); + values = new MetricValueSet<CountValue>(); + flags = LOG_IF_UNSET; + } + + public CountMetric(CountMetric other, CopyType copyType, MetricSet owner) { + super(other, owner); + values = new MetricValueSet<CountValue>(other.values, copyType == CopyType.CLONE ? other.values.size() : 1); + flags = other.flags; + } + + private CountValue getValues() { + return values.getValue(); + } + + public long getValue() { + CountValue val = getValues(); + return (val == null ? 0 : val.value); + } + + @SuppressWarnings("UnusedDeclaration") + public void logOnlyIfSet() { + flags &= LOG_IF_UNSET; + } + + public void set(long value) { + while (!values.setValue(new CountValue(value))) { + // try again + } + } + + public void inc() { + inc(1); + } + + public void dec() { + dec(1); + } + + public void inc(long i) { + boolean overflow; + CountValue val; + do { + val = getValues(); + if (val == null) { + val = new CountValue(0); + } + overflow = (val.value + i < val.value); + val.value += i; + } while (!values.setValue(val)); + + if (overflow) { + reset(); + log.fine("Overflow in metric " + getName() + ". Resetting it."); + } + } + + public void dec(long i) { + boolean underflow; + CountValue val; + do { + val = getValues(); + if (val == null) { + val = new CountValue(0); + } + underflow = (val.value - i > val.value); + val.value -= i; + } while (!values.setValue(val)); + + if (underflow) { + reset(); + log.fine("Underflow in metric " + getName() + ". Resetting it."); + } + } + + @Override + public void reset() { + values.reset(); + } + + @Override + public boolean logFromTotalMetrics() { + return true; + } + + @Override + public void logEvent(EventLogger logger, String fullName) { + CountValue val = getValues(); + + if ((flags & LOG_IF_UNSET) != 0 || val != null) { + logger.count(fullName, val == null ? 0 : val.value); + } + } + + @Override + public void printXml(XMLWriter writer, + int secondsPassed, + int verbosity) + { + CountValue valRef = getValues(); + if (valRef == null && verbosity < 2) { + return; + } + long val = valRef != null ? valRef.value : 0; + openXMLTag(writer, verbosity); + writer.attribute(COUNT, String.valueOf(val)); + + if (secondsPassed > 0) { + writer.attribute(AVERAGE_CHANGE_PER_SECOND, + String.format(Locale.US, "%.2f", (double)val / secondsPassed)); + } + + writer.closeTag(); + } + + // Only one metric in valuemetric, so return it on any id. + @Override + public long getLongValue(String id) { + CountValue val = getValues(); + return (val == null ? 0 : val.value); + } + + @Override + public double getDoubleValue(String id) { + CountValue val = getValues(); + return (val == null ? 0 : val.value); + } + + @Override + public boolean used() { + return getValues() != null; + } + + @Override + public void addToSnapshot(Metric m) { + CountValue val = getValues(); + if (val != null) { + ((CountMetric)m).inc(val.value); + } + } + + @Override + public void addToPart(Metric m) { + CountValue val = getValues(); + if (val != null) { + ((CountMetric)m).inc(val.value); + } + } + + @Override + public Metric clone(CopyType type, MetricSet owner, boolean includeUnused) { + return new CountMetric(this, type, owner); + } + + private static class CountValue implements ValueType, HasCopy<CountValue> { + + long value; + + private CountValue(long value) { + this.value = value; + } + + public CountValue clone() { + try { + return (CountValue)super.clone(); + } catch (CloneNotSupportedException e) { + return null; + } + } + + public void add(ValueType other) { + value += ((CountValue)other).value; + } + + public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior) { + CountValue result = new CountValue(0); + for (ValueType t : sources) { + result.add(t); + } + if (joinBehavior == JoinBehavior.AVERAGE_ON_JOIN) { + result.value /= sources.size(); + } + return result; + } + + public String toString() { + return Long.toString(value); + } + + public CountValue copyObject() { + return new CountValue(value); + } + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/DoubleValue.java b/metrics/src/main/java/com/yahoo/metrics/DoubleValue.java new file mode 100644 index 00000000000..6309e4a9ea2 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/DoubleValue.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.metrics.util.ValueType; +import com.yahoo.metrics.util.HasCopy; + +import java.util.Collection; +import java.util.Locale; + +public class DoubleValue implements ValueMetric.Value<Double> { + private int count; + private double min, max, last; + private double total; + + public DoubleValue() { + count = 0; + min = Double.POSITIVE_INFINITY; + max = Double.NEGATIVE_INFINITY; + last = 0; + total = 0; + } + + public String toString() { + return "(count " + count + ", min " + min + ", max " + max + ", last " + last + ", total " + total + ")"; + } + + public void add(Double v) { + count = count + 1; + total = total + v; + min = Math.min(min, v); + max = Math.max(max, v); + last = v; + } + + public void join(ValueMetric.Value<Double> v2, boolean createAverageOnJoin) { + //StringBuffer sb = new StringBuffer(); + //sb.append("Adding " + this + " to " + v2); + if (createAverageOnJoin) { + count += v2.getCount(); + total += v2.getTotal(); + last = v2.getLast(); + + } else { + double totalAverage = getAverage() + v2.getAverage(); + count += v2.getCount(); + total = totalAverage * count; + last += v2.getLast(); + } + min = Math.min(min, v2.getMin()); + max = Math.max(max, v2.getMax()); + //sb.append(" and got " + this); + //System.err.println(sb.toString()); + } + + public void add(ValueType other) { + DoubleValue dv = (DoubleValue) other; + count = count + dv.count; + total = total + dv.total; + min = Math.min(min, dv.min); + max = Math.max(max, dv.max); + last = dv.last; + } + + public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior) { + DoubleValue result = new DoubleValue(); + for (ValueType t : sources) { + DoubleValue dv = (DoubleValue) t; + result.count = result.count + dv.count; + result.total = result.total + dv.total; + result.min = Math.min(result.min, dv.min); + result.max = Math.max(result.max, dv.max); + result.last += dv.last; + } + if (joinBehavior == JoinBehavior.AVERAGE_ON_JOIN) { + result.last /= sources.size(); + } else { + result.total *= sources.size(); + } + return result; + } + + public boolean overflow(ValueMetric.Value<Double> v2) { + if (count > (count + v2.getCount())) { + return true; + } + if (v2.getTotal() > 0 && getTotal() > getTotal() + v2.getTotal()) { + return true; + } + if (v2.getTotal() < 0 && getTotal() < getTotal() + v2.getTotal()) { + return true; + } + + return false; + } + + public int getCount() { return count; } + public Double getMin() { return (count > 0) ? min : 0; } + public Double getMax() { return (count > 0) ? max : 0; } + public Double getLast() { return last; } + public Double getTotal() { return total; } + + public Double getAverage() { + if (count == 0) { + return 0.0; + } + + return total / count; + } + + public String valueToString(Double val) { + if (val == Double.MIN_VALUE || val == Double.MAX_VALUE) { + return "0.00"; + } + + return String.format(Locale.US, "%.2f", val); + } + + public DoubleValue clone() { + try{ + return (DoubleValue) super.clone(); + } catch (CloneNotSupportedException e) { return null; } + } + + public ValueMetric.Value<Double> copyObject() { + return clone(); + } + +} diff --git a/metrics/src/main/java/com/yahoo/metrics/EventLogger.java b/metrics/src/main/java/com/yahoo/metrics/EventLogger.java new file mode 100644 index 00000000000..15ebb2ed0ef --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/EventLogger.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +public interface EventLogger { + public void value(String name, double value); + public void count(String name, long value); +} diff --git a/metrics/src/main/java/com/yahoo/metrics/JoinBehavior.java b/metrics/src/main/java/com/yahoo/metrics/JoinBehavior.java new file mode 100644 index 00000000000..eb8f5a540aa --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/JoinBehavior.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +/** + * When joining multiple metrics as a result of dimension + * removal. Should the result be an average or a sum? As an example, + * a latency metric should likely be averaged, while a number of + * pending metric should likely be summed. This join behavior property + * lets the metric framework know how to remove dimensions. + **/ +public enum JoinBehavior { AVERAGE_ON_JOIN, SUM_ON_JOIN } diff --git a/metrics/src/main/java/com/yahoo/metrics/LongValue.java b/metrics/src/main/java/com/yahoo/metrics/LongValue.java new file mode 100644 index 00000000000..29a8202f845 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/LongValue.java @@ -0,0 +1,143 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.metrics.util.ValueType; +import com.yahoo.metrics.util.HasCopy; + +import java.util.Collection; + +/** + * @author thomasg + */ +public class LongValue + implements ValueMetric.Value<Long> +{ + private int count; + private long min, max, last; + private long total; + + public LongValue() { + count = 0; + min = Long.MAX_VALUE; + max = Long.MIN_VALUE; + last = 0; + total = 0; + } + + @Override + public void add(Long v) { + LongValue val = this; + val.count = count + 1; + val.total = total + v; + val.min = Math.min(min, v); + val.max = Math.max(max, v); + val.last = v; + } + + @Override + public void join(ValueMetric.Value<Long> v2, boolean createAverageOnJoin) { + LongValue value = this; + + if (createAverageOnJoin) { + value.count = count + v2.getCount(); + value.total = total + v2.getTotal(); + value.last = v2.getLast(); + } else { + double totalAverage = getAverage() + v2.getAverage(); + value.count = count + v2.getCount(); + value.total = (long) (totalAverage * value.count); // Total is "wrong" I guess. + value.last = last + v2.getLast(); + } + + value.min = Math.min(min, v2.getMin()); + value.max = Math.max(max, v2.getMax()); + } + + @Override + public void add(ValueType other) { + LongValue dv = (LongValue) other; + count = count + dv.count; + total = total + dv.total; + min = Math.min(min, dv.min); + max = Math.max(max, dv.max); + last = dv.last; + } + + @Override + public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior) { + LongValue result = new LongValue(); + for (ValueType t : sources) { + LongValue dv = (LongValue) t; + result.count = result.count + dv.count; + result.total = result.total + dv.total; + result.min = Math.min(result.min, dv.min); + result.max = Math.max(result.max, dv.max); + result.last += dv.last; + } + if (joinBehavior == JoinBehavior.AVERAGE_ON_JOIN) { + result.last /= sources.size(); + } else { + result.total *= sources.size(); + } + return result; + } + + @Override + public boolean overflow(ValueMetric.Value<Long> v2) { + if (count > (count + v2.getCount())) { + return true; + } + if (v2.getTotal() > 0 && getTotal() > getTotal() + v2.getTotal()) { + return true; + } + if (v2.getTotal() < 0 && getTotal() < getTotal() + v2.getTotal()) { + return true; + } + + return false; + } + + @Override + public int getCount() { return count; } + + @Override + public Long getMin() { return (count > 0) ? min : 0; } + + @Override + public Long getMax() { return (count > 0) ? max : 0; } + + @Override + public Long getLast() { return last; } + + @Override + public Long getTotal() { return total; } + + @Override + public Double getAverage() { + if (count == 0) { + return 0.0; + } + return ((double) total) / count; + } + + @Override + public String valueToString(Long val) { + if (val == Long.MIN_VALUE || val == Long.MAX_VALUE) { + return valueToString((long)0); + } + + return val.toString(); + } + + @Override + public LongValue clone() { + try{ + return (LongValue) super.clone(); + } catch (CloneNotSupportedException e) { return null; } + } + + @Override + public ValueMetric.Value<Long> copyObject() { + return clone(); + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/Metric.java b/metrics/src/main/java/com/yahoo/metrics/Metric.java new file mode 100644 index 00000000000..0ef7dfbcd0a --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/Metric.java @@ -0,0 +1,256 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.text.XMLWriter; +import com.yahoo.text.Utf8String; + +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; + +public abstract class Metric +{ + private String name; + private String tags; + private String description; + + public String getXMLTag() { + return getName(); + } + + public void setName(String name) { + this.name = name; + } + + public void setTags(String tags) { + this.tags = tags; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getTags() { + return tags; + } + + MetricSet owner; + + public Metric(String name, String tags, String description) { + this(name, tags, description, null); + } + + public Metric(String name, String tags, String description, MetricSet owner) { + this.name = name; + this.tags = tags; + this.description = description; + + if (owner != null) { + owner.registerMetric(this); + } + } + + public Metric(Metric other, MetricSet owner) { + this(other.name, other.tags, other.description, owner); + } + + public String getName() { return name; } + + public String getPath() { + if (owner == null || owner.owner == null) { + return getName(); + } + + return owner.getPath() + "." + getName(); + } + + public List<String> getPathVector() { + List<String> result = new ArrayList<String>(); + result.add(getName()); + MetricSet owner = this.owner; + while (owner != null) { + result.add(0, owner.getName()); + owner = owner.owner; + } + return result; + } + + public String getDescription() { return description; } + + public String[] getTagVector() { + return getTags().split("[ \r\t\f]"); + } + + /** + * Returns true if the given tag exists in this metric's tag list. + * + * @return true if tag exists in tag list + */ + public boolean hasTag(String tag) { + for (String s : getTagVector()) { + if (s.equals(tag)) { + return true; + } + } + return false; + } + + public enum CopyType { CLONE, INACTIVE } + + /** + * The clone function will clone metrics to an identical subtree of + * metrics. Clone is primarily used for load metrics that wants to clone + * a template metric for each loadtype. But it should work generically. + * + * @param type If set to inactive, sum metrics will evaluate to primitives + * and metrics can save memory by knowing no updates are coming. + * @param includeUnused When creating snapshots we do not want to include + * unused metrics, but while generating sum metric sum in active + * metrics we want to. This has no affect if type is CLONE. + */ + public abstract Metric clone(CopyType type, MetricSet owner, boolean includeUnused); + + /** + * Utility function for assigning values from one metric of identical type + * to this metric. For simplicity sake it does a const cast and calls + * addToSnapshot, which should not alter source if reset is false. This can + * not be used to copy between active metrics and inactive copies. + * + * @return Returns itself. + */ + public Metric assignValues(Metric m) { + m.addToSnapshot(this); + // As this should only be called among active metrics, all metrics + // should exist and owner list should thus always end up empty. + return this; + } + + /** Reset all metric values. */ + public abstract void reset(); + + public boolean logFromTotalMetrics() { return false; } + + /** Implement to make metric able to log event. + * + * @param logger An event logger to use for logging. + * @param fullName The name to use for the event. + */ + public abstract void logEvent(EventLogger logger, String fullName); + + public static final Utf8String TAG_NAME = new Utf8String("name"); + public static final Utf8String TAG_TAGS = new Utf8String("tags"); + public static final Utf8String TAG_DESC = new Utf8String("description"); + + void openXMLTag(XMLWriter writer, int verbosity) { + String[] tags = getTagVector(); + + writer.openTag(getXMLTag()); + + if (getXMLTag() != getName()) { + writer.attribute(TAG_NAME, getName()); + } + + if (verbosity >= 3 && tags.length > 0) { + String tagStr = ""; + for (String tag : tags) { + if (!tagStr.isEmpty()) { + tagStr = ","; + } + tagStr += tag; + } + + writer.attribute(TAG_TAGS, tagStr); + } + + if (verbosity >= 1 && !getDescription().isEmpty()) { + writer.attribute(TAG_DESC, getDescription()); + } + } + + /** + * The verbosity says how much to print. + * At verbosity level 0, only the most critical parts are printed. + * At verbosity level 1, descriptions are added. + * At verbosity level 2, metrics without data is added. + * At verbosity level 3, tags are included too. + */ + public abstract void printXml(XMLWriter writer, + int secondsPassed, + int verbosity); + + public String toXml(int secondsPassed, int verbosity) { + StringWriter writer = new StringWriter(); + printXml(new XMLWriter(writer), secondsPassed, verbosity); + return writer.toString(); + } + + /** + * Most metrics report numbers of some kind. To be able to report numbers + * without having code to handle each possible metric type, these functions + * exist to extract raw data to present easily. + * @param id The part of the metric to extract. For instance, an average + * metric have average, + */ + public abstract long getLongValue(String id); + public abstract double getDoubleValue(String id); + + /** + * When snapshotting we need to be able to join data from one set of metrics + * to another set of metrics taken at another time. MetricSet doesn't know + * the type of the metrics it contains, so we need a generic function for + * doing this. This function assumes metric given as input is of the exact + * same type as the one it is called on for simplicity. This is true when + * adding to snapshots as they have been created with clone and is thus + * always exactly equal. + * + * @param m Metric of exact same type as this one. (Will core if wrong) + */ + abstract void addToSnapshot(Metric m); + + /** + * For sum metrics to work with metric sets, metric sets need operator+=. + * To implement this, we need a function to join any metric type together. + * This is different from adding to snapshot. When adding to snapshots we + * join different time periods to the same metric, but when adding parts + * together we join different metrics for the same time. For instance, an + * average metric of queuesize, should just join new values to create new + * average when adding to snapshot, but when adding parts, the averages + * themselves should be added together. + * + * @param m Metric of exact same type as this one. (Will core if wrong) + */ + abstract void addToPart(Metric m); + + public boolean visit(MetricVisitor visitor, boolean tagAsAutoGenerated) { + return visitor.visitPrimitiveMetric(this, tagAsAutoGenerated); + } + + /** Set whether metrics have ever been set. */ + public abstract boolean used(); + + /** Returns true if this metric is registered in a metric set. */ + public boolean isRegistered() { return (owner != null); } + + /** + * If this metric is registered with an owner, remove itself from that owner. + */ + public void unregister() { + if (isRegistered()) { + getOwner().unregisterMetric(this); + } + } + + public MetricSet getOwner() { return owner; } + + public MetricSet getRoot() { + if (owner == null) { + if (this instanceof MetricSet) { + return (MetricSet)this; + } else { + return null; + } + } else { + return owner.getRoot(); + } + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricManager.java b/metrics/src/main/java/com/yahoo/metrics/MetricManager.java new file mode 100644 index 00000000000..b44205660de --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/MetricManager.java @@ -0,0 +1,704 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.collections.Pair; +import com.yahoo.text.XMLWriter; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.logging.Logger; + +/** + * A metrics-enabled application should have a single MetricManager. You can register a number of MetricSets in the + * MetricManager. Each metric in the metrics sets can be used by zero or more consumers, configurable using + * readConfig(). + * + * The consumers get their data by calling the getMetrics() method, which gives them a snapshot of all the current + * metrics which are configured for the given name. + * + * Locking strategy: + * + * There are three locks in this class: + * + * Config lock: - This protects the class on config changes. It protects the _config and _consumerConfig members. + * + * Thread monitor (waiter): - This lock is kept by the worker thread while it is doing a work cycle, and it uses this + * monitor to sleep. It is used to make shutdown quick by interrupting thread, and to let functions called by clients be + * able to do a change while the worker thread is idle. - The log period is protected by the thread monitor. - The + * update hooks is protected by the thread monitor. + * + * Metric lock: - The metric log protects the active metric set when adding or removing metrics. Clients need to grab + * this lock before altering active metrics. The metric manager needs to grab this lock everytime it visits active + * metrics. - The metric log protects the snapshots. The snapshot writer is the metric worker thread and will grab the + * lock while editing them. Readers that aren't the worker thread itself must grab lock to be sure. + * + * If multiple locks is taken, the allowed locking order is: 1. Thread monitor. 2. Metric lock. 3. Config lock. + */ +public class MetricManager implements Runnable { + + private static final int STATE_CREATED = 0; + private static final int STATE_RUNNING = 1; + private static final int STATE_STOPPED = 2; + private static final Logger log = Logger.getLogger(MetricManager.class.getName()); + private final CountDownLatch termination = new CountDownLatch(1); + private final MetricSnapshot activeMetrics = new MetricSnapshot("Active metrics showing updates since " + + "last snapshot"); + private final Map<String, ConsumerSpec> consumerConfig = new HashMap<String, ConsumerSpec>(); + private final List<UpdateHook> periodicUpdateHooks = new ArrayList<UpdateHook>(); + private final List<UpdateHook> snapshotUpdateHooks = new ArrayList<UpdateHook>(); + private final Timer timer; + private Pair<Integer, Integer> logPeriod; + private List<MetricSnapshotSet> snapshots = new ArrayList<MetricSnapshotSet>(); + private MetricSnapshot totalMetrics = new MetricSnapshot("Empty metrics before init", 0, + activeMetrics.getMetrics(), false); + private int state = STATE_CREATED; + private int lastProcessedTime = 0; + private boolean forceEventLogging = false; + private boolean snapshotUnsetMetrics = false; // TODO: add to config + + public MetricManager() { + this(new Timer()); + } + + MetricManager(Timer timer) { + this.timer = timer; + initializeSnapshots(); + logPeriod = new Pair<Integer, Integer>(snapshots.get(0).getPeriod(), 0); + } + + void initializeSnapshots() { + int currentTime = timer.secs(); + + List<Pair<Integer, String>> snapshotPeriods = new ArrayList<Pair<Integer, String>>(); + snapshotPeriods.add(new Pair<Integer, String>(60 * 5, "5 minute")); + snapshotPeriods.add(new Pair<Integer, String>(60 * 60, "1 hour")); + snapshotPeriods.add(new Pair<Integer, String>(60 * 60 * 24, "1 day")); + snapshotPeriods.add(new Pair<Integer, String>(60 * 60 * 24 * 7, "1 week")); + + int count = 1; + for (int i = 0; i < snapshotPeriods.size(); ++i) { + int nextCount = 1; + if (i + 1 < snapshotPeriods.size()) { + nextCount = snapshotPeriods.get(i + 1).getFirst() + / snapshotPeriods.get(i).getFirst(); + if (snapshotPeriods.get(i + 1).getFirst() % snapshotPeriods.get(i).getFirst() != 0) { + throw new IllegalStateException("Snapshot periods must be multiplum of each other"); + } + } + snapshots.add(new MetricSnapshotSet(snapshotPeriods.get(i).getSecond(), + snapshotPeriods.get(i).getFirst(), + count, + activeMetrics.getMetrics(), + snapshotUnsetMetrics)); + count = nextCount; + } + // Add all time snapshot. + totalMetrics = new MetricSnapshot("All time snapshot", + 0, activeMetrics.getMetrics(), + snapshotUnsetMetrics); + totalMetrics.reset(currentTime); + } + + public void stop() { + synchronized (this) { + int prevState = state; + state = STATE_STOPPED; + if (prevState == STATE_CREATED) { + return; + } + notifyAll(); + } + try { + termination.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @SuppressWarnings("UnusedDeclaration") + void setSnapshotUnsetMetrics(boolean value) { + snapshotUnsetMetrics = value; + } + + /** + * Add a metric update hook. This will always be called prior to snapshotting and metric logging, to make the + * metrics the best as they can be at those occasions. + * + * @param hook The hook to add. + * @param period Period in seconds for how often callback should be called. The default value of 0, means only + * before snapshotting or logging, while another value will give callbacks each period seconds. + * Expensive metrics to calculate will typically only want to do it before snapshotting, while + * inexpensive metrics might want to log their value every 5 seconds or so. Any value of period >= the + * smallest snapshot time will behave identically as if period is set to 0. + */ + @SuppressWarnings("UnusedDeclaration") + public synchronized void addMetricUpdateHook(UpdateHook hook, int period) { + hook.period = period; + + // If we've already initialized manager, log period has been set. + // In this case. Call first time after period + hook.nextCall = (logPeriod.getSecond() == 0 ? 0 : timer.secs() + period); + if (period == 0) { + if (!snapshotUpdateHooks.contains(hook)) { + snapshotUpdateHooks.add(hook); + } + } else { + if (!periodicUpdateHooks.contains(hook)) { + periodicUpdateHooks.add(hook); + } + } + } + + @SuppressWarnings("UnusedDeclaration") + public synchronized void removeMetricUpdateHook(UpdateHook hook) { + if (hook.period == 0) { + snapshotUpdateHooks.remove(hook); + } else { + periodicUpdateHooks.remove(hook); + } + } + + /** + * Force a metric update for all update hooks. Useful if you want to ensure nice values before reporting something. + * This function can not be called from an update hook callback. + * + * @param includeSnapshotOnlyHooks True to also run snapshot hooks. + */ + @SuppressWarnings("UnusedDeclaration") + public synchronized void updateMetrics(boolean includeSnapshotOnlyHooks) { + log.fine("Giving " + periodicUpdateHooks.size() + " periodic update hooks."); + + updatePeriodicMetrics(0, true); + + if (includeSnapshotOnlyHooks) { + log.fine("Giving " + snapshotUpdateHooks.size() + " snapshot update hooks."); + updateSnapshotMetrics(); + } + } + + /** + * Force event logging to happen now. This function can not be called from an update hook callback. + */ + @SuppressWarnings("UnusedDeclaration") + public void forceEventLogging() { + log.fine("Forcing event logging to happen."); + // Ensure background thread is not in a current cycle during change. + + synchronized (this) { + forceEventLogging = true; + this.notifyAll(); + } + } + + /** + * Register a new metric to be included in the active metric set. You need to have grabbed the metric lock in order + * to do this. (You also need to grab that lock if you alter registration of already registered metric set.) This + * function can not be called from an update hook callback. + * + * @param m The metric to register. + */ + public void registerMetric(Metric m) { + activeMetrics.getMetrics().registerMetric(m); + } + + /** + * Unregister a metric from the active metric set. You need to have grabbed the metric lock in order to do this. + * (You also need to grab that lock if you alter registration of already registered metric set.) This function can + * not be called from an update hook callback. + * + * @param m The Metric to unregister. + */ + @SuppressWarnings("UnusedDeclaration") + public void unregisterMetric(Metric m) { + activeMetrics.getMetrics().unregisterMetric(m); + } + + /** + * Reset all metrics including all snapshots. This function can not be called from an update hook callback. + * + * @param currentTime The current time. + */ + public synchronized void reset(int currentTime) { + activeMetrics.reset(currentTime); + + for (MetricSnapshotSet m : snapshots) { + m.reset(currentTime); + } + totalMetrics.reset(currentTime); + } + + /** + * Read configuration. Before reading config, all metrics should be set up first. By doing this, the metrics manager + * can optimize reporting of consumers. readConfig() will start a config subscription. It should not be called + * multiple times. + */ +/* public synchronized void init(String configId, ThreadPool pool) { + log.fine("Initializing metric manager") + + LOG(debug, "Initializing metric manager."); + _configSubscription = Config::subscribe(configId, *this); + LOG(debug, "Starting worker thread, waiting for first " + "iteration to complete."); + Runnable::start(pool); + // Wait for first iteration to have completed, such that it is safe + // to access snapshots afterwards. + vespalib::MonitorGuard sync(_waiter); + while (_lastProcessedTime == 0) { + sync.wait(1); + } + LOG(debug, "Metric manager completed initialization."); +} + +*/ + + class ConsumerMetricVisitor extends MetricVisitor { + + ConsumerSpec metricsToMatch; + MetricVisitor clientVisitor; + + ConsumerMetricVisitor(ConsumerSpec spec, + MetricVisitor clientVisitor) + { + metricsToMatch = spec; + this.clientVisitor = clientVisitor; + log.fine("Consuming metrics: " + spec.includedMetrics); + } + + public boolean visitMetricSet(MetricSet metricSet, boolean autoGenerated) { + if (metricSet.getOwner() == null) { + return true; + } + + if (!metricsToMatch.contains(metricSet)) { + log.fine("Metric doesn't match " + metricSet.getPath()); + return false; + } + + return clientVisitor.visitMetricSet(metricSet, autoGenerated); + } + + public void doneVisitingMetricSet(MetricSet metricSet) { + if (metricSet.getOwner() != null) { + clientVisitor.doneVisitingMetricSet(metricSet); + } + } + + public boolean visitPrimitiveMetric(Metric metric, boolean autoGenerated) { + if (metricsToMatch.contains(metric)) { + return clientVisitor.visitPrimitiveMetric(metric, autoGenerated); + } else { + log.fine("Metric doesn't match " + metric.getPath()); + } + return true; + } + } + + public synchronized void visit(MetricSet metrics, MetricVisitor visitor, String consumer) { + if (consumer.isEmpty()) { + metrics.visit(visitor, false); + return; + } + + ConsumerSpec spec = getConsumerSpec(consumer); + + if (spec != null) { + ConsumerMetricVisitor consumerVis = new ConsumerMetricVisitor(spec, visitor); + metrics.visit(consumerVis, false); + } else { + log.warning("Requested metrics for non-defined consumer " + consumer); + } + } + + class XmlWriterMetricVisitor extends MetricVisitor { + + int period; + XMLWriter writer; + int verbosity; + + XmlWriterMetricVisitor(XMLWriter writer, int period, int verbosity) { + this.period = period; + this.verbosity = verbosity; + this.writer = writer; + } + + public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { + if (set.used() || verbosity >= 2) { + set.openXMLTag(writer, verbosity); + return true; + } + return false; + } + + public void doneVisitingMetricSet(MetricSet set) { + writer.closeTag(); + } + + public boolean visitPrimitiveMetric(Metric metric, boolean autoGenerated) { + metric.printXml(writer, period, verbosity); + return true; + } + } + + void printXml(MetricSet set, XMLWriter writer, int period, String consumer, int verbosity) { + visit(set, new XmlWriterMetricVisitor(writer, period, verbosity), consumer); + } + + /** + * Synchronize over this while the returned object + * + * @return The MetricSnapshot of all active metrics. + */ + public MetricSnapshot getActiveMetrics() { + return activeMetrics; + } + + /** + * Synchronize over this while the returned object + * + * @return The MetricSnapshot for the total metric. + */ + public MetricSnapshot getTotalMetricSnapshot() { + return totalMetrics; + } + + public synchronized List<Integer> getSnapshotPeriods() { + List<Integer> retVal = new ArrayList<Integer>(); + + for (MetricSnapshotSet m : snapshots) { + retVal.add(m.getPeriod()); + } + return retVal; + } + + /** + * While accessing snapshots you should synchronize over this + * + * @param period The id of the snapshot period to access. + * @param getInProgressSet True to retrieve the snapshot currently being built. + * @return The appropriate MetricSnapshot. + */ + MetricSnapshot getMetricSnapshot(int period, boolean getInProgressSet) { + return getMetricSnapshotSet(period).getSnapshot(getInProgressSet); + } + + MetricSnapshot getMetricSnapshot(int period) { + return getMetricSnapshot(period, false); + } + + public MetricSnapshotSet getMetricSnapshotSet(int period) { + for (MetricSnapshotSet m : snapshots) { + if (m.getPeriod() == period) { + return m; + } + } + + throw new IllegalArgumentException("No snapshot for period of length " + period + " exists."); + } + + @SuppressWarnings("UnusedDeclaration") + public synchronized boolean hasTemporarySnapshot(int period) { + return getMetricSnapshotSet(period).hasTemporarySnapshot(); + } + + public synchronized void addMetricToConsumer(String consumerName, String metricPath) { + ConsumerSpec spec = getConsumerSpec(consumerName); + if (spec == null) { + spec = new ConsumerSpec(); + consumerConfig.put(consumerName, spec); + } + spec.register(metricPath); + } + + public synchronized ConsumerSpec getConsumerSpec(String consumer) { + return consumerConfig.get(consumer); + } + + /** + * If you join or remove metrics from the active metric sets, normally, snapshots will be recreated next snapshot + * period. However, if you want to see the effects of such changes in status pages ahead of that, you can call this + * function in order to check whether snapshots needs to be regenerated and regenerate them if needed. + */ + public synchronized void checkMetricsAltered() { + if (activeMetrics.getMetrics().isRegistrationAltered()) { + handleMetricsAltered(); + } + } + + /** + * Used by unit tests to verify that we have processed for a given time. + * + * @return Returns the timestamp of the previous tick. + */ + @SuppressWarnings("UnusedDeclaration") + public int getLastProcessedTime() { + return lastProcessedTime; + } + + class LogMetricVisitor extends MetricVisitor { + + boolean total; + EventLogger logger; + + LogMetricVisitor(boolean totalVals, EventLogger logger) { + total = totalVals; + this.logger = logger; + } + + public boolean visitPrimitiveMetric(Metric metric, boolean autoGenerated) { + if (metric.logFromTotalMetrics() == total) { + String logName = metric.getPath().replace('.', '_'); + metric.logEvent(logger, logName); + } + return true; + } + } + + public void logTotal(int currentTime, EventLogger logger) { + LogMetricVisitor totalVisitor = new LogMetricVisitor(true, logger); + LogMetricVisitor fiveMinVisitor = new LogMetricVisitor(false, logger); + + if (logPeriod.getSecond() <= currentTime) { + log.fine("Logging total metrics."); + visit(totalMetrics.getMetrics(), totalVisitor, "log"); + visit(snapshots.get(0).getSnapshot().getMetrics(), fiveMinVisitor, "log"); + if (logPeriod.getSecond() + logPeriod.getFirst() < currentTime) { + logPeriod = new Pair<Integer, Integer>(logPeriod.getFirst(), + snapshots.get(0).getFromTime() + logPeriod.getFirst()); + } else { + logPeriod = + new Pair<Integer, Integer>(logPeriod.getFirst(), logPeriod.getSecond() + logPeriod.getFirst()); + } + } + } + + public void logOutOfSequence(int currentTime, EventLogger logger) { + LogMetricVisitor totalVisitor = new LogMetricVisitor(true, logger); + LogMetricVisitor fiveMinVisitor = new LogMetricVisitor(false, logger); + + log.fine("Logging total metrics out of sequence."); + MetricSnapshot snapshot = new MetricSnapshot( + "Total out of sequence metrics from start until current time", + 0, + totalMetrics.getMetrics(), + snapshotUnsetMetrics); + + activeMetrics.addToSnapshot(snapshot, currentTime, false); + snapshot.setFromTime(totalMetrics.getFromTime()); + visit(snapshot.getMetrics(), totalVisitor, "log"); + visit(snapshot.getMetrics(), fiveMinVisitor, "log"); + } + + /** + * Runs one iteration of the thread activity. + * + * @param logger An event logger to use for any new events generated. + * @return The number of milliseconds to sleep until waking up again + */ + public synchronized int tick(EventLogger logger) { + int currentTime = timer.secs(); + + log.finest("Worker thread starting to process for time " + currentTime); + + boolean firstIteration = (logPeriod.getSecond() == 0); + // For a slow system to still be doing metrics tasks each n'th + // second, rather than each n'th + time to do something seconds, + // we constantly join next time to do something from the last timer. + // For that to work, we need to initialize timers on first iteration + // to set them to current time. + if (firstIteration) { + // Setting next log period to now, such that we log metrics + // straight away + logPeriod = new Pair<Integer, Integer>(logPeriod.getFirst(), currentTime); + for (MetricSnapshotSet m : snapshots) { + m.setFromTime(currentTime); + } + for (UpdateHook h : periodicUpdateHooks) { + h.nextCall = currentTime; + } + } + + // If metrics have changed since last time we did a snapshot, + // work that out before taking the snapshot, such that new + // metric can be included + checkMetricsAltered(); + + // Set next work time to the time we want to take next snapshot. + int nextWorkTime = snapshots.get(0).getPeriod() + snapshots.get(0).getFromTime(); + + int nextUpdateHookTime; + + if (nextWorkTime <= currentTime) { + log.fine("Time to do snapshot. Calling update hooks"); + nextUpdateHookTime = updatePeriodicMetrics(currentTime, true); + updateSnapshotMetrics(); + takeSnapshots(nextWorkTime); + } else if (forceEventLogging) { + log.fine("Out of sequence event logging. Calling update hooks"); + nextUpdateHookTime = updatePeriodicMetrics(currentTime, true); + updateSnapshotMetrics(); + } else { + // If not taking a new snapshot. Only give update hooks to + // periodic hooks wanting it. + nextUpdateHookTime = updatePeriodicMetrics(currentTime, false); + } + + // Log if it is time + if (logPeriod.getSecond() <= currentTime || forceEventLogging) { + logTotal(currentTime, logger); + } else { + logOutOfSequence(currentTime, logger); + } + + forceEventLogging = false; + lastProcessedTime = (nextWorkTime <= currentTime ? nextWorkTime : currentTime); + log.fine("Worker thread done with processing for time " + lastProcessedTime); + + int next = Math.min(logPeriod.getSecond(), snapshots.get(0).getPeriod() + snapshots.get(0).getFromTime()); + next = Math.min(next, nextUpdateHookTime); + if (currentTime < next) { + return (next - currentTime) * 1000; + } + + return 0; + } + + @Override + public synchronized void run() { + if (state != STATE_CREATED) { + throw new IllegalStateException(); + } + try { + for (state = STATE_RUNNING; state == STATE_RUNNING; ) { + int timeout = tick(new VespaLogEventLogger()); + if (timeout > 0) { + wait(timeout); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + termination.countDown(); + } + } + + public synchronized void takeSnapshots(int timeToProcess) { + // If not time to do dump data from active snapshot yet, nothing to do + if (!snapshots.get(0).timeForAnotherSnapshot(timeToProcess)) { + return; + } + + log.fine("Updating " + snapshots.get(0).getName() + " snapshot from active metrics"); + // int fromTime = snapshots.get(0).getSnapshot().getToTime(); + MetricSnapshot firstTarget = (snapshots.get(0).getNextTarget()); + firstTarget.reset(timeToProcess); + activeMetrics.addToSnapshot(firstTarget, timeToProcess, false); + log.fine("Updating total metrics with five minute period of active metrics"); + activeMetrics.addToSnapshot(totalMetrics, timeToProcess, false); + activeMetrics.reset(timeToProcess); + + for (int i = 1; i < snapshots.size(); ++i) { + MetricSnapshotSet s = snapshots.get(i); + + log.fine("Adding data from last snapshot to building snapshot of " + + "next period snapshot " + s.getName()); + + MetricSnapshot target = s.getNextTarget(); + snapshots.get(i - 1).getSnapshot().addToSnapshot(target, timeToProcess, false); + target.setToTime(timeToProcess); + + if (!snapshots.get(i).haveCompletedNewPeriod(timeToProcess)) { + log.fine("Not time to roll snapshot " + s.getName() + " yet. " + + s.getBuilderCount() + " of " + s.getCount() + " snapshot " + + "taken at time" + (s.getBuilderCount() * s.getPeriod() + s.getFromTime()) + + ", and period of " + s.getPeriod() + " is not up " + + "yet as we're currently processing for time " + timeToProcess); + break; + } else { + log.fine("Rolled snapshot " + s.getName() + " at time " + timeToProcess); + } + } + } + + /** + * Utility function for updating periodic metrics. + * + * @param updateTime Update metrics timed to update at this time. + * @param outOfSchedule Force calls to all hooks. Don't screw up normal schedule though. If not time to update yet, + * update without adjusting schedule for next update. + * @return Time of next hook to be called in the future. + */ + int updatePeriodicMetrics(int updateTime, boolean outOfSchedule) { + int nextUpdateTime = Integer.MAX_VALUE; + for (UpdateHook h : periodicUpdateHooks) { + if (h.nextCall <= updateTime) { + h.updateMetrics(); + if (h.nextCall + h.period < updateTime) { + h.nextCall = updateTime + h.period; + } else { + h.nextCall += h.period; + } + } else if (outOfSchedule) { + h.updateMetrics(); + } + nextUpdateTime = Math.min(nextUpdateTime, h.nextCall); + } + return nextUpdateTime; + } + + void updateSnapshotMetrics() { + for (UpdateHook h : snapshotUpdateHooks) { + h.updateMetrics(); + } + } + + synchronized void handleMetricsAltered() { +/* if (consumerConfig.isEmpty()) { + log.fine("Setting up consumers for the first time."); + } else { + log.info("Metrics registration changes detected. Handling changes."); + } + + Map<String, ConsumerSpec> configMap = new HashMap<String, ConsumerSpec>(); + activeMetrics.getMetrics().clearRegistrationAltered(); + + + for (<config::MetricsmanagerConfig::Consumer>::const_iterator it + = _config.consumer.begin(); it != _config.consumer.end(); ++it) + { + ConsumerMetricBuilder consumerMetricBuilder(*it); + _activeMetrics.getMetrics().visit(consumerMetricBuilder); + configMap[it->name] = ConsumerSpec::SP( + new ConsumerSpec(consumerMetricBuilder._matchedMetrics)); + } + _consumerConfig.swap(configMap); + */ + log.fine("Recreating snapshots to include altered metrics"); + totalMetrics.recreateSnapshot(activeMetrics.getMetrics(), snapshotUnsetMetrics); + + for (MetricSnapshotSet set : snapshots) { + set.recreateSnapshot(activeMetrics.getMetrics(), snapshotUnsetMetrics); + } + } + + abstract class UpdateHook { + + String name; + int nextCall; + int period; + + public UpdateHook(String name) { + this.name = name; + nextCall = 0; + period = 0; + } + + public abstract void updateMetrics(); + + public String getName() { + return name; + } + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricSet.java b/metrics/src/main/java/com/yahoo/metrics/MetricSet.java new file mode 100644 index 00000000000..f10834d93cc --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/MetricSet.java @@ -0,0 +1,239 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.text.XMLWriter; + +import java.util.*; +import java.util.logging.Logger; + +public abstract class MetricSet extends Metric +{ + private static Logger log = Logger.getLogger(MetricSet.class.getName()); + + List<Metric> metricOrder = new ArrayList<Metric>(); // Keep added order for reporting + boolean registrationAltered; // Set to true if metrics have been + // registered/unregistered since last time + // it was reset + + public MetricSet(String name, String tags, String description, MetricSet owner) { + super(name, tags, description, owner); + } + + public MetricSet(MetricSet other, CopyType copyType, MetricSet owner, boolean includeUnused) { + super(other, owner); + + for (Metric m : other.metricOrder) { + if (copyType != CopyType.INACTIVE || includeUnused || m.used()) { + m.clone(copyType, this, includeUnused); + } + } + } + + /** + * @return Returns true if registration has been altered since it was last + * cleared. Used by the metric manager to know when it needs to recalculate + * which consumers will see what. + */ + public boolean isRegistrationAltered() { return registrationAltered; } + + /** Clear all registration altered flags. */ + void clearRegistrationAltered() { + visit(new MetricVisitor() { + public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { + if (autoGenerated) { + return false; + } + + set.registrationAltered = false; + return true; + } + }, false); + } + + public void registerMetric(Metric m) { + if (m.isRegistered()) { + throw new IllegalStateException("Metric " + m.getName() + + " is already registered in a metric set. Cannot register it twice."); + } + + if (getMetricInternal(m.getName()) != null) { + throw new IllegalStateException("A metric named " + m.getName() + " is already registered " + + "in metric set " + getPath()); + } + + metricOrder.add(m); + m.owner = this; + tagRegistrationAltered(); + } + + public void unregisterMetric(Metric m) { + // In case of abrubt shutdowns, don't die hard on attempts to unregister + // non-registered metrics. Just warn and ignore. + if (!metricOrder.remove(m)) { + log.warning("Attempt to unregister metric " + m.getName() + " in metric set " + getPath() + + ", where it wasn't registered to begin with."); + return; + } + + m.owner = null; + tagRegistrationAltered(); + + log.finest("Unregistered metric " + m.getName() + " from metric set " + getPath() + "."); + } + + @Override + public void reset() { + for (Metric m : metricOrder) { + m.reset(); + } + } + + @Override + public boolean visit(MetricVisitor visitor, boolean tagAsAutoGenerated) { + if (!visitor.visitMetricSet(this, tagAsAutoGenerated)) { + return true; + } + + for (Metric m : metricOrder) { + if (!m.visit(visitor, tagAsAutoGenerated)) { + break; + } + } + + visitor.doneVisitingMetricSet(this); + return true; + } + + @Override + public void logEvent(EventLogger logger, String fullName) { + throw new IllegalStateException("Can't log event from a MetricsSet: " + fullName); + } + + // These should never be called on metrics set. + @Override + public long getLongValue(String id) { + throw new IllegalStateException("Tried to get long from metricset"); + } + + @Override + public double getDoubleValue(String id) { + throw new IllegalStateException("Tried to get double from metricset"); + } + + public Metric getMetric(String name) { + int pos = name.indexOf('.'); + if (pos == -1) { + return getMetricInternal(name); + } else { + String child = name.substring(0, pos); + String rest = name.substring(pos + 1); + + Metric m = getMetricInternal(child); + if (m == null || !(m instanceof MetricSet)) { + return null; + } else { + return ((MetricSet)m).getMetric(rest); + } + } + } + + private Metric getMetricInternal(String name) { + for (Metric m : metricOrder) { + if (m.getName().equals(name)) { + return m; + } + } + return null; + } + + Map<String, Metric> createMetricMap() { + Map<String, Metric> map = new TreeMap<String, Metric>(); + + for (Metric m : metricOrder) { + map.put(m.getName(), m); + } + + return map; + } + + @Override + public void addToSnapshot(Metric snapshotMetric) { + MetricSet o = (MetricSet)snapshotMetric; + + Map<String, Metric> map1 = createMetricMap(); + Set<String> seen = new HashSet<String>(); + + // For all the metrics in the other's order, join ours to the snapshot. + for (Metric m : o.metricOrder) { + Metric myCopy = map1.get(m.getName()); + + if (myCopy != null) { + seen.add(m.getName()); + myCopy.addToSnapshot(m); + } + } + + // For all the remaining metrics, just join them to the other one. + for (Metric m : metricOrder) { + if (!seen.contains(m.getName())) { + m.clone(CopyType.INACTIVE, o, false); + } + } + } + + List<Metric> getRegisteredMetrics() + { return metricOrder; } + + @Override + public boolean used() { + for (Metric m : metricOrder) { + if (m.used()) { + return true; + } + } + + return false; + } + + @Override + public void addToPart(Metric partMetric) { + MetricSet o = (MetricSet)partMetric; + + Map<String, Metric> map2 = o.createMetricMap(); + + for (Metric m : metricOrder) { + Metric other = map2.get(m.getName()); + if (other != null) { + m.addToPart(other); + } else { + m.clone(CopyType.INACTIVE, o, false); + } + } + } + + private void tagRegistrationAltered() { + registrationAltered = true; + if (owner != null) { + owner.tagRegistrationAltered(); + } + } + + @Override + public void printXml(XMLWriter writer, + int secondsPassed, int verbosity) + { + if (!used() && verbosity < 3) { + return; + } + + openXMLTag(writer, verbosity); + + for (Metric m : metricOrder) { + m.printXml(writer, secondsPassed, verbosity); + } + + writer.closeTag(); + } + + +} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricSnapshot.java b/metrics/src/main/java/com/yahoo/metrics/MetricSnapshot.java new file mode 100644 index 00000000000..cd02b170b4a --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/MetricSnapshot.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.text.XMLWriter; +import com.yahoo.text.Utf8String; + +import java.io.StringWriter; + +public class MetricSnapshot +{ + String name; + int period; + int fromTime; + int toTime; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public int getPeriod() { + return period; + } + + public void setPeriod(int period) { + this.period = period; + } + + public int getFromTime() { + return fromTime; + } + + public void setFromTime(int fromTime) { + this.fromTime = fromTime; + } + + public int getToTime() { + return toTime; + } + + public void setToTime(int toTime) { + this.toTime = toTime; + } + + public MetricSet getMetrics() { + return snapshot; + } + + MetricSet snapshot; + + public MetricSnapshot(String name) { + this.name = name; + this.period = 0; + this.fromTime = 0; + this.toTime = 0; + snapshot = new SimpleMetricSet("metrics", "", ""); + } + + public MetricSnapshot(String name, + int period, + MetricSet source, + boolean copyUnset) { + this(name); + this.period = period; + snapshot = (MetricSet)source.clone(Metric.CopyType.INACTIVE, null, copyUnset); + } + + void reset(int currentTime) { + fromTime = currentTime; + toTime = 0; + snapshot.reset(); + } + + public void recreateSnapshot(MetricSet source, boolean copyUnset) { + MetricSet newSnapshot = (MetricSet)source.clone(Metric.CopyType.INACTIVE, null, copyUnset); + newSnapshot.reset(); + snapshot.addToSnapshot(newSnapshot); + snapshot = newSnapshot; + } + + public static final Utf8String TAG_NAME = new Utf8String("name"); + public static final Utf8String TAG_FROM = new Utf8String("from"); + public static final Utf8String TAG_TO = new Utf8String("to"); + public static final Utf8String TAG_PERIOD = new Utf8String("period"); + + public void printXml(MetricManager man, String consumer, int verbosity, XMLWriter writer) { + writer.openTag("snapshot"); + writer.attribute(TAG_NAME, name); + writer.attribute(TAG_FROM, fromTime); + writer.attribute(TAG_TO, toTime); + writer.attribute(TAG_PERIOD, period); + + for (Metric m : snapshot.getRegisteredMetrics()) { + if (m instanceof MetricSet) { + man.printXml((MetricSet)m, writer, toTime > fromTime ? (toTime - fromTime) : period, consumer, verbosity); + } + } + + writer.closeTag(); + } + + public String toXml(MetricManager man, String consumer, int verbosity) { + StringWriter str = new StringWriter(); + XMLWriter writer = new XMLWriter(str); + printXml(man, consumer, verbosity, writer); + return str.toString(); + } + + public void addToSnapshot(MetricSnapshot other, int currentTime, boolean reset) { + snapshot.addToSnapshot(other.getMetrics()); + if (reset) { + reset(currentTime); + } + other.toTime = currentTime; + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricSnapshotSet.java b/metrics/src/main/java/com/yahoo/metrics/MetricSnapshotSet.java new file mode 100644 index 00000000000..e65f1db8fb4 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/MetricSnapshotSet.java @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +/** + * Represents two snapshots for the same time period. + */ +public class MetricSnapshotSet { + int count; // Number of times we need to join to building period + // before we have a full time window. + int builderCount; // Number of times we've currently added to the + // building instance. + MetricSnapshot current = null; // The last full period + MetricSnapshot building = null; // The building period + + MetricSnapshotSet(String name, int period, int count, MetricSet source, boolean snapshotUnsetMetrics) { + this.count = count; + this.builderCount = 0; + current = new MetricSnapshot(name, period, source, snapshotUnsetMetrics); + current.reset(0); + if (count != 1) { + building = new MetricSnapshot(name, period, source, snapshotUnsetMetrics); + building.reset(0); + } + } + + MetricSnapshot getNextTarget() { + if (count == 1) { + return current; + } else { + return building; + } + } + + public boolean haveCompletedNewPeriod(int newFromTime) { + if (count == 1) { + current.setToTime(newFromTime); + return true; + } + building.setToTime(newFromTime); + + // If not time to roll yet, just return + if (++builderCount < count) return false; + // Building buffer done. Use that as current and reset current. + MetricSnapshot tmp = current; + current = building; + building = tmp; + building.setFromTime(newFromTime); + building.setToTime(0); + builderCount = 0; + return true; + } + + public boolean timeForAnotherSnapshot(int currentTime) { + int lastTime = getFromTime() + builderCount * getPeriod(); + return currentTime >= lastTime + getPeriod(); + } + + public void reset(int currentTime) { + if (count != 1) building.reset(currentTime); + current.reset(currentTime); + builderCount = 0; + } + + public void recreateSnapshot(MetricSet metrics, boolean copyUnset) { + if (count != 1) building.recreateSnapshot(metrics, copyUnset); + current.recreateSnapshot(metrics, copyUnset); + } + + public void setFromTime(int fromTime) + { + if (count != 1) { + building.setFromTime(fromTime); + } else { + current.setFromTime(fromTime); + } + } + + public int getPeriod() { + return current.getPeriod(); + } + + public int getFromTime() { + return current.getFromTime(); + } + + public int getCount() { + return count; + } + + public MetricSnapshot getSnapshot() { + return getSnapshot(false); + } + + public MetricSnapshot getSnapshot(boolean getBuilding) { + if (getBuilding) { + if (count == 1) { + throw new IllegalStateException("No temporary snapshot for set " + current.name); + } + return building; + } + + return current; + } + + public boolean hasTemporarySnapshot() { + return count > 1; + } + + public String getName() { + return current.getName(); + } + + public int getBuilderCount() { + return builderCount; + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java b/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java new file mode 100644 index 00000000000..4a9ea225bdd --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +public abstract class MetricVisitor { + /** + * Visit a metric set. + * + * @param autoGenerated True for metric sets that are generated on the + * fly such as in sum metrics. + * @return True if you want to visit the content of this metric set. + */ + public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { + return true; + } + + /** + * Callback visitors can use if they need to know the tree traversal of + * metric sets. This function is not called if visitMetricSet returned + * false. + */ + public void doneVisitingMetricSet(MetricSet set) { + } + + /** + * Visit a primitive metric within an accepted metric set. + * + * @return True if you want to continue visiting, false to abort. + */ + public boolean visitPrimitiveMetric(Metric m, boolean autoGenerated) { + return true; + } +}
\ No newline at end of file diff --git a/metrics/src/main/java/com/yahoo/metrics/SimpleMetricSet.java b/metrics/src/main/java/com/yahoo/metrics/SimpleMetricSet.java new file mode 100644 index 00000000000..2eab3c6635e --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/SimpleMetricSet.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +/** + * A final metric set. + */ +public final class SimpleMetricSet extends MetricSet { + + public SimpleMetricSet(String name, String tags, String description) { + this(name, tags, description, null); + } + + public SimpleMetricSet(String name, String tags, String description, MetricSet owner) { + super(name, tags, description, owner); + } + + public SimpleMetricSet(SimpleMetricSet other, Metric.CopyType copyType, MetricSet owner, boolean includeUnused) { + super(other, copyType, owner, includeUnused); + } + + @Override + public Metric clone(Metric.CopyType type, MetricSet owner, boolean includeUnused) + { return new SimpleMetricSet(this, type, owner, includeUnused); } + +} diff --git a/metrics/src/main/java/com/yahoo/metrics/SumMetric.java b/metrics/src/main/java/com/yahoo/metrics/SumMetric.java new file mode 100644 index 00000000000..6335d329836 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/SumMetric.java @@ -0,0 +1,238 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class metrics::CounterMetric + * @ingroup metrics + * + * @brief Counts a value that only moves upwards. + * + * NB! If you have a MetricSet subclass you want to create a sum for, use + * MetricSet itself as the template argument. Otherwise you'll need to override + * clone(...) in order to make it return the correct type for your + * implementation. + */ + +package com.yahoo.metrics; + +import com.yahoo.text.XMLWriter; + +import java.util.ArrayList; +import java.util.List; + +public class SumMetric extends Metric +{ + ArrayList<Metric> metricsToSum = new ArrayList<Metric>(); + + public SumMetric(String name, String tags, String description, MetricSet owner) { + super(name, tags, description, owner); + metricsToSum = new ArrayList<Metric>(); + } + + public SumMetric(SumMetric other, MetricSet owner) { + super(other, owner); + + + if (other.owner == null) { + throw new IllegalStateException( + "Cannot copy a sum metric not registered in a metric set, as " + + "we need to use parent to detect new metrics to point to."); + } + if (owner == null) { + throw new IllegalStateException( + "Cannot copy a sum metric directly. One needs to at least " + + "include metric set above it in order to include metrics " + + "summed."); + } + + metricsToSum.ensureCapacity(other.metricsToSum.size()); + List<String> parentPath = other.owner.getPathVector(); + + for (Metric metric : other.metricsToSum) { + List<String> addendPath = metric.getPathVector(); + MetricSet newAddendParent = owner; + + for (int i = parentPath.size(); i < addendPath.size() - 1; ++i) { + String path = addendPath.get(i); + Metric child = newAddendParent.getMetric(path); + if (child == null) { + throw new IllegalStateException( + "Metric " + path + " in metric set " + + newAddendParent.getPath() + " was expected to " + + "exist. This sounds like a bug."); + } + + newAddendParent = (MetricSet)child; + } + + String path = addendPath.get(addendPath.size() - 1); + Metric child = newAddendParent.getMetric(path); + if (child == null) { + throw new IllegalStateException( + "Metric " + path + " in metric set " + + newAddendParent.getPath() + " was expected to " + + "exist. This sounds like a bug."); + } + + metricsToSum.add(child); + } + } + + @Override + public boolean visit(MetricVisitor visitor, boolean tagAsAutoGenerated) { + if (metricsToSum.isEmpty()) return true; + + Metric sum = generateSum(); + + if (sum == null) { + return true; + } + + if (sum instanceof MetricSet) { + sum.visit(visitor, true); + return true; + } else { + return visitor.visitPrimitiveMetric(sum, true); + } + } + + @Override + public Metric clone(CopyType copyType, MetricSet owner, boolean includeUnused) { + if (copyType == CopyType.CLONE) { + return new SumMetric(this, owner); + } + + Metric sum = null; + for (Metric metric : metricsToSum) { + if (sum == null) { + sum = metric.clone(CopyType.INACTIVE, null, includeUnused); + sum.setName(getName()); + sum.setDescription(getDescription()); + sum.setTags(getTags()); + + if (owner != null) { + owner.registerMetric(sum); + } + } else { + metric.addToPart(sum); + } + } + + return sum; + } + + @Override + public void addToPart(Metric partMetric) { + Metric m = generateSum(); + if (m != null) { + m.addToPart(partMetric); + } + } + + @Override + public void addToSnapshot(Metric snapshotMetric) { + Metric m = generateSum(); + if (m != null) { + m.addToSnapshot(snapshotMetric); + } + } + + public void addMetricToSum(Metric metric) { + if (owner == null) { + throw new IllegalStateException( + "Sum metric needs to be registered in a parent metric set " + + "prior to adding metrics to sum."); + } + if (!metricsToSum.isEmpty() && !(metric.getClass().equals(metricsToSum.get(0).getClass()))) { + throw new IllegalStateException( + "All metrics in a metric set must be of the same type."); + } + + List<String> sumParentPath = owner.getPathVector(); + List<String> addedPath = metric.getPathVector(); + + boolean error = false; + if (addedPath.size() <= sumParentPath.size()) { + error = true; + } else for (int i=0; i<sumParentPath.size(); ++i) { + if (!sumParentPath.get(i).equals(addedPath.get(i))) { + error = true; + break; + } + } + if (error) { + throw new IllegalStateException( + "Metric added to sum is required to be a child of the sum's " + + "direct parent metric set. (Need not be a direct child) " + + "Metric set " + metric.getPath() + " is not a child of " + + owner.getPath()); + } + + ArrayList<Metric> metrics = new ArrayList<Metric>(metricsToSum.size() + 1); + for (int i = 0; i < metricsToSum.size(); ++i) { + metrics.add(metricsToSum.get(i)); + } + metrics.add(metric); + metricsToSum = metrics; + } + + public void removeMetricFromSum(Metric metric) { + metricsToSum.remove(metric); + } + + public Metric generateSum() { + Metric m = clone(CopyType.INACTIVE, null, true); + + if (m != null) { + m.owner = owner; + } + return m; + } + + @Override + public long getLongValue(String id) { + Metric s = generateSum(); + if (s == null) { + return 0; + } + return s.getLongValue(id); + } + + @Override + public double getDoubleValue(String id) { + Metric s = generateSum(); + if (s == null) { + return 0.0; + } + return generateSum().getDoubleValue(id); + } + + @Override + public void logEvent(EventLogger logger, String fullName) { + Metric s = generateSum(); + if (s != null) { + s.logEvent(logger, fullName); + } + } + + @Override + public void printXml(XMLWriter writer, int secondsPassed, int verbosity) { + Metric s = generateSum(); + if (s == null) { + openXMLTag(writer, verbosity); + writer.closeTag(); + return; + } + + generateSum().printXml(writer, secondsPassed, verbosity); + } + + @Override + public boolean used() { + for (Metric m : metricsToSum) { + if (m.used()) return true; + } + return false; + } + + @Override + public void reset() {} +}
\ No newline at end of file diff --git a/metrics/src/main/java/com/yahoo/metrics/Timer.java b/metrics/src/main/java/com/yahoo/metrics/Timer.java new file mode 100644 index 00000000000..12258ebfeb1 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/Timer.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +/** +* @author thomasg +*/ +class Timer { + int secs() { return (int)(milliSecs() / 1000); } + long milliSecs() { return System.currentTimeMillis(); } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/ValueMetric.java b/metrics/src/main/java/com/yahoo/metrics/ValueMetric.java new file mode 100644 index 00000000000..42cea298439 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/ValueMetric.java @@ -0,0 +1,261 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.log.LogLevel; +import com.yahoo.metrics.util.*; +import com.yahoo.text.XMLWriter; +import com.yahoo.text.Utf8String; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Logger; + +/** + * A metric that represents the value of something. + */ +public class ValueMetric<N extends Number> + extends Metric +{ + private static Logger log = Logger.getLogger(ValueMetric.class.getName()); + private static AtomicBoolean hasWarnedOnNonFinite = new AtomicBoolean(false); + + public static interface Value<N extends Number> extends ValueType, HasCopy<Value<N>> { + void add(N v); + void join(Value<N> v2, boolean createAverageOnJoin); + boolean overflow(Value<N> v2); + + int getCount(); + N getMin(); + N getMax(); + N getLast(); + N getTotal(); + Double getAverage(); + + String valueToString(N value); + } + + public boolean hasFlag(int flag) { return (flags & flag) != 0; } + public ValueMetric<N> setFlag(int flag) { flags |= flag; return this; } + + MetricValueKeeper<Value<N>> values; + int flags = 0; + + static int AVERAGE_METRIC = 1; + static int CREATE_AVERAGE_ON_JOIN = 2; + static int UNSET_ON_ZERO_VALUE = 4; + static int LOG_IF_UNSET = 8; + + boolean isAverageMetric() { return hasFlag(AVERAGE_METRIC); } + boolean doCreateAverageOnJoin() { return hasFlag(CREATE_AVERAGE_ON_JOIN); } + boolean isUnsetOnZeroValue() { return hasFlag(UNSET_ON_ZERO_VALUE); } + boolean doLogIfUnset() { return hasFlag(LOG_IF_UNSET); } + JoinBehavior getJoinBehavior() { + return hasFlag(CREATE_AVERAGE_ON_JOIN) ? JoinBehavior.AVERAGE_ON_JOIN + : JoinBehavior.SUM_ON_JOIN; + } + + public ValueMetric<N> averageMetric() { return setFlag(AVERAGE_METRIC); } + public ValueMetric<N> createAverageOnJoin() { return setFlag(CREATE_AVERAGE_ON_JOIN); } + public ValueMetric<N> unsetOnZeroValue() { return setFlag(UNSET_ON_ZERO_VALUE); } + public ValueMetric<N> logIfUnset() { return setFlag(LOG_IF_UNSET); } + + public ValueMetric(String name, String tags, String description, MetricSet owner) { + super(name, tags, description, owner); + //values = new MetricValueSet(); + values = new ThreadLocalDirectoryValueKeeper<Value<N>>(); + } + + public ValueMetric(ValueMetric<N> other, CopyType copyType, MetricSet owner) { + super(other, owner); + if (copyType == CopyType.INACTIVE || other.values instanceof SimpleMetricValueKeeper) { + values = new SimpleMetricValueKeeper<Value<N>>(); + values.set(other.values.get(getJoinBehavior())); + } else { + //values = new MetricValueSet((MetricValueSet) other.values, ((MetricValueSet) other.values).size()); + values = new ThreadLocalDirectoryValueKeeper<Value<N>>(other.values); + } + this.flags = other.flags; + } + + private void logNonFiniteValueWarning() { + if (!hasWarnedOnNonFinite.getAndSet(true)) { + log.log(LogLevel.WARNING, + "Metric '" + getPath() + "' attempted updated with a value that is NaN or " + + "Infinity; update ignored! No further warnings will be printed for " + + "such updates on any metrics, but they can be observed with debug " + + "logging enabled on component " + log.getName()); + } else if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, + "Metric '" + getPath() + "' attempted updated with a value that is NaN or " + + "Infinity; update ignored!"); + } + } + + public void addValue(N v) { + if (v == null) throw new NullPointerException("Cannot have null value"); + if (v instanceof Long) { + LongValue lv = new LongValue(); + lv.add((Long) v); + values.add((Value<N>) lv); + } else { + Double d = (Double)v; + if (d.isNaN() || d.isInfinite()) { + logNonFiniteValueWarning(); + return; + } + DoubleValue dv = new DoubleValue(); + dv.add(d); + values.add((Value<N>) dv); + } + } + + private Value<N> getValues() { + return (Value<N>) values.get(getJoinBehavior()); + } + + @Override + public void addToSnapshot(Metric snapshotMetric) { + Value<N> v = getValues(); + if (v != null) ((ValueMetric<N>)snapshotMetric).join(v, true); + } + + @Override + void addToPart(Metric m) { + Value<N> v = getValues(); + if (v != null) ((ValueMetric<N>) m).join(v, doCreateAverageOnJoin()); + } + + public void join(Value<N> v2, boolean createAverageOnJoin) { + Value<N> tmpVals = getValues(); + if (tmpVals == null) { + if (v2 instanceof LongValue) { + tmpVals = (Value<N>) new LongValue(); + } else { + tmpVals = (Value<N>) new DoubleValue(); + } + } + if (tmpVals.overflow(v2)) { + this.values.reset(); + log.fine("Metric " + getPath() + " overflowed, resetting it."); + return; + } + if (tmpVals.getCount() == 0) { + tmpVals = v2; + } else if (v2.getCount() == 0) { + // Do nothing + } else { + tmpVals.join(v2, createAverageOnJoin); + } + this.values.set(tmpVals); + } + + public static final Utf8String TAG_AVG = new Utf8String("average"); + public static final Utf8String TAG_LAST = new Utf8String("last"); + public static final Utf8String TAG_MIN = new Utf8String("min"); + public static final Utf8String TAG_MAX = new Utf8String("max"); + public static final Utf8String TAG_CNT = new Utf8String("count"); + public static final Utf8String TAG_TOT = new Utf8String("total"); + + @Override + public void printXml(XMLWriter writer, + int timePassed, + int verbosity) + { + Value<N> val = getValues(); + if (!inUse(val) && verbosity < 2) { + return; + } + if (val == null) val = (Value<N>) new LongValue(); + + openXMLTag(writer, verbosity); + writer.attribute(TAG_AVG, new DoubleValue().valueToString(val.getAverage())); + writer.attribute(TAG_LAST, val.valueToString(val.getLast())); + + if (val.getCount() > 0) { + writer.attribute(TAG_MIN, val.valueToString(val.getMin())); + writer.attribute(TAG_MAX, val.valueToString(val.getMax())); + } + writer.attribute(TAG_CNT, val.getCount()); + if (verbosity >= 2) { + writer.attribute(TAG_TOT, val.valueToString(val.getTotal())); + } + + writer.closeTag(); + } + + public Number getValue(String id) { + Value<N> val = getValues(); + if (val == null) return 0; + + if (id.equals("last") || (!isAverageMetric() && id.equals("value"))) { + return val.getLast(); + } else if (id.equals("average") || (isAverageMetric() && id.equals("value"))) { + return val.getAverage(); + } else if (id.equals("count")) { + return val.getCount(); + } else if (id.equals("total")) { + return val.getTotal(); + } else if (id.equals("min")) { + return val.getMin(); + } else if (id.equals("max")) { + return val.getMax(); + } else { + throw new IllegalArgumentException("No id " + id + " in value metric " + getName()); + } + } + + @Override + public long getLongValue(String id) { + return getValue(id).longValue(); + } + + @Override + public double getDoubleValue(String id) { + return getValue(id).doubleValue(); + } + + @Override + public ValueMetric<N> clone(CopyType type, MetricSet owner, boolean includeUnused) { + return new ValueMetric<N>(this, type, owner); + } + + @Override + public void reset() { values.reset(); } + + @Override + public void logEvent(EventLogger logger, String fullName) { + Value<N> val = getValues(); + if (!doLogIfUnset() && !inUse(val)) { + return; + } + logger.value(fullName, val == null ? 0 + : isAverageMetric() ? val.getAverage().doubleValue() + : val.getLast().doubleValue()); + } + + public boolean inUse(Value<?> value) { + return (value != null + && (value.getTotal().longValue() != 0 + || (value.getCount() != 0 && !isUnsetOnZeroValue()))); + } + + @Override + public boolean used() { + return inUse(getValues()); + } + + @Override + public String toString() { + Value<N> tmpVals = getValues(); + if (tmpVals == null) { + tmpVals = (Value<N>) new LongValue(); + } + return ("count=\"" + tmpVals.getCount() + + "\" min=\"" + tmpVals.valueToString(tmpVals.getMin()) + + "\" max=\"" + tmpVals.valueToString(tmpVals.getMax()) + + "\" last=\""+ tmpVals.valueToString(tmpVals.getLast()) + + "\" total=\"" + tmpVals.valueToString(tmpVals.getTotal()) + + "\" average=\"" + new DoubleValue().valueToString(tmpVals.getAverage()) + + "\""); + } + +} diff --git a/metrics/src/main/java/com/yahoo/metrics/VespaLogEventLogger.java b/metrics/src/main/java/com/yahoo/metrics/VespaLogEventLogger.java new file mode 100644 index 00000000000..aa2698c92fb --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/VespaLogEventLogger.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.log.event.Event; + +/** + * @author thomasg + */ +public class VespaLogEventLogger implements EventLogger { + public void value(String name, double value) { + Event.value(name, value); + } + + public void count(String name, long value) { + Event.count(name, value); + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/HasCopy.java b/metrics/src/main/java/com/yahoo/metrics/util/HasCopy.java new file mode 100644 index 00000000000..497f640e0fe --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/util/HasCopy.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.util; + +public interface HasCopy<T> { + public T copyObject(); +} + diff --git a/metrics/src/main/java/com/yahoo/metrics/util/MetricValueKeeper.java b/metrics/src/main/java/com/yahoo/metrics/util/MetricValueKeeper.java new file mode 100644 index 00000000000..65bd5abdd3e --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/util/MetricValueKeeper.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.util; + +import com.yahoo.metrics.JoinBehavior; +import java.util.Collection; + +/** + * + */ +public interface MetricValueKeeper<Value> { + /** Add a value to a given value. Used when metrics are written to by metric writer. */ + public void add(Value v); + /** Set the value to exactly this, forgetting old value. This operation needs only be supported by value keepers used in inactive snapshots. */ + public void set(Value v); + /** Get the value of the metrics written. If multiple values exist, use given join behavior. */ + public Value get(JoinBehavior joinBehavior); + /** Reset the value of the metrics to zero. */ + public void reset(); + /** Return copy of current view */ + public Collection<Value> getDirectoryView(); +} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/MetricValueSet.java b/metrics/src/main/java/com/yahoo/metrics/util/MetricValueSet.java new file mode 100644 index 00000000000..e04c4b4eb25 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/util/MetricValueSet.java @@ -0,0 +1,141 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class MetricValueSet + * \ingroup metrics + * + * \brief Utility for doing lockless metric updates and reads. + * + * We don't want to use regular locking while updating metrics due to overhead. + * We use this class to make metric updates as safe as possible without + * requiring locks. + * + * It keeps the set of values a metric wants to set. Thus it is templated on + * the class keeping the values. All that is required of this class is that it + * has an empty constructor and a copy constructor. + * + * The locking works, by keeping a set of values, with an active pointer into + * the value vector. Assuming only one thread calls setValues(), it can update + * the active pointer safely. We assume updating the active pointer is a + * non-interruptable operations, such that other threads will see either the new + * or the old value correctly. This should be the case on our platforms. + * + * Due to the reset functionality, it is possible to miss out on a metrics + * added during the reset, but this is very unlikely. For that to happen, when + * someone sets the reset flag, the writer thread must be in setValues(), + * having already passed the check for the reset flag, but not finished setting + * the values yet. + */ + +package com.yahoo.metrics.util; + +import java.util.Collection; +import com.yahoo.metrics.JoinBehavior; + +public class MetricValueSet<Value extends HasCopy<Value> & ValueType> + implements MetricValueKeeper<Value> +{ + private final ValueType[] values; + + private void setIndexedValue(int idx, Value v) { + values[idx] = v; + } + + @SuppressWarnings("unchecked") + private Value getIndexedValue(int idx) { + return (Value) values[idx]; + } + + + private volatile int activeValueIndex; + private volatile boolean reset = false; + + public MetricValueSet() { + values = new ValueType[3]; + activeValueIndex = 0; + } + + public MetricValueSet(MetricValueSet<Value> other, int copyCount) { + values = new ValueType[copyCount]; + activeValueIndex = 0; + setValue(other.getValue()); + } + + /** Get the current values. */ + public Value getValue() { + if (reset) return null; + Value v = getIndexedValue(activeValueIndex); + return (v == null ? null : v.copyObject()); + } + + /** + * Get the current values from the metric. This function should not be + * called in parallel. Only call it from a single thread or use external + * locking. If it returns false, it means the metric have just been reset. + * In which case, redo getValues(), apply the update again, and call + * setValues() again. + */ + public boolean setValue(Value value) { + int nextIndex = (activeValueIndex + 1) % values.length; + if (reset) { + reset = false; + setIndexedValue(nextIndex, null); + activeValueIndex = nextIndex; + return false; + } else { + setIndexedValue(nextIndex, value); + activeValueIndex = nextIndex; + return true; + } + } + + /** + * Retrieve and reset in a single operation, to minimize chance of + * alteration in the process. + */ + public Value getValuesAndReset() { + Value result = getValue(); + reset = true; + return result; + } + + public String toString() { + String retVal = "MetricValueSet(reset=" + reset + + ", active " + activeValueIndex + "["; + for (ValueType n : values) retVal += n + ","; + retVal += "])"; + return retVal; + } + + public int size() { return values.length; } + + public void add(Value v) { + while (true) { + Value current = getValue(); + if (current != null) { + current.add(v); + if (setValue(current)) { + return; + } + } else { + if (setValue(v)) { + return; + } + } + } + } + + public void set(Value v) { + setValue(v); + } + + public Value get(JoinBehavior joinBehavior) { + return getValue(); + } + + public void reset() { reset = true; } + + public Collection<Value> getDirectoryView() { + return null; + } + +} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/SimpleMetricValueKeeper.java b/metrics/src/main/java/com/yahoo/metrics/util/SimpleMetricValueKeeper.java new file mode 100644 index 00000000000..f3a3e64beb6 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/util/SimpleMetricValueKeeper.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.util; + +import java.util.Collection; +import com.yahoo.metrics.JoinBehavior; + +/** + * Simple implementation of the metric value keeper for use with snapshots. Only keeps + * one instance of data, does not need to worry about threads, and should have a small + * memory footprint. + */ +public class SimpleMetricValueKeeper<Value> implements MetricValueKeeper<Value> { + private Value value; + + public SimpleMetricValueKeeper() { + } + + public void add(Value v) { + throw new UnsupportedOperationException("Not supported. This value keeper is not intended to be used for live metrics."); + } + + public void set(Value v) { + this.value = v; + } + + public Value get(JoinBehavior joinBehavior) { + return value; + } + + public void reset() { + value = null; + } + + public Collection<Value> getDirectoryView() { + return null; + } + +} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/ThreadLocalDirectoryValueKeeper.java b/metrics/src/main/java/com/yahoo/metrics/util/ThreadLocalDirectoryValueKeeper.java new file mode 100644 index 00000000000..72dcef377d0 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/util/ThreadLocalDirectoryValueKeeper.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.util; + +import com.yahoo.concurrent.ThreadLocalDirectory; +import com.yahoo.metrics.DoubleValue; +import com.yahoo.metrics.LongValue; +import com.yahoo.metrics.JoinBehavior; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +/** + * Value keeper class using ThreadLocalDirectory to maintain thread safety from multiple threads writing. + */ +public class ThreadLocalDirectoryValueKeeper<Value extends ValueType & HasCopy<Value>> + implements MetricValueKeeper<Value>, ThreadLocalDirectory.ObservableUpdater<Value, Value> +{ + private final ThreadLocalDirectory<Value, Value> directory = new ThreadLocalDirectory<Value, Value>(this); + private List<Value> initialValues = null; + + public ThreadLocalDirectoryValueKeeper() { + } + + public ThreadLocalDirectoryValueKeeper(MetricValueKeeper<Value> other) { + initialValues = new LinkedList<Value>(); + for (Value v : other.getDirectoryView()) initialValues.add(v.copyObject()); + } + + public Value createGenerationInstance(Value value) { + return null; + } + + public Value update(Value current, Value newValue) { + if (current == null) { + if (newValue instanceof DoubleValue) { + current = (Value) new DoubleValue(); + } else { + current = (Value) new LongValue(); + } + } + current.add(newValue); + return current; + } + + public Value copy(Value value) { + return value.copyObject(); + } + + public void add(Value v) { + directory.update(v, directory.getLocalInstance()); + } + + public void set(Value v) { + throw new UnsupportedOperationException("This value keeper is only intended to use with active metrics. Set operation not supported."); + } + + public Value get(JoinBehavior joinBehavior) { + List<Value> values = directory.view(); + if (initialValues != null) values.addAll(initialValues); + if (values.isEmpty()) { + return null; + } + return (Value) values.get(0).join(Collections.<ValueType>unmodifiableList(values), joinBehavior); + } + + public void reset() { + directory.fetch(); + initialValues = null; + } + + public Collection<Value> getDirectoryView() { + return directory.view(); + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/ValueType.java b/metrics/src/main/java/com/yahoo/metrics/util/ValueType.java new file mode 100644 index 00000000000..b777e213406 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/util/ValueType.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.util; + +import com.yahoo.metrics.JoinBehavior; + +import java.util.Collection; + +public interface ValueType extends Cloneable { + public void add(ValueType other); + public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior); +} diff --git a/metrics/src/main/metrics-with-dependencies.xml b/metrics/src/main/metrics-with-dependencies.xml new file mode 100644 index 00000000000..f16f31e0398 --- /dev/null +++ b/metrics/src/main/metrics-with-dependencies.xml @@ -0,0 +1,19 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<assembly> + <id>jar-with-dependencies</id> + <formats> + <format>jar</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <dependencySets> + <dependencySet> + <unpack>true</unpack> + <scope>runtime</scope> + </dependencySet> + </dependencySets> + <fileSets> + <fileSet> + <directory>${project.build.outputDirectory}</directory> + </fileSet> + </fileSets> +</assembly> |