diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /metrics |
Publish
Diffstat (limited to 'metrics')
83 files changed, 12515 insertions, 0 deletions
diff --git a/metrics/.gitignore b/metrics/.gitignore new file mode 100644 index 00000000000..cb87b051cf2 --- /dev/null +++ b/metrics/.gitignore @@ -0,0 +1,5 @@ +/libexec +/target +/pom.xml.build +Makefile +Testing diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt new file mode 100644 index 00000000000..03da6f2ee64 --- /dev/null +++ b/metrics/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_define_module( + DEPENDS + fastos + vespalog + vespalib + staging_vespalib + config_cloudconfig + + TEST_EXTERNAL_DEPENDS + cppunit + + TESTS + src/tests + src/vespa/metrics +) diff --git a/metrics/OWNERS b/metrics/OWNERS new file mode 100644 index 00000000000..dbcff24b338 --- /dev/null +++ b/metrics/OWNERS @@ -0,0 +1 @@ +vekterli diff --git a/metrics/pom.xml b/metrics/pom.xml new file mode 100644 index 00000000000..0aa81dff64b --- /dev/null +++ b/metrics/pom.xml @@ -0,0 +1,155 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>metrics</artifactId> + <packaging>jar</packaging> + <version>6-SNAPSHOT</version> + <name>metrics</name> + <description></description> + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-bundle</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <version>1.4</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-di</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:rawtypes</arg> + <arg>-Xlint:unchecked</arg> + <arg>-Xlint:deprecation</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <archive> + <manifest> + <mainClass>com.yahoo.metrics.foo</mainClass> + </manifest> + </archive> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <!-- append to the packaging phase. --> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>javacc-maven-plugin</artifactId> + <version>2.6</version> + <executions> + <execution> + <id>javacc</id> + <goals> + <goal>javacc</goal> + </goals> + <configuration> + <lookAhead>1</lookAhead> + <isStatic>false</isStatic> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-class-plugin</artifactId> + <version>${project.version}</version> + <configuration> + <defFilesDirectories> + src/vespa/metrics, + src/main/resources/configdefinitions + </defFilesDirectories> + </configuration> + <executions> + <execution> + <id>config-gen</id> + <goals> + <goal>config-gen</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <configuration> + <updateReleaseInfo>true</updateReleaseInfo> + </configuration> + </plugin> + <plugin> + <!-- Remove when v2.1 is the default + - it is required by maven-project-info-reports-plugin v2.2 --> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-site-plugin</artifactId> + <version>2.1</version> + </plugin> + </plugins> + </build> +</project> diff --git a/metrics/src/.gitignore b/metrics/src/.gitignore new file mode 100644 index 00000000000..698739d7bbc --- /dev/null +++ b/metrics/src/.gitignore @@ -0,0 +1,4 @@ +Makefile.ini +config_command.sh +project.dsw +/metrics.mak diff --git a/metrics/src/main/java/com/yahoo/metrics/ConsumerSpec.java b/metrics/src/main/java/com/yahoo/metrics/ConsumerSpec.java new file mode 100644 index 00000000000..ec6963bef30 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/ConsumerSpec.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import java.util.HashSet; +import java.util.Set; +import java.util.StringTokenizer; + +/** + * Spec saved from config. If metricSetChildren has content, metric pointed + * to is a metric set. + */ +class ConsumerSpec { + Set<String> includedMetrics = new HashSet<String>(); + + public boolean contains(Metric m) { + return includedMetrics.contains(m.getPath()); + } + + public void register(String path) { + StringTokenizer tokenizer = new StringTokenizer(path, "."); + + String total = ""; + + while (tokenizer.hasMoreTokens()) { + if (!total.isEmpty()) { + total += "."; + } + total += tokenizer.nextToken(); + includedMetrics.add(total); + } + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/CountMetric.java b/metrics/src/main/java/com/yahoo/metrics/CountMetric.java new file mode 100644 index 00000000000..409866e4040 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/CountMetric.java @@ -0,0 +1,222 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.metrics.util.MetricValueSet; +import com.yahoo.metrics.util.ValueType; +import com.yahoo.metrics.util.HasCopy; +import com.yahoo.text.Utf8String; +import com.yahoo.text.XMLWriter; + +import java.util.Collection; +import java.util.Locale; +import java.util.logging.Logger; + +/** + * A metric that counts something. The value should always be positive. + */ +@SuppressWarnings("unchecked") +public class CountMetric extends Metric { + + public static final int LOG_IF_UNSET = 2; + + private static final Utf8String AVERAGE_CHANGE_PER_SECOND = new Utf8String("average_change_per_second"); + private static final Utf8String COUNT = new Utf8String("count"); + private static final Logger log = Logger.getLogger(CountMetric.class.getName()); + private final MetricValueSet<CountValue> values; + private int flags; + + public CountMetric(String name, String tags, String description, MetricSet owner) { + super(name, tags, description, owner); + values = new MetricValueSet<CountValue>(); + flags = LOG_IF_UNSET; + } + + public CountMetric(CountMetric other, CopyType copyType, MetricSet owner) { + super(other, owner); + values = new MetricValueSet<CountValue>(other.values, copyType == CopyType.CLONE ? other.values.size() : 1); + flags = other.flags; + } + + private CountValue getValues() { + return values.getValue(); + } + + public long getValue() { + CountValue val = getValues(); + return (val == null ? 0 : val.value); + } + + @SuppressWarnings("UnusedDeclaration") + public void logOnlyIfSet() { + flags &= LOG_IF_UNSET; + } + + public void set(long value) { + while (!values.setValue(new CountValue(value))) { + // try again + } + } + + public void inc() { + inc(1); + } + + public void dec() { + dec(1); + } + + public void inc(long i) { + boolean overflow; + CountValue val; + do { + val = getValues(); + if (val == null) { + val = new CountValue(0); + } + overflow = (val.value + i < val.value); + val.value += i; + } while (!values.setValue(val)); + + if (overflow) { + reset(); + log.fine("Overflow in metric " + getName() + ". Resetting it."); + } + } + + public void dec(long i) { + boolean underflow; + CountValue val; + do { + val = getValues(); + if (val == null) { + val = new CountValue(0); + } + underflow = (val.value - i > val.value); + val.value -= i; + } while (!values.setValue(val)); + + if (underflow) { + reset(); + log.fine("Underflow in metric " + getName() + ". Resetting it."); + } + } + + @Override + public void reset() { + values.reset(); + } + + @Override + public boolean logFromTotalMetrics() { + return true; + } + + @Override + public void logEvent(EventLogger logger, String fullName) { + CountValue val = getValues(); + + if ((flags & LOG_IF_UNSET) != 0 || val != null) { + logger.count(fullName, val == null ? 0 : val.value); + } + } + + @Override + public void printXml(XMLWriter writer, + int secondsPassed, + int verbosity) + { + CountValue valRef = getValues(); + if (valRef == null && verbosity < 2) { + return; + } + long val = valRef != null ? valRef.value : 0; + openXMLTag(writer, verbosity); + writer.attribute(COUNT, String.valueOf(val)); + + if (secondsPassed > 0) { + writer.attribute(AVERAGE_CHANGE_PER_SECOND, + String.format(Locale.US, "%.2f", (double)val / secondsPassed)); + } + + writer.closeTag(); + } + + // Only one metric in valuemetric, so return it on any id. + @Override + public long getLongValue(String id) { + CountValue val = getValues(); + return (val == null ? 0 : val.value); + } + + @Override + public double getDoubleValue(String id) { + CountValue val = getValues(); + return (val == null ? 0 : val.value); + } + + @Override + public boolean used() { + return getValues() != null; + } + + @Override + public void addToSnapshot(Metric m) { + CountValue val = getValues(); + if (val != null) { + ((CountMetric)m).inc(val.value); + } + } + + @Override + public void addToPart(Metric m) { + CountValue val = getValues(); + if (val != null) { + ((CountMetric)m).inc(val.value); + } + } + + @Override + public Metric clone(CopyType type, MetricSet owner, boolean includeUnused) { + return new CountMetric(this, type, owner); + } + + private static class CountValue implements ValueType, HasCopy<CountValue> { + + long value; + + private CountValue(long value) { + this.value = value; + } + + public CountValue clone() { + try { + return (CountValue)super.clone(); + } catch (CloneNotSupportedException e) { + return null; + } + } + + public void add(ValueType other) { + value += ((CountValue)other).value; + } + + public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior) { + CountValue result = new CountValue(0); + for (ValueType t : sources) { + result.add(t); + } + if (joinBehavior == JoinBehavior.AVERAGE_ON_JOIN) { + result.value /= sources.size(); + } + return result; + } + + public String toString() { + return Long.toString(value); + } + + public CountValue copyObject() { + return new CountValue(value); + } + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/DoubleValue.java b/metrics/src/main/java/com/yahoo/metrics/DoubleValue.java new file mode 100644 index 00000000000..6309e4a9ea2 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/DoubleValue.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.metrics.util.ValueType; +import com.yahoo.metrics.util.HasCopy; + +import java.util.Collection; +import java.util.Locale; + +public class DoubleValue implements ValueMetric.Value<Double> { + private int count; + private double min, max, last; + private double total; + + public DoubleValue() { + count = 0; + min = Double.POSITIVE_INFINITY; + max = Double.NEGATIVE_INFINITY; + last = 0; + total = 0; + } + + public String toString() { + return "(count " + count + ", min " + min + ", max " + max + ", last " + last + ", total " + total + ")"; + } + + public void add(Double v) { + count = count + 1; + total = total + v; + min = Math.min(min, v); + max = Math.max(max, v); + last = v; + } + + public void join(ValueMetric.Value<Double> v2, boolean createAverageOnJoin) { + //StringBuffer sb = new StringBuffer(); + //sb.append("Adding " + this + " to " + v2); + if (createAverageOnJoin) { + count += v2.getCount(); + total += v2.getTotal(); + last = v2.getLast(); + + } else { + double totalAverage = getAverage() + v2.getAverage(); + count += v2.getCount(); + total = totalAverage * count; + last += v2.getLast(); + } + min = Math.min(min, v2.getMin()); + max = Math.max(max, v2.getMax()); + //sb.append(" and got " + this); + //System.err.println(sb.toString()); + } + + public void add(ValueType other) { + DoubleValue dv = (DoubleValue) other; + count = count + dv.count; + total = total + dv.total; + min = Math.min(min, dv.min); + max = Math.max(max, dv.max); + last = dv.last; + } + + public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior) { + DoubleValue result = new DoubleValue(); + for (ValueType t : sources) { + DoubleValue dv = (DoubleValue) t; + result.count = result.count + dv.count; + result.total = result.total + dv.total; + result.min = Math.min(result.min, dv.min); + result.max = Math.max(result.max, dv.max); + result.last += dv.last; + } + if (joinBehavior == JoinBehavior.AVERAGE_ON_JOIN) { + result.last /= sources.size(); + } else { + result.total *= sources.size(); + } + return result; + } + + public boolean overflow(ValueMetric.Value<Double> v2) { + if (count > (count + v2.getCount())) { + return true; + } + if (v2.getTotal() > 0 && getTotal() > getTotal() + v2.getTotal()) { + return true; + } + if (v2.getTotal() < 0 && getTotal() < getTotal() + v2.getTotal()) { + return true; + } + + return false; + } + + public int getCount() { return count; } + public Double getMin() { return (count > 0) ? min : 0; } + public Double getMax() { return (count > 0) ? max : 0; } + public Double getLast() { return last; } + public Double getTotal() { return total; } + + public Double getAverage() { + if (count == 0) { + return 0.0; + } + + return total / count; + } + + public String valueToString(Double val) { + if (val == Double.MIN_VALUE || val == Double.MAX_VALUE) { + return "0.00"; + } + + return String.format(Locale.US, "%.2f", val); + } + + public DoubleValue clone() { + try{ + return (DoubleValue) super.clone(); + } catch (CloneNotSupportedException e) { return null; } + } + + public ValueMetric.Value<Double> copyObject() { + return clone(); + } + +} diff --git a/metrics/src/main/java/com/yahoo/metrics/EventLogger.java b/metrics/src/main/java/com/yahoo/metrics/EventLogger.java new file mode 100644 index 00000000000..15ebb2ed0ef --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/EventLogger.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +public interface EventLogger { + public void value(String name, double value); + public void count(String name, long value); +} diff --git a/metrics/src/main/java/com/yahoo/metrics/JoinBehavior.java b/metrics/src/main/java/com/yahoo/metrics/JoinBehavior.java new file mode 100644 index 00000000000..eb8f5a540aa --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/JoinBehavior.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +/** + * When joining multiple metrics as a result of dimension + * removal. Should the result be an average or a sum? As an example, + * a latency metric should likely be averaged, while a number of + * pending metric should likely be summed. This join behavior property + * lets the metric framework know how to remove dimensions. + **/ +public enum JoinBehavior { AVERAGE_ON_JOIN, SUM_ON_JOIN } diff --git a/metrics/src/main/java/com/yahoo/metrics/LongValue.java b/metrics/src/main/java/com/yahoo/metrics/LongValue.java new file mode 100644 index 00000000000..29a8202f845 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/LongValue.java @@ -0,0 +1,143 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.metrics.util.ValueType; +import com.yahoo.metrics.util.HasCopy; + +import java.util.Collection; + +/** + * @author thomasg + */ +public class LongValue + implements ValueMetric.Value<Long> +{ + private int count; + private long min, max, last; + private long total; + + public LongValue() { + count = 0; + min = Long.MAX_VALUE; + max = Long.MIN_VALUE; + last = 0; + total = 0; + } + + @Override + public void add(Long v) { + LongValue val = this; + val.count = count + 1; + val.total = total + v; + val.min = Math.min(min, v); + val.max = Math.max(max, v); + val.last = v; + } + + @Override + public void join(ValueMetric.Value<Long> v2, boolean createAverageOnJoin) { + LongValue value = this; + + if (createAverageOnJoin) { + value.count = count + v2.getCount(); + value.total = total + v2.getTotal(); + value.last = v2.getLast(); + } else { + double totalAverage = getAverage() + v2.getAverage(); + value.count = count + v2.getCount(); + value.total = (long) (totalAverage * value.count); // Total is "wrong" I guess. + value.last = last + v2.getLast(); + } + + value.min = Math.min(min, v2.getMin()); + value.max = Math.max(max, v2.getMax()); + } + + @Override + public void add(ValueType other) { + LongValue dv = (LongValue) other; + count = count + dv.count; + total = total + dv.total; + min = Math.min(min, dv.min); + max = Math.max(max, dv.max); + last = dv.last; + } + + @Override + public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior) { + LongValue result = new LongValue(); + for (ValueType t : sources) { + LongValue dv = (LongValue) t; + result.count = result.count + dv.count; + result.total = result.total + dv.total; + result.min = Math.min(result.min, dv.min); + result.max = Math.max(result.max, dv.max); + result.last += dv.last; + } + if (joinBehavior == JoinBehavior.AVERAGE_ON_JOIN) { + result.last /= sources.size(); + } else { + result.total *= sources.size(); + } + return result; + } + + @Override + public boolean overflow(ValueMetric.Value<Long> v2) { + if (count > (count + v2.getCount())) { + return true; + } + if (v2.getTotal() > 0 && getTotal() > getTotal() + v2.getTotal()) { + return true; + } + if (v2.getTotal() < 0 && getTotal() < getTotal() + v2.getTotal()) { + return true; + } + + return false; + } + + @Override + public int getCount() { return count; } + + @Override + public Long getMin() { return (count > 0) ? min : 0; } + + @Override + public Long getMax() { return (count > 0) ? max : 0; } + + @Override + public Long getLast() { return last; } + + @Override + public Long getTotal() { return total; } + + @Override + public Double getAverage() { + if (count == 0) { + return 0.0; + } + return ((double) total) / count; + } + + @Override + public String valueToString(Long val) { + if (val == Long.MIN_VALUE || val == Long.MAX_VALUE) { + return valueToString((long)0); + } + + return val.toString(); + } + + @Override + public LongValue clone() { + try{ + return (LongValue) super.clone(); + } catch (CloneNotSupportedException e) { return null; } + } + + @Override + public ValueMetric.Value<Long> copyObject() { + return clone(); + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/Metric.java b/metrics/src/main/java/com/yahoo/metrics/Metric.java new file mode 100644 index 00000000000..0ef7dfbcd0a --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/Metric.java @@ -0,0 +1,256 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.text.XMLWriter; +import com.yahoo.text.Utf8String; + +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; + +public abstract class Metric +{ + private String name; + private String tags; + private String description; + + public String getXMLTag() { + return getName(); + } + + public void setName(String name) { + this.name = name; + } + + public void setTags(String tags) { + this.tags = tags; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getTags() { + return tags; + } + + MetricSet owner; + + public Metric(String name, String tags, String description) { + this(name, tags, description, null); + } + + public Metric(String name, String tags, String description, MetricSet owner) { + this.name = name; + this.tags = tags; + this.description = description; + + if (owner != null) { + owner.registerMetric(this); + } + } + + public Metric(Metric other, MetricSet owner) { + this(other.name, other.tags, other.description, owner); + } + + public String getName() { return name; } + + public String getPath() { + if (owner == null || owner.owner == null) { + return getName(); + } + + return owner.getPath() + "." + getName(); + } + + public List<String> getPathVector() { + List<String> result = new ArrayList<String>(); + result.add(getName()); + MetricSet owner = this.owner; + while (owner != null) { + result.add(0, owner.getName()); + owner = owner.owner; + } + return result; + } + + public String getDescription() { return description; } + + public String[] getTagVector() { + return getTags().split("[ \r\t\f]"); + } + + /** + * Returns true if the given tag exists in this metric's tag list. + * + * @return true if tag exists in tag list + */ + public boolean hasTag(String tag) { + for (String s : getTagVector()) { + if (s.equals(tag)) { + return true; + } + } + return false; + } + + public enum CopyType { CLONE, INACTIVE } + + /** + * The clone function will clone metrics to an identical subtree of + * metrics. Clone is primarily used for load metrics that wants to clone + * a template metric for each loadtype. But it should work generically. + * + * @param type If set to inactive, sum metrics will evaluate to primitives + * and metrics can save memory by knowing no updates are coming. + * @param includeUnused When creating snapshots we do not want to include + * unused metrics, but while generating sum metric sum in active + * metrics we want to. This has no affect if type is CLONE. + */ + public abstract Metric clone(CopyType type, MetricSet owner, boolean includeUnused); + + /** + * Utility function for assigning values from one metric of identical type + * to this metric. For simplicity sake it does a const cast and calls + * addToSnapshot, which should not alter source if reset is false. This can + * not be used to copy between active metrics and inactive copies. + * + * @return Returns itself. + */ + public Metric assignValues(Metric m) { + m.addToSnapshot(this); + // As this should only be called among active metrics, all metrics + // should exist and owner list should thus always end up empty. + return this; + } + + /** Reset all metric values. */ + public abstract void reset(); + + public boolean logFromTotalMetrics() { return false; } + + /** Implement to make metric able to log event. + * + * @param logger An event logger to use for logging. + * @param fullName The name to use for the event. + */ + public abstract void logEvent(EventLogger logger, String fullName); + + public static final Utf8String TAG_NAME = new Utf8String("name"); + public static final Utf8String TAG_TAGS = new Utf8String("tags"); + public static final Utf8String TAG_DESC = new Utf8String("description"); + + void openXMLTag(XMLWriter writer, int verbosity) { + String[] tags = getTagVector(); + + writer.openTag(getXMLTag()); + + if (getXMLTag() != getName()) { + writer.attribute(TAG_NAME, getName()); + } + + if (verbosity >= 3 && tags.length > 0) { + String tagStr = ""; + for (String tag : tags) { + if (!tagStr.isEmpty()) { + tagStr = ","; + } + tagStr += tag; + } + + writer.attribute(TAG_TAGS, tagStr); + } + + if (verbosity >= 1 && !getDescription().isEmpty()) { + writer.attribute(TAG_DESC, getDescription()); + } + } + + /** + * The verbosity says how much to print. + * At verbosity level 0, only the most critical parts are printed. + * At verbosity level 1, descriptions are added. + * At verbosity level 2, metrics without data is added. + * At verbosity level 3, tags are included too. + */ + public abstract void printXml(XMLWriter writer, + int secondsPassed, + int verbosity); + + public String toXml(int secondsPassed, int verbosity) { + StringWriter writer = new StringWriter(); + printXml(new XMLWriter(writer), secondsPassed, verbosity); + return writer.toString(); + } + + /** + * Most metrics report numbers of some kind. To be able to report numbers + * without having code to handle each possible metric type, these functions + * exist to extract raw data to present easily. + * @param id The part of the metric to extract. For instance, an average + * metric have average, + */ + public abstract long getLongValue(String id); + public abstract double getDoubleValue(String id); + + /** + * When snapshotting we need to be able to join data from one set of metrics + * to another set of metrics taken at another time. MetricSet doesn't know + * the type of the metrics it contains, so we need a generic function for + * doing this. This function assumes metric given as input is of the exact + * same type as the one it is called on for simplicity. This is true when + * adding to snapshots as they have been created with clone and is thus + * always exactly equal. + * + * @param m Metric of exact same type as this one. (Will core if wrong) + */ + abstract void addToSnapshot(Metric m); + + /** + * For sum metrics to work with metric sets, metric sets need operator+=. + * To implement this, we need a function to join any metric type together. + * This is different from adding to snapshot. When adding to snapshots we + * join different time periods to the same metric, but when adding parts + * together we join different metrics for the same time. For instance, an + * average metric of queuesize, should just join new values to create new + * average when adding to snapshot, but when adding parts, the averages + * themselves should be added together. + * + * @param m Metric of exact same type as this one. (Will core if wrong) + */ + abstract void addToPart(Metric m); + + public boolean visit(MetricVisitor visitor, boolean tagAsAutoGenerated) { + return visitor.visitPrimitiveMetric(this, tagAsAutoGenerated); + } + + /** Set whether metrics have ever been set. */ + public abstract boolean used(); + + /** Returns true if this metric is registered in a metric set. */ + public boolean isRegistered() { return (owner != null); } + + /** + * If this metric is registered with an owner, remove itself from that owner. + */ + public void unregister() { + if (isRegistered()) { + getOwner().unregisterMetric(this); + } + } + + public MetricSet getOwner() { return owner; } + + public MetricSet getRoot() { + if (owner == null) { + if (this instanceof MetricSet) { + return (MetricSet)this; + } else { + return null; + } + } else { + return owner.getRoot(); + } + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricManager.java b/metrics/src/main/java/com/yahoo/metrics/MetricManager.java new file mode 100644 index 00000000000..b44205660de --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/MetricManager.java @@ -0,0 +1,704 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.collections.Pair; +import com.yahoo.text.XMLWriter; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.logging.Logger; + +/** + * A metrics-enabled application should have a single MetricManager. You can register a number of MetricSets in the + * MetricManager. Each metric in the metrics sets can be used by zero or more consumers, configurable using + * readConfig(). + * + * The consumers get their data by calling the getMetrics() method, which gives them a snapshot of all the current + * metrics which are configured for the given name. + * + * Locking strategy: + * + * There are three locks in this class: + * + * Config lock: - This protects the class on config changes. It protects the _config and _consumerConfig members. + * + * Thread monitor (waiter): - This lock is kept by the worker thread while it is doing a work cycle, and it uses this + * monitor to sleep. It is used to make shutdown quick by interrupting thread, and to let functions called by clients be + * able to do a change while the worker thread is idle. - The log period is protected by the thread monitor. - The + * update hooks is protected by the thread monitor. + * + * Metric lock: - The metric log protects the active metric set when adding or removing metrics. Clients need to grab + * this lock before altering active metrics. The metric manager needs to grab this lock everytime it visits active + * metrics. - The metric log protects the snapshots. The snapshot writer is the metric worker thread and will grab the + * lock while editing them. Readers that aren't the worker thread itself must grab lock to be sure. + * + * If multiple locks is taken, the allowed locking order is: 1. Thread monitor. 2. Metric lock. 3. Config lock. + */ +public class MetricManager implements Runnable { + + private static final int STATE_CREATED = 0; + private static final int STATE_RUNNING = 1; + private static final int STATE_STOPPED = 2; + private static final Logger log = Logger.getLogger(MetricManager.class.getName()); + private final CountDownLatch termination = new CountDownLatch(1); + private final MetricSnapshot activeMetrics = new MetricSnapshot("Active metrics showing updates since " + + "last snapshot"); + private final Map<String, ConsumerSpec> consumerConfig = new HashMap<String, ConsumerSpec>(); + private final List<UpdateHook> periodicUpdateHooks = new ArrayList<UpdateHook>(); + private final List<UpdateHook> snapshotUpdateHooks = new ArrayList<UpdateHook>(); + private final Timer timer; + private Pair<Integer, Integer> logPeriod; + private List<MetricSnapshotSet> snapshots = new ArrayList<MetricSnapshotSet>(); + private MetricSnapshot totalMetrics = new MetricSnapshot("Empty metrics before init", 0, + activeMetrics.getMetrics(), false); + private int state = STATE_CREATED; + private int lastProcessedTime = 0; + private boolean forceEventLogging = false; + private boolean snapshotUnsetMetrics = false; // TODO: add to config + + public MetricManager() { + this(new Timer()); + } + + MetricManager(Timer timer) { + this.timer = timer; + initializeSnapshots(); + logPeriod = new Pair<Integer, Integer>(snapshots.get(0).getPeriod(), 0); + } + + void initializeSnapshots() { + int currentTime = timer.secs(); + + List<Pair<Integer, String>> snapshotPeriods = new ArrayList<Pair<Integer, String>>(); + snapshotPeriods.add(new Pair<Integer, String>(60 * 5, "5 minute")); + snapshotPeriods.add(new Pair<Integer, String>(60 * 60, "1 hour")); + snapshotPeriods.add(new Pair<Integer, String>(60 * 60 * 24, "1 day")); + snapshotPeriods.add(new Pair<Integer, String>(60 * 60 * 24 * 7, "1 week")); + + int count = 1; + for (int i = 0; i < snapshotPeriods.size(); ++i) { + int nextCount = 1; + if (i + 1 < snapshotPeriods.size()) { + nextCount = snapshotPeriods.get(i + 1).getFirst() + / snapshotPeriods.get(i).getFirst(); + if (snapshotPeriods.get(i + 1).getFirst() % snapshotPeriods.get(i).getFirst() != 0) { + throw new IllegalStateException("Snapshot periods must be multiplum of each other"); + } + } + snapshots.add(new MetricSnapshotSet(snapshotPeriods.get(i).getSecond(), + snapshotPeriods.get(i).getFirst(), + count, + activeMetrics.getMetrics(), + snapshotUnsetMetrics)); + count = nextCount; + } + // Add all time snapshot. + totalMetrics = new MetricSnapshot("All time snapshot", + 0, activeMetrics.getMetrics(), + snapshotUnsetMetrics); + totalMetrics.reset(currentTime); + } + + public void stop() { + synchronized (this) { + int prevState = state; + state = STATE_STOPPED; + if (prevState == STATE_CREATED) { + return; + } + notifyAll(); + } + try { + termination.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @SuppressWarnings("UnusedDeclaration") + void setSnapshotUnsetMetrics(boolean value) { + snapshotUnsetMetrics = value; + } + + /** + * Add a metric update hook. This will always be called prior to snapshotting and metric logging, to make the + * metrics the best as they can be at those occasions. + * + * @param hook The hook to add. + * @param period Period in seconds for how often callback should be called. The default value of 0, means only + * before snapshotting or logging, while another value will give callbacks each period seconds. + * Expensive metrics to calculate will typically only want to do it before snapshotting, while + * inexpensive metrics might want to log their value every 5 seconds or so. Any value of period >= the + * smallest snapshot time will behave identically as if period is set to 0. + */ + @SuppressWarnings("UnusedDeclaration") + public synchronized void addMetricUpdateHook(UpdateHook hook, int period) { + hook.period = period; + + // If we've already initialized manager, log period has been set. + // In this case. Call first time after period + hook.nextCall = (logPeriod.getSecond() == 0 ? 0 : timer.secs() + period); + if (period == 0) { + if (!snapshotUpdateHooks.contains(hook)) { + snapshotUpdateHooks.add(hook); + } + } else { + if (!periodicUpdateHooks.contains(hook)) { + periodicUpdateHooks.add(hook); + } + } + } + + @SuppressWarnings("UnusedDeclaration") + public synchronized void removeMetricUpdateHook(UpdateHook hook) { + if (hook.period == 0) { + snapshotUpdateHooks.remove(hook); + } else { + periodicUpdateHooks.remove(hook); + } + } + + /** + * Force a metric update for all update hooks. Useful if you want to ensure nice values before reporting something. + * This function can not be called from an update hook callback. + * + * @param includeSnapshotOnlyHooks True to also run snapshot hooks. + */ + @SuppressWarnings("UnusedDeclaration") + public synchronized void updateMetrics(boolean includeSnapshotOnlyHooks) { + log.fine("Giving " + periodicUpdateHooks.size() + " periodic update hooks."); + + updatePeriodicMetrics(0, true); + + if (includeSnapshotOnlyHooks) { + log.fine("Giving " + snapshotUpdateHooks.size() + " snapshot update hooks."); + updateSnapshotMetrics(); + } + } + + /** + * Force event logging to happen now. This function can not be called from an update hook callback. + */ + @SuppressWarnings("UnusedDeclaration") + public void forceEventLogging() { + log.fine("Forcing event logging to happen."); + // Ensure background thread is not in a current cycle during change. + + synchronized (this) { + forceEventLogging = true; + this.notifyAll(); + } + } + + /** + * Register a new metric to be included in the active metric set. You need to have grabbed the metric lock in order + * to do this. (You also need to grab that lock if you alter registration of already registered metric set.) This + * function can not be called from an update hook callback. + * + * @param m The metric to register. + */ + public void registerMetric(Metric m) { + activeMetrics.getMetrics().registerMetric(m); + } + + /** + * Unregister a metric from the active metric set. You need to have grabbed the metric lock in order to do this. + * (You also need to grab that lock if you alter registration of already registered metric set.) This function can + * not be called from an update hook callback. + * + * @param m The Metric to unregister. + */ + @SuppressWarnings("UnusedDeclaration") + public void unregisterMetric(Metric m) { + activeMetrics.getMetrics().unregisterMetric(m); + } + + /** + * Reset all metrics including all snapshots. This function can not be called from an update hook callback. + * + * @param currentTime The current time. + */ + public synchronized void reset(int currentTime) { + activeMetrics.reset(currentTime); + + for (MetricSnapshotSet m : snapshots) { + m.reset(currentTime); + } + totalMetrics.reset(currentTime); + } + + /** + * Read configuration. Before reading config, all metrics should be set up first. By doing this, the metrics manager + * can optimize reporting of consumers. readConfig() will start a config subscription. It should not be called + * multiple times. + */ +/* public synchronized void init(String configId, ThreadPool pool) { + log.fine("Initializing metric manager") + + LOG(debug, "Initializing metric manager."); + _configSubscription = Config::subscribe(configId, *this); + LOG(debug, "Starting worker thread, waiting for first " + "iteration to complete."); + Runnable::start(pool); + // Wait for first iteration to have completed, such that it is safe + // to access snapshots afterwards. + vespalib::MonitorGuard sync(_waiter); + while (_lastProcessedTime == 0) { + sync.wait(1); + } + LOG(debug, "Metric manager completed initialization."); +} + +*/ + + class ConsumerMetricVisitor extends MetricVisitor { + + ConsumerSpec metricsToMatch; + MetricVisitor clientVisitor; + + ConsumerMetricVisitor(ConsumerSpec spec, + MetricVisitor clientVisitor) + { + metricsToMatch = spec; + this.clientVisitor = clientVisitor; + log.fine("Consuming metrics: " + spec.includedMetrics); + } + + public boolean visitMetricSet(MetricSet metricSet, boolean autoGenerated) { + if (metricSet.getOwner() == null) { + return true; + } + + if (!metricsToMatch.contains(metricSet)) { + log.fine("Metric doesn't match " + metricSet.getPath()); + return false; + } + + return clientVisitor.visitMetricSet(metricSet, autoGenerated); + } + + public void doneVisitingMetricSet(MetricSet metricSet) { + if (metricSet.getOwner() != null) { + clientVisitor.doneVisitingMetricSet(metricSet); + } + } + + public boolean visitPrimitiveMetric(Metric metric, boolean autoGenerated) { + if (metricsToMatch.contains(metric)) { + return clientVisitor.visitPrimitiveMetric(metric, autoGenerated); + } else { + log.fine("Metric doesn't match " + metric.getPath()); + } + return true; + } + } + + public synchronized void visit(MetricSet metrics, MetricVisitor visitor, String consumer) { + if (consumer.isEmpty()) { + metrics.visit(visitor, false); + return; + } + + ConsumerSpec spec = getConsumerSpec(consumer); + + if (spec != null) { + ConsumerMetricVisitor consumerVis = new ConsumerMetricVisitor(spec, visitor); + metrics.visit(consumerVis, false); + } else { + log.warning("Requested metrics for non-defined consumer " + consumer); + } + } + + class XmlWriterMetricVisitor extends MetricVisitor { + + int period; + XMLWriter writer; + int verbosity; + + XmlWriterMetricVisitor(XMLWriter writer, int period, int verbosity) { + this.period = period; + this.verbosity = verbosity; + this.writer = writer; + } + + public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { + if (set.used() || verbosity >= 2) { + set.openXMLTag(writer, verbosity); + return true; + } + return false; + } + + public void doneVisitingMetricSet(MetricSet set) { + writer.closeTag(); + } + + public boolean visitPrimitiveMetric(Metric metric, boolean autoGenerated) { + metric.printXml(writer, period, verbosity); + return true; + } + } + + void printXml(MetricSet set, XMLWriter writer, int period, String consumer, int verbosity) { + visit(set, new XmlWriterMetricVisitor(writer, period, verbosity), consumer); + } + + /** + * Synchronize over this while the returned object + * + * @return The MetricSnapshot of all active metrics. + */ + public MetricSnapshot getActiveMetrics() { + return activeMetrics; + } + + /** + * Synchronize over this while the returned object + * + * @return The MetricSnapshot for the total metric. + */ + public MetricSnapshot getTotalMetricSnapshot() { + return totalMetrics; + } + + public synchronized List<Integer> getSnapshotPeriods() { + List<Integer> retVal = new ArrayList<Integer>(); + + for (MetricSnapshotSet m : snapshots) { + retVal.add(m.getPeriod()); + } + return retVal; + } + + /** + * While accessing snapshots you should synchronize over this + * + * @param period The id of the snapshot period to access. + * @param getInProgressSet True to retrieve the snapshot currently being built. + * @return The appropriate MetricSnapshot. + */ + MetricSnapshot getMetricSnapshot(int period, boolean getInProgressSet) { + return getMetricSnapshotSet(period).getSnapshot(getInProgressSet); + } + + MetricSnapshot getMetricSnapshot(int period) { + return getMetricSnapshot(period, false); + } + + public MetricSnapshotSet getMetricSnapshotSet(int period) { + for (MetricSnapshotSet m : snapshots) { + if (m.getPeriod() == period) { + return m; + } + } + + throw new IllegalArgumentException("No snapshot for period of length " + period + " exists."); + } + + @SuppressWarnings("UnusedDeclaration") + public synchronized boolean hasTemporarySnapshot(int period) { + return getMetricSnapshotSet(period).hasTemporarySnapshot(); + } + + public synchronized void addMetricToConsumer(String consumerName, String metricPath) { + ConsumerSpec spec = getConsumerSpec(consumerName); + if (spec == null) { + spec = new ConsumerSpec(); + consumerConfig.put(consumerName, spec); + } + spec.register(metricPath); + } + + public synchronized ConsumerSpec getConsumerSpec(String consumer) { + return consumerConfig.get(consumer); + } + + /** + * If you join or remove metrics from the active metric sets, normally, snapshots will be recreated next snapshot + * period. However, if you want to see the effects of such changes in status pages ahead of that, you can call this + * function in order to check whether snapshots needs to be regenerated and regenerate them if needed. + */ + public synchronized void checkMetricsAltered() { + if (activeMetrics.getMetrics().isRegistrationAltered()) { + handleMetricsAltered(); + } + } + + /** + * Used by unit tests to verify that we have processed for a given time. + * + * @return Returns the timestamp of the previous tick. + */ + @SuppressWarnings("UnusedDeclaration") + public int getLastProcessedTime() { + return lastProcessedTime; + } + + class LogMetricVisitor extends MetricVisitor { + + boolean total; + EventLogger logger; + + LogMetricVisitor(boolean totalVals, EventLogger logger) { + total = totalVals; + this.logger = logger; + } + + public boolean visitPrimitiveMetric(Metric metric, boolean autoGenerated) { + if (metric.logFromTotalMetrics() == total) { + String logName = metric.getPath().replace('.', '_'); + metric.logEvent(logger, logName); + } + return true; + } + } + + public void logTotal(int currentTime, EventLogger logger) { + LogMetricVisitor totalVisitor = new LogMetricVisitor(true, logger); + LogMetricVisitor fiveMinVisitor = new LogMetricVisitor(false, logger); + + if (logPeriod.getSecond() <= currentTime) { + log.fine("Logging total metrics."); + visit(totalMetrics.getMetrics(), totalVisitor, "log"); + visit(snapshots.get(0).getSnapshot().getMetrics(), fiveMinVisitor, "log"); + if (logPeriod.getSecond() + logPeriod.getFirst() < currentTime) { + logPeriod = new Pair<Integer, Integer>(logPeriod.getFirst(), + snapshots.get(0).getFromTime() + logPeriod.getFirst()); + } else { + logPeriod = + new Pair<Integer, Integer>(logPeriod.getFirst(), logPeriod.getSecond() + logPeriod.getFirst()); + } + } + } + + public void logOutOfSequence(int currentTime, EventLogger logger) { + LogMetricVisitor totalVisitor = new LogMetricVisitor(true, logger); + LogMetricVisitor fiveMinVisitor = new LogMetricVisitor(false, logger); + + log.fine("Logging total metrics out of sequence."); + MetricSnapshot snapshot = new MetricSnapshot( + "Total out of sequence metrics from start until current time", + 0, + totalMetrics.getMetrics(), + snapshotUnsetMetrics); + + activeMetrics.addToSnapshot(snapshot, currentTime, false); + snapshot.setFromTime(totalMetrics.getFromTime()); + visit(snapshot.getMetrics(), totalVisitor, "log"); + visit(snapshot.getMetrics(), fiveMinVisitor, "log"); + } + + /** + * Runs one iteration of the thread activity. + * + * @param logger An event logger to use for any new events generated. + * @return The number of milliseconds to sleep until waking up again + */ + public synchronized int tick(EventLogger logger) { + int currentTime = timer.secs(); + + log.finest("Worker thread starting to process for time " + currentTime); + + boolean firstIteration = (logPeriod.getSecond() == 0); + // For a slow system to still be doing metrics tasks each n'th + // second, rather than each n'th + time to do something seconds, + // we constantly join next time to do something from the last timer. + // For that to work, we need to initialize timers on first iteration + // to set them to current time. + if (firstIteration) { + // Setting next log period to now, such that we log metrics + // straight away + logPeriod = new Pair<Integer, Integer>(logPeriod.getFirst(), currentTime); + for (MetricSnapshotSet m : snapshots) { + m.setFromTime(currentTime); + } + for (UpdateHook h : periodicUpdateHooks) { + h.nextCall = currentTime; + } + } + + // If metrics have changed since last time we did a snapshot, + // work that out before taking the snapshot, such that new + // metric can be included + checkMetricsAltered(); + + // Set next work time to the time we want to take next snapshot. + int nextWorkTime = snapshots.get(0).getPeriod() + snapshots.get(0).getFromTime(); + + int nextUpdateHookTime; + + if (nextWorkTime <= currentTime) { + log.fine("Time to do snapshot. Calling update hooks"); + nextUpdateHookTime = updatePeriodicMetrics(currentTime, true); + updateSnapshotMetrics(); + takeSnapshots(nextWorkTime); + } else if (forceEventLogging) { + log.fine("Out of sequence event logging. Calling update hooks"); + nextUpdateHookTime = updatePeriodicMetrics(currentTime, true); + updateSnapshotMetrics(); + } else { + // If not taking a new snapshot. Only give update hooks to + // periodic hooks wanting it. + nextUpdateHookTime = updatePeriodicMetrics(currentTime, false); + } + + // Log if it is time + if (logPeriod.getSecond() <= currentTime || forceEventLogging) { + logTotal(currentTime, logger); + } else { + logOutOfSequence(currentTime, logger); + } + + forceEventLogging = false; + lastProcessedTime = (nextWorkTime <= currentTime ? nextWorkTime : currentTime); + log.fine("Worker thread done with processing for time " + lastProcessedTime); + + int next = Math.min(logPeriod.getSecond(), snapshots.get(0).getPeriod() + snapshots.get(0).getFromTime()); + next = Math.min(next, nextUpdateHookTime); + if (currentTime < next) { + return (next - currentTime) * 1000; + } + + return 0; + } + + @Override + public synchronized void run() { + if (state != STATE_CREATED) { + throw new IllegalStateException(); + } + try { + for (state = STATE_RUNNING; state == STATE_RUNNING; ) { + int timeout = tick(new VespaLogEventLogger()); + if (timeout > 0) { + wait(timeout); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + termination.countDown(); + } + } + + public synchronized void takeSnapshots(int timeToProcess) { + // If not time to do dump data from active snapshot yet, nothing to do + if (!snapshots.get(0).timeForAnotherSnapshot(timeToProcess)) { + return; + } + + log.fine("Updating " + snapshots.get(0).getName() + " snapshot from active metrics"); + // int fromTime = snapshots.get(0).getSnapshot().getToTime(); + MetricSnapshot firstTarget = (snapshots.get(0).getNextTarget()); + firstTarget.reset(timeToProcess); + activeMetrics.addToSnapshot(firstTarget, timeToProcess, false); + log.fine("Updating total metrics with five minute period of active metrics"); + activeMetrics.addToSnapshot(totalMetrics, timeToProcess, false); + activeMetrics.reset(timeToProcess); + + for (int i = 1; i < snapshots.size(); ++i) { + MetricSnapshotSet s = snapshots.get(i); + + log.fine("Adding data from last snapshot to building snapshot of " + + "next period snapshot " + s.getName()); + + MetricSnapshot target = s.getNextTarget(); + snapshots.get(i - 1).getSnapshot().addToSnapshot(target, timeToProcess, false); + target.setToTime(timeToProcess); + + if (!snapshots.get(i).haveCompletedNewPeriod(timeToProcess)) { + log.fine("Not time to roll snapshot " + s.getName() + " yet. " + + s.getBuilderCount() + " of " + s.getCount() + " snapshot " + + "taken at time" + (s.getBuilderCount() * s.getPeriod() + s.getFromTime()) + + ", and period of " + s.getPeriod() + " is not up " + + "yet as we're currently processing for time " + timeToProcess); + break; + } else { + log.fine("Rolled snapshot " + s.getName() + " at time " + timeToProcess); + } + } + } + + /** + * Utility function for updating periodic metrics. + * + * @param updateTime Update metrics timed to update at this time. + * @param outOfSchedule Force calls to all hooks. Don't screw up normal schedule though. If not time to update yet, + * update without adjusting schedule for next update. + * @return Time of next hook to be called in the future. + */ + int updatePeriodicMetrics(int updateTime, boolean outOfSchedule) { + int nextUpdateTime = Integer.MAX_VALUE; + for (UpdateHook h : periodicUpdateHooks) { + if (h.nextCall <= updateTime) { + h.updateMetrics(); + if (h.nextCall + h.period < updateTime) { + h.nextCall = updateTime + h.period; + } else { + h.nextCall += h.period; + } + } else if (outOfSchedule) { + h.updateMetrics(); + } + nextUpdateTime = Math.min(nextUpdateTime, h.nextCall); + } + return nextUpdateTime; + } + + void updateSnapshotMetrics() { + for (UpdateHook h : snapshotUpdateHooks) { + h.updateMetrics(); + } + } + + synchronized void handleMetricsAltered() { +/* if (consumerConfig.isEmpty()) { + log.fine("Setting up consumers for the first time."); + } else { + log.info("Metrics registration changes detected. Handling changes."); + } + + Map<String, ConsumerSpec> configMap = new HashMap<String, ConsumerSpec>(); + activeMetrics.getMetrics().clearRegistrationAltered(); + + + for (<config::MetricsmanagerConfig::Consumer>::const_iterator it + = _config.consumer.begin(); it != _config.consumer.end(); ++it) + { + ConsumerMetricBuilder consumerMetricBuilder(*it); + _activeMetrics.getMetrics().visit(consumerMetricBuilder); + configMap[it->name] = ConsumerSpec::SP( + new ConsumerSpec(consumerMetricBuilder._matchedMetrics)); + } + _consumerConfig.swap(configMap); + */ + log.fine("Recreating snapshots to include altered metrics"); + totalMetrics.recreateSnapshot(activeMetrics.getMetrics(), snapshotUnsetMetrics); + + for (MetricSnapshotSet set : snapshots) { + set.recreateSnapshot(activeMetrics.getMetrics(), snapshotUnsetMetrics); + } + } + + abstract class UpdateHook { + + String name; + int nextCall; + int period; + + public UpdateHook(String name) { + this.name = name; + nextCall = 0; + period = 0; + } + + public abstract void updateMetrics(); + + public String getName() { + return name; + } + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricSet.java b/metrics/src/main/java/com/yahoo/metrics/MetricSet.java new file mode 100644 index 00000000000..f10834d93cc --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/MetricSet.java @@ -0,0 +1,239 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.text.XMLWriter; + +import java.util.*; +import java.util.logging.Logger; + +public abstract class MetricSet extends Metric +{ + private static Logger log = Logger.getLogger(MetricSet.class.getName()); + + List<Metric> metricOrder = new ArrayList<Metric>(); // Keep added order for reporting + boolean registrationAltered; // Set to true if metrics have been + // registered/unregistered since last time + // it was reset + + public MetricSet(String name, String tags, String description, MetricSet owner) { + super(name, tags, description, owner); + } + + public MetricSet(MetricSet other, CopyType copyType, MetricSet owner, boolean includeUnused) { + super(other, owner); + + for (Metric m : other.metricOrder) { + if (copyType != CopyType.INACTIVE || includeUnused || m.used()) { + m.clone(copyType, this, includeUnused); + } + } + } + + /** + * @return Returns true if registration has been altered since it was last + * cleared. Used by the metric manager to know when it needs to recalculate + * which consumers will see what. + */ + public boolean isRegistrationAltered() { return registrationAltered; } + + /** Clear all registration altered flags. */ + void clearRegistrationAltered() { + visit(new MetricVisitor() { + public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { + if (autoGenerated) { + return false; + } + + set.registrationAltered = false; + return true; + } + }, false); + } + + public void registerMetric(Metric m) { + if (m.isRegistered()) { + throw new IllegalStateException("Metric " + m.getName() + + " is already registered in a metric set. Cannot register it twice."); + } + + if (getMetricInternal(m.getName()) != null) { + throw new IllegalStateException("A metric named " + m.getName() + " is already registered " + + "in metric set " + getPath()); + } + + metricOrder.add(m); + m.owner = this; + tagRegistrationAltered(); + } + + public void unregisterMetric(Metric m) { + // In case of abrubt shutdowns, don't die hard on attempts to unregister + // non-registered metrics. Just warn and ignore. + if (!metricOrder.remove(m)) { + log.warning("Attempt to unregister metric " + m.getName() + " in metric set " + getPath() + + ", where it wasn't registered to begin with."); + return; + } + + m.owner = null; + tagRegistrationAltered(); + + log.finest("Unregistered metric " + m.getName() + " from metric set " + getPath() + "."); + } + + @Override + public void reset() { + for (Metric m : metricOrder) { + m.reset(); + } + } + + @Override + public boolean visit(MetricVisitor visitor, boolean tagAsAutoGenerated) { + if (!visitor.visitMetricSet(this, tagAsAutoGenerated)) { + return true; + } + + for (Metric m : metricOrder) { + if (!m.visit(visitor, tagAsAutoGenerated)) { + break; + } + } + + visitor.doneVisitingMetricSet(this); + return true; + } + + @Override + public void logEvent(EventLogger logger, String fullName) { + throw new IllegalStateException("Can't log event from a MetricsSet: " + fullName); + } + + // These should never be called on metrics set. + @Override + public long getLongValue(String id) { + throw new IllegalStateException("Tried to get long from metricset"); + } + + @Override + public double getDoubleValue(String id) { + throw new IllegalStateException("Tried to get double from metricset"); + } + + public Metric getMetric(String name) { + int pos = name.indexOf('.'); + if (pos == -1) { + return getMetricInternal(name); + } else { + String child = name.substring(0, pos); + String rest = name.substring(pos + 1); + + Metric m = getMetricInternal(child); + if (m == null || !(m instanceof MetricSet)) { + return null; + } else { + return ((MetricSet)m).getMetric(rest); + } + } + } + + private Metric getMetricInternal(String name) { + for (Metric m : metricOrder) { + if (m.getName().equals(name)) { + return m; + } + } + return null; + } + + Map<String, Metric> createMetricMap() { + Map<String, Metric> map = new TreeMap<String, Metric>(); + + for (Metric m : metricOrder) { + map.put(m.getName(), m); + } + + return map; + } + + @Override + public void addToSnapshot(Metric snapshotMetric) { + MetricSet o = (MetricSet)snapshotMetric; + + Map<String, Metric> map1 = createMetricMap(); + Set<String> seen = new HashSet<String>(); + + // For all the metrics in the other's order, join ours to the snapshot. + for (Metric m : o.metricOrder) { + Metric myCopy = map1.get(m.getName()); + + if (myCopy != null) { + seen.add(m.getName()); + myCopy.addToSnapshot(m); + } + } + + // For all the remaining metrics, just join them to the other one. + for (Metric m : metricOrder) { + if (!seen.contains(m.getName())) { + m.clone(CopyType.INACTIVE, o, false); + } + } + } + + List<Metric> getRegisteredMetrics() + { return metricOrder; } + + @Override + public boolean used() { + for (Metric m : metricOrder) { + if (m.used()) { + return true; + } + } + + return false; + } + + @Override + public void addToPart(Metric partMetric) { + MetricSet o = (MetricSet)partMetric; + + Map<String, Metric> map2 = o.createMetricMap(); + + for (Metric m : metricOrder) { + Metric other = map2.get(m.getName()); + if (other != null) { + m.addToPart(other); + } else { + m.clone(CopyType.INACTIVE, o, false); + } + } + } + + private void tagRegistrationAltered() { + registrationAltered = true; + if (owner != null) { + owner.tagRegistrationAltered(); + } + } + + @Override + public void printXml(XMLWriter writer, + int secondsPassed, int verbosity) + { + if (!used() && verbosity < 3) { + return; + } + + openXMLTag(writer, verbosity); + + for (Metric m : metricOrder) { + m.printXml(writer, secondsPassed, verbosity); + } + + writer.closeTag(); + } + + +} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricSnapshot.java b/metrics/src/main/java/com/yahoo/metrics/MetricSnapshot.java new file mode 100644 index 00000000000..cd02b170b4a --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/MetricSnapshot.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.text.XMLWriter; +import com.yahoo.text.Utf8String; + +import java.io.StringWriter; + +public class MetricSnapshot +{ + String name; + int period; + int fromTime; + int toTime; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public int getPeriod() { + return period; + } + + public void setPeriod(int period) { + this.period = period; + } + + public int getFromTime() { + return fromTime; + } + + public void setFromTime(int fromTime) { + this.fromTime = fromTime; + } + + public int getToTime() { + return toTime; + } + + public void setToTime(int toTime) { + this.toTime = toTime; + } + + public MetricSet getMetrics() { + return snapshot; + } + + MetricSet snapshot; + + public MetricSnapshot(String name) { + this.name = name; + this.period = 0; + this.fromTime = 0; + this.toTime = 0; + snapshot = new SimpleMetricSet("metrics", "", ""); + } + + public MetricSnapshot(String name, + int period, + MetricSet source, + boolean copyUnset) { + this(name); + this.period = period; + snapshot = (MetricSet)source.clone(Metric.CopyType.INACTIVE, null, copyUnset); + } + + void reset(int currentTime) { + fromTime = currentTime; + toTime = 0; + snapshot.reset(); + } + + public void recreateSnapshot(MetricSet source, boolean copyUnset) { + MetricSet newSnapshot = (MetricSet)source.clone(Metric.CopyType.INACTIVE, null, copyUnset); + newSnapshot.reset(); + snapshot.addToSnapshot(newSnapshot); + snapshot = newSnapshot; + } + + public static final Utf8String TAG_NAME = new Utf8String("name"); + public static final Utf8String TAG_FROM = new Utf8String("from"); + public static final Utf8String TAG_TO = new Utf8String("to"); + public static final Utf8String TAG_PERIOD = new Utf8String("period"); + + public void printXml(MetricManager man, String consumer, int verbosity, XMLWriter writer) { + writer.openTag("snapshot"); + writer.attribute(TAG_NAME, name); + writer.attribute(TAG_FROM, fromTime); + writer.attribute(TAG_TO, toTime); + writer.attribute(TAG_PERIOD, period); + + for (Metric m : snapshot.getRegisteredMetrics()) { + if (m instanceof MetricSet) { + man.printXml((MetricSet)m, writer, toTime > fromTime ? (toTime - fromTime) : period, consumer, verbosity); + } + } + + writer.closeTag(); + } + + public String toXml(MetricManager man, String consumer, int verbosity) { + StringWriter str = new StringWriter(); + XMLWriter writer = new XMLWriter(str); + printXml(man, consumer, verbosity, writer); + return str.toString(); + } + + public void addToSnapshot(MetricSnapshot other, int currentTime, boolean reset) { + snapshot.addToSnapshot(other.getMetrics()); + if (reset) { + reset(currentTime); + } + other.toTime = currentTime; + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricSnapshotSet.java b/metrics/src/main/java/com/yahoo/metrics/MetricSnapshotSet.java new file mode 100644 index 00000000000..e65f1db8fb4 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/MetricSnapshotSet.java @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +/** + * Represents two snapshots for the same time period. + */ +public class MetricSnapshotSet { + int count; // Number of times we need to join to building period + // before we have a full time window. + int builderCount; // Number of times we've currently added to the + // building instance. + MetricSnapshot current = null; // The last full period + MetricSnapshot building = null; // The building period + + MetricSnapshotSet(String name, int period, int count, MetricSet source, boolean snapshotUnsetMetrics) { + this.count = count; + this.builderCount = 0; + current = new MetricSnapshot(name, period, source, snapshotUnsetMetrics); + current.reset(0); + if (count != 1) { + building = new MetricSnapshot(name, period, source, snapshotUnsetMetrics); + building.reset(0); + } + } + + MetricSnapshot getNextTarget() { + if (count == 1) { + return current; + } else { + return building; + } + } + + public boolean haveCompletedNewPeriod(int newFromTime) { + if (count == 1) { + current.setToTime(newFromTime); + return true; + } + building.setToTime(newFromTime); + + // If not time to roll yet, just return + if (++builderCount < count) return false; + // Building buffer done. Use that as current and reset current. + MetricSnapshot tmp = current; + current = building; + building = tmp; + building.setFromTime(newFromTime); + building.setToTime(0); + builderCount = 0; + return true; + } + + public boolean timeForAnotherSnapshot(int currentTime) { + int lastTime = getFromTime() + builderCount * getPeriod(); + return currentTime >= lastTime + getPeriod(); + } + + public void reset(int currentTime) { + if (count != 1) building.reset(currentTime); + current.reset(currentTime); + builderCount = 0; + } + + public void recreateSnapshot(MetricSet metrics, boolean copyUnset) { + if (count != 1) building.recreateSnapshot(metrics, copyUnset); + current.recreateSnapshot(metrics, copyUnset); + } + + public void setFromTime(int fromTime) + { + if (count != 1) { + building.setFromTime(fromTime); + } else { + current.setFromTime(fromTime); + } + } + + public int getPeriod() { + return current.getPeriod(); + } + + public int getFromTime() { + return current.getFromTime(); + } + + public int getCount() { + return count; + } + + public MetricSnapshot getSnapshot() { + return getSnapshot(false); + } + + public MetricSnapshot getSnapshot(boolean getBuilding) { + if (getBuilding) { + if (count == 1) { + throw new IllegalStateException("No temporary snapshot for set " + current.name); + } + return building; + } + + return current; + } + + public boolean hasTemporarySnapshot() { + return count > 1; + } + + public String getName() { + return current.getName(); + } + + public int getBuilderCount() { + return builderCount; + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java b/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java new file mode 100644 index 00000000000..4a9ea225bdd --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +public abstract class MetricVisitor { + /** + * Visit a metric set. + * + * @param autoGenerated True for metric sets that are generated on the + * fly such as in sum metrics. + * @return True if you want to visit the content of this metric set. + */ + public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { + return true; + } + + /** + * Callback visitors can use if they need to know the tree traversal of + * metric sets. This function is not called if visitMetricSet returned + * false. + */ + public void doneVisitingMetricSet(MetricSet set) { + } + + /** + * Visit a primitive metric within an accepted metric set. + * + * @return True if you want to continue visiting, false to abort. + */ + public boolean visitPrimitiveMetric(Metric m, boolean autoGenerated) { + return true; + } +}
\ No newline at end of file diff --git a/metrics/src/main/java/com/yahoo/metrics/SimpleMetricSet.java b/metrics/src/main/java/com/yahoo/metrics/SimpleMetricSet.java new file mode 100644 index 00000000000..2eab3c6635e --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/SimpleMetricSet.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +/** + * A final metric set. + */ +public final class SimpleMetricSet extends MetricSet { + + public SimpleMetricSet(String name, String tags, String description) { + this(name, tags, description, null); + } + + public SimpleMetricSet(String name, String tags, String description, MetricSet owner) { + super(name, tags, description, owner); + } + + public SimpleMetricSet(SimpleMetricSet other, Metric.CopyType copyType, MetricSet owner, boolean includeUnused) { + super(other, copyType, owner, includeUnused); + } + + @Override + public Metric clone(Metric.CopyType type, MetricSet owner, boolean includeUnused) + { return new SimpleMetricSet(this, type, owner, includeUnused); } + +} diff --git a/metrics/src/main/java/com/yahoo/metrics/SumMetric.java b/metrics/src/main/java/com/yahoo/metrics/SumMetric.java new file mode 100644 index 00000000000..6335d329836 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/SumMetric.java @@ -0,0 +1,238 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class metrics::CounterMetric + * @ingroup metrics + * + * @brief Counts a value that only moves upwards. + * + * NB! If you have a MetricSet subclass you want to create a sum for, use + * MetricSet itself as the template argument. Otherwise you'll need to override + * clone(...) in order to make it return the correct type for your + * implementation. + */ + +package com.yahoo.metrics; + +import com.yahoo.text.XMLWriter; + +import java.util.ArrayList; +import java.util.List; + +public class SumMetric extends Metric +{ + ArrayList<Metric> metricsToSum = new ArrayList<Metric>(); + + public SumMetric(String name, String tags, String description, MetricSet owner) { + super(name, tags, description, owner); + metricsToSum = new ArrayList<Metric>(); + } + + public SumMetric(SumMetric other, MetricSet owner) { + super(other, owner); + + + if (other.owner == null) { + throw new IllegalStateException( + "Cannot copy a sum metric not registered in a metric set, as " + + "we need to use parent to detect new metrics to point to."); + } + if (owner == null) { + throw new IllegalStateException( + "Cannot copy a sum metric directly. One needs to at least " + + "include metric set above it in order to include metrics " + + "summed."); + } + + metricsToSum.ensureCapacity(other.metricsToSum.size()); + List<String> parentPath = other.owner.getPathVector(); + + for (Metric metric : other.metricsToSum) { + List<String> addendPath = metric.getPathVector(); + MetricSet newAddendParent = owner; + + for (int i = parentPath.size(); i < addendPath.size() - 1; ++i) { + String path = addendPath.get(i); + Metric child = newAddendParent.getMetric(path); + if (child == null) { + throw new IllegalStateException( + "Metric " + path + " in metric set " + + newAddendParent.getPath() + " was expected to " + + "exist. This sounds like a bug."); + } + + newAddendParent = (MetricSet)child; + } + + String path = addendPath.get(addendPath.size() - 1); + Metric child = newAddendParent.getMetric(path); + if (child == null) { + throw new IllegalStateException( + "Metric " + path + " in metric set " + + newAddendParent.getPath() + " was expected to " + + "exist. This sounds like a bug."); + } + + metricsToSum.add(child); + } + } + + @Override + public boolean visit(MetricVisitor visitor, boolean tagAsAutoGenerated) { + if (metricsToSum.isEmpty()) return true; + + Metric sum = generateSum(); + + if (sum == null) { + return true; + } + + if (sum instanceof MetricSet) { + sum.visit(visitor, true); + return true; + } else { + return visitor.visitPrimitiveMetric(sum, true); + } + } + + @Override + public Metric clone(CopyType copyType, MetricSet owner, boolean includeUnused) { + if (copyType == CopyType.CLONE) { + return new SumMetric(this, owner); + } + + Metric sum = null; + for (Metric metric : metricsToSum) { + if (sum == null) { + sum = metric.clone(CopyType.INACTIVE, null, includeUnused); + sum.setName(getName()); + sum.setDescription(getDescription()); + sum.setTags(getTags()); + + if (owner != null) { + owner.registerMetric(sum); + } + } else { + metric.addToPart(sum); + } + } + + return sum; + } + + @Override + public void addToPart(Metric partMetric) { + Metric m = generateSum(); + if (m != null) { + m.addToPart(partMetric); + } + } + + @Override + public void addToSnapshot(Metric snapshotMetric) { + Metric m = generateSum(); + if (m != null) { + m.addToSnapshot(snapshotMetric); + } + } + + public void addMetricToSum(Metric metric) { + if (owner == null) { + throw new IllegalStateException( + "Sum metric needs to be registered in a parent metric set " + + "prior to adding metrics to sum."); + } + if (!metricsToSum.isEmpty() && !(metric.getClass().equals(metricsToSum.get(0).getClass()))) { + throw new IllegalStateException( + "All metrics in a metric set must be of the same type."); + } + + List<String> sumParentPath = owner.getPathVector(); + List<String> addedPath = metric.getPathVector(); + + boolean error = false; + if (addedPath.size() <= sumParentPath.size()) { + error = true; + } else for (int i=0; i<sumParentPath.size(); ++i) { + if (!sumParentPath.get(i).equals(addedPath.get(i))) { + error = true; + break; + } + } + if (error) { + throw new IllegalStateException( + "Metric added to sum is required to be a child of the sum's " + + "direct parent metric set. (Need not be a direct child) " + + "Metric set " + metric.getPath() + " is not a child of " + + owner.getPath()); + } + + ArrayList<Metric> metrics = new ArrayList<Metric>(metricsToSum.size() + 1); + for (int i = 0; i < metricsToSum.size(); ++i) { + metrics.add(metricsToSum.get(i)); + } + metrics.add(metric); + metricsToSum = metrics; + } + + public void removeMetricFromSum(Metric metric) { + metricsToSum.remove(metric); + } + + public Metric generateSum() { + Metric m = clone(CopyType.INACTIVE, null, true); + + if (m != null) { + m.owner = owner; + } + return m; + } + + @Override + public long getLongValue(String id) { + Metric s = generateSum(); + if (s == null) { + return 0; + } + return s.getLongValue(id); + } + + @Override + public double getDoubleValue(String id) { + Metric s = generateSum(); + if (s == null) { + return 0.0; + } + return generateSum().getDoubleValue(id); + } + + @Override + public void logEvent(EventLogger logger, String fullName) { + Metric s = generateSum(); + if (s != null) { + s.logEvent(logger, fullName); + } + } + + @Override + public void printXml(XMLWriter writer, int secondsPassed, int verbosity) { + Metric s = generateSum(); + if (s == null) { + openXMLTag(writer, verbosity); + writer.closeTag(); + return; + } + + generateSum().printXml(writer, secondsPassed, verbosity); + } + + @Override + public boolean used() { + for (Metric m : metricsToSum) { + if (m.used()) return true; + } + return false; + } + + @Override + public void reset() {} +}
\ No newline at end of file diff --git a/metrics/src/main/java/com/yahoo/metrics/Timer.java b/metrics/src/main/java/com/yahoo/metrics/Timer.java new file mode 100644 index 00000000000..12258ebfeb1 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/Timer.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +/** +* @author thomasg +*/ +class Timer { + int secs() { return (int)(milliSecs() / 1000); } + long milliSecs() { return System.currentTimeMillis(); } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/ValueMetric.java b/metrics/src/main/java/com/yahoo/metrics/ValueMetric.java new file mode 100644 index 00000000000..42cea298439 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/ValueMetric.java @@ -0,0 +1,261 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.log.LogLevel; +import com.yahoo.metrics.util.*; +import com.yahoo.text.XMLWriter; +import com.yahoo.text.Utf8String; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Logger; + +/** + * A metric that represents the value of something. + */ +public class ValueMetric<N extends Number> + extends Metric +{ + private static Logger log = Logger.getLogger(ValueMetric.class.getName()); + private static AtomicBoolean hasWarnedOnNonFinite = new AtomicBoolean(false); + + public static interface Value<N extends Number> extends ValueType, HasCopy<Value<N>> { + void add(N v); + void join(Value<N> v2, boolean createAverageOnJoin); + boolean overflow(Value<N> v2); + + int getCount(); + N getMin(); + N getMax(); + N getLast(); + N getTotal(); + Double getAverage(); + + String valueToString(N value); + } + + public boolean hasFlag(int flag) { return (flags & flag) != 0; } + public ValueMetric<N> setFlag(int flag) { flags |= flag; return this; } + + MetricValueKeeper<Value<N>> values; + int flags = 0; + + static int AVERAGE_METRIC = 1; + static int CREATE_AVERAGE_ON_JOIN = 2; + static int UNSET_ON_ZERO_VALUE = 4; + static int LOG_IF_UNSET = 8; + + boolean isAverageMetric() { return hasFlag(AVERAGE_METRIC); } + boolean doCreateAverageOnJoin() { return hasFlag(CREATE_AVERAGE_ON_JOIN); } + boolean isUnsetOnZeroValue() { return hasFlag(UNSET_ON_ZERO_VALUE); } + boolean doLogIfUnset() { return hasFlag(LOG_IF_UNSET); } + JoinBehavior getJoinBehavior() { + return hasFlag(CREATE_AVERAGE_ON_JOIN) ? JoinBehavior.AVERAGE_ON_JOIN + : JoinBehavior.SUM_ON_JOIN; + } + + public ValueMetric<N> averageMetric() { return setFlag(AVERAGE_METRIC); } + public ValueMetric<N> createAverageOnJoin() { return setFlag(CREATE_AVERAGE_ON_JOIN); } + public ValueMetric<N> unsetOnZeroValue() { return setFlag(UNSET_ON_ZERO_VALUE); } + public ValueMetric<N> logIfUnset() { return setFlag(LOG_IF_UNSET); } + + public ValueMetric(String name, String tags, String description, MetricSet owner) { + super(name, tags, description, owner); + //values = new MetricValueSet(); + values = new ThreadLocalDirectoryValueKeeper<Value<N>>(); + } + + public ValueMetric(ValueMetric<N> other, CopyType copyType, MetricSet owner) { + super(other, owner); + if (copyType == CopyType.INACTIVE || other.values instanceof SimpleMetricValueKeeper) { + values = new SimpleMetricValueKeeper<Value<N>>(); + values.set(other.values.get(getJoinBehavior())); + } else { + //values = new MetricValueSet((MetricValueSet) other.values, ((MetricValueSet) other.values).size()); + values = new ThreadLocalDirectoryValueKeeper<Value<N>>(other.values); + } + this.flags = other.flags; + } + + private void logNonFiniteValueWarning() { + if (!hasWarnedOnNonFinite.getAndSet(true)) { + log.log(LogLevel.WARNING, + "Metric '" + getPath() + "' attempted updated with a value that is NaN or " + + "Infinity; update ignored! No further warnings will be printed for " + + "such updates on any metrics, but they can be observed with debug " + + "logging enabled on component " + log.getName()); + } else if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, + "Metric '" + getPath() + "' attempted updated with a value that is NaN or " + + "Infinity; update ignored!"); + } + } + + public void addValue(N v) { + if (v == null) throw new NullPointerException("Cannot have null value"); + if (v instanceof Long) { + LongValue lv = new LongValue(); + lv.add((Long) v); + values.add((Value<N>) lv); + } else { + Double d = (Double)v; + if (d.isNaN() || d.isInfinite()) { + logNonFiniteValueWarning(); + return; + } + DoubleValue dv = new DoubleValue(); + dv.add(d); + values.add((Value<N>) dv); + } + } + + private Value<N> getValues() { + return (Value<N>) values.get(getJoinBehavior()); + } + + @Override + public void addToSnapshot(Metric snapshotMetric) { + Value<N> v = getValues(); + if (v != null) ((ValueMetric<N>)snapshotMetric).join(v, true); + } + + @Override + void addToPart(Metric m) { + Value<N> v = getValues(); + if (v != null) ((ValueMetric<N>) m).join(v, doCreateAverageOnJoin()); + } + + public void join(Value<N> v2, boolean createAverageOnJoin) { + Value<N> tmpVals = getValues(); + if (tmpVals == null) { + if (v2 instanceof LongValue) { + tmpVals = (Value<N>) new LongValue(); + } else { + tmpVals = (Value<N>) new DoubleValue(); + } + } + if (tmpVals.overflow(v2)) { + this.values.reset(); + log.fine("Metric " + getPath() + " overflowed, resetting it."); + return; + } + if (tmpVals.getCount() == 0) { + tmpVals = v2; + } else if (v2.getCount() == 0) { + // Do nothing + } else { + tmpVals.join(v2, createAverageOnJoin); + } + this.values.set(tmpVals); + } + + public static final Utf8String TAG_AVG = new Utf8String("average"); + public static final Utf8String TAG_LAST = new Utf8String("last"); + public static final Utf8String TAG_MIN = new Utf8String("min"); + public static final Utf8String TAG_MAX = new Utf8String("max"); + public static final Utf8String TAG_CNT = new Utf8String("count"); + public static final Utf8String TAG_TOT = new Utf8String("total"); + + @Override + public void printXml(XMLWriter writer, + int timePassed, + int verbosity) + { + Value<N> val = getValues(); + if (!inUse(val) && verbosity < 2) { + return; + } + if (val == null) val = (Value<N>) new LongValue(); + + openXMLTag(writer, verbosity); + writer.attribute(TAG_AVG, new DoubleValue().valueToString(val.getAverage())); + writer.attribute(TAG_LAST, val.valueToString(val.getLast())); + + if (val.getCount() > 0) { + writer.attribute(TAG_MIN, val.valueToString(val.getMin())); + writer.attribute(TAG_MAX, val.valueToString(val.getMax())); + } + writer.attribute(TAG_CNT, val.getCount()); + if (verbosity >= 2) { + writer.attribute(TAG_TOT, val.valueToString(val.getTotal())); + } + + writer.closeTag(); + } + + public Number getValue(String id) { + Value<N> val = getValues(); + if (val == null) return 0; + + if (id.equals("last") || (!isAverageMetric() && id.equals("value"))) { + return val.getLast(); + } else if (id.equals("average") || (isAverageMetric() && id.equals("value"))) { + return val.getAverage(); + } else if (id.equals("count")) { + return val.getCount(); + } else if (id.equals("total")) { + return val.getTotal(); + } else if (id.equals("min")) { + return val.getMin(); + } else if (id.equals("max")) { + return val.getMax(); + } else { + throw new IllegalArgumentException("No id " + id + " in value metric " + getName()); + } + } + + @Override + public long getLongValue(String id) { + return getValue(id).longValue(); + } + + @Override + public double getDoubleValue(String id) { + return getValue(id).doubleValue(); + } + + @Override + public ValueMetric<N> clone(CopyType type, MetricSet owner, boolean includeUnused) { + return new ValueMetric<N>(this, type, owner); + } + + @Override + public void reset() { values.reset(); } + + @Override + public void logEvent(EventLogger logger, String fullName) { + Value<N> val = getValues(); + if (!doLogIfUnset() && !inUse(val)) { + return; + } + logger.value(fullName, val == null ? 0 + : isAverageMetric() ? val.getAverage().doubleValue() + : val.getLast().doubleValue()); + } + + public boolean inUse(Value<?> value) { + return (value != null + && (value.getTotal().longValue() != 0 + || (value.getCount() != 0 && !isUnsetOnZeroValue()))); + } + + @Override + public boolean used() { + return inUse(getValues()); + } + + @Override + public String toString() { + Value<N> tmpVals = getValues(); + if (tmpVals == null) { + tmpVals = (Value<N>) new LongValue(); + } + return ("count=\"" + tmpVals.getCount() + + "\" min=\"" + tmpVals.valueToString(tmpVals.getMin()) + + "\" max=\"" + tmpVals.valueToString(tmpVals.getMax()) + + "\" last=\""+ tmpVals.valueToString(tmpVals.getLast()) + + "\" total=\"" + tmpVals.valueToString(tmpVals.getTotal()) + + "\" average=\"" + new DoubleValue().valueToString(tmpVals.getAverage()) + + "\""); + } + +} diff --git a/metrics/src/main/java/com/yahoo/metrics/VespaLogEventLogger.java b/metrics/src/main/java/com/yahoo/metrics/VespaLogEventLogger.java new file mode 100644 index 00000000000..aa2698c92fb --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/VespaLogEventLogger.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import com.yahoo.log.event.Event; + +/** + * @author thomasg + */ +public class VespaLogEventLogger implements EventLogger { + public void value(String name, double value) { + Event.value(name, value); + } + + public void count(String name, long value) { + Event.count(name, value); + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/HasCopy.java b/metrics/src/main/java/com/yahoo/metrics/util/HasCopy.java new file mode 100644 index 00000000000..497f640e0fe --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/util/HasCopy.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.util; + +public interface HasCopy<T> { + public T copyObject(); +} + diff --git a/metrics/src/main/java/com/yahoo/metrics/util/MetricValueKeeper.java b/metrics/src/main/java/com/yahoo/metrics/util/MetricValueKeeper.java new file mode 100644 index 00000000000..65bd5abdd3e --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/util/MetricValueKeeper.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.util; + +import com.yahoo.metrics.JoinBehavior; +import java.util.Collection; + +/** + * + */ +public interface MetricValueKeeper<Value> { + /** Add a value to a given value. Used when metrics are written to by metric writer. */ + public void add(Value v); + /** Set the value to exactly this, forgetting old value. This operation needs only be supported by value keepers used in inactive snapshots. */ + public void set(Value v); + /** Get the value of the metrics written. If multiple values exist, use given join behavior. */ + public Value get(JoinBehavior joinBehavior); + /** Reset the value of the metrics to zero. */ + public void reset(); + /** Return copy of current view */ + public Collection<Value> getDirectoryView(); +} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/MetricValueSet.java b/metrics/src/main/java/com/yahoo/metrics/util/MetricValueSet.java new file mode 100644 index 00000000000..e04c4b4eb25 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/util/MetricValueSet.java @@ -0,0 +1,141 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class MetricValueSet + * \ingroup metrics + * + * \brief Utility for doing lockless metric updates and reads. + * + * We don't want to use regular locking while updating metrics due to overhead. + * We use this class to make metric updates as safe as possible without + * requiring locks. + * + * It keeps the set of values a metric wants to set. Thus it is templated on + * the class keeping the values. All that is required of this class is that it + * has an empty constructor and a copy constructor. + * + * The locking works, by keeping a set of values, with an active pointer into + * the value vector. Assuming only one thread calls setValues(), it can update + * the active pointer safely. We assume updating the active pointer is a + * non-interruptable operations, such that other threads will see either the new + * or the old value correctly. This should be the case on our platforms. + * + * Due to the reset functionality, it is possible to miss out on a metrics + * added during the reset, but this is very unlikely. For that to happen, when + * someone sets the reset flag, the writer thread must be in setValues(), + * having already passed the check for the reset flag, but not finished setting + * the values yet. + */ + +package com.yahoo.metrics.util; + +import java.util.Collection; +import com.yahoo.metrics.JoinBehavior; + +public class MetricValueSet<Value extends HasCopy<Value> & ValueType> + implements MetricValueKeeper<Value> +{ + private final ValueType[] values; + + private void setIndexedValue(int idx, Value v) { + values[idx] = v; + } + + @SuppressWarnings("unchecked") + private Value getIndexedValue(int idx) { + return (Value) values[idx]; + } + + + private volatile int activeValueIndex; + private volatile boolean reset = false; + + public MetricValueSet() { + values = new ValueType[3]; + activeValueIndex = 0; + } + + public MetricValueSet(MetricValueSet<Value> other, int copyCount) { + values = new ValueType[copyCount]; + activeValueIndex = 0; + setValue(other.getValue()); + } + + /** Get the current values. */ + public Value getValue() { + if (reset) return null; + Value v = getIndexedValue(activeValueIndex); + return (v == null ? null : v.copyObject()); + } + + /** + * Get the current values from the metric. This function should not be + * called in parallel. Only call it from a single thread or use external + * locking. If it returns false, it means the metric have just been reset. + * In which case, redo getValues(), apply the update again, and call + * setValues() again. + */ + public boolean setValue(Value value) { + int nextIndex = (activeValueIndex + 1) % values.length; + if (reset) { + reset = false; + setIndexedValue(nextIndex, null); + activeValueIndex = nextIndex; + return false; + } else { + setIndexedValue(nextIndex, value); + activeValueIndex = nextIndex; + return true; + } + } + + /** + * Retrieve and reset in a single operation, to minimize chance of + * alteration in the process. + */ + public Value getValuesAndReset() { + Value result = getValue(); + reset = true; + return result; + } + + public String toString() { + String retVal = "MetricValueSet(reset=" + reset + + ", active " + activeValueIndex + "["; + for (ValueType n : values) retVal += n + ","; + retVal += "])"; + return retVal; + } + + public int size() { return values.length; } + + public void add(Value v) { + while (true) { + Value current = getValue(); + if (current != null) { + current.add(v); + if (setValue(current)) { + return; + } + } else { + if (setValue(v)) { + return; + } + } + } + } + + public void set(Value v) { + setValue(v); + } + + public Value get(JoinBehavior joinBehavior) { + return getValue(); + } + + public void reset() { reset = true; } + + public Collection<Value> getDirectoryView() { + return null; + } + +} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/SimpleMetricValueKeeper.java b/metrics/src/main/java/com/yahoo/metrics/util/SimpleMetricValueKeeper.java new file mode 100644 index 00000000000..f3a3e64beb6 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/util/SimpleMetricValueKeeper.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.util; + +import java.util.Collection; +import com.yahoo.metrics.JoinBehavior; + +/** + * Simple implementation of the metric value keeper for use with snapshots. Only keeps + * one instance of data, does not need to worry about threads, and should have a small + * memory footprint. + */ +public class SimpleMetricValueKeeper<Value> implements MetricValueKeeper<Value> { + private Value value; + + public SimpleMetricValueKeeper() { + } + + public void add(Value v) { + throw new UnsupportedOperationException("Not supported. This value keeper is not intended to be used for live metrics."); + } + + public void set(Value v) { + this.value = v; + } + + public Value get(JoinBehavior joinBehavior) { + return value; + } + + public void reset() { + value = null; + } + + public Collection<Value> getDirectoryView() { + return null; + } + +} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/ThreadLocalDirectoryValueKeeper.java b/metrics/src/main/java/com/yahoo/metrics/util/ThreadLocalDirectoryValueKeeper.java new file mode 100644 index 00000000000..72dcef377d0 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/util/ThreadLocalDirectoryValueKeeper.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.util; + +import com.yahoo.concurrent.ThreadLocalDirectory; +import com.yahoo.metrics.DoubleValue; +import com.yahoo.metrics.LongValue; +import com.yahoo.metrics.JoinBehavior; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +/** + * Value keeper class using ThreadLocalDirectory to maintain thread safety from multiple threads writing. + */ +public class ThreadLocalDirectoryValueKeeper<Value extends ValueType & HasCopy<Value>> + implements MetricValueKeeper<Value>, ThreadLocalDirectory.ObservableUpdater<Value, Value> +{ + private final ThreadLocalDirectory<Value, Value> directory = new ThreadLocalDirectory<Value, Value>(this); + private List<Value> initialValues = null; + + public ThreadLocalDirectoryValueKeeper() { + } + + public ThreadLocalDirectoryValueKeeper(MetricValueKeeper<Value> other) { + initialValues = new LinkedList<Value>(); + for (Value v : other.getDirectoryView()) initialValues.add(v.copyObject()); + } + + public Value createGenerationInstance(Value value) { + return null; + } + + public Value update(Value current, Value newValue) { + if (current == null) { + if (newValue instanceof DoubleValue) { + current = (Value) new DoubleValue(); + } else { + current = (Value) new LongValue(); + } + } + current.add(newValue); + return current; + } + + public Value copy(Value value) { + return value.copyObject(); + } + + public void add(Value v) { + directory.update(v, directory.getLocalInstance()); + } + + public void set(Value v) { + throw new UnsupportedOperationException("This value keeper is only intended to use with active metrics. Set operation not supported."); + } + + public Value get(JoinBehavior joinBehavior) { + List<Value> values = directory.view(); + if (initialValues != null) values.addAll(initialValues); + if (values.isEmpty()) { + return null; + } + return (Value) values.get(0).join(Collections.<ValueType>unmodifiableList(values), joinBehavior); + } + + public void reset() { + directory.fetch(); + initialValues = null; + } + + public Collection<Value> getDirectoryView() { + return directory.view(); + } +} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/ValueType.java b/metrics/src/main/java/com/yahoo/metrics/util/ValueType.java new file mode 100644 index 00000000000..b777e213406 --- /dev/null +++ b/metrics/src/main/java/com/yahoo/metrics/util/ValueType.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics.util; + +import com.yahoo.metrics.JoinBehavior; + +import java.util.Collection; + +public interface ValueType extends Cloneable { + public void add(ValueType other); + public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior); +} diff --git a/metrics/src/main/metrics-with-dependencies.xml b/metrics/src/main/metrics-with-dependencies.xml new file mode 100644 index 00000000000..f16f31e0398 --- /dev/null +++ b/metrics/src/main/metrics-with-dependencies.xml @@ -0,0 +1,19 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<assembly> + <id>jar-with-dependencies</id> + <formats> + <format>jar</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <dependencySets> + <dependencySet> + <unpack>true</unpack> + <scope>runtime</scope> + </dependencySet> + </dependencySets> + <fileSets> + <fileSet> + <directory>${project.build.outputDirectory}</directory> + </fileSet> + </fileSets> +</assembly> diff --git a/metrics/src/test/java/com/yahoo/metrics/AveragedDoubleValueMetric.java b/metrics/src/test/java/com/yahoo/metrics/AveragedDoubleValueMetric.java new file mode 100644 index 00000000000..933f1ec5fa0 --- /dev/null +++ b/metrics/src/test/java/com/yahoo/metrics/AveragedDoubleValueMetric.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +class AveragedDoubleValueMetric extends ValueMetric<Double> { + + public AveragedDoubleValueMetric(String name, String tags, String description, MetricSet owner) { + super(name, tags, description, owner); + averageMetric(); + createAverageOnJoin(); + } + + public AveragedDoubleValueMetric(AveragedDoubleValueMetric other, CopyType copyType, MetricSet owner) { + super(other, copyType, owner); + } +} diff --git a/metrics/src/test/java/com/yahoo/metrics/AveragedLongValueMetric.java b/metrics/src/test/java/com/yahoo/metrics/AveragedLongValueMetric.java new file mode 100644 index 00000000000..a1abfced548 --- /dev/null +++ b/metrics/src/test/java/com/yahoo/metrics/AveragedLongValueMetric.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +class AveragedLongValueMetric extends ValueMetric<Long> { + + public AveragedLongValueMetric(String name, String tags, String description, MetricSet owner) { + super(name, tags, description, owner); + averageMetric(); + createAverageOnJoin(); + } + + public AveragedLongValueMetric(AveragedLongValueMetric other, CopyType copyType, MetricSet owner) { + super(other, copyType, owner); + } +} diff --git a/metrics/src/test/java/com/yahoo/metrics/CountMetricTest.java b/metrics/src/test/java/com/yahoo/metrics/CountMetricTest.java new file mode 100644 index 00000000000..ad09e289b0f --- /dev/null +++ b/metrics/src/test/java/com/yahoo/metrics/CountMetricTest.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import junit.framework.TestCase; + +/** + * @author thomasg + */ +public class CountMetricTest extends TestCase { + + public void testCountMetric() { + CountMetric m = new CountMetric("test", "tag", "description", null); + assertEquals(false, m.used()); + m.set(100); + assertEquals(true, m.used()); + assertEquals(100, m.getValue()); + m.inc(5); + assertEquals(105, m.getValue()); + m.dec(15); + assertEquals(90, m.getValue()); + + CountMetric m2 = new CountMetric(m, Metric.CopyType.CLONE, null); + m.reset(); + assertEquals(90, m2.getValue()); + assertEquals(0, m.getValue()); + + CountMetric n = new CountMetric("m2", "", "desc", null); + n.set(6); + + n.addToSnapshot(m2); + assertEquals(96, m2.getValue()); + n.addToPart(m); + assertEquals(6, m.getValue()); + + assertEquals(6, n.getValue()); + + assertEquals("<test description=\"description\" count=\"96\"/>\n", m2.toXml(0, 2)); + assertEquals("<test description=\"description\" count=\"96\" average_change_per_second=\"9.60\"/>\n", m2.toXml(10, 2)); + assertEquals(96.0, m2.getDoubleValue("value")); + assertEquals(96, m2.getLongValue("value")); + } + +} diff --git a/metrics/src/test/java/com/yahoo/metrics/DummyTimer.java b/metrics/src/test/java/com/yahoo/metrics/DummyTimer.java new file mode 100644 index 00000000000..8264d8e00eb --- /dev/null +++ b/metrics/src/test/java/com/yahoo/metrics/DummyTimer.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +/** + * @author thomasg + */ +public class DummyTimer extends Timer { + long ms = 0; + + public long milliSecs() { return ms; } +} diff --git a/metrics/src/test/java/com/yahoo/metrics/MetricManagerTest.java b/metrics/src/test/java/com/yahoo/metrics/MetricManagerTest.java new file mode 100644 index 00000000000..dad476d2bdb --- /dev/null +++ b/metrics/src/test/java/com/yahoo/metrics/MetricManagerTest.java @@ -0,0 +1,192 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import org.junit.Test; + +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.*; + +/** + * @author thomasg + */ +public class MetricManagerTest { + + @Test + public void requireThatManagerCanBeStopped() throws InterruptedException { + final MetricManager manager = new MetricManager(new DummyTimer()); + Thread managerThread = new Thread(manager); + managerThread.start(); + Thread stopThread = new Thread(new Runnable() { + + @Override + public void run() { + manager.stop(); + } + }); + stopThread.start(); + stopThread.join(TimeUnit.SECONDS.toMillis(60)); + assertFalse(stopThread.isAlive()); + managerThread.join(TimeUnit.SECONDS.toMillis(60)); + assertFalse(managerThread.isAlive()); + } + + @Test + public void requireThatManagerCanBeStoppedBeforeStarting() throws InterruptedException { + final MetricManager manager = new MetricManager(new DummyTimer()); + Thread thread = new Thread(new Runnable() { + + @Override + public void run() { + manager.stop(); + } + }); + thread.start(); + thread.join(TimeUnit.SECONDS.toMillis(60)); + assertFalse(thread.isAlive()); + } + + @Test + public void requireThatManagerCanNotBeStartedAfterItHasBeenStopped() { + final MetricManager manager = new MetricManager(new DummyTimer()); + manager.stop(); + try { + manager.run(); + fail(); + } catch (IllegalStateException e) { + + } + } + + @Test + public void testSimpleLogging() { + DummyTimer timer = new DummyTimer(); + MetricManager manager = new MetricManager(timer); + SimpleMetricSet sms = new SimpleMetricSet("foo", "", "", null); + manager.registerMetric(sms); + + CountMetric bar = new CountMetric("bar", "", "", sms); + bar.inc(3); + + manager.addMetricToConsumer("log", "foo.bar"); + + DummyEventLogger logger = new DummyEventLogger(); + manager.tick(logger); + //assertEquals("CNT foo_bar: 0\n", logger.output); + + timer.ms = 300000; + + manager.tick(logger); + assertEquals("CNT foo_bar: 0\nCNT foo_bar: 3\n", logger.output.toString()); + } + + @Test + public void testSumLogging() { + DummyTimer timer = new DummyTimer(); + MetricManager manager = new MetricManager(timer); + SimpleMetricSet sms = new SimpleMetricSet("foo", "", "", null); + manager.registerMetric(sms); + + CountMetric v1 = new CountMetric("v1", "", "", sms); + CountMetric v2 = new CountMetric("v2", "", "", sms); + CountMetric v3 = new CountMetric("v3", "", "", sms); + SumMetric sum = new SumMetric("sum", "", "", sms); + + sum.addMetricToSum(v1); + sum.addMetricToSum(v2); + sum.addMetricToSum(v3); + + v1.inc(3); + v2.inc(1); + v3.inc(2); + + manager.addMetricToConsumer("log", "foo.sum"); + + DummyEventLogger logger = new DummyEventLogger(); + manager.tick(logger); + assertEquals("CNT foo_sum: 0\n", logger.output.toString()); + + timer.ms = 300000; + + manager.tick(logger); + assertEquals("CNT foo_sum: 0\nCNT foo_sum: 6\n", logger.output.toString()); + } + + @Test + public void testSumAndSetLogging() { + DummyTimer timer = new DummyTimer(); + timer.ms = 1000000; + MetricManager manager = new MetricManager(timer); + SimpleMetricSet sms = new SimpleMetricSet("foo", "", "", null); + manager.registerMetric(sms); + + SimpleMetricSet v1 = new SimpleMetricSet("v1", "", "", sms); + SimpleMetricSet v2 = new SimpleMetricSet("v2", "", "", sms); + + CountMetric v1_a = new CountMetric("a", "", "", v1); + SummedLongValueMetric v1_b = new SummedLongValueMetric("b", "", "", v1); + AveragedLongValueMetric v1_c = new AveragedLongValueMetric("c", "", "", v1); + + CountMetric v2_a = new CountMetric("a", "", "", v2); + SummedLongValueMetric v2_b = new SummedLongValueMetric("b", "", "", v2); + AveragedLongValueMetric v2_c = new AveragedLongValueMetric("c", "", "", v2); + + SumMetric sum = new SumMetric("sum", "", "", sms); + + sum.addMetricToSum(v1); + sum.addMetricToSum(v2); + + v1_a.inc(3); + v1_b.addValue((long)1); + v1_c.addValue((long)4); + v2_a.inc(2); + v2_b.addValue((long)2); + v2_c.addValue((long)8); + + manager.addMetricToConsumer("log", "foo.sum.a"); + manager.addMetricToConsumer("log", "foo.sum.b"); + manager.addMetricToConsumer("log", "foo.sum.c"); + + { + DummyEventLogger logger = new DummyEventLogger(); + assertEquals(300000, manager.tick(logger)); + assertEquals("CNT foo_sum_a: 0\n", logger.output.toString()); + } + + timer.ms = 1300000; + + { + DummyEventLogger logger = new DummyEventLogger(); + assertEquals(300000, manager.tick(logger)); + assertEquals("CNT foo_sum_a: 5\nVAL foo_sum_b: 3.0\nVAL foo_sum_c: 6.0\n", logger.output.toString()); + } + + v1_a.inc(4); + v1_b.addValue((long)2); + v1_b.addValue((long)6); + v2_b.addValue((long)4); + + timer.ms = 1600000; + + { + DummyEventLogger logger = new DummyEventLogger(); + assertEquals(300000, manager.tick(logger)); + assertEquals("CNT foo_sum_a: 9\nVAL foo_sum_b: 10.0\n", logger.output.toString()); + } + } + + private static class DummyEventLogger implements EventLogger { + + StringBuilder output = new StringBuilder(); + + @Override + public void value(String name, double value) { + output.append("VAL " + name + ": " + value + "\n"); + } + + @Override + public void count(String name, long value) { + output.append("CNT " + name + ": " + value + "\n"); + } + } +} diff --git a/metrics/src/test/java/com/yahoo/metrics/MetricSetTest.java b/metrics/src/test/java/com/yahoo/metrics/MetricSetTest.java new file mode 100644 index 00000000000..c9b85566cea --- /dev/null +++ b/metrics/src/test/java/com/yahoo/metrics/MetricSetTest.java @@ -0,0 +1,338 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import junit.framework.TestCase; + + +public class MetricSetTest extends TestCase { + + class TestMetricVisitor extends MetricVisitor { + String output = ""; + int setsToVisit; + + TestMetricVisitor(int setsToVisit) { + this.setsToVisit = setsToVisit; + } + + public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { + output += "[" + (autoGenerated ? "*" : "") + set.getName() + "]\n"; + if (setsToVisit > 0) { + --setsToVisit; + return true; + } + return false; + } + public boolean visitPrimitiveMetric(Metric m, boolean autoGenerated) { + output += (autoGenerated ? "*" : "") + m.getName() + "\n"; + return true; + } + } + + public void testNormalUsage() + { + // Set up some metrics to test.. + MetricSet set = new SimpleMetricSet("a", "foo", ""); + SummedDoubleValueMetric v1 = new SummedDoubleValueMetric("c", "foo", "", set); + AveragedLongValueMetric v2 = new AveragedLongValueMetric("b", "", "", set); + CountMetric v3 = new CountMetric("d", "bar", "", set); + MetricSet set2 = new SimpleMetricSet("e", "bar", "", set); + CountMetric v4 = new CountMetric("f", "foo", "", set2); + + // Give them some values + v1.addValue(4.2); + v2.addValue((long)8); + v3.inc(); + v4.inc(3); + + // Check that we can register through registerMetric function too. + CountMetric v5 = new CountMetric("g", "", "", null); + set.registerMetric(v5); + v5.inc(3); + v5.dec(); + + // Check that getMetric works, and doesn't return copy. + AveragedLongValueMetric v2copy = (AveragedLongValueMetric)set.getMetric("b"); + assertNotNull(v2copy); + v2copy.addValue((long)9); + + assertNull(set.getMetric("nonexisting")); + assertNull(set.getMetric("non.existing")); + + // Check that paths are set + MetricSet topSet = new SimpleMetricSet("top", "", "", null); + topSet.registerMetric(set); + + assertEquals("a", set.getPath()); + assertEquals("a.c", v1.getPath()); + assertEquals("a.b", v2.getPath()); + assertEquals("a.d", v3.getPath()); + assertEquals("a.e", set2.getPath()); + assertEquals("a.e.f", v4.getPath()); + assertEquals("a.g", v5.getPath()); + assertEquals("a.b", v2copy.getPath()); + + // Verify XML output. Should be in register order. + assertEquals( + "<a>\n\n" + + " <c average=\"4.20\" last=\"4.20\" min=\"4.20\" max=\"4.20\" count=\"1\" total=\"4.20\"/>\n\n" + + " <b average=\"8.50\" last=\"9\" min=\"8\" max=\"9\" count=\"2\" total=\"17\"/>\n\n" + + " <d count=\"1\"/>\n\n" + + " <e>\n"+ + " <f count=\"3\"/>\n" + + " </e>\n\n" + + " <g count=\"2\"/>\n\n"+ + "</a>\n", set.toXml(0,2)); + + // Verify that visiting works. That you get all metrics if you answer + // true to all sets, and that you don't get members of sets you answer + // false to get. + { + TestMetricVisitor visitor = new TestMetricVisitor(2); + set.visit(visitor, false); + assertEquals("[a]\nc\nb\nd\n[e]\nf\ng\n", visitor.output); + } + { + TestMetricVisitor visitor = new TestMetricVisitor(1); + set.visit(visitor, false); + assertEquals("[a]\nc\nb\nd\n[e]\ng\n", visitor.output); + } + { + TestMetricVisitor visitor = new TestMetricVisitor(0); + set.visit(visitor, false); + assertEquals("[a]\n", visitor.output); + } + } + + class MyTimer extends Timer { + long timeNow = 0; + + @Override + long milliSecs() { return timeNow; } + } + + @SuppressWarnings("rawtypes") + public void testSumInSet() { + MetricSet ms_a = new SimpleMetricSet("a", "", "", null); + MyTimer timer = new MyTimer(); + timer.timeNow = 1; + + MetricManager manager = new MetricManager(timer); + manager.registerMetric(ms_a); + + manager.takeSnapshots(1); + + SumMetric s_a = new SumMetric("sum", "", "", ms_a); + MetricSet ms_b = new SimpleMetricSet("b", "", "", ms_a); + s_a.addMetricToSum(ms_b); + MetricSet ms_c = new SimpleMetricSet("c", "", "", ms_a); + s_a.addMetricToSum(ms_c); + + SumMetric s_b_a = new SumMetric("sum", "", "", ms_b); + SumMetric s_c_a = new SumMetric("sum", "", "", ms_c); + + AveragedDoubleValueMetric v1_b = new AveragedDoubleValueMetric("v1", "", "", ms_b); + s_b_a.addMetricToSum(v1_b); + AveragedDoubleValueMetric v3_b = new AveragedDoubleValueMetric("v3", "", "", ms_b); + s_b_a.addMetricToSum(v3_b); + + AveragedDoubleValueMetric v1_c = new AveragedDoubleValueMetric("v1", "", "", ms_c); + s_c_a.addMetricToSum(v1_c); + AveragedDoubleValueMetric v2_c = new AveragedDoubleValueMetric("v2", "", "", ms_c); + s_c_a.addMetricToSum(v2_c); + + v1_b.addValue(1.0); + v3_b.addValue(2.0); + v1_c.addValue(3.0); + v2_c.addValue(4.0); + + SimpleMetricSet sms = (SimpleMetricSet)s_a.generateSum(); + ValueMetric av_sum = (ValueMetric)sms.getMetric("sum"); + ValueMetric av_1 = (ValueMetric)sms.getMetric("v1"); + ValueMetric av_2 = (ValueMetric)sms.getMetric("v2"); + ValueMetric av_3 = (ValueMetric)sms.getMetric("v3"); + + assertEquals(10.0, av_sum.getDoubleValue("total")); + assertEquals(4, av_sum.getLongValue("count")); + assertEquals(4.0, av_1.getDoubleValue("total")); + assertEquals(4.0, av_2.getDoubleValue("total")); + assertEquals(2.0, av_3.getDoubleValue("total")); + + timer.timeNow = 301; + manager.takeSnapshots(301); + + MetricSnapshot snapshot = manager.getMetricSnapshot(300); + + assertEquals("<snapshot name=\"5 minute\" from=\"301\" to=\"301\" period=\"300\">\n" + + "\n" + + " <a>\n" + + " <sum>\n" + + " <sum average=\"2.50\" last=\"4.00\" min=\"1.00\" max=\"4.00\" count=\"4\" total=\"10.00\"/>\n" + + " <v1 average=\"2.00\" last=\"3.00\" min=\"1.00\" max=\"3.00\" count=\"2\" total=\"4.00\"/>\n" + + " <v3 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" + + " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" + + " </sum>\n" + + " <b>\n" + + " <sum average=\"1.50\" last=\"2.00\" min=\"1.00\" max=\"2.00\" count=\"2\" total=\"3.00\"/>\n" + + " <v1 average=\"1.00\" last=\"1.00\" min=\"1.00\" max=\"1.00\" count=\"1\" total=\"1.00\"/>\n" + + " <v3 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" + + " </b>\n" + + " <c>\n" + + " <sum average=\"3.50\" last=\"4.00\" min=\"3.00\" max=\"4.00\" count=\"2\" total=\"7.00\"/>\n" + + " <v1 average=\"3.00\" last=\"3.00\" min=\"3.00\" max=\"3.00\" count=\"1\" total=\"3.00\"/>\n" + + " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" + + " </c>\n" + + " </a>\n" + + "\n" + + "</snapshot>\n", snapshot.toXml(manager, "", 2)); + + assertEquals("<snapshot name=\"1 hour\" from=\"0\" to=\"301\" period=\"3600\">\n" + + "\n" + + " <a>\n" + + " <sum>\n" + + " <sum average=\"2.50\" last=\"4.00\" min=\"1.00\" max=\"4.00\" count=\"4\" total=\"10.00\"/>\n" + + " <v1 average=\"2.00\" last=\"3.00\" min=\"1.00\" max=\"3.00\" count=\"2\" total=\"4.00\"/>\n" + + " <v3 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" + + " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" + + " </sum>\n" + + " <b>\n" + + " <sum average=\"1.50\" last=\"2.00\" min=\"1.00\" max=\"2.00\" count=\"2\" total=\"3.00\"/>\n" + + " <v1 average=\"1.00\" last=\"1.00\" min=\"1.00\" max=\"1.00\" count=\"1\" total=\"1.00\"/>\n" + + " <v3 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" + + " </b>\n" + + " <c>\n" + + " <sum average=\"3.50\" last=\"4.00\" min=\"3.00\" max=\"4.00\" count=\"2\" total=\"7.00\"/>\n" + + " <v1 average=\"3.00\" last=\"3.00\" min=\"3.00\" max=\"3.00\" count=\"1\" total=\"3.00\"/>\n" + + " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" + + " </c>\n" + + " </a>\n" + + "\n" + + "</snapshot>\n", manager.getMetricSnapshot(3600, true).toXml(manager, "", 2)); + + v1_b.addValue(2.0); + v3_b.addValue(3.0); + v1_c.addValue(4.0); + v2_c.addValue(8.0); + + CountMetric c1_b = new CountMetric("cnt", "", "", ms_b); + c1_b.inc(500); + + timer.timeNow = 601; + manager.takeSnapshots(601); + + // THIS IS WRONG. See ticket + assertEquals("<snapshot name=\"5 minute\" from=\"601\" to=\"601\" period=\"300\">\n" + + "\n" + + " <a>\n" + + " <sum>\n" + + " <sum average=\"4.25\" last=\"8.00\" min=\"2.00\" max=\"8.00\" count=\"4\" total=\"17.00\"/>\n" + + " <v1 average=\"3.00\" last=\"4.00\" min=\"2.00\" max=\"4.00\" count=\"2\" total=\"6.00\"/>\n" + + " <v3 average=\"3.00\" last=\"3.00\" min=\"3.00\" max=\"3.00\" count=\"1\" total=\"3.00\"/>\n" + + " <v2 average=\"8.00\" last=\"8.00\" min=\"8.00\" max=\"8.00\" count=\"1\" total=\"8.00\"/>\n" + + " <cnt count=\"500\" average_change_per_second=\"1.67\"/>\n" + + " </sum>\n" + + " <b>\n" + + " <sum average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" + + " <v1 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" + + " <v3 average=\"3.00\" last=\"3.00\" min=\"3.00\" max=\"3.00\" count=\"1\" total=\"3.00\"/>\n" + + " <cnt count=\"500\" average_change_per_second=\"1.67\"/>\n" + + " </b>\n" + + " <c>\n" + + " <sum average=\"6.00\" last=\"8.00\" min=\"4.00\" max=\"8.00\" count=\"2\" total=\"12.00\"/>\n" + + " <v1 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" + + " <v2 average=\"8.00\" last=\"8.00\" min=\"8.00\" max=\"8.00\" count=\"1\" total=\"8.00\"/>\n" + + " </c>\n" + + " </a>\n" + + "\n" + + "</snapshot>\n", manager.getMetricSnapshot(300, false).toXml(manager, "", 2)); + + assertEquals("<snapshot name=\"1 hour\" from=\"0\" to=\"601\" period=\"3600\">\n" + + "\n" + + " <a>\n" + + " <sum>\n" + + " <sum average=\"3.38\" last=\"8.00\" min=\"1.00\" max=\"8.00\" count=\"8\" total=\"27.00\"/>\n" + + " <v1 average=\"2.50\" last=\"4.00\" min=\"1.00\" max=\"4.00\" count=\"4\" total=\"10.00\"/>\n" + + " <v3 average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" + + " <v2 average=\"6.00\" last=\"8.00\" min=\"4.00\" max=\"8.00\" count=\"2\" total=\"12.00\"/>\n" + + " <cnt count=\"500\" average_change_per_second=\"0.83\"/>\n" + + " </sum>\n" + + " <b>\n" + + " <sum average=\"2.00\" last=\"3.00\" min=\"1.00\" max=\"3.00\" count=\"4\" total=\"8.00\"/>\n" + + " <v1 average=\"1.50\" last=\"2.00\" min=\"1.00\" max=\"2.00\" count=\"2\" total=\"3.00\"/>\n" + + " <v3 average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" + + " <cnt count=\"500\" average_change_per_second=\"0.83\"/>\n" + + " </b>\n" + + " <c>\n" + + " <sum average=\"4.75\" last=\"8.00\" min=\"3.00\" max=\"8.00\" count=\"4\" total=\"19.00\"/>\n" + + " <v1 average=\"3.50\" last=\"4.00\" min=\"3.00\" max=\"4.00\" count=\"2\" total=\"7.00\"/>\n" + + " <v2 average=\"6.00\" last=\"8.00\" min=\"4.00\" max=\"8.00\" count=\"2\" total=\"12.00\"/>\n" + + " </c>\n" + + " </a>\n" + + "\n" + + "</snapshot>\n", manager.getMetricSnapshot(3600, true).toXml(manager, "", 2)); + + AveragedDoubleValueMetric v4_c = new AveragedDoubleValueMetric("v4", "", "", ms_c); + s_c_a.addMetricToSum(v4_c); + + v2_c.addValue(4.0); + v4_c.addValue(12.0); + c1_b.inc(3000); + + timer.timeNow = 901; + manager.takeSnapshots(901); + + assertEquals("<snapshot name=\"5 minute\" from=\"901\" to=\"901\" period=\"300\">\n" + + "\n" + + " <a>\n" + + " <sum>\n" + + " <sum average=\"8.00\" last=\"12.00\" min=\"4.00\" max=\"12.00\" count=\"2\" total=\"16.00\"/>\n" + + " <v1 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" + + " <v3 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" + + " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" + + " <cnt count=\"3000\" average_change_per_second=\"10.00\"/>\n" + + " <v4 average=\"12.00\" last=\"12.00\" min=\"12.00\" max=\"12.00\" count=\"1\" total=\"12.00\"/>\n" + + " </sum>\n" + + " <b>\n" + + " <sum average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" + + " <v1 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" + + " <v3 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" + + " <cnt count=\"3000\" average_change_per_second=\"10.00\"/>\n" + + " </b>\n" + + " <c>\n" + + " <sum average=\"8.00\" last=\"12.00\" min=\"4.00\" max=\"12.00\" count=\"2\" total=\"16.00\"/>\n" + + " <v1 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" + + " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" + + " <v4 average=\"12.00\" last=\"12.00\" min=\"12.00\" max=\"12.00\" count=\"1\" total=\"12.00\"/>\n" + + " </c>\n" + + " </a>\n" + + "\n" + + "</snapshot>\n", manager.getMetricSnapshot(300, false).toXml(manager, "", 2)); + + assertEquals("<snapshot name=\"1 hour\" from=\"0\" to=\"901\" period=\"3600\">\n" + + "\n" + + " <a>\n" + + " <sum>\n" + + " <sum average=\"4.30\" last=\"12.00\" min=\"1.00\" max=\"12.00\" count=\"10\" total=\"43.00\"/>\n" + + " <v1 average=\"2.50\" last=\"4.00\" min=\"1.00\" max=\"4.00\" count=\"4\" total=\"10.00\"/>\n" + + " <v3 average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" + + " <v2 average=\"5.33\" last=\"4.00\" min=\"4.00\" max=\"8.00\" count=\"3\" total=\"16.00\"/>\n" + + " <cnt count=\"3500\" average_change_per_second=\"3.88\"/>\n" + + " <v4 average=\"12.00\" last=\"12.00\" min=\"12.00\" max=\"12.00\" count=\"1\" total=\"12.00\"/>\n" + + " </sum>\n" + + " <b>\n" + + " <sum average=\"2.00\" last=\"3.00\" min=\"1.00\" max=\"3.00\" count=\"4\" total=\"8.00\"/>\n" + + " <v1 average=\"1.50\" last=\"2.00\" min=\"1.00\" max=\"2.00\" count=\"2\" total=\"3.00\"/>\n" + + " <v3 average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" + + " <cnt count=\"3500\" average_change_per_second=\"3.88\"/>\n" + + " </b>\n" + + " <c>\n" + + " <sum average=\"5.83\" last=\"12.00\" min=\"3.00\" max=\"12.00\" count=\"6\" total=\"35.00\"/>\n" + + " <v1 average=\"3.50\" last=\"4.00\" min=\"3.00\" max=\"4.00\" count=\"2\" total=\"7.00\"/>\n" + + " <v2 average=\"5.33\" last=\"4.00\" min=\"4.00\" max=\"8.00\" count=\"3\" total=\"16.00\"/>\n" + + " <v4 average=\"12.00\" last=\"12.00\" min=\"12.00\" max=\"12.00\" count=\"1\" total=\"12.00\"/>\n" + + " </c>\n" + + " </a>\n" + + "\n" + + "</snapshot>\n", manager.getMetricSnapshot(3600, true).toXml(manager, "", 2)); + } + +} // metrics diff --git a/metrics/src/test/java/com/yahoo/metrics/SumMetricTest.java b/metrics/src/test/java/com/yahoo/metrics/SumMetricTest.java new file mode 100644 index 00000000000..09204a66454 --- /dev/null +++ b/metrics/src/test/java/com/yahoo/metrics/SumMetricTest.java @@ -0,0 +1,153 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import junit.framework.TestCase; + +/** + * @author thomasg + */ +public class SumMetricTest extends TestCase { + public void testSummedCountMetric() { + MetricSet parent = new SimpleMetricSet("parent", "", ""); + SumMetric sum = new SumMetric("foo", "", "foodesc", parent); + + CountMetric v1 = new CountMetric("aa", "", "", parent); + CountMetric v2 = new CountMetric("bb", "", "", parent); + CountMetric v3 = new CountMetric("cc", "", "", parent); + + sum.addMetricToSum(v1); + sum.addMetricToSum(v2); + sum.addMetricToSum(v3); + + // Give them some values + v1.inc(3); + v2.inc(7); + v3.inc(5); + + assertEquals("<foo description=\"foodesc\" count=\"15\"/>\n", sum.toXml(0, 2)); + assertEquals(15, sum.getLongValue("value")); + + v3.inc(5); + + assertEquals("<foo description=\"foodesc\" count=\"20\"/>\n", sum.toXml(0, 2)); + assertEquals(20, sum.getLongValue("value")); + } + + public void testSummedValueMetric() { + MetricSet parent = new SimpleMetricSet("parent", "", ""); + SumMetric sum = new SumMetric("foo", "", "foodesc", parent); + + SummedDoubleValueMetric v1 = new SummedDoubleValueMetric("aa", "", "", parent); + SummedDoubleValueMetric v2 = new SummedDoubleValueMetric("bb", "", "", parent); + SummedDoubleValueMetric v3 = new SummedDoubleValueMetric("cc", "", "", parent); + + sum.addMetricToSum(v1); + sum.addMetricToSum(v2); + sum.addMetricToSum(v3); + + v1.addValue(3.0); + v1.addValue(2.0); + + v2.addValue(7.0); + + v3.addValue(5.0); + v3.addValue(10.0); + + assertEquals("<foo description=\"foodesc\" average=\"17.00\" last=\"19.00\" min=\"2.00\" max=\"10.00\" count=\"5\" total=\"85.00\"/>\n", sum.toXml(0,2)); + + assertEquals(19, sum.getLongValue("value")); + assertEquals(2, sum.getLongValue("min")); + assertEquals(10, sum.getLongValue("max")); + } + + + public void testAveragedValueMetric() { + MetricSet parent = new SimpleMetricSet("parent", "", ""); + SumMetric sum = new SumMetric("foo", "", "foodesc", parent); + + AveragedDoubleValueMetric v1 = new AveragedDoubleValueMetric("aa", "", "", parent); + AveragedDoubleValueMetric v2 = new AveragedDoubleValueMetric("bb", "", "", parent); + AveragedDoubleValueMetric v3 = new AveragedDoubleValueMetric("cc", "", "", parent); + + sum.addMetricToSum(v1); + sum.addMetricToSum(v2); + sum.addMetricToSum(v3); + + v1.addValue(3.0); + v1.addValue(2.0); + + v2.addValue(7.0); + + v3.addValue(5.0); + v3.addValue(10.0); + + assertEquals("<foo description=\"foodesc\" average=\"5.40\" last=\"10.00\" min=\"2.00\" max=\"10.00\" count=\"5\" total=\"27.00\"/>\n", sum.toXml(0,2)); + + assertEquals(5.40, sum.getDoubleValue("value")); + assertEquals(2.0, sum.getDoubleValue("min")); + assertEquals(10.0, sum.getDoubleValue("max")); + } + + public void testMetricSet() { + MetricSet parent = new SimpleMetricSet("parent", "", ""); + SumMetric sum = new SumMetric("foo", "", "bar", parent); + + MetricSet set1 = new SimpleMetricSet("a", "", "", parent); + MetricSet set2 = new SimpleMetricSet("b", "", "", parent); + + SummedLongValueMetric v1 = new SummedLongValueMetric("c", "", "", set1); + SummedLongValueMetric v2 = new SummedLongValueMetric("c", "", "", set2); + CountMetric v3 = new CountMetric("e", "", "", set1); + CountMetric v4 = new CountMetric("e", "", "", set2); + + sum.addMetricToSum(set1); + sum.addMetricToSum(set2); + + // Give them some values + v1.addValue((long)3); + v2.addValue((long)7); + v3.inc(2); + v4.inc(); + + // Verify XML output. Should be in register order. + assertEquals( + "<foo description=\"bar\">\n\n" + + " <c average=\"10.00\" last=\"10\" min=\"3\" max=\"7\" count=\"2\" total=\"20\"/>\n\n" + + " <e count=\"3\"/>\n\n" + + "</foo>\n", sum.toXml(0,2)); + + } + + + public void testRemove() + { + MetricSet parent = new SimpleMetricSet("parent", "", ""); + SumMetric sum = new SumMetric("foo", "", "foodesc", parent); + + SummedDoubleValueMetric v1 = new SummedDoubleValueMetric("aa", "", "", parent); + SummedDoubleValueMetric v2 = new SummedDoubleValueMetric("bb", "", "", parent); + SummedDoubleValueMetric v3 = new SummedDoubleValueMetric("cc", "", "", parent); + + sum.addMetricToSum(v1); + sum.addMetricToSum(v2); + sum.addMetricToSum(v3); + + v1.addValue(3.0); + v1.addValue(2.0); + + v2.addValue(7.0); + + v3.addValue(5.0); + v3.addValue(10.0); + + sum.removeMetricFromSum(v1); + + assertEquals("<foo description=\"foodesc\" average=\"14.50\" last=\"17.00\" min=\"5.00\" max=\"10.00\" count=\"3\" total=\"43.50\"/>\n", sum.toXml(0,2)); + } + + public void testEmpty() { + SumMetric sum = new SumMetric("foo", "", "foodesc", null); + assertEquals("<foo description=\"foodesc\"/>\n", sum.toXml(0,2)); + } + +} diff --git a/metrics/src/test/java/com/yahoo/metrics/SummedDoubleValueMetric.java b/metrics/src/test/java/com/yahoo/metrics/SummedDoubleValueMetric.java new file mode 100644 index 00000000000..a869348b332 --- /dev/null +++ b/metrics/src/test/java/com/yahoo/metrics/SummedDoubleValueMetric.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +class SummedDoubleValueMetric extends ValueMetric<Double> { + + public SummedDoubleValueMetric(String name, String tags, String description, MetricSet owner) { + super(name, tags, description, owner); + } + + public SummedDoubleValueMetric(SummedDoubleValueMetric other, CopyType copyType, MetricSet owner) { + super(other, copyType, owner); + } +} diff --git a/metrics/src/test/java/com/yahoo/metrics/SummedLongValueMetric.java b/metrics/src/test/java/com/yahoo/metrics/SummedLongValueMetric.java new file mode 100644 index 00000000000..d08abe6f242 --- /dev/null +++ b/metrics/src/test/java/com/yahoo/metrics/SummedLongValueMetric.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +class SummedLongValueMetric extends ValueMetric<Long> { + + public SummedLongValueMetric(String name, String tags, String description, MetricSet owner) { + super(name, tags, description, owner); + } + + public SummedLongValueMetric(SummedLongValueMetric other, CopyType copyType, MetricSet owner) { + super(other, copyType, owner); + } +} + diff --git a/metrics/src/test/java/com/yahoo/metrics/ValueMetricTest.java b/metrics/src/test/java/com/yahoo/metrics/ValueMetricTest.java new file mode 100644 index 00000000000..4758888b0c8 --- /dev/null +++ b/metrics/src/test/java/com/yahoo/metrics/ValueMetricTest.java @@ -0,0 +1,182 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.metrics; + +import junit.framework.TestCase; + +public class ValueMetricTest extends TestCase { + public void testAveragedDoubleValueMetric() + { + AveragedDoubleValueMetric m = new AveragedDoubleValueMetric("test", "tag", "description", null); + m.addValue(100.0); + assertEquals("count=\"1\" min=\"100.00\" max=\"100.00\" last=\"100.00\" total=\"100.00\" average=\"100.00\"", m.toString()); + m.addValue(100.0); + assertEquals("count=\"2\" min=\"100.00\" max=\"100.00\" last=\"100.00\" total=\"200.00\" average=\"100.00\"", m.toString()); + m.addValue(40.0); + assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m.toString()); + + AveragedDoubleValueMetric m2 = new AveragedDoubleValueMetric(m, Metric.CopyType.CLONE, null); + assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m.toString()); + assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m2.toString()); + m.reset(); + assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString()); + assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m2.toString()); + + AveragedDoubleValueMetric m3 = new AveragedDoubleValueMetric("test", "tag", "description", null); + m3.addValue(200.0); + m3.addValue(100.0); + m3.addValue(400.0); + + AveragedDoubleValueMetric m4 = new AveragedDoubleValueMetric("test", "tag", "description", null); + m3.addValue(50.0); + m3.addValue(100.0); + m3.addValue(2000.0); + + AveragedDoubleValueMetric sum = new AveragedDoubleValueMetric(m2, Metric.CopyType.INACTIVE, null); + m3.addToPart(sum); + m4.addToPart(sum); + assertEquals("count=\"9\" min=\"40.00\" max=\"2000.00\" last=\"2000.00\" total=\"3090.00\" average=\"343.33\"", sum.toString()); + + AveragedDoubleValueMetric snapshot = new AveragedDoubleValueMetric(m2, Metric.CopyType.INACTIVE, null); + m3.addToSnapshot(snapshot); + m4.addToSnapshot(snapshot); + assertEquals("count=\"9\" min=\"40.00\" max=\"2000.00\" last=\"2000.00\" total=\"3090.00\" average=\"343.33\"", snapshot.toString()); + + assertEquals("<test description=\"description\" average=\"343.33\" last=\"2000.00\" min=\"40.00\" max=\"2000.00\" count=\"9\" total=\"3090.00\"/>\n", sum.toXml(0, 2)); + + assertEquals(80.0, m2.getDoubleValue("value")); + assertEquals(80.0, m2.getDoubleValue("average")); + assertEquals(40.0, m2.getDoubleValue("min")); + assertEquals(100.0, m2.getDoubleValue("max")); + assertEquals(40.0, m2.getDoubleValue("last")); + assertEquals(3.0, m2.getDoubleValue("count")); + assertEquals(240.0, m2.getDoubleValue("total")); + + assertEquals(80, m2.getLongValue("value")); + assertEquals(80, m2.getLongValue("average")); + assertEquals(40, m2.getLongValue("min")); + assertEquals(100, m2.getLongValue("max")); + assertEquals(40, m2.getLongValue("last")); + assertEquals(3, m2.getLongValue("count")); + assertEquals(240, m2.getLongValue("total")); + } + + public void testDoubleValueMetricNotUpdatedOnNaN() { + AveragedDoubleValueMetric m = new AveragedDoubleValueMetric("test", "tag", "description", null); + m.addValue(Double.NaN); + assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString()); + } + + public void testDoubleValueMetricNotUpdatedOnInfinity() { + AveragedDoubleValueMetric m = new AveragedDoubleValueMetric("test", "tag", "description", null); + m.addValue(Double.POSITIVE_INFINITY); + assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString()); + } + + public void testSummedDoubleValueMetric() + { + SummedDoubleValueMetric m = new SummedDoubleValueMetric("test", "tag", "description", null); + m.addValue(100.0); + assertEquals("count=\"1\" min=\"100.00\" max=\"100.00\" last=\"100.00\" total=\"100.00\" average=\"100.00\"", m.toString()); + m.addValue(100.0); + assertEquals("count=\"2\" min=\"100.00\" max=\"100.00\" last=\"100.00\" total=\"200.00\" average=\"100.00\"", m.toString()); + m.addValue(40.0); + assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m.toString()); + + SummedDoubleValueMetric m2 = new SummedDoubleValueMetric(m, Metric.CopyType.CLONE, null); + assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m.toString()); + assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m2.toString()); + m.reset(); + assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString()); + assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m2.toString()); + + SummedDoubleValueMetric m3 = new SummedDoubleValueMetric("test", "tag", "description", null); + m3.addValue(200.0); + m3.addValue(100.0); + m3.addValue(400.0); + + SummedDoubleValueMetric m4 = new SummedDoubleValueMetric("test", "tag", "description", null); + m4.addValue(2000.0); + + SummedDoubleValueMetric sum = new SummedDoubleValueMetric(m2, Metric.CopyType.INACTIVE, null); + m3.addToPart(sum); + m4.addToPart(sum); + assertEquals("count=\"7\" min=\"40.00\" max=\"2000.00\" last=\"2440.00\" total=\"16193.33\" average=\"2313.33\"", sum.toString()); + + SummedDoubleValueMetric snapshot = new SummedDoubleValueMetric(m2, Metric.CopyType.INACTIVE, null); + m3.addToSnapshot(snapshot); + m4.addToSnapshot(snapshot); + assertEquals("count=\"7\" min=\"40.00\" max=\"2000.00\" last=\"2000.00\" total=\"2940.00\" average=\"420.00\"", snapshot.toString()); + + assertEquals("<test description=\"description\" average=\"2313.33\" last=\"2440.00\" min=\"40.00\" max=\"2000.00\" count=\"7\" total=\"16193.33\"/>\n", sum.toXml(0, 2)); + + assertEquals(40.0, m2.getDoubleValue("value")); + assertEquals(80.0, m2.getDoubleValue("average")); + assertEquals(40.0, m2.getDoubleValue("min")); + assertEquals(100.0, m2.getDoubleValue("max")); + assertEquals(40.0, m2.getDoubleValue("last")); + assertEquals(3.0, m2.getDoubleValue("count")); + assertEquals(240.0, m2.getDoubleValue("total")); + + assertEquals(40, m2.getLongValue("value")); + assertEquals(80, m2.getLongValue("average")); + assertEquals(40, m2.getLongValue("min")); + assertEquals(100, m2.getLongValue("max")); + assertEquals(40, m2.getLongValue("last")); + assertEquals(3, m2.getLongValue("count")); + assertEquals(240, m2.getLongValue("total")); + } + + public void testAveragedLongValueMetric() + { + AveragedLongValueMetric m = new AveragedLongValueMetric("test", "tag", "description", null); + + assertEquals(0l, m.getLongValue("max")); + assertEquals(0l, m.getLongValue("min")); + + m.addValue((long)100); + assertEquals("count=\"1\" min=\"100\" max=\"100\" last=\"100\" total=\"100\" average=\"100.00\"", m.toString()); + m.addValue((long)100); + assertEquals("count=\"2\" min=\"100\" max=\"100\" last=\"100\" total=\"200\" average=\"100.00\"", m.toString()); + m.addValue((long)40); + assertEquals("count=\"3\" min=\"40\" max=\"100\" last=\"40\" total=\"240\" average=\"80.00\"", m.toString()); + + AveragedLongValueMetric m2 = new AveragedLongValueMetric(m, Metric.CopyType.CLONE, null); + assertEquals("count=\"3\" min=\"40\" max=\"100\" last=\"40\" total=\"240\" average=\"80.00\"", m.toString()); + assertEquals("count=\"3\" min=\"40\" max=\"100\" last=\"40\" total=\"240\" average=\"80.00\"", m2.toString()); + m.reset(); + assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString()); + assertEquals("count=\"3\" min=\"40\" max=\"100\" last=\"40\" total=\"240\" average=\"80.00\"", m2.toString()); + + AveragedLongValueMetric m3 = new AveragedLongValueMetric("test", "tag", "description", null); + m3.addValue((long)200); + m3.addValue((long)100); + m3.addValue((long)400); + + AveragedLongValueMetric m4 = new AveragedLongValueMetric("test", "tag", "description", null); + m3.addValue((long)50); + m3.addValue((long)100); + m3.addValue((long)2000); + + AveragedLongValueMetric sum = new AveragedLongValueMetric(m2, Metric.CopyType.INACTIVE, null); + m3.addToPart(sum); + m4.addToPart(sum); + assertEquals("count=\"9\" min=\"40\" max=\"2000\" last=\"2000\" total=\"3090\" average=\"343.33\"", sum.toString()); + + AveragedLongValueMetric snapshot = new AveragedLongValueMetric(m2, Metric.CopyType.INACTIVE, null); + m3.addToSnapshot(snapshot); + m4.addToSnapshot(snapshot); + assertEquals("count=\"9\" min=\"40\" max=\"2000\" last=\"2000\" total=\"3090\" average=\"343.33\"", snapshot.toString()); + + assertEquals("<test description=\"description\" average=\"343.33\" last=\"2000\" min=\"40\" max=\"2000\" count=\"9\" total=\"3090\"/>\n", sum.toXml(0, 2)); + + assertEquals(80, m2.getLongValue("value")); + assertEquals(80, m2.getLongValue("average")); + assertEquals(40, m2.getLongValue("min")); + assertEquals(100, m2.getLongValue("max")); + assertEquals(40, m2.getLongValue("last")); + assertEquals(3, m2.getLongValue("count")); + assertEquals(240, m2.getLongValue("total")); + } + + +} diff --git a/metrics/src/tests/.gitignore b/metrics/src/tests/.gitignore new file mode 100644 index 00000000000..453dd9e4ab3 --- /dev/null +++ b/metrics/src/tests/.gitignore @@ -0,0 +1,7 @@ +.*.swp +.depend +.~ +Makefile +testrunner +/test.vlog +metrics_testrunner_app diff --git a/metrics/src/tests/CMakeLists.txt b/metrics/src/tests/CMakeLists.txt new file mode 100644 index 00000000000..b84701e9487 --- /dev/null +++ b/metrics/src/tests/CMakeLists.txt @@ -0,0 +1,18 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(metrics_testrunner_app + SOURCES + testrunner.cpp + countmetrictest.cpp + valuemetrictest.cpp + metricsettest.cpp + summetrictest.cpp + metricmanagertest.cpp + loadmetrictest.cpp + snapshottest.cpp + stresstest.cpp + metrictest.cpp + DEPENDS + metrics + vdstestlib +) +vespa_add_test(NAME metrics_testrunner_app COMMAND metrics_testrunner_app) diff --git a/metrics/src/tests/countmetrictest.cpp b/metrics/src/tests/countmetrictest.cpp new file mode 100644 index 00000000000..1f39113474b --- /dev/null +++ b/metrics/src/tests/countmetrictest.cpp @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/objects/floatingpointtype.h> +#include <vespa/metrics/countmetric.h> + +using vespalib::Double; + +namespace metrics { + +struct CountMetricTest : public CppUnit::TestFixture { + void testLongCountMetric(); + + CPPUNIT_TEST_SUITE(CountMetricTest); + CPPUNIT_TEST(testLongCountMetric); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(CountMetricTest); + +void CountMetricTest::testLongCountMetric() +{ + LongCountMetric m("test", "tag", "description"); + m.set(100); + CPPUNIT_ASSERT_EQUAL(uint64_t(100), m.getValue()); + m.inc(5); + CPPUNIT_ASSERT_EQUAL(uint64_t(105), m.getValue()); + m.dec(15); + CPPUNIT_ASSERT_EQUAL(uint64_t(90), m.getValue()); + LongCountMetric m2(m); + CPPUNIT_ASSERT_EQUAL(uint64_t(90), m2.getValue()); + m.reset(); + CPPUNIT_ASSERT_EQUAL(uint64_t(0), m.getValue()); + + LongCountMetric n("m2", "", "desc"); + n.set(6); + CPPUNIT_ASSERT_EQUAL(uint64_t(6), n.getValue()); + + LongCountMetric o = m2 + n; + CPPUNIT_ASSERT_EQUAL(uint64_t(96), o.getValue()); + + o = m2 - n; + CPPUNIT_ASSERT_EQUAL(uint64_t(84), o.getValue()); + + std::string expected( + "test count=84"); + CPPUNIT_ASSERT_EQUAL(expected, o.toString()); + + CPPUNIT_ASSERT_EQUAL(Double(84), Double(o.getDoubleValue("value"))); + CPPUNIT_ASSERT_EQUAL(int64_t(84), o.getLongValue("value")); +} + +} diff --git a/metrics/src/tests/loadmetrictest.cpp b/metrics/src/tests/loadmetrictest.cpp new file mode 100644 index 00000000000..b0755fa46e9 --- /dev/null +++ b/metrics/src/tests/loadmetrictest.cpp @@ -0,0 +1,138 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/metrics/loadmetric.h> +#include <vespa/metrics/valuemetric.h> +#include <vespa/vdstestlib/cppunit/macros.h> + +namespace metrics { + +struct LoadTypeSetImpl : public LoadTypeSet { + LoadTypeSetImpl() { + push_back(LoadType(0, "default")); + } + LoadTypeSetImpl& add(uint32_t id, const char* name) { + push_back(LoadType(id, name)); + return *this; + } + const LoadType& operator[](const std::string& name) const { + for (uint32_t i=0; i<size(); ++i) { + const LoadType& lt(LoadTypeSet::operator[](i)); + if (lt.getName() == name) return lt; + } + CPPUNIT_FAIL("No load type with name " + name); + return operator[](0); // Should never get here + } +}; + +struct LoadMetricTest : public CppUnit::TestFixture { + void testNormalUsage(); + void testClone(Metric::CopyType); + void testInactiveCopy() { testClone(Metric::INACTIVE); } + void testActiveCopy() { testClone(Metric::CLONE); } + void testAdding(); + + CPPUNIT_TEST_SUITE(LoadMetricTest); + CPPUNIT_TEST(testNormalUsage); + CPPUNIT_TEST(testActiveCopy); + CPPUNIT_TEST(testInactiveCopy); + CPPUNIT_TEST(testAdding); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(LoadMetricTest); + +void +LoadMetricTest::testNormalUsage() +{ + LoadTypeSetImpl loadTypes; + loadTypes.add(32, "foo").add(1000, "bar"); + LoadMetric<LongValueMetric> metric( + loadTypes, LongValueMetric("put", "", "Put")); +} + +namespace { + struct MyMetricSet : public MetricSet { + LongAverageMetric metric; + + MyMetricSet(MetricSet* owner = 0) + : MetricSet("tick", "", "", owner), + metric("tack", "", "", this) + { + } + + Metric* clone(std::vector<Metric::LP>& ownerList, + CopyType copyType, + MetricSet* owner, + bool includeUnused = false) const + { + if (copyType != CLONE) { + return MetricSet::clone(ownerList, copyType, owner, + includeUnused); + } + return (new MyMetricSet(owner))->assignValues(*this); + } + }; +} + +void +LoadMetricTest::testClone(Metric::CopyType copyType) +{ + LoadTypeSetImpl loadTypes; + loadTypes.add(32, "foo").add(1000, "bar"); + MetricSet top("top", "", ""); + MyMetricSet myset; + LoadMetric<MyMetricSet> metric(loadTypes, myset, &top); + metric[loadTypes["foo"]].metric.addValue(5); + + std::vector<Metric::LP> ownerList; + MetricSet::UP copy(dynamic_cast<MetricSet*>( + top.clone(ownerList, copyType, 0, true))); + CPPUNIT_ASSERT(copy.get()); + + std::string expected = + "top:\n" + " tick:\n" + " sum:\n" + " tack average=5 last=5 min=5 max=5 count=1 total=5\n" + " default:\n" + " tack average=0 last=0 count=0 total=0\n" + " foo:\n" + " tack average=5 last=5 min=5 max=5 count=1 total=5\n" + " bar:\n" + " tack average=0 last=0 count=0 total=0"; + + CPPUNIT_ASSERT_EQUAL(expected, std::string(copy->toString(true))); +} + +void +LoadMetricTest::testAdding() +{ + LoadTypeSetImpl loadTypes; + loadTypes.add(32, "foo").add(1000, "bar"); + MetricSet top("top", "", ""); + MyMetricSet myset; + LoadMetric<MyMetricSet> metric(loadTypes, myset, &top); + metric[loadTypes["foo"]].metric.addValue(5); + + std::vector<Metric::LP> ownerList; + MetricSet::UP copy(dynamic_cast<MetricSet*>( + top.clone(ownerList, Metric::INACTIVE, 0, false))); + CPPUNIT_ASSERT(copy.get()); + + top.reset(); + + top.addToSnapshot(*copy, ownerList); + + std::string expected = + "top:\n" + " tick:\n" + " sum:\n" + " tack average=5 last=5 min=5 max=5 count=1 total=5\n" + " foo:\n" + " tack average=5 last=5 min=5 max=5 count=1 total=5"; + + CPPUNIT_ASSERT_EQUAL(expected, std::string(copy->toString(true))); + +} + +} // documentapi diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp new file mode 100644 index 00000000000..31471cc40ed --- /dev/null +++ b/metrics/src/tests/metricmanagertest.cpp @@ -0,0 +1,1280 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/metrics/metrics.h> +#include <vespa/metrics/xmlwriter.h> +#include <vespa/metrics/jsonwriter.h> +#include <vespa/metrics/textwriter.h> +#include <vespa/metrics/printutils.h> +#include <vespa/metrics/state_api_adapter.h> +#include <vespa/vdstestlib/cppunit/macros.h> + +#include <fstream> +#include <sstream> + +LOG_SETUP(".test.metricmanager"); + +namespace metrics { + +struct MetricManagerTest : public CppUnit::TestFixture { + void testConsumerVisitor(); + void testSnapshots(); + void testPrintUtils(); + void testXmlOutput(); + void testJsonOutput(); + void jsonOutputSupportsMultipleDimensions(); + void jsonOutputCanNestDimensionsFromMultipleMetricSets(); + void jsonOutputCanHaveMultipleSetsWithSameName(); + void testTextOutput(); + void textOutputSupportsDimensions(); + void testUpdateHooks(); + + CPPUNIT_TEST_SUITE(MetricManagerTest); + CPPUNIT_TEST(testConsumerVisitor); + CPPUNIT_TEST(testSnapshots); + CPPUNIT_TEST(testPrintUtils); + CPPUNIT_TEST(testXmlOutput); + CPPUNIT_TEST(testJsonOutput); + CPPUNIT_TEST(jsonOutputSupportsMultipleDimensions); + CPPUNIT_TEST(jsonOutputCanNestDimensionsFromMultipleMetricSets); + CPPUNIT_TEST(jsonOutputCanHaveMultipleSetsWithSameName); + CPPUNIT_TEST(testTextOutput); + CPPUNIT_TEST(textOutputSupportsDimensions); + CPPUNIT_TEST(testUpdateHooks); + CPPUNIT_TEST_SUITE_END(); + +public: + // Ugly indirection hack caused by some tests using friended internals of + // MetricManager that aren't accessible to "freestanding" fixtures. So we + // get the test to do the necessary poking and prodding for us instead. + void takeSnapshots(MetricManager& mm, time_t timeToProcess) { + mm.takeSnapshots(mm.getMetricLock(), timeToProcess); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(MetricManagerTest); + +namespace { + +struct SubMetricSet : public MetricSet +{ + DoubleValueMetric val1; + DoubleValueMetric val2; + SumMetric<DoubleValueMetric> valsum; + + SubMetricSet(const Metric::String & name, MetricSet* owner) + : MetricSet(name, "sub", "sub desc", owner, "sub"), + val1("val1", "tag4 snaptest", "val1 desc", this), + val2("val2", "tag5", "val2 desc", this), + valsum("valsum", "tag4 snaptest", "valsum desc", this) + { + valsum.addMetricToSum(val1); + valsum.addMetricToSum(val2); + } +}; + +struct MultiSubMetricSet +{ + MetricSet set; + LongCountMetric count; + SubMetricSet a; + SubMetricSet b; + SumMetric<MetricSet> sum; + + MultiSubMetricSet(MetricSet* owner) + : set("multisub", "multisub", "", owner), + count("count", "snaptest", "counter", &set), + a("a", &set), + b("b", &set), + sum("sum", "snaptest", "", &set) + { + sum.addMetricToSum(a); + sum.addMetricToSum(b); + } + +}; + +struct TestMetricSet +{ + MetricSet set; + DoubleValueMetric val1; + DoubleValueMetric val2; + DoubleValueMetric val3; + DoubleValueMetric val4; + DoubleValueMetric val5; + DoubleValueMetric val6; + DoubleValueMetric val7; + DoubleValueMetric val8; + SubMetricSet val9; + MultiSubMetricSet val10; + + TestMetricSet() + : set("temp", "test", "desc of test set"), + val1("val1", "tag1", "val1 desc", &set), + val2("val2", "tag1 tag2", "val2 desc", &set), + val3("val3", "tag2 tag3", "val3 desc", &set), + val4("val4", "tag3", "val4 desc", &set), + val5("val5", "tag2", "val5 desc", &set), + val6("val6", "tag4 snaptest", "val6 desc", &set), + val7("val7", "", "val7 desc", &set), + val8("val8", "tag6", "val8 desc", &set), + val9("sub", &set), + val10(&set) + { + } + +}; + +struct MetricNameVisitor : public MetricVisitor { + std::ostringstream ost; + bool debug; + + MetricNameVisitor(bool debug_ = false) : debug(debug_) {} + + virtual bool visitMetricSet(const MetricSet& metricSet, + bool autoGenerated) + { + if (debug) { + ost << "<" << (autoGenerated ? "*" : "") + << metricSet.getPath() << ">\n"; + } + return true; + } + virtual void doneVisitingMetricSet(const MetricSet& metricSet) { + if (debug) { + ost << "</" << metricSet.getPath() << ">\n"; + } + } + + virtual bool visitMetric(const Metric& m, bool autoGenerated) { + ost << (autoGenerated ? "*" : "") << m.getPath() << "\n"; + return true; + } + + std::string toString() const { return ost.str(); } +}; + +} + +namespace { + std::pair<std::string, std::string> getMatchedMetrics( + const vespalib::string& config) + { + FastOS_ThreadPool pool(256 * 1024); + TestMetricSet mySet; + MetricManager mm; + mm.registerMetric(mm.getMetricLock(), mySet.set); + mm.init(config, pool); + MetricNameVisitor visitor; + + /** Take a copy to verify clone works. + std::list<Metric::SP> ownerList; + MetricSet::UP copy(dynamic_cast<MetricSet*>( + mm.getMetrics().clone(ownerList))); + mm.visit(*copy, visitor, "consumer"); + */ + + MetricLockGuard g(mm.getMetricLock()); + mm.visit(g, mm.getActiveMetrics(g), visitor, "consumer"); + MetricManager::ConsumerSpec::SP consumerSpec( + mm.getConsumerSpec(g, "consumer")); + return std::pair<std::string, std::string>( + visitor.toString(), + consumerSpec.get() ? consumerSpec->toString() + : "Non-existing consumer"); + } +} + +#define ASSERT_CONSUMER_MATCH(name, expected, config) \ +{ \ + std::pair<std::string, std::string> consumerMatch( \ + getMatchedMetrics(config)); \ + CPPUNIT_ASSERT_EQUAL_MSG(name + std::string(": ") + consumerMatch.second, \ + "\n" + expected, "\n" + consumerMatch.first); \ +} + +void MetricManagerTest::testConsumerVisitor() +{ + // Add one tag and a name, check that we get all three. + ASSERT_CONSUMER_MATCH("testAddTagAndName", std::string( + "temp.val1\n" + "temp.val2\n" + "temp.val4\n" + "*temp.multisub.sum.val1\n" + ), + "raw:" + "consumer[1]\n" + "consumer[0].name consumer\n" + "consumer[0].tags[1]\n" + "consumer[0].tags[0] tag1\n" + "consumer[0].addedmetrics[2]\n" + "consumer[0].addedmetrics[0] temp.val4\n" + "consumer[0].addedmetrics[1] temp.multisub.sum.val1\n" + ); + // Add two tags, remove one + ASSERT_CONSUMER_MATCH("testAddAndRemoveTag", std::string( + "temp.val1\n" + "temp.val4\n" + ), + "raw:" + "consumer[1]\n" + "consumer[0].name consumer\n" + "consumer[0].tags[2]\n" + "consumer[0].tags[0] tag1\n" + "consumer[0].tags[1] tag3\n" + "consumer[0].removedtags[1]\n" + "consumer[0].removedtags[0] tag2\n" + ); + // Test simple wildcards + ASSERT_CONSUMER_MATCH("testWildCards", std::string( + "temp.val1\n" + "temp.val2\n" + "temp.val3\n" + "temp.val4\n" + "temp.val5\n" + "temp.val6\n" + "temp.val7\n" + "temp.val8\n" + ), + "raw:" + "consumer[1]\n" + "consumer[0].name consumer\n" + "consumer[0].addedmetrics[1]\n" + "consumer[0].addedmetrics[0] temp.*\n" + "consumer[0].removedmetrics[2]\n" + "consumer[0].removedmetrics[0] temp.sub.*\n" + "consumer[0].removedmetrics[1] temp.multisub.*\n" + ); + // Test more wildcards + ASSERT_CONSUMER_MATCH("testWildCards2", std::string( + "temp.sub.val1\n" + ), + "raw:" + "consumer[1]\n" + "consumer[0].name consumer\n" + "consumer[0].addedmetrics[1]\n" + "consumer[0].addedmetrics[0] temp.*.val1\n" + ); + // test adding all + ASSERT_CONSUMER_MATCH("testAddAll", std::string( + "metricmanager.periodichooklatency\n" + "metricmanager.snapshothooklatency\n" + "metricmanager.resetlatency\n" + "metricmanager.snapshotlatency\n" + "metricmanager.sleeptime\n" + "temp.val1\n" + "temp.val2\n" + "temp.val3\n" + "temp.val4\n" + "temp.val5\n" + "temp.val6\n" + "temp.val7\n" + "temp.val8\n" + "temp.sub.val1\n" + "temp.sub.val2\n" + "*temp.sub.valsum\n" + "temp.multisub.count\n" + "temp.multisub.a.val1\n" + "temp.multisub.a.val2\n" + "*temp.multisub.a.valsum\n" + "temp.multisub.b.val1\n" + "temp.multisub.b.val2\n" + "*temp.multisub.b.valsum\n" + "*temp.multisub.sum.val1\n" + "*temp.multisub.sum.val2\n" + "*temp.multisub.sum.valsum\n" + ), + "raw:" + "consumer[1]\n" + "consumer[0].name consumer\n" + "consumer[0].addedmetrics[1]\n" + "consumer[0].addedmetrics[0] *\n" + ); + // test adding all using tags + ASSERT_CONSUMER_MATCH("testAddAll2", std::string( + "temp.val1\n" + "temp.val2\n" + "temp.val3\n" + "temp.val4\n" + "temp.val5\n" + "temp.val6\n" + "temp.val7\n" + "temp.val8\n" + "temp.sub.val1\n" + "temp.sub.val2\n" + "*temp.sub.valsum\n" + "temp.multisub.count\n" + "temp.multisub.a.val1\n" + "temp.multisub.a.val2\n" + "*temp.multisub.a.valsum\n" + "temp.multisub.b.val1\n" + "temp.multisub.b.val2\n" + "*temp.multisub.b.valsum\n" + "*temp.multisub.sum.val1\n" + "*temp.multisub.sum.val2\n" + "*temp.multisub.sum.valsum\n" + ), + "raw:" + "consumer[1]\n" + "consumer[0].name consumer\n" + "consumer[0].tags[1]\n" + "consumer[0].tags[0] *\n" + ); + // Test that all metrics are added when a metricsset is added by name + ASSERT_CONSUMER_MATCH("testSpecifiedSetName", std::string( + "temp.sub.val1\n" + "temp.sub.val2\n" + "*temp.sub.valsum\n" + ), + "raw:" + "consumer[1]\n" + "consumer[0].name consumer\n" + "consumer[0].addedmetrics[1]\n" + "consumer[0].addedmetrics[0] temp.sub\n" + ); + // Test that all metrics are added when a metricsset is added by tag + ASSERT_CONSUMER_MATCH("testSpecifiedSetTag", std::string( + "temp.sub.val1\n" + "temp.sub.val2\n" + "*temp.sub.valsum\n" + "temp.multisub.a.val1\n" + "temp.multisub.a.val2\n" + "*temp.multisub.a.valsum\n" + "temp.multisub.b.val1\n" + "temp.multisub.b.val2\n" + "*temp.multisub.b.valsum\n" + ), + "raw:" + "consumer[1]\n" + "consumer[0].name consumer\n" + "consumer[0].tags[1]\n" + "consumer[0].tags[0] sub\n" + ); + // Test that all metrics are added from the set except those with a + // certain tag. + ASSERT_CONSUMER_MATCH("testSpecifiedSetTagWithExceptionTags", std::string( + "temp.val1\n" + "temp.val4\n" + "temp.val7\n" + "temp.val8\n" + "temp.sub.val2\n" + "temp.multisub.count\n" + "temp.multisub.a.val2\n" + "temp.multisub.b.val2\n" + "*temp.multisub.sum.val2\n" + ), + "raw:" + "consumer[1]\n" + "consumer[0].name consumer\n" + "consumer[0].tags[1]\n" + "consumer[0].tags[0] test\n" + "consumer[0].removedtags[2]\n" + "consumer[0].removedtags[0] tag2\n" + "consumer[0].removedtags[1] tag4\n" + ); +}; + +namespace { + struct FakeTimer : public MetricManager::Timer { + time_t _time; + FakeTimer(time_t startTime = 0) : _time(startTime) {} + virtual time_t getTime() const { return _time; } + }; + + struct BriefValuePrinter : public MetricVisitor { + uint32_t count; + std::ostringstream ost; + + BriefValuePrinter() : count(0), ost() {} + + virtual bool visitMetric(const Metric& metric, bool) { + if (++count > 1) ost << ","; + //ost << metric.getPath() << ":"; + ost << metric.getDoubleValue("value"); + return true; + } + }; + + bool waitForTimeProcessed(const MetricManager& mm, + time_t processtime, uint32_t timeout = 120) + { + uint32_t lastchance = time(0) + timeout; + while (time(0) < lastchance) { + if (mm.getLastProcessedTime() >= processtime) return true; + mm.timeChangedNotification(); + FastOS_Thread::Sleep(10); + } + return false; + } + + std::string dumpAllSnapshots(const MetricManager& mm, + const std::string& consumer) + { + std::ostringstream ost; + ost << "\n"; + { + MetricLockGuard metricLock(mm.getMetricLock()); + BriefValuePrinter briefValuePrinter; + mm.visit(metricLock, mm.getActiveMetrics(metricLock), + briefValuePrinter, consumer); + ost << "Current: " << briefValuePrinter.ost.str() << "\n"; + } + { + MetricLockGuard metricLock(mm.getMetricLock()); + BriefValuePrinter briefValuePrinter; + mm.visit(metricLock, mm.getTotalMetricSnapshot(metricLock), + briefValuePrinter, consumer); + ost << "Total: " << briefValuePrinter.ost.str() << "\n"; + } + std::vector<uint32_t> periods; + { + MetricLockGuard metricLock(mm.getMetricLock()); + periods = mm.getSnapshotPeriods(metricLock); + } + for (uint32_t i=0; i<periods.size(); ++i) { + MetricLockGuard metricLock(mm.getMetricLock()); + const MetricSnapshotSet& set(mm.getMetricSnapshotSet( + metricLock, periods[i])); + ost << set.getName() << "\n"; + uint32_t count = 0; + for (uint32_t j=0; j<2; ++j) { + if (set.getCount() == 1 && j == 1) continue; + const MetricSnapshot& snap(set.getSnapshot(j == 1)); + BriefValuePrinter briefValuePrinter; + mm.visit(metricLock, snap, briefValuePrinter, consumer); + ost << " " << count++ << " " << &snap.getMetrics() << ": " + << briefValuePrinter.ost.str() << "\n"; + } + } + return ost.str(); + } + +} + +#define ASSERT_VALUES(mm, period, expected) \ +{ \ + MetricLockGuard lockGuard(mm.getMetricLock()); \ + BriefValuePrinter briefValuePrinter; \ + if (period == -1) { \ + mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), \ + briefValuePrinter, "snapper"); \ + } else if (period == 0) { \ + mm.visit(lockGuard, mm.getTotalMetricSnapshot(lockGuard), \ + briefValuePrinter, "snapper"); \ + } else { \ + mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, period), \ + briefValuePrinter, "snapper"); \ + } \ + CPPUNIT_ASSERT_EQUAL_MSG(dumpAllSnapshots(mm, "snapper"), \ + std::string(expected), briefValuePrinter.ost.str()); \ +} + +#define ASSERT_PROCESS_TIME(mm, time) \ +{ \ + LOG(info, "Waiting for processed time %u.", time); \ + bool gotToCorrectProgress = waitForTimeProcessed(mm, time); \ + if (!gotToCorrectProgress) \ + CPPUNIT_FAIL("Failed to get to processed time within timeout"); \ +} + +void MetricManagerTest::testSnapshots() +{ + FastOS_ThreadPool pool(256 * 1024); + FakeTimer* timer = new FakeTimer(1000); + std::unique_ptr<MetricManager::Timer> timerImpl(timer); + TestMetricSet mySet; + MetricManager mm(std::move(timerImpl)); + { + MetricLockGuard lockGuard(mm.getMetricLock()); + mm.registerMetric(lockGuard, mySet.set); + } + mm.init("raw:" + "consumer[2]\n" + "consumer[0].name snapper\n" + "consumer[0].tags[1]\n" + "consumer[0].tags[0] snaptest\n" + "consumer[1].name log\n" + "consumer[1].tags[1]\n" + "consumer[1].tags[0] snaptest\n", + pool); + MetricNameVisitor visitor; + { + MetricLockGuard lockGuard(mm.getMetricLock()); + mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), visitor, "snapper"); + MetricManager::ConsumerSpec::SP consumerSpec( + mm.getConsumerSpec(lockGuard, "snapper")); + CPPUNIT_ASSERT_EQUAL_MSG(consumerSpec.get() ? consumerSpec->toString() + : "Non-existing consumer", + std::string("\n" + "temp.val6\n" + "temp.sub.val1\n" + "*temp.sub.valsum\n" + "temp.multisub.count\n" + "temp.multisub.a.val1\n" + "*temp.multisub.a.valsum\n" + "temp.multisub.b.val1\n" + "*temp.multisub.b.valsum\n" + "*temp.multisub.sum.val1\n" + "*temp.multisub.sum.val2\n" + "*temp.multisub.sum.valsum\n" + ), + "\n" + visitor.toString()); + } + // Initially, there should be no metrics logged + ASSERT_PROCESS_TIME(mm, 1000); + ASSERT_VALUES(mm, 5 * 60, ""); + + // Adding metrics done in first five minutes. + mySet.val6.addValue(2); + mySet.val9.val1.addValue(4); + mySet.val10.count.inc(); + mySet.val10.a.val1.addValue(7); + mySet.val10.a.val2.addValue(2); + mySet.val10.b.val1.addValue(1); + timer->_time += 5 * 60; + ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60); + ASSERT_VALUES(mm, 5 * 60, "2,4,4,1,7,9,1,1,8,2,10"); + ASSERT_VALUES(mm, 60 * 60, ""); + ASSERT_VALUES(mm, 0 * 60, "2,4,4,1,7,9,1,1,8,2,10"); + + // Adding metrics done in second five minute period. Total should + // be updated to account for both + mySet.val6.addValue(4); + mySet.val9.val1.addValue(5); + mySet.val10.count.inc(); + mySet.val10.a.val1.addValue(8); + mySet.val10.a.val2.addValue(3); + mySet.val10.b.val1.addValue(2); + timer->_time += 5 * 60; + ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 2); + ASSERT_VALUES(mm, 5 * 60, "4,5,5,1,8,11,2,2,10,3,13"); + ASSERT_VALUES(mm, 60 * 60, ""); + ASSERT_VALUES(mm, 0 * 60, "4,5,5,2,8,11,2,2,10,3,13"); + + //std::cerr << dumpAllSnapshots(mm, "snapper") << "\n"; + + // Adding another five minute period where nothing have happened. + // Metric for last 5 minutes should be 0. + timer->_time += 5 * 60; + ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 3); + ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); + ASSERT_VALUES(mm, 60 * 60, ""); + ASSERT_VALUES(mm, 0 * 60, "4,5,5,2,8,11,2,2,10,3,13"); + + // Advancing time to 60 minute period, we should create a proper + // 60 minute period timer. + mySet.val6.addValue(6); + for (uint32_t i=0; i<9; ++i) { // 9 x 5 minutes. Avoid snapshot bumping + // due to taking snapshots in the past + timer->_time += 5 * 60; + ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * (4 + i)); + } + ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); + ASSERT_VALUES(mm, 60 * 60, "6,5,5,2,8,11,2,2,10,3,13"); + ASSERT_VALUES(mm, 0 * 60, "6,5,5,2,8,11,2,2,10,3,13"); + + // Test that reset works + mm.reset(1000); + ASSERT_VALUES(mm, -1, "0,0,0,0,0,0,0,0,0,0,0"); + ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); + ASSERT_VALUES(mm, 60 * 60, "0,0,0,0,0,0,0,0,0,0,0"); + ASSERT_VALUES(mm, 0 * 60, "0,0,0,0,0,0,0,0,0,0,0"); +} + +void MetricManagerTest::testPrintUtils() +{ + FastOS_ThreadPool pool(256 * 1024); + FakeTimer* timer = new FakeTimer(1000); + std::unique_ptr<MetricManager::Timer> timerImpl(timer); + MetricManager mm(std::move(timerImpl)); + TestMetricSet mySet; + { + MetricLockGuard lockGuard(mm.getMetricLock()); + mm.registerMetric(lockGuard, mySet.set); + } + // Adding metrics to have some values in them + mySet.val6.addValue(2); + mySet.val9.val1.addValue(4); + mySet.val10.count.inc(); + mySet.val10.a.val1.addValue(7); + mySet.val10.a.val2.addValue(2); + mySet.val10.b.val1.addValue(1); + // Initialize metric manager to get snapshots created. + mm.init("raw:" + "consumer[2]\n" + "consumer[0].name snapper\n" + "consumer[0].tags[1]\n" + "consumer[0].tags[0] snaptest\n" + "consumer[1].name log\n" + "consumer[1].tags[1]\n" + "consumer[1].tags[0] snaptest\n", + pool); + using namespace printutils; + MetricLockGuard lockGuard(mm.getMetricLock()); + MetricSource source(mm.getActiveMetrics(lockGuard), + "temp.multisub"); + CPPUNIT_ASSERT(source.getMetric("count") != 0); + CPPUNIT_ASSERT(source.getMetric("a.val1") != 0); + CPPUNIT_ASSERT(source.getMetric("a.valsum") != 0); + CPPUNIT_ASSERT(source.getMetric("sum.val1") != 0); + CPPUNIT_ASSERT(source.getMetric("sum.valsum") != 0); + + std::vector<Metric::String> metrics( + source.getPathsMatchingPrefix("a.val")); + std::vector<Metric::String> expected; + expected.push_back("val1"); + expected.push_back("val2"); + expected.push_back("valsum"); + CPPUNIT_ASSERT_EQUAL(expected, metrics); + + HttpTable table("mytable", "stuff"); + table.colNames.push_back("values"); + table.rowNames.push_back("valsum"); + table[0][0] = getValueString( + getLongMetric("a.val1.value", source) + + getLongMetric("a.val2.value", source)); + std::ostringstream ost; + table.print(ost); + CPPUNIT_ASSERT_EQUAL(std::string( + "<h3>mytable</h3>\n" + "<table border=\"1\">\n" + "<tr><th>stuff</th><th>values</th></tr>\n" + "<tr><td>valsum</td><td align=\"right\">9</td></tr>\n" + "</table>\n" + ), ost.str()); +} + +void MetricManagerTest::testXmlOutput() +{ + FastOS_ThreadPool pool(256 * 1024); + FakeTimer* timer = new FakeTimer(1000); + std::unique_ptr<MetricManager::Timer> timerImpl(timer); + MetricManager mm(std::move(timerImpl)); + TestMetricSet mySet; + { + MetricLockGuard lockGuard(mm.getMetricLock()); + mm.registerMetric(lockGuard, mySet.set); + } + + // Initialize metric manager to get snapshots created. + mm.init("raw:" + "consumer[2]\n" + "consumer[0].name snapper\n" + "consumer[0].tags[1]\n" + "consumer[0].tags[0] snaptest\n" + "consumer[1].name log\n" + "consumer[1].tags[1]\n" + "consumer[1].tags[0] snaptest\n", + pool); + + mm.takeSnapshots(mm.getMetricLock(), 1000); + + // Adding metrics to have some values in them + mySet.val6.addValue(2); + mySet.val9.val1.addValue(4); + mySet.val10.count.inc(); + mySet.val10.a.val1.addValue(7); + mySet.val10.a.val2.addValue(2); + mySet.val10.b.val1.addValue(1); + + timer->_time = 1300; + mm.takeSnapshots(mm.getMetricLock(), 1300); + + std::string expected( + "'<snapshot name=\"5 minute\" from=\"1000\" to=\"1300\" period=\"300\">\n" + " <temp>\n" + " <val6 average=\"2\" last=\"2\" min=\"2\" max=\"2\" count=\"1\"/>\n" + " <sub>\n" + " <val1 average=\"4\" last=\"4\" min=\"4\" max=\"4\" count=\"1\"/>\n" + " <valsum average=\"4\" last=\"4\" min=\"4\" max=\"4\" count=\"1\"/>\n" + " </sub>\n" + " <multisub>\n" + " <count count=\"1\"/>\n" + " <a>\n" + " <val1 average=\"7\" last=\"7\" min=\"7\" max=\"7\" count=\"1\"/>\n" + " <valsum average=\"9\" last=\"9\"/>\n" + " </a>\n" + " <b>\n" + " <val1 average=\"1\" last=\"1\" min=\"1\" max=\"1\" count=\"1\"/>\n" + " <valsum average=\"1\" last=\"1\" min=\"1\" max=\"1\" count=\"1\"/>\n" + " </b>\n" + " <sum>\n" + " <val1 average=\"8\" last=\"8\"/>\n" + " <val2 average=\"2\" last=\"2\" min=\"2\" max=\"2\" count=\"1\"/>\n" + " <valsum average=\"10\" last=\"10\"/>\n" + " </sum>\n" + " </multisub>\n" + " </temp>\n" + "</snapshot>'"); + + std::ostringstream ost; + vespalib::XmlOutputStream xos(ost, " "); + XmlWriter writer(xos, 300, 0); + { + MetricLockGuard lockGuard(mm.getMetricLock()); + mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false), + writer, "snapper"); + } + std::string actual(ost.str()); + // Not bothering to match all the nitty gritty details as it will test + // more than it needs to. Just be here in order to check on XML output + // easily if needed. + CPPUNIT_ASSERT_EQUAL(expected, "'" + actual + "'"); +} + +void MetricManagerTest::testJsonOutput() +{ + FastOS_ThreadPool pool(256 * 1024); + FakeTimer* timer = new FakeTimer(1000); + std::unique_ptr<MetricManager::Timer> timerImpl(timer); + MetricManager mm(std::move(timerImpl)); + TestMetricSet mySet; + { + MetricLockGuard lockGuard(mm.getMetricLock()); + mm.registerMetric(lockGuard, mySet.set); + } + + // Initialize metric manager to get snapshots created. + mm.init("raw:" + "consumer[1]\n" + "consumer[0].name snapper\n" + "consumer[0].tags[1]\n" + "consumer[0].tags[0] snaptest\n", + pool); + + mm.takeSnapshots(mm.getMetricLock(), 1000); + + // Adding metrics to have some values in them + mySet.val6.addValue(2); + mySet.val9.val1.addValue(4); + mySet.val10.count.inc(); + mySet.val10.a.val1.addValue(7); + mySet.val10.a.val2.addValue(2); + mySet.val10.b.val1.addValue(1); + + timer->_time = 1300; + mm.takeSnapshots(mm.getMetricLock(), 1300); + + // Create json output + vespalib::asciistream as; + vespalib::JsonStream jsonStream(as); + JsonWriter writer(jsonStream); + { + MetricLockGuard lockGuard(mm.getMetricLock()); + mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false), + writer, "snapper"); + } + jsonStream.finalize(); + std::string jsonData = as.str(); + // Parse it back + using namespace vespalib::slime; + vespalib::Slime slime; + size_t parsed = JsonFormat::decode(Memory(jsonData), slime); + if (jsonData.size() != parsed) { + SimpleBuffer buffer; + JsonFormat::encode(slime, buffer, false); + std::ostringstream ost; + ost << "Failed to parse JSON: '\n" + << jsonData << "'\n:" << buffer.get().make_string() << "\n"; + CPPUNIT_ASSERT_EQUAL_MSG(ost.str(), jsonData.size(), parsed); + } + // Verify some content + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 1000.0, + slime.get()["snapshot"]["from"].asDouble()); + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 1300.0, + slime.get()["snapshot"]["to"].asDouble()); + + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, vespalib::string("temp.val6"), + slime.get()["values"][0]["name"].asString().make_string()); + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, vespalib::string("val6 desc"), + slime.get()["values"][0]["description"].asString().make_string()); + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 2.0, + slime.get()["values"][0]["values"]["average"].asDouble()); + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 1.0, + slime.get()["values"][0]["values"]["count"].asDouble()); + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 2.0, + slime.get()["values"][0]["values"]["min"].asDouble()); + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 2.0, + slime.get()["values"][0]["values"]["max"].asDouble()); + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 2.0, + slime.get()["values"][0]["values"]["last"].asDouble()); + + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, vespalib::string("temp.multisub.sum.valsum"), + slime.get()["values"][10]["name"].asString().make_string()); + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, vespalib::string("valsum desc"), + slime.get()["values"][10]["description"].asString().make_string()); + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 10.0, + slime.get()["values"][10]["values"]["average"].asDouble()); + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 3.0, + slime.get()["values"][10]["values"]["count"].asDouble()); + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 1.0, + slime.get()["values"][10]["values"]["min"].asDouble()); + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 7.0, + slime.get()["values"][10]["values"]["max"].asDouble()); + CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 10.0, + slime.get()["values"][10]["values"]["last"].asDouble()); + + + metrics::StateApiAdapter adapter(mm); + vespalib::string normal = adapter.getMetrics("snapper"); + CPPUNIT_ASSERT_EQUAL(vespalib::string(jsonData), normal); + vespalib::string total = adapter.getTotalMetrics("snapper"); + CPPUNIT_ASSERT(total.size() > 0); + CPPUNIT_ASSERT(total != normal); +} + +namespace { + +struct MetricSnapshotTestFixture +{ + static const size_t DEFAULT_THREAD_STACK_SIZE = 256 * 1024; + + MetricManagerTest& test; + FastOS_ThreadPool pool; + FakeTimer* timer; + MetricManager manager; + MetricSet& mset; + + MetricSnapshotTestFixture(MetricManagerTest& callerTest, MetricSet& metricSet) + : test(callerTest), + pool(DEFAULT_THREAD_STACK_SIZE), + timer(new FakeTimer(1000)), + manager(std::unique_ptr<MetricManager::Timer>(timer)), + mset(metricSet) + { + { + MetricLockGuard lockGuard(manager.getMetricLock()); + manager.registerMetric(lockGuard, mset); + } + + // Initialize metric manager to get snapshots created. + manager.init( + "raw:" + "consumer[1]\n" + "consumer[0].name snapper\n" + "consumer[0].addedmetrics[1]\n" + "consumer[0].addedmetrics[0] *\n", + pool); + + test.takeSnapshots(manager, 1000); + } + + // Take snapshot of metric values from time 1000 to time 1300 + void takeSnapshotsOnce() { + timer->_time = 1300; + test.takeSnapshots(manager, 1300); + } + + std::string renderLastSnapshotAsJson() const { + vespalib::asciistream as; + vespalib::JsonStream jsonStream(as, true); + JsonWriter writer(jsonStream); + { + MetricLockGuard lockGuard(manager.getMetricLock()); + manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), + writer, "snapper"); + } + jsonStream.finalize(); + return as.str(); + } + + std::string renderLastSnapshotAsText( + const std::string& matchPattern = ".*") const + { + std::ostringstream ss; + TextWriter writer(ss, 300, matchPattern, true); + { + MetricLockGuard lockGuard(manager.getMetricLock()); + manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), + writer, "snapper"); + } + return ss.str(); + } +}; + +class JsonMetricWrapper +{ + std::string _jsonText; + vespalib::Slime _tree; +public: + JsonMetricWrapper(const std::string& jsonText) + : _jsonText(jsonText) + { + vespalib::slime::JsonFormat::decode( + vespalib::slime::Memory(jsonText), _tree); + } + + // XXX ideally all these should be const, but is not clear how/if it's + // possible to get a const cursor into the underlying tree. + vespalib::slime::Cursor& root() { return _tree.get(); } + + vespalib::slime::Cursor& nthMetric(size_t metricIndex) { + return root()["values"][metricIndex]; + } + + size_t nthMetricDimensionCount(size_t metricIndex) { + return nthMetric(metricIndex)["dimensions"].children(); + } + + std::string nthMetricName(size_t metricIndex) { + return nthMetric(metricIndex)["name"].asString().make_string(); + } + + std::string nthMetricDimension(size_t metricIndex, const std::string& key) { + return nthMetric(metricIndex)["dimensions"][key] + .asString().make_string(); + } + + // Verify that the nth metric has the given name and the given set of + // dimension key-values. Cannot use name alone to check, as multiple metrics + // may have same name but different dimensions and output ordering of + // metrics is well defined as being that of the insertion order. + void verifyDimensions(size_t metricIndex, + const std::string& name, + const Metric::Tags& dimensions) { + // Works to do this outside of main test body because cppunit uses + // exceptions for its failures. + CPPUNIT_ASSERT_EQUAL_MSG(_jsonText, name, nthMetricName(metricIndex)); + CPPUNIT_ASSERT_EQUAL_MSG(_jsonText, dimensions.size(), + nthMetricDimensionCount(metricIndex)); + for (auto& dim : dimensions) { + CPPUNIT_ASSERT_EQUAL_MSG(_jsonText, std::string(dim.value), + nthMetricDimension(metricIndex, dim.key)); + } + } +}; + +struct DimensionTestMetricSet : MetricSet +{ + DoubleValueMetric val1; + LongCountMetric val2; + + DimensionTestMetricSet(MetricSet* owner = nullptr) + : MetricSet("temp", {{"foo", "megafoo"}, {"bar", "hyperbar"}}, + "", owner), + val1("val1", "tag1", "val1 desc", this), // Legacy tag; not dimension + val2("val2", {{"baz", "superbaz"}}, "val2 desc", this) + { + } +}; + +} // anon ns + +void +MetricManagerTest::jsonOutputSupportsMultipleDimensions() +{ + DimensionTestMetricSet mset; + MetricSnapshotTestFixture fixture(*this, mset); + + mset.val1.addValue(2); + mset.val2.inc(); + + fixture.takeSnapshotsOnce(); + std::string actual = fixture.renderLastSnapshotAsJson(); + JsonMetricWrapper json(actual); + + json.verifyDimensions(0, "temp.val1", + {{"foo", "megafoo"}, {"bar", "hyperbar"}}); + json.verifyDimensions(1, "temp.val2", + {{"foo", "megafoo"}, {"bar", "hyperbar"}, + {"baz", "superbaz"}}); +} + +namespace { + +struct NestedDimensionTestMetricSet : MetricSet +{ + DimensionTestMetricSet nestedSet; + + NestedDimensionTestMetricSet() + : MetricSet("outer", {{"fancy", "stuff"}}, ""), + nestedSet(this) + { + } +}; + +} // anon ns + +void +MetricManagerTest::jsonOutputCanNestDimensionsFromMultipleMetricSets() +{ + NestedDimensionTestMetricSet mset; + MetricSnapshotTestFixture fixture(*this, mset); + + mset.nestedSet.val1.addValue(2); + mset.nestedSet.val2.inc(); + + fixture.takeSnapshotsOnce(); + std::string actual = fixture.renderLastSnapshotAsJson(); + JsonMetricWrapper json(actual); + + json.verifyDimensions(0, "outer.temp.val1", + {{"foo", "megafoo"}, {"bar", "hyperbar"}, + {"fancy", "stuff"}}); + json.verifyDimensions(1, "outer.temp.val2", + {{"foo", "megafoo"}, {"bar", "hyperbar"}, + {"baz", "superbaz"}, {"fancy", "stuff"}}); +} + +namespace { + +struct DimensionOverridableTestMetricSet : MetricSet +{ + DoubleValueMetric val; + + DimensionOverridableTestMetricSet(const std::string& dimValue, + MetricSet* owner = nullptr) + : MetricSet("temp", {{"foo", dimValue}}, "", owner), + val("val", "", "val desc", this) + { + } +}; + +struct SameNamesTestMetricSet : MetricSet +{ + DimensionOverridableTestMetricSet set1; + DimensionOverridableTestMetricSet set2; + + SameNamesTestMetricSet() + : MetricSet("outer", {{"fancy", "stuff"}}, ""), + set1("bar", this), + set2("baz", this) + { + } +}; + +} // anon ns + +void +MetricManagerTest::jsonOutputCanHaveMultipleSetsWithSameName() +{ + SameNamesTestMetricSet mset; + MetricSnapshotTestFixture fixture(*this, mset); + + mset.set1.val.addValue(2); + mset.set2.val.addValue(5); + + fixture.takeSnapshotsOnce(); + std::string actual = fixture.renderLastSnapshotAsJson(); + JsonMetricWrapper json(actual); + + // Note the identical names. Only difference is the dimensions per set. + json.verifyDimensions(0, "outer.temp.val", + {{"foo", "bar"}, {"fancy", "stuff"}}); + json.verifyDimensions(1, "outer.temp.val", + {{"foo", "baz"}, {"fancy", "stuff"}}); +} + +void MetricManagerTest::testTextOutput() +{ + FastOS_ThreadPool pool(256 * 1024); + FakeTimer* timer = new FakeTimer(1000); + std::unique_ptr<MetricManager::Timer> timerImpl(timer); + MetricManager mm(std::move(timerImpl)); + TestMetricSet mySet; + { + MetricLockGuard lockGuard(mm.getMetricLock()); + mm.registerMetric(lockGuard, mySet.set); + } + // Adding metrics to have some values in them + mySet.val6.addValue(2); + mySet.val9.val1.addValue(4); + mySet.val10.count.inc(); + mySet.val10.a.val1.addValue(7); + mySet.val10.a.val2.addValue(2); + mySet.val10.b.val1.addValue(1); + // Initialize metric manager to get snapshots created. + mm.init("raw:" + "consumer[2]\n" + "consumer[0].name snapper\n" + "consumer[0].tags[1]\n" + "consumer[0].tags[0] snaptest\n" + "consumer[1].name log\n" + "consumer[1].tags[1]\n" + "consumer[1].tags[0] snaptest\n", + pool); + std::string expected( + "snapshot \"Active metrics showing updates since last snapshot\" from 1000 to 0 period 0\n" + "temp.val6 average=2 last=2 min=2 max=2 count=1 total=2\n" + "temp.sub.val1 average=4 last=4 min=4 max=4 count=1 total=4\n" + "temp.sub.valsum average=4 last=4 min=4 max=4 count=1 total=4\n" + "temp.multisub.count count=1\n" + "temp.multisub.a.val1 average=7 last=7 min=7 max=7 count=1 total=7\n" + "temp.multisub.a.valsum average=9 last=9\n" + "temp.multisub.b.val1 average=1 last=1 min=1 max=1 count=1 total=1\n" + "temp.multisub.b.valsum average=1 last=1 min=1 max=1 count=1 total=1\n" + "temp.multisub.sum.val1 average=8 last=8\n" + "temp.multisub.sum.val2 average=2 last=2 min=2 max=2 count=1 total=2\n" + "temp.multisub.sum.valsum average=10 last=10"); + std::ostringstream ost; + TextWriter writer(ost, 300, ".*", true); + { + MetricLockGuard lockGuard(mm.getMetricLock()); + mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), writer, "snapper"); + } + std::string actual(ost.str()); + // Not bothering to match all the nitty gritty details as it will test + // more than it needs to. Just be here in order to check on XML output + // easily if needed. + CPPUNIT_ASSERT_EQUAL(expected, actual); +} + +void +MetricManagerTest::textOutputSupportsDimensions() +{ + NestedDimensionTestMetricSet mset; + MetricSnapshotTestFixture fixture(*this, mset); + + mset.nestedSet.val1.addValue(2); + mset.nestedSet.val2.inc(); + + fixture.takeSnapshotsOnce(); + std::string actual = fixture.renderLastSnapshotAsText("outer.*temp.*val"); + std::string expected( + "snapshot \"5 minute\" from 1000 to 1300 period 300\n" + "outer{fancy:stuff}.temp{bar:hyperbar,foo:megafoo}.val1 " + "average=2 last=2 min=2 max=2 count=1 total=2\n" + "outer{fancy:stuff}.temp{bar:hyperbar,foo:megafoo}.val2" + "{baz:superbaz} count=1"); + CPPUNIT_ASSERT_EQUAL(expected, actual); +} + +namespace { + struct MyUpdateHook : public MetricManager::UpdateHook { + std::ostringstream& _output; + FakeTimer& _timer; + + MyUpdateHook(std::ostringstream& output, const char* name, + FakeTimer& timer) + : MetricManager::UpdateHook(name), _output(output), _timer(timer) {} + + void updateMetrics(const MetricLockGuard & ) override { + _output << _timer.getTime() << ": " << getName() << " called\n"; + } + }; +} + +void MetricManagerTest::testUpdateHooks() +{ + std::ostringstream output; + FastOS_ThreadPool pool(256 * 1024); + FakeTimer* timer = new FakeTimer(1000); + std::unique_ptr<MetricManager::Timer> timerImpl(timer); + // Add a metric set just so one exist + TestMetricSet mySet; + MetricManager mm(std::move(timerImpl)); + { + MetricLockGuard lockGuard(mm.getMetricLock()); + mm.registerMetric(lockGuard, mySet.set); + } + + MyUpdateHook preInitShort(output, "BIS", *timer); + MyUpdateHook preInitLong(output, "BIL", *timer); + MyUpdateHook preInitInfinite(output, "BII", *timer); + mm.addMetricUpdateHook(preInitShort, 5); + mm.addMetricUpdateHook(preInitLong, 300); + mm.addMetricUpdateHook(preInitInfinite, 0); + + // All hooks should get called during initialization + + // Initialize metric manager to get snapshots created. + output << "Running init\n"; + mm.init("raw:" + "consumer[2]\n" + "consumer[0].name snapper\n" + "consumer[0].tags[1]\n" + "consumer[0].tags[0] snaptest\n" + "consumer[1].name log\n" + "consumer[1].tags[1]\n" + "consumer[1].tags[0] snaptest\n", + pool); + output << "Init done\n"; + + MyUpdateHook postInitShort(output, "AIS", *timer); + MyUpdateHook postInitLong(output, "AIL", *timer); + MyUpdateHook postInitInfinite(output, "AII", *timer); + mm.addMetricUpdateHook(postInitShort, 5); + mm.addMetricUpdateHook(postInitLong, 400); + mm.addMetricUpdateHook(postInitInfinite, 0); + + // After 5 seconds the short ones should get another. + + timer->_time = 1006; + waitForTimeProcessed(mm, 1006); + + // After 4 more seconds the short ones should get another + // since last update was a second late. (Stable periods, process time + // should not affect how often they are updated) + + timer->_time = 1010; + waitForTimeProcessed(mm, 1010); + + // Bumping considerably ahead, such that next update is in the past, + // we should only get one update called in this period. + + timer->_time = 1200; + waitForTimeProcessed(mm, 1200); + + // No updates at this time. + timer->_time = 1204; + waitForTimeProcessed(mm, 1204); + + // Give all hooks an update + mm.updateMetrics(true); + + // Last update should not have interfered with periods + timer->_time = 1205; + waitForTimeProcessed(mm, 1205); + + // Time is just ahead of a snapshot. + timer->_time = 1299; + waitForTimeProcessed(mm, 1299); + + // At time 1300 we are at a 5 minute snapshot bump + // All hooks should thus get an update. The one with matching period + // should only get one + timer->_time = 1300; + waitForTimeProcessed(mm, 1300); + + // The snapshot time currently doesn't count for the metric at period + // 400. It will get an event at this time. + timer->_time = 1450; + waitForTimeProcessed(mm, 1450); + + std::string expected( + "Running init\n" + "1000: BIS called\n" + "1000: BIL called\n" + "Init done\n" + "1006: BIS called\n" + "1006: AIS called\n" + "1010: BIS called\n" + "1010: AIS called\n" + "1200: BIS called\n" + "1200: AIS called\n" + "1204: BIS called\n" + "1204: BIL called\n" + "1204: AIS called\n" + "1204: AIL called\n" + "1204: BII called\n" + "1204: AII called\n" + "1205: BIS called\n" + "1205: AIS called\n" + "1299: BIS called\n" + "1299: AIS called\n" + "1300: BIS called\n" + "1300: BIL called\n" + "1300: AIS called\n" + "1300: AIL called\n" + "1300: BII called\n" + "1300: AII called\n" + "1450: BIS called\n" + "1450: AIS called\n" + "1450: AIL called\n" + ); + std::string actual(output.str()); + CPPUNIT_ASSERT_EQUAL(expected, actual); +} + +} // metrics diff --git a/metrics/src/tests/metricsettest.cpp b/metrics/src/tests/metricsettest.cpp new file mode 100644 index 00000000000..9ce141feaf3 --- /dev/null +++ b/metrics/src/tests/metricsettest.cpp @@ -0,0 +1,157 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/objects/floatingpointtype.h> +#include <vespa/metrics/metrics.h> + +namespace metrics { + +struct MetricSetTest : public CppUnit::TestFixture { + void testNormalUsage(); + void supportMultipleMetricsWithSameNameDifferentDimensions(); + void uniqueTargetMetricsAreAddedToMetricSet(); + + CPPUNIT_TEST_SUITE(MetricSetTest); + CPPUNIT_TEST(testNormalUsage); + CPPUNIT_TEST(supportMultipleMetricsWithSameNameDifferentDimensions); + CPPUNIT_TEST(uniqueTargetMetricsAreAddedToMetricSet); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(MetricSetTest); + +namespace { + struct TestMetricVisitor : public MetricVisitor { + std::ostringstream ost; + uint32_t setsToVisit; + + TestMetricVisitor(uint32_t setsToVisit_ = 100) + : ost(), setsToVisit(setsToVisit_) {} + + virtual bool visitMetricSet(const MetricSet& set, bool autoGenerated) { + ost << "[" << (autoGenerated ? "*" : "") << set.getName() << "]\n"; + if (setsToVisit > 0) { + --setsToVisit; + return true; + } + return false; + } + virtual bool visitMetric(const Metric& m, bool autoGenerated) { + ost << (autoGenerated ? "*" : "") << m.getName() << "\n"; + return true; + } + }; +} + +void +MetricSetTest::testNormalUsage() +{ + // Set up some metrics to test.. + MetricSet set("a", "foo", ""); + DoubleValueMetric v1("c", "foo", "", &set); + LongAverageMetric v2("b", "", "", &set); + LongCountMetric v3("d", "bar", "", &set); + MetricSet set2("e", "bar", "", &set); + LongCountMetric v4("f", "foo", "", &set2); + + // Give them some values + v1.addValue(4.2); + v2.addValue(8); + v3.inc(); + v4.inc(3); + + // Check that we can register through registerMetric function too. + LongCountMetric v5("g", "", ""); + set.registerMetric(v5); + v5.inc(3); + v5.dec(); + + // Check that getMetric works, and doesn't return copy. + LongAverageMetric* v2copy( + dynamic_cast<LongAverageMetric*>(set.getMetric("b"))); + CPPUNIT_ASSERT(v2copy != 0); + v2copy->addValue(9); + Metric* nonExistingCopy = set.getMetric("nonexisting"); + CPPUNIT_ASSERT(nonExistingCopy == 0); + nonExistingCopy = set.getMetric("non.existing"); + CPPUNIT_ASSERT(nonExistingCopy == 0); + + // Check that paths are set + MetricSet topSet("top", "", ""); + topSet.registerMetric(set); + CPPUNIT_ASSERT_EQUAL(vespalib::string("a"), set.getPath()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("a.c"), v1.getPath()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("a.b"), v2.getPath()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("a.d"), v3.getPath()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("a.e"), set2.getPath()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("a.e.f"), v4.getPath()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("a.g"), v5.getPath()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("a.b"), v2copy->getPath()); + + // Verify XML output. Should be in register order. + std::string expected("'\n" + "a:\n" + " c average=4.2 last=4.2 min=4.2 max=4.2 count=1 total=4.2\n" + " b average=8.5 last=9 min=8 max=9 count=2 total=17\n" + " d count=1\n" + " e:\n" + " f count=3\n" + " g count=2'" + ); + CPPUNIT_ASSERT_EQUAL(expected, "'\n" + set.toString() + "'"); + + // Verify that visiting works. That you get all metrics if you answer + // true to all sets, and that you don't get members of sets you answer + // false to get. + { + TestMetricVisitor visitor(2); + set.visit(visitor); + expected = "[a]\nc\nb\nd\n[e]\nf\ng\n"; + CPPUNIT_ASSERT_EQUAL("\n" + expected, "\n" + visitor.ost.str()); + } + { + TestMetricVisitor visitor(1); + set.visit(visitor); + expected = "[a]\nc\nb\nd\n[e]\ng\n"; + CPPUNIT_ASSERT_EQUAL("\n" + expected, "\n" + visitor.ost.str()); + } + { + TestMetricVisitor visitor(0); + set.visit(visitor); + expected = "[a]\n"; + CPPUNIT_ASSERT_EQUAL("\n" + expected, "\n" + visitor.ost.str()); + } +} + +void +MetricSetTest::supportMultipleMetricsWithSameNameDifferentDimensions() +{ + MetricSet set("dimset", {{"foo", "bar"}}, ""); + DoubleValueMetric v1("stuff", {{"baz", "blarg"}}, "", &set); + LongAverageMetric v2("stuff", {{"flarn", "yarn"}}, "", &set); + + CPPUNIT_ASSERT_EQUAL(size_t(2), set.getRegisteredMetrics().size()); +} + +void +MetricSetTest::uniqueTargetMetricsAreAddedToMetricSet() +{ + MetricSet set1("a", "foo", ""); + LongCountMetric v1("wow", "foo", "", &set1); + MetricSet set2("e", "bar", ""); + LongCountMetric v2("doge", "foo", "", &set2); + + // Have to actually assign a value to metrics or they won't be carried over. + v1.inc(); + v2.inc(); + + // 'doge' metric in set2 must be preserved even though it does not exist + // in set1. + std::vector<Metric::LP> ownerList; + set1.addToSnapshot(set2, ownerList); + + CPPUNIT_ASSERT(set2.getMetric("wow") != nullptr); + CPPUNIT_ASSERT(set2.getMetric("doge") != nullptr); +} + +} // metrics diff --git a/metrics/src/tests/metrictest.cpp b/metrics/src/tests/metrictest.cpp new file mode 100644 index 00000000000..67dd608051a --- /dev/null +++ b/metrics/src/tests/metrictest.cpp @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/metrics/valuemetric.h> +#include <vespa/metrics/countmetric.h> + +namespace metrics { + +struct MetricTest : public CppUnit::TestFixture +{ + template <typename MetricImpl> + void testMetricsGetDimensionsAsPartOfMangledNameImpl(); + template <typename MetricImpl> + void testMangledNameMayContainMultipleDimensionsImpl(); + + void valueMetricsGetDimensionsAsPartOfMangledName(); + void countMetricsGetDimensionsAsPartOfMangledName(); + void valueMetricMangledNameMayContainMultipleDimensions(); + void countMetricMangledNameMayContainMultipleDimensions(); + void mangledNameListsDimensionsInLexicographicOrder(); + void manglingDoesNotChangeOriginalMetricName(); + void legacyTagsDoNotCreateMangledName(); + + CPPUNIT_TEST_SUITE(MetricTest); + CPPUNIT_TEST(valueMetricsGetDimensionsAsPartOfMangledName); + CPPUNIT_TEST(countMetricsGetDimensionsAsPartOfMangledName); + CPPUNIT_TEST(valueMetricMangledNameMayContainMultipleDimensions); + CPPUNIT_TEST(countMetricMangledNameMayContainMultipleDimensions); + CPPUNIT_TEST(mangledNameListsDimensionsInLexicographicOrder); + CPPUNIT_TEST(manglingDoesNotChangeOriginalMetricName); + CPPUNIT_TEST(legacyTagsDoNotCreateMangledName); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(MetricTest); + +// Metric subclasses have the same constructor parameters, so we template +// our way from having to duplicate code. Templated GTest fixtures would be +// a far more elegant solution. +template <typename MetricImpl> +void +MetricTest::testMetricsGetDimensionsAsPartOfMangledNameImpl() +{ + MetricImpl m("test", {{"foo", "bar"}}, "description goes here"); + CPPUNIT_ASSERT_EQUAL(std::string("test{foo:bar}"), m.getMangledName()); +} + +template <typename MetricImpl> +void +MetricTest::testMangledNameMayContainMultipleDimensionsImpl() +{ + MetricImpl m("test", + {{"flarn", "yarn"}, {"foo", "bar"}}, + "description goes here"); + CPPUNIT_ASSERT_EQUAL(std::string("test{flarn:yarn,foo:bar}"), + m.getMangledName()); +} + +void +MetricTest::valueMetricsGetDimensionsAsPartOfMangledName() +{ + testMetricsGetDimensionsAsPartOfMangledNameImpl<LongValueMetric>(); +} + +void +MetricTest::countMetricsGetDimensionsAsPartOfMangledName() +{ + testMetricsGetDimensionsAsPartOfMangledNameImpl<LongCountMetric>(); +} + +void +MetricTest::valueMetricMangledNameMayContainMultipleDimensions() +{ + testMangledNameMayContainMultipleDimensionsImpl<LongValueMetric>(); +} + +void +MetricTest::countMetricMangledNameMayContainMultipleDimensions() +{ + testMangledNameMayContainMultipleDimensionsImpl<LongCountMetric>(); +} + +// Assuming the above tests pass, we simplify by not requiring all subclasses +// to be tested since propagation down to the base class has already been +// verified. +void +MetricTest::mangledNameListsDimensionsInLexicographicOrder() +{ + LongValueMetric m("test", + {{"xyz", "bar"}, {"abc", "foo"}, {"def", "baz"}}, + ""); + CPPUNIT_ASSERT_EQUAL(std::string("test{abc:foo,def:baz,xyz:bar}"), + m.getMangledName()); +} + +void +MetricTest::manglingDoesNotChangeOriginalMetricName() +{ + LongValueMetric m("test", {{"foo", "bar"}}, ""); + CPPUNIT_ASSERT_EQUAL(std::string("test"), m.getName()); +} + +void +MetricTest::legacyTagsDoNotCreateMangledName() +{ + LongValueMetric m("test", "foo bar", ""); + CPPUNIT_ASSERT_EQUAL(std::string("test"), m.getName()); + CPPUNIT_ASSERT_EQUAL(std::string("test"), m.getMangledName()); +} + +} // metrics + diff --git a/metrics/src/tests/snapshottest.cpp b/metrics/src/tests/snapshottest.cpp new file mode 100644 index 00000000000..800d9f8465a --- /dev/null +++ b/metrics/src/tests/snapshottest.cpp @@ -0,0 +1,354 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/metrics/metrics.h> +#include <vespa/vdstestlib/cppunit/macros.h> + +LOG_SETUP(".test.snapshot"); + +namespace metrics { + +struct SnapshotTest : public CppUnit::TestFixture { + void testSnapshotTwoDays(); + + CPPUNIT_TEST_SUITE(SnapshotTest); + CPPUNIT_TEST(testSnapshotTwoDays); + CPPUNIT_TEST_SUITE_END(); + +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(SnapshotTest); + +namespace { + +struct SubSubMetricSet : public MetricSet { + const LoadTypeSet& loadTypes; + int incVal; + LongCountMetric count1; + LongCountMetric count2; + LoadMetric<LongCountMetric> loadCount; + SumMetric<LongCountMetric> countSum; + DoubleValueMetric value1; + DoubleValueMetric value2; + LoadMetric<DoubleValueMetric> loadValue; + SumMetric<DoubleValueMetric> valueSum; + DoubleAverageMetric average1; + DoubleAverageMetric average2; + LoadMetric<DoubleAverageMetric> loadAverage; + SumMetric<DoubleAverageMetric> averageSum; + + SubSubMetricSet(vespalib::stringref name, const LoadTypeSet& loadTypes_, + MetricSet* owner = 0) + : MetricSet(name, "", "", owner), + loadTypes(loadTypes_), + incVal(1), + count1("count1", "", "", this), + count2("count2", "", "", this), + loadCount(loadTypes, LongCountMetric("loadCount", "", ""), this), + countSum("countSum", "", "", this), + value1("value1", "", "", this), + value2("value2", "", "", this), + loadValue(loadTypes, DoubleValueMetric("loadValue", "", ""), this), + valueSum("valueSum", "", "", this), + average1("average1", "", "", this), + average2("average2", "", "", this), + loadAverage(loadTypes, DoubleAverageMetric("loadAverage", "", ""), + this), + averageSum("averageSum", "", "", this) + { + countSum.addMetricToSum(count1); + countSum.addMetricToSum(count2); + valueSum.addMetricToSum(value1); + valueSum.addMetricToSum(value2); + averageSum.addMetricToSum(average1); + averageSum.addMetricToSum(average2); + } + + virtual Metric* clone(std::vector<Metric::LP>& ownerList, + CopyType copyType, metrics::MetricSet* owner, + bool includeUnused) const + { + if (copyType == INACTIVE) { + return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused); + } + return (SubSubMetricSet*) (new SubSubMetricSet( + getName(), loadTypes, owner)) + ->assignValues(*this); + } + + void incValues() { + count1.inc(incVal); + count2.inc(incVal); + for (uint32_t i=0; i<loadTypes.size(); ++i) { + loadCount[loadTypes[i]].inc(incVal); + } + value1.set(incVal); + value2.set(incVal); + for (uint32_t i=0; i<loadTypes.size(); ++i) { + loadValue[loadTypes[i]].set(incVal); + } + average1.set(incVal); + average2.set(incVal); + for (uint32_t i=0; i<loadTypes.size(); ++i) { + loadAverage[loadTypes[i]].set(incVal); + } + } +}; + +struct SubMetricSet : public MetricSet { + const LoadTypeSet& loadTypes; + SubSubMetricSet set1; + SubSubMetricSet set2; + LoadMetric<SubSubMetricSet> loadSet; + SumMetric<SubSubMetricSet> setSum; + + SubMetricSet(vespalib::stringref name, const LoadTypeSet& loadTypes_, + MetricSet* owner = 0) + : MetricSet(name, "", "", owner), + loadTypes(loadTypes_), + set1("set1", loadTypes, this), + set2("set2", loadTypes, this), + loadSet(loadTypes, *std::unique_ptr<SubSubMetricSet>( + new SubSubMetricSet("loadSet", loadTypes)), this), + setSum("setSum", "", "", this) + { + setSum.addMetricToSum(set1); + setSum.addMetricToSum(set2); + } + + virtual Metric* clone(std::vector<Metric::LP>& ownerList, + CopyType copyType, metrics::MetricSet* owner, + bool includeUnused) const + { + if (copyType == INACTIVE) { + return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused); + } + return (SubMetricSet*) (new SubMetricSet(getName(), loadTypes, owner)) + ->assignValues(*this); + } + + void incValues() { + set1.incValues(); + set2.incValues(); + for (uint32_t i=0; i<loadTypes.size(); ++i) { + loadSet[loadTypes[i]].incValues(); + } + } +}; + +struct TestMetricSet : public MetricSet { + const LoadTypeSet& loadTypes; + SubMetricSet set1; + SubMetricSet set2; + LoadMetric<SubMetricSet> loadSet; + SumMetric<SubMetricSet> setSum; + + TestMetricSet(vespalib::stringref name, const LoadTypeSet& loadTypes_, + MetricSet* owner = 0) + : MetricSet(name, "", "", owner), + loadTypes(loadTypes_), + set1("set1", loadTypes, this), + set2("set2", loadTypes, this), + loadSet(loadTypes, *std::unique_ptr<SubMetricSet>( + new SubMetricSet("loadSet", loadTypes)), this), + setSum("setSum", "", "", this) + { + setSum.addMetricToSum(set1); + setSum.addMetricToSum(set2); + } + + void incValues() { + set1.incValues(); + set2.incValues(); + for (uint32_t i=0; i<loadTypes.size(); ++i) { + loadSet[loadTypes[i]].incValues(); + } + } +}; + +struct FakeTimer : public MetricManager::Timer { + uint32_t _timeInSecs; + + FakeTimer() : _timeInSecs(1) {} + + virtual time_t getTime() const { return _timeInSecs; } +}; + +} // End of anonymous namespace + +#define ASSERT_VALUE(value, snapshot, name) \ +{ \ + const Metric* _metricValue_((snapshot).getMetrics().getMetric(name)); \ + if (_metricValue_ == 0) { \ + CPPUNIT_FAIL("Metric value '" + std::string(name) \ + + "' not found in snapshot"); \ + } \ + CPPUNIT_ASSERT_EQUAL(value, \ + int32_t(_metricValue_->getLongValue("value"))); \ +} + +void SnapshotTest::testSnapshotTwoDays() +{ + // Create load types + LoadTypeSet loadTypes; + loadTypes.push_back(LoadType(1, "foo")); + loadTypes.push_back(LoadType(2, "bar")); + + TestMetricSet set("test", loadTypes); + + FakeTimer* timer; + FastOS_ThreadPool threadPool(256 * 1024); + MetricManager mm( + std::unique_ptr<MetricManager::Timer>(timer = new FakeTimer)); + { + MetricLockGuard lockGuard(mm.getMetricLock()); + mm.registerMetric(lockGuard, set); + } + mm.init("raw:consumer[1]\n" + "consumer[0].name \"log\"", threadPool, false); + mm.tick(mm.getMetricLock(), timer->_timeInSecs * 1000); + + for (uint32_t days=0; days<2; ++days) { + for (uint32_t hour=0; hour<24; ++hour) { + for (uint32_t fiveMin=0; fiveMin<12; ++fiveMin) { + set.incValues(); + timer->_timeInSecs += 5 * 60; + mm.tick(mm.getMetricLock(), timer->_timeInSecs * 1000); + } + } + } + + // Print all data. Useful for debugging. It's too much to verify + // everything, so test will just verify some parts. Add more parts if you + // find a failure. + /* + std::string regex = ".*"; + bool verbose = true; + std::cout << "\n" << mm.getActiveMetrics().toString(mm, "", regex, verbose) + << "\n\n"; + std::vector<uint32_t> periods(mm.getSnapshotPeriods()); + for (uint32_t i=0; i<periods.size(); ++i) { + const MetricSnapshot& snap(mm.getMetricSnapshot(periods[i])); + std::cout << snap.toString(mm, "", regex, verbose) << "\n\n"; + } + + std::cout << mm.getTotalMetricSnapshot().toString(mm, "", regex, verbose) + << "\n"; + */ + + const MetricSnapshot* snap = 0; + // active snapshot + MetricLockGuard lockGuard(mm.getMetricLock()); + snap = &mm.getActiveMetrics(lockGuard); + ASSERT_VALUE(0, *snap, "test.set1.set1.count1"); + ASSERT_VALUE(0, *snap, "test.set1.set1.loadCount.foo"); + ASSERT_VALUE(0, *snap, "test.set1.set1.loadCount.sum"); + ASSERT_VALUE(0, *snap, "test.set1.set1.countSum"); + ASSERT_VALUE(0, *snap, "test.set1.loadSet.foo.count1"); + ASSERT_VALUE(0, *snap, "test.set1.loadSet.foo.countSum"); +/* Current test procedure for fetching values, don't work in active sums of sets + ASSERT_VALUE(0, *snap, "test.set1.loadSet.sum.count1"); + ASSERT_VALUE(0, *snap, "test.set1.loadSet.sum.loadCount.foo"); + ASSERT_VALUE(0, *snap, "test.set1.loadSet.sum.loadCount.sum"); + ASSERT_VALUE(0, *snap, "test.set1.loadSet.sum.countSum"); +*/ + + // 5 minute snapshot + snap = &mm.getMetricSnapshot(lockGuard, 5 * 60); + ASSERT_VALUE(1, *snap, "test.set1.set1.count1"); + ASSERT_VALUE(1, *snap, "test.set1.set1.loadCount.foo"); + ASSERT_VALUE(2, *snap, "test.set1.set1.loadCount.sum"); + ASSERT_VALUE(2, *snap, "test.set1.set1.countSum"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.count1"); + ASSERT_VALUE(2, *snap, "test.set1.loadSet.foo.countSum"); + ASSERT_VALUE(2, *snap, "test.set1.loadSet.sum.count1"); + ASSERT_VALUE(2, *snap, "test.set1.loadSet.sum.loadCount.foo"); + ASSERT_VALUE(4, *snap, "test.set1.loadSet.sum.loadCount.sum"); + ASSERT_VALUE(4, *snap, "test.set1.loadSet.sum.countSum"); + + ASSERT_VALUE(1, *snap, "test.set1.set1.average1"); + ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.foo"); + ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.sum"); + ASSERT_VALUE(1, *snap, "test.set1.set1.averageSum"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.average1"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.averageSum"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.average1"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.foo"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.sum"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.averageSum"); + + // 1 hour snapshot + snap = &mm.getMetricSnapshot(lockGuard, 60 * 60); + ASSERT_VALUE(12, *snap, "test.set1.set1.count1"); + ASSERT_VALUE(12, *snap, "test.set1.set1.loadCount.foo"); + ASSERT_VALUE(24, *snap, "test.set1.set1.loadCount.sum"); + ASSERT_VALUE(24, *snap, "test.set1.set1.countSum"); + ASSERT_VALUE(12, *snap, "test.set1.loadSet.foo.count1"); + ASSERT_VALUE(24, *snap, "test.set1.loadSet.foo.countSum"); + ASSERT_VALUE(24, *snap, "test.set1.loadSet.sum.count1"); + ASSERT_VALUE(24, *snap, "test.set1.loadSet.sum.loadCount.foo"); + ASSERT_VALUE(48, *snap, "test.set1.loadSet.sum.loadCount.sum"); + ASSERT_VALUE(48, *snap, "test.set1.loadSet.sum.countSum"); + + ASSERT_VALUE(1, *snap, "test.set1.set1.average1"); + ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.foo"); + ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.sum"); + ASSERT_VALUE(1, *snap, "test.set1.set1.averageSum"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.average1"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.averageSum"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.average1"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.foo"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.sum"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.averageSum"); + + // 1 day snapshot + snap = &mm.getMetricSnapshot(lockGuard, 24 * 60 * 60); + ASSERT_VALUE(288, *snap, "test.set1.set1.count1"); + ASSERT_VALUE(288, *snap, "test.set1.set1.loadCount.foo"); + ASSERT_VALUE(576, *snap, "test.set1.set1.loadCount.sum"); + ASSERT_VALUE(576, *snap, "test.set1.set1.countSum"); + ASSERT_VALUE(288, *snap, "test.set1.loadSet.foo.count1"); + ASSERT_VALUE(576, *snap, "test.set1.loadSet.foo.countSum"); + ASSERT_VALUE(576, *snap, "test.set1.loadSet.sum.count1"); + ASSERT_VALUE(576, *snap, "test.set1.loadSet.sum.loadCount.foo"); + ASSERT_VALUE(1152, *snap, "test.set1.loadSet.sum.loadCount.sum"); + ASSERT_VALUE(1152, *snap, "test.set1.loadSet.sum.countSum"); + + ASSERT_VALUE(1, *snap, "test.set1.set1.average1"); + ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.foo"); + ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.sum"); + ASSERT_VALUE(1, *snap, "test.set1.set1.averageSum"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.average1"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.averageSum"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.average1"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.foo"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.sum"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.averageSum"); + + // total snapshot (2 days currently, not testing weeks) + snap = &mm.getTotalMetricSnapshot(lockGuard); + ASSERT_VALUE(576, *snap, "test.set1.set1.count1"); + ASSERT_VALUE(576, *snap, "test.set1.set1.loadCount.foo"); + ASSERT_VALUE(1152, *snap, "test.set1.set1.loadCount.sum"); + ASSERT_VALUE(1152, *snap, "test.set1.set1.countSum"); + ASSERT_VALUE(576, *snap, "test.set1.loadSet.foo.count1"); + ASSERT_VALUE(1152, *snap, "test.set1.loadSet.foo.countSum"); + ASSERT_VALUE(1152, *snap, "test.set1.loadSet.sum.count1"); + ASSERT_VALUE(1152, *snap, "test.set1.loadSet.sum.loadCount.foo"); + ASSERT_VALUE(2304, *snap, "test.set1.loadSet.sum.loadCount.sum"); + ASSERT_VALUE(2304, *snap, "test.set1.loadSet.sum.countSum"); + + ASSERT_VALUE(1, *snap, "test.set1.set1.average1"); + ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.foo"); + ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.sum"); + ASSERT_VALUE(1, *snap, "test.set1.set1.averageSum"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.average1"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.averageSum"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.average1"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.foo"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.sum"); + ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.averageSum"); +} + +} // metrics diff --git a/metrics/src/tests/stresstest.cpp b/metrics/src/tests/stresstest.cpp new file mode 100644 index 00000000000..f88c3a4944c --- /dev/null +++ b/metrics/src/tests/stresstest.cpp @@ -0,0 +1,144 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/metrics/metrics.h> +#include <vespa/vdstestlib/cppunit/macros.h> + +LOG_SETUP(".metrics.test.stress"); + +namespace metrics { + +struct StressTest : public CppUnit::TestFixture { + void testStress(); + + CPPUNIT_TEST_SUITE(StressTest); + CPPUNIT_TEST(testStress); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(StressTest); + +namespace { + struct InnerMetricSet : public MetricSet { + const LoadTypeSet& _loadTypes; + LongCountMetric _count; + LongAverageMetric _value1; + LongAverageMetric _value2; + SumMetric<LongAverageMetric> _valueSum; + LoadMetric<LongAverageMetric> _load; + + InnerMetricSet(const char* name, const LoadTypeSet& lt, + MetricSet* owner = 0) + : MetricSet(name, "", "", owner), + _loadTypes(lt), + _count("count", "", "", this), + _value1("value1", "", "", this), + _value2("value2", "", "", this), + _valueSum("valuesum", "", "", this), + _load(lt, LongAverageMetric("load", "", ""), this) + { + _valueSum.addMetricToSum(_value1); + _valueSum.addMetricToSum(_value2); + } + + Metric* clone(std::vector<Metric::LP>& ownerList, CopyType copyType, + MetricSet* owner, bool includeUnused) const + { + if (copyType != CLONE) { + return MetricSet::clone(ownerList, copyType, owner, + includeUnused); + } + return (new InnerMetricSet(getName().c_str(), _loadTypes, owner)) + ->assignValues(*this); + } + }; + struct OuterMetricSet : public MetricSet { + InnerMetricSet _inner1; + InnerMetricSet _inner2; + SumMetric<InnerMetricSet> _innerSum; + InnerMetricSet _tmp; + LoadMetric<InnerMetricSet> _load; + + OuterMetricSet(const LoadTypeSet& lt, MetricSet* owner = 0) + : MetricSet("outer", "", "", owner), + _inner1("inner1", lt, this), + _inner2("inner2", lt, this), + _innerSum("innersum", "", "", this), + _tmp("innertmp", lt, 0), + _load(lt, _tmp, this) + { + _innerSum.addMetricToSum(_inner1); + _innerSum.addMetricToSum(_inner2); + } + }; + struct Hammer : public document::Runnable { + typedef vespalib::LinkedPtr<Hammer> LP; + + OuterMetricSet& _metrics; + const LoadTypeSet& _loadTypes; + LoadType _nonexistingLoadType; + + Hammer(OuterMetricSet& metrics, const LoadTypeSet& lt, + FastOS_ThreadPool& threadPool) + : _metrics(metrics), _loadTypes(lt), + _nonexistingLoadType(123, "nonexisting") + { + start(threadPool); + } + ~Hammer() { + stop(); + join(); + //std::cerr << "Loadgiver thread joined\n"; + } + + virtual void run() { + uint64_t i = 0; + while (running()) { + ++i; + setMetrics(i, _metrics._inner1); + setMetrics(i + 3, _metrics._inner2); + const LoadType& loadType(_loadTypes[i % _loadTypes.size()]); + setMetrics(i + 5, _metrics._load[loadType]); + } + } + + void setMetrics(uint64_t val, InnerMetricSet& set) { + set._count.inc(val); + set._value1.addValue(val); + set._value2.addValue(val + 10); + set._load[_loadTypes[val % _loadTypes.size()]].addValue(val); + } + }; +} + + +void +StressTest::testStress() +{ + LoadTypeSet loadTypes; + loadTypes.push_back(LoadType(0, "default")); + loadTypes.push_back(LoadType(2, "foo")); + loadTypes.push_back(LoadType(1, "bar")); + + OuterMetricSet metrics(loadTypes); + + LOG(info, "Starting load givers"); + FastOS_ThreadPool threadPool(256 * 1024); + std::vector<Hammer::LP> hammers; + for (uint32_t i=0; i<10; ++i) { + hammers.push_back(Hammer::LP( + new Hammer(metrics, loadTypes, threadPool))); + } + LOG(info, "Waiting to let loadgivers hammer a while"); + FastOS_Thread::Sleep(5 * 1000); + + LOG(info, "Removing loadgivers"); + hammers.clear(); + + LOG(info, "Printing end state"); + std::ostringstream ost; + metrics.print(ost, true, "", 5); + // std::cerr << ost.str() << "\n"; +} + +} // metrics diff --git a/metrics/src/tests/summetrictest.cpp b/metrics/src/tests/summetrictest.cpp new file mode 100644 index 00000000000..fd4d67d7f3b --- /dev/null +++ b/metrics/src/tests/summetrictest.cpp @@ -0,0 +1,150 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/metrics/metrics.h> + +namespace metrics { + +struct SumMetricTest : public CppUnit::TestFixture { + void testLongCountMetric(); + void testAverageMetric(); + void testMetricSet(); + void testRemove(); + void testStartValue(); + + CPPUNIT_TEST_SUITE(SumMetricTest); + CPPUNIT_TEST(testLongCountMetric); + CPPUNIT_TEST(testAverageMetric); + CPPUNIT_TEST(testMetricSet); + CPPUNIT_TEST(testRemove); + CPPUNIT_TEST(testStartValue); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(SumMetricTest); + +void +SumMetricTest::testLongCountMetric() +{ + MetricSet parent("parent", "", ""); + SumMetric<LongCountMetric> sum("foo", "", "foodesc", &parent); + + LongCountMetric v1("ff", "", "", &parent); + LongCountMetric v2("aa", "", "", &parent); + + sum.addMetricToSum(v1); + sum.addMetricToSum(v2); + + // Give them some values + v1.inc(3); + v2.inc(7); + + // Verify XML output. Should be in register order. + std::string expected("foo count=10"); + CPPUNIT_ASSERT_EQUAL(expected, sum.toString()); + CPPUNIT_ASSERT_EQUAL(int64_t(10), sum.getLongValue("value")); +} + +void +SumMetricTest::testAverageMetric() { + MetricSet parent("parent", "", ""); + SumMetric<LongAverageMetric> sum("foo", "", "foodesc", &parent); + + LongAverageMetric v1("ff", "", "", &parent); + LongAverageMetric v2("aa", "", "", &parent); + + sum.addMetricToSum(v1); + sum.addMetricToSum(v2); + + // Give them some values + v1.addValue(3); + v2.addValue(7); + + // Verify XML output. Should be in register order. + std::string expected("foo average=5 last=7 min=3 max=7 count=2 total=10"); + CPPUNIT_ASSERT_EQUAL(expected, sum.toString()); + CPPUNIT_ASSERT_EQUAL(int64_t(5), sum.getLongValue("value")); + CPPUNIT_ASSERT_EQUAL(int64_t(3), sum.getLongValue("min")); + CPPUNIT_ASSERT_EQUAL(int64_t(7), sum.getLongValue("max")); +} + +void +SumMetricTest::testMetricSet() { + MetricSet parent("parent", "", ""); + SumMetric<MetricSet> sum("foo", "", "bar", &parent); + + MetricSet set1("a", "", "", &parent); + MetricSet set2("b", "", "", &parent); + LongValueMetric v1("c", "", "", &set1); + LongValueMetric v2("d", "", "", &set2); + LongCountMetric v3("e", "", "", &set1); + LongCountMetric v4("f", "", "", &set2); + + sum.addMetricToSum(set1); + sum.addMetricToSum(set2); + + // Give them some values + v1.addValue(3); + v2.addValue(7); + v3.inc(2); + v4.inc(); + + // Verify XML output. Should be in register order. + std::string expected("'\n" + "foo:\n" + " c average=3 last=3 min=3 max=3 count=1 total=3\n" + " e count=2'" + ); + CPPUNIT_ASSERT_EQUAL(expected, "'\n" + sum.toString() + "'"); +} + +void +SumMetricTest::testRemove() +{ + MetricSet parent("parent", "", ""); + SumMetric<LongCountMetric> sum("foo", "", "foodesc", &parent); + + LongCountMetric v1("ff", "", "", &parent); + LongCountMetric v2("aa", "", "", &parent); + LongCountMetric v3("zz", "", "", &parent); + + sum.addMetricToSum(v1); + sum.addMetricToSum(v2); + sum.addMetricToSum(v3); + + // Give them some values + v1.inc(3); + v2.inc(7); + v3.inc(10); + + CPPUNIT_ASSERT_EQUAL(int64_t(20), sum.getLongValue("value")); + sum.removeMetricFromSum(v2); + CPPUNIT_ASSERT_EQUAL(int64_t(13), sum.getLongValue("value")); +} + +void +SumMetricTest::testStartValue() +{ + MetricSnapshot snapshot("active"); + SumMetric<LongValueMetric> sum("foo", "", "foodesc", + &snapshot.getMetrics()); + LongValueMetric start("start", "", "", 0); + start.set(50); + sum.setStartValue(start); + + // without children + CPPUNIT_ASSERT_EQUAL(int64_t(50), sum.getLongValue("value")); + + MetricSnapshot copy("copy"); + copy.recreateSnapshot(snapshot.getMetrics(), true); + snapshot.addToSnapshot(copy, 100); + + LongValueMetric value("value", "", "", &snapshot.getMetrics()); + sum.addMetricToSum(value); + value.set(10); + + // with children + CPPUNIT_ASSERT_EQUAL(int64_t(60), sum.getLongValue("value")); +} + +} // metrics diff --git a/metrics/src/tests/testrunner.cpp b/metrics/src/tests/testrunner.cpp new file mode 100644 index 00000000000..5d8dc8d4c1f --- /dev/null +++ b/metrics/src/tests/testrunner.cpp @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <iostream> +#include <vespa/log/log.h> +#include <vespa/vdstestlib/cppunit/cppunittestrunner.h> + +LOG_SETUP("storagecppunittests"); + +int +main(int argc, char **argv) +{ + vdstestlib::CppUnitTestRunner testRunner; + return testRunner.run(argc, argv); +} diff --git a/metrics/src/tests/valuemetrictest.cpp b/metrics/src/tests/valuemetrictest.cpp new file mode 100644 index 00000000000..afe4221481e --- /dev/null +++ b/metrics/src/tests/valuemetrictest.cpp @@ -0,0 +1,281 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/objects/floatingpointtype.h> +#include <vespa/metrics/jsonwriter.h> +#include <vespa/metrics/metricmanager.h> +#include <vespa/metrics/valuemetric.h> + +using vespalib::Double; + +namespace metrics { + +struct ValueMetricTest : public CppUnit::TestFixture { + void testDoubleValueMetric(); + void testDoubleValueMetricNotUpdatedOnNaN(); + void testDoubleValueMetricNotUpdatedOnInfinity(); + void testLongValueMetric(); + void testSmallAverage(); + void testAddValueBatch(); + void testJson(); + + CPPUNIT_TEST_SUITE(ValueMetricTest); + CPPUNIT_TEST(testDoubleValueMetric); + CPPUNIT_TEST(testDoubleValueMetricNotUpdatedOnNaN); + CPPUNIT_TEST(testDoubleValueMetricNotUpdatedOnInfinity); + CPPUNIT_TEST(testLongValueMetric); + CPPUNIT_TEST(testSmallAverage); + CPPUNIT_TEST(testAddValueBatch); + CPPUNIT_TEST(testJson); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(ValueMetricTest); + +#define ASSERT_AVERAGE(metric, avg, min, max, count, last) \ + CPPUNIT_ASSERT_EQUAL_MSG("avg", Double(avg), Double(metric.getAverage())); \ + CPPUNIT_ASSERT_EQUAL_MSG("cnt", Double(count), Double(metric.getCount())); \ + CPPUNIT_ASSERT_EQUAL_MSG("last", Double(last), Double(metric.getLast())); \ + if (metric.getCount() > 0) { \ + CPPUNIT_ASSERT_EQUAL_MSG("min", Double(min), Double(metric.getMinimum())); \ + CPPUNIT_ASSERT_EQUAL_MSG("max", Double(max), Double(metric.getMaximum())); \ + } + +void ValueMetricTest::testDoubleValueMetric() +{ + DoubleValueMetric m("test", "tag", "description"); + m.addValue(100); + ASSERT_AVERAGE(m, 100, 100, 100, 1, 100); + m.addValue(100); + ASSERT_AVERAGE(m, 100, 100, 100, 2, 100); + m.addValue(40); + ASSERT_AVERAGE(m, 80, 40, 100, 3, 40); + DoubleValueMetric m2(m); + ASSERT_AVERAGE(m, 80, 40, 100, 3, 40); + m.reset(); + ASSERT_AVERAGE(m, 0, 0, 0, 0, 0); + + DoubleValueMetric n("m2", "", "desc"); + n.addValue(60); + ASSERT_AVERAGE(n, 60, 60, 60, 1, 60); + + DoubleValueMetric o = m2 + n; + ASSERT_AVERAGE(o, 140, 40, 100, 4, 100); + + o = n + m2; + ASSERT_AVERAGE(o, 140, 40, 100, 4, 100); + + std::string expected( + "test average=80 last=40 min=40 max=100 count=3 total=240"); + CPPUNIT_ASSERT_EQUAL(expected, m2.toString()); + expected = "m2 average=140 last=100"; + CPPUNIT_ASSERT_EQUAL(expected, o.toString()); + + CPPUNIT_ASSERT_EQUAL(Double(40), Double(m2.getDoubleValue("value"))); + CPPUNIT_ASSERT_EQUAL(Double(80), Double(m2.getDoubleValue("average"))); + CPPUNIT_ASSERT_EQUAL(Double(40), Double(m2.getDoubleValue("min"))); + CPPUNIT_ASSERT_EQUAL(Double(100), Double(m2.getDoubleValue("max"))); + CPPUNIT_ASSERT_EQUAL(Double(40), Double(m2.getDoubleValue("last"))); + CPPUNIT_ASSERT_EQUAL(Double(3), Double(m2.getDoubleValue("count"))); + CPPUNIT_ASSERT_EQUAL(Double(240), Double(m2.getDoubleValue("total"))); + + CPPUNIT_ASSERT_EQUAL(int64_t(40), m2.getLongValue("value")); + CPPUNIT_ASSERT_EQUAL(int64_t(80), m2.getLongValue("average")); + CPPUNIT_ASSERT_EQUAL(int64_t(40), m2.getLongValue("min")); + CPPUNIT_ASSERT_EQUAL(int64_t(100), m2.getLongValue("max")); + CPPUNIT_ASSERT_EQUAL(int64_t(40), m2.getLongValue("last")); + CPPUNIT_ASSERT_EQUAL(int64_t(3), m2.getLongValue("count")); + CPPUNIT_ASSERT_EQUAL(int64_t(240), m2.getLongValue("total")); +} + +void +ValueMetricTest::testDoubleValueMetricNotUpdatedOnNaN() +{ + DoubleValueMetric m("test", "tag", "description"); + m.addValue(std::numeric_limits<double>::quiet_NaN()); + CPPUNIT_ASSERT_EQUAL(std::string(), m.toString()); + + m.addAvgValueWithCount(std::numeric_limits<double>::quiet_NaN(), 123); + CPPUNIT_ASSERT_EQUAL(std::string(), m.toString()); + + m.inc(std::numeric_limits<double>::quiet_NaN()); + CPPUNIT_ASSERT_EQUAL(std::string(), m.toString()); + + m.dec(std::numeric_limits<double>::quiet_NaN()); + CPPUNIT_ASSERT_EQUAL(std::string(), m.toString()); +} + +void +ValueMetricTest::testDoubleValueMetricNotUpdatedOnInfinity() +{ + DoubleValueMetric m("test", "tag", "description"); + m.addValue(std::numeric_limits<double>::infinity()); + CPPUNIT_ASSERT_EQUAL(std::string(), m.toString()); + + m.addAvgValueWithCount(std::numeric_limits<double>::quiet_NaN(), 123); + CPPUNIT_ASSERT_EQUAL(std::string(), m.toString()); + + m.inc(std::numeric_limits<double>::infinity()); + CPPUNIT_ASSERT_EQUAL(std::string(), m.toString()); + + m.dec(std::numeric_limits<double>::infinity()); + CPPUNIT_ASSERT_EQUAL(std::string(), m.toString()); +} + +void ValueMetricTest::testLongValueMetric() +{ + LongValueMetric m("test", "tag", "description"); + m.addValue(100); + ASSERT_AVERAGE(m, 100, 100, 100, 1, 100); + m.addValue(100); + ASSERT_AVERAGE(m, 100, 100, 100, 2, 100); + m.addValue(41); + ASSERT_AVERAGE(m, 241.0 / 3, 41, 100, 3, 41); + LongValueMetric m2(m); + ASSERT_AVERAGE(m, 241.0 / 3, 41, 100, 3, 41); + m.reset(); + ASSERT_AVERAGE(m, 0, 0, 0, 0, 0); + + LongValueMetric n("m2", "", "desc"); + n.addValue(60); + ASSERT_AVERAGE(n, 60, 60, 60, 1, 60); + + LongValueMetric o = m2 + n; + ASSERT_AVERAGE(o, 140.25, 41, 100, 4, 101); + + o = n + m2; + ASSERT_AVERAGE(o, 140.25, 41, 100, 4, 101); + + std::string expected( + "test average=80.3333 last=41 min=41 max=100 count=3 total=241"); + CPPUNIT_ASSERT_EQUAL(expected, m2.toString()); + expected = "m2 average=140.25 last=101"; + CPPUNIT_ASSERT_EQUAL(expected, o.toString()); + + CPPUNIT_ASSERT_EQUAL(Double(41), Double(m2.getDoubleValue("value"))); + CPPUNIT_ASSERT_EQUAL(Double(241.0/3), Double(m2.getDoubleValue("average"))); + CPPUNIT_ASSERT_EQUAL(Double(41), Double(m2.getDoubleValue("min"))); + CPPUNIT_ASSERT_EQUAL(Double(100), Double(m2.getDoubleValue("max"))); + CPPUNIT_ASSERT_EQUAL(Double(41), Double(m2.getDoubleValue("last"))); + CPPUNIT_ASSERT_EQUAL(Double(3), Double(m2.getDoubleValue("count"))); + CPPUNIT_ASSERT_EQUAL(Double(241), Double(m2.getDoubleValue("total"))); + + CPPUNIT_ASSERT_EQUAL(int64_t(41), m2.getLongValue("value")); + CPPUNIT_ASSERT_EQUAL(int64_t(80), m2.getLongValue("average")); + CPPUNIT_ASSERT_EQUAL(int64_t(41), m2.getLongValue("min")); + CPPUNIT_ASSERT_EQUAL(int64_t(100), m2.getLongValue("max")); + CPPUNIT_ASSERT_EQUAL(int64_t(41), m2.getLongValue("last")); + CPPUNIT_ASSERT_EQUAL(int64_t(3), m2.getLongValue("count")); + CPPUNIT_ASSERT_EQUAL(int64_t(241), m2.getLongValue("total")); +} + +void ValueMetricTest::testSmallAverage() +{ + DoubleValueMetric m("test", "tag", "description"); + m.addValue(0.0001); + m.addValue(0.0002); + m.addValue(0.0003); + std::vector<Metric::LP> ownerList; + Metric::UP c(m.clone(ownerList, Metric::INACTIVE, 0, false)); + std::string expect("test average=0.0002 last=0.0003 min=0.0001 max=0.0003 count=3 total=0.0006"); + CPPUNIT_ASSERT_EQUAL(expect, m.toString()); + CPPUNIT_ASSERT_EQUAL(expect, c->toString()); +} + +void ValueMetricTest::testAddValueBatch() { + DoubleValueMetric m("test", "tag", "description"); + m.addValueBatch(100, 3); + ASSERT_AVERAGE(m, 100, 100, 100, 3, 100); + m.addValueBatch(0, 0); + ASSERT_AVERAGE(m, 100, 100, 100, 3, 100); +} + +namespace { + vespalib::string extractMetricJson(vespalib::stringref s) { + vespalib::StringTokenizer st(s, "\n", ""); + for (uint32_t i = st.size() - 1; i < st.size(); --i) { + if (st[i].find("\"name\":\"") != std::string::npos) { + vespalib::asciistream as; + as << "'\n"; + for (uint32_t j=i-1; j<st.size() - 2; ++j) { + as << st[j].substr(4) << "\n"; + } + as << "'"; + return as.str(); + } + } + throw vespalib::IllegalArgumentException("Didn't find metric"); + } + vespalib::string getJson(MetricManager& mm) { + vespalib::asciistream as; + vespalib::JsonStream stream(as, true); + JsonWriter writer(stream); + MetricLockGuard guard(mm.getMetricLock()); + mm.visit(guard, mm.getActiveMetrics(guard), writer, ""); + stream.finalize(); + return as.str(); + } +} + +void ValueMetricTest::testJson() { + MetricManager mm; + DoubleValueMetric m("test", "tag", "description"); + mm.registerMetric(mm.getMetricLock(), m); + + vespalib::string expected("'\n" + "{\n" + " \"name\":\"test\",\n" + " \"description\":\"description\",\n" + " \"values\":\n" + " {\n" + " \"average\":0.0,\n" + " \"count\":0,\n" + " \"min\":0.0,\n" + " \"max\":0.0,\n" + " \"last\":0.0\n" + " },\n" + " \"dimensions\":\n" + " {\n" + " }\n" + "}\n'" + ); + CPPUNIT_ASSERT_EQUAL(expected, extractMetricJson(getJson(mm))); + m.addValue(100); + expected = "'\n" + "{\n" + " \"name\":\"test\",\n" + " \"description\":\"description\",\n" + " \"values\":\n" + " {\n" + " \"average\":100.0,\n" + " \"count\":1,\n" + " \"min\":100.0,\n" + " \"max\":100.0,\n" + " \"last\":100.0\n" + " },\n" + " \"dimensions\":\n" + " {\n" + " }\n" + "}\n'"; + CPPUNIT_ASSERT_EQUAL(expected, extractMetricJson(getJson(mm))); + m.addValue(500); + expected = "'\n" + "{\n" + " \"name\":\"test\",\n" + " \"description\":\"description\",\n" + " \"values\":\n" + " {\n" + " \"average\":300.0,\n" + " \"count\":2,\n" + " \"min\":100.0,\n" + " \"max\":500.0,\n" + " \"last\":500.0\n" + " },\n" + " \"dimensions\":\n" + " {\n" + " }\n" + "}\n'"; + CPPUNIT_ASSERT_EQUAL(expected, extractMetricJson(getJson(mm))); +} + +} // namespace metrics diff --git a/metrics/src/vespa/metrics/.gitignore b/metrics/src/vespa/metrics/.gitignore new file mode 100644 index 00000000000..d6b11486175 --- /dev/null +++ b/metrics/src/vespa/metrics/.gitignore @@ -0,0 +1,10 @@ +.depend +.*.swp +.~ +*.o +*.So +Makefile +config-metricsmanager.h +config-metricsmanager.cpp + +/libmetrics.so.5.1 diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt new file mode 100644 index 00000000000..7a1813d7e18 --- /dev/null +++ b/metrics/src/vespa/metrics/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(metrics + SOURCES + metric.cpp + countmetric.cpp + valuemetric.cpp + metrictimer.cpp + metricset.cpp + metricmanager.cpp + metricsnapshot.cpp + jsonwriter.cpp + textwriter.cpp + xmlwriter.cpp + state_api_adapter.cpp + INSTALL lib64 + DEPENDS +) +vespa_generate_config(metrics metricsmanager.def) +install(FILES metricsmanager.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/metrics/src/vespa/metrics/countmetric.cpp b/metrics/src/vespa/metrics/countmetric.cpp new file mode 100644 index 00000000000..d152c09a242 --- /dev/null +++ b/metrics/src/vespa/metrics/countmetric.cpp @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/metrics/countmetric.h> + +#include <vespa/log/log.h> + +LOG_SETUP(".metrics.metric.count"); + +namespace metrics { + +void +AbstractCountMetric::logWarning(const char* msg) const +{ + LOG(warning, "%s", msg); +} + +void +AbstractCountMetric::sendLogCountEvent( + Metric::String name, uint64_t value) const +{ + EV_COUNT(name.c_str(), value); +} + +} // metrics diff --git a/metrics/src/vespa/metrics/countmetric.h b/metrics/src/vespa/metrics/countmetric.h new file mode 100644 index 00000000000..c9eef00f6ea --- /dev/null +++ b/metrics/src/vespa/metrics/countmetric.h @@ -0,0 +1,357 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class CountMetric + * \ingroup metrics + * + * \brief Metric representing a count. + * + * A counter metric have the following properties: + * - It can never decrease, only increase. + * - Logs its value as a count event. + * - When summing counts, the counts are added together. + */ + +#pragma once + +#include <sstream> +#include <vespa/metrics/metric.h> +#include <vespa/metrics/metricvalueset.h> +#include <vespa/vespalib/util/jsonstream.h> +#include <atomic> + +namespace metrics { + +struct AbstractCountMetric : public Metric { + virtual bool visit(MetricVisitor& visitor, + bool tagAsAutoGenerated = false) const + { + return visitor.visitCountMetric(*this, tagAsAutoGenerated); + } + virtual MetricValueClass::UP getValues() const = 0; + virtual bool sumOnAdd() const = 0; + virtual bool inUse(const MetricValueClass& v) const = 0; + +protected: + AbstractCountMetric(const String& name, const String& tags, + const String& description, MetricSet* owner = 0) + : Metric(name, tags, description, owner) + { + } + + AbstractCountMetric(const String& name, Tags dimensions, + const String& description, MetricSet* owner = 0) + : Metric(name, std::move(dimensions), description, owner) + { + } + + AbstractCountMetric(const AbstractCountMetric& other, MetricSet* owner) + : Metric(other, owner) + { + } + + void logWarning(const char* msg) const; + void sendLogCountEvent(Metric::String name, uint64_t value) const; +}; + +template<typename T, bool SumOnAdd> +class CountMetric : public AbstractCountMetric +{ + struct Values : public MetricValueClass { + T _value; + + struct AtomicImpl { + std::atomic<T> _value {0}; + }; + + void relaxedStoreInto(AtomicImpl& target) const noexcept { + target._value.store(_value, std::memory_order_relaxed); + } + + void relaxedLoadFrom(const AtomicImpl& source) noexcept { + _value = source._value.load(std::memory_order_relaxed); + } + + Values() : _value(0) {} + + std::string toString() const { + std::ostringstream ost; + ost << _value; + return ost.str(); + } + double getDoubleValue(const stringref &) const + { return static_cast<double>(_value); } + uint64_t getLongValue(const stringref &) const + { return static_cast<uint64_t>(_value); } + void output(const std::string&, std::ostream& out) const + { out << _value; } + void output(const std::string&, vespalib::JsonStream& stream) const + { stream << _value; } + bool inUse() const { return (_value != 0); } + }; + MetricValueSet<Values> _values; + + enum Flag { LOG_IF_UNSET = 2 }; + + bool logIfUnset() const { return _values.hasFlag(LOG_IF_UNSET); } + +public: + CountMetric(const String& name, const String& tags, + const String& description, MetricSet* owner = 0); + + CountMetric(const String& name, Tags dimensions, + const String& description, MetricSet* owner = 0); + + CountMetric(const CountMetric<T, SumOnAdd>& other, + CopyType, MetricSet* owner); + + virtual MetricValueClass::UP getValues() const + { return MetricValueClass::UP(new Values(_values.getValues())); } + void logOnlyIfSet() { _values.removeFlag(LOG_IF_UNSET); } + + void set(T value); + void inc(T value = 1); + void dec(T value = 1); + CountMetric & operator++() { inc(); return *this; } + CountMetric & operator--() { dec(); return *this; } + + CountMetric operator++(int) { CountMetric tmp(*this); inc(); return tmp; } + CountMetric operator--(int) { CountMetric tmp(*this); inc(); return tmp; } + + CountMetric & operator+=(const CountMetric &); + CountMetric & operator-=(const CountMetric &); + friend CountMetric operator+(const CountMetric & a, const CountMetric & b) { + CountMetric t(a); t += b; return t; + } + friend CountMetric operator-(const CountMetric & a, const CountMetric & b) { + CountMetric t(a); t -= b; return t; + } + + virtual CountMetric * clone( + std::vector<Metric::LP>&, CopyType type, MetricSet* owner, + bool /*includeUnused*/) const + { return new CountMetric<T, SumOnAdd>(*this, type, owner); } + + T getValue() const { return _values.getValues()._value; } + + void reset() { + _values.reset(); + } + + virtual bool logFromTotalMetrics() const { return true; } + virtual bool logEvent(const String& fullName) const; + + virtual void print(std::ostream&, bool verbose, + const std::string& indent, uint64_t secondsPassed) const; + + // Only one metric in valuemetric, so return it on any id. + virtual int64_t getLongValue(const stringref & id) const + { (void) id; return static_cast<int64_t>(getValue()); } + virtual double getDoubleValue(const stringref & id) const + { (void) id; return static_cast<double>(getValue()); } + + virtual bool inUse(const MetricValueClass& v) const + { return static_cast<const Values&>(v).inUse(); } + virtual bool used() const { return _values.getValues().inUse(); } + virtual bool sumOnAdd() const { return SumOnAdd; } + + virtual void addMemoryUsage(MemoryConsumption&) const; + + virtual void printDebug(std::ostream&, const std::string& indent) const; + + virtual void addToPart(Metric&) const; + virtual void addToSnapshot(Metric&, std::vector<Metric::LP>&) const; + +}; + +typedef CountMetric<uint64_t, true> LongCountMetric; + +template <typename T, bool SumOnAdd> +CountMetric<T, SumOnAdd>::CountMetric(const String& name, const String& tags, + const String& desc, MetricSet* owner) + : AbstractCountMetric(name, tags, desc, owner), + _values() +{ + _values.setFlag(LOG_IF_UNSET); +} + +template <typename T, bool SumOnAdd> +CountMetric<T, SumOnAdd>::CountMetric(const String& name, Tags dimensions, + const String& desc, MetricSet* owner) + : AbstractCountMetric(name, std::move(dimensions), desc, owner), + _values() +{ + _values.setFlag(LOG_IF_UNSET); +} + +template <typename T, bool SumOnAdd> +CountMetric<T, SumOnAdd>::CountMetric(const CountMetric<T, SumOnAdd>& other, + CopyType copyType, MetricSet* owner) + : AbstractCountMetric(other, owner), + _values(other._values, copyType == CLONE ? other._values.size() : 1) +{ +} + +template <typename T, bool SumOnAdd> +CountMetric<T, SumOnAdd>& +CountMetric<T, SumOnAdd>::operator+=(const CountMetric<T, SumOnAdd>& other) +{ + T otherValues(other.getValue()); + bool overflow; + Values values; + do { + values = _values.getValues(); + overflow = (values._value + otherValues < values._value); + values._value += otherValues; + } while (!_values.setValues(values)); + if (overflow) { + _values.reset(); + std::ostringstream ost; + ost << "Overflow in metric " << getPath() << " op +=. Resetting it."; + logWarning(ost.str().c_str()); + } + return *this; +} + +template <typename T, bool SumOnAdd> +CountMetric<T, SumOnAdd>& +CountMetric<T, SumOnAdd>::operator-=(const CountMetric<T, SumOnAdd>& other) +{ + T otherValues(other.getValue()); + bool underflow; + Values values; + do { + values = _values.getValues(); + underflow = (values._value - otherValues > values._value); + values._value -= otherValues; + } while (!_values.setValues(values)); + if (underflow) { + _values.reset(); + std::ostringstream ost; + ost << "Underflow in metric " << getPath() << " op -=. Resetting it."; + logWarning(ost.str().c_str()); + } + return *this; +} + +template <typename T, bool SumOnAdd> +void +CountMetric<T, SumOnAdd>::set(T value) +{ + Values values; + values._value = value; + while (!_values.setValues(values)) {} +} + +template <typename T, bool SumOnAdd> +void +CountMetric<T, SumOnAdd>::inc(T value) +{ + bool overflow; + Values values; + do { + values = _values.getValues(); + overflow = (values._value + value < values._value); + values._value += value; + } while (!_values.setValues(values)); + if (overflow) { + _values.reset(); + std::ostringstream ost; + ost << "Overflow in metric " << getPath() << ". Resetting it."; + logWarning(ost.str().c_str()); + } +} + +template <typename T, bool SumOnAdd> +void +CountMetric<T, SumOnAdd>::dec(T value) +{ + bool underflow; + Values values; + do { + values = _values.getValues(); + underflow = (values._value - value > values._value); + values._value -= value; + } while (!_values.setValues(values)); + if (underflow) { + _values.reset(); + std::ostringstream ost; + ost << "Underflow in metric " << getPath() << ". Resetting it."; + logWarning(ost.str().c_str()); + } +} + +template <typename T, bool SumOnAdd> +void +CountMetric<T, SumOnAdd>::addToSnapshot( + Metric& other, std::vector<Metric::LP>&) const +{ + CountMetric<T, SumOnAdd>& o( + reinterpret_cast<CountMetric<T, SumOnAdd>&>(other)); + o.inc(_values.getValues()._value); +} + +template <typename T, bool SumOnAdd> +void +CountMetric<T, SumOnAdd>::addToPart(Metric& other) const +{ + CountMetric<T, SumOnAdd>& o( + reinterpret_cast<CountMetric<T, SumOnAdd>&>(other)); + if (SumOnAdd) { + o.inc(_values.getValues()._value); + } else { + o.set((_values.getValues()._value + o._values.getValues()._value) / 2); + } +} + +template <typename T, bool SumOnAdd> +bool +CountMetric<T, SumOnAdd>::logEvent(const String& fullName) const +{ + Values values(_values.getValues()); + if (!logIfUnset() && values._value == 0) return false; + sendLogCountEvent( + fullName, static_cast<uint64_t>(values._value)); + return true; +} + +template <typename T, bool SumOnAdd> +void +CountMetric<T, SumOnAdd>::print(std::ostream& out, bool verbose, + const std::string& indent, + uint64_t secondsPassed) const +{ + (void) indent; + Values values(_values.getValues()); + if (values._value == 0 && !verbose) return; + out << this->_name << (SumOnAdd ? " count=" : " value=") << values._value; + if (SumOnAdd) { + if (secondsPassed != 0) { + double avgDiff = values._value / ((double) secondsPassed); + out << " average_change_per_second=" << avgDiff; + } + } +} + +template <typename T, bool SumOnAdd> +void +CountMetric<T, SumOnAdd>::addMemoryUsage(MemoryConsumption& mc) const +{ + ++mc._countMetricCount; + mc._countMetricValues += _values.getMemoryUsageAllocatedInternally(); + mc._countMetricMeta += sizeof(CountMetric<T, SumOnAdd>) + - sizeof(Metric); + Metric::addMemoryUsage(mc); +} + +template <typename T, bool SumOnAdd> +void +CountMetric<T, SumOnAdd>::printDebug(std::ostream& out, + const std::string& indent) const +{ + Values values(_values.getValues()); + out << "count=" << values._value << " "; + Metric::printDebug(out, indent); +} + +} // metrics + diff --git a/metrics/src/vespa/metrics/jsonwriter.cpp b/metrics/src/vespa/metrics/jsonwriter.cpp new file mode 100644 index 00000000000..0847a5a1624 --- /dev/null +++ b/metrics/src/vespa/metrics/jsonwriter.cpp @@ -0,0 +1,179 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/metrics/jsonwriter.h> + +#include <vespa/metrics/countmetric.h> +#include <vespa/metrics/valuemetric.h> +#include <vespa/metrics/metricsnapshot.h> +#include <vespa/vespalib/util/jsonstream.h> + +#include <algorithm> +#include <iterator> + +namespace metrics { + +JsonWriter::JsonWriter(vespalib::JsonStream& stream) + : _stream(stream), + _flag(NOT_STARTED), + _dimensionStack(), + _period(0) +{ +} + +bool +JsonWriter::visitSnapshot(const MetricSnapshot& snapshot) +{ + _stream << Object() + << "snapshot" << Object() + << "from" << snapshot.getFromTime() + << "to" << snapshot.getToTime() + << End() + << "values" << Array(); + _flag = SNAPSHOT_STARTED; + _period = snapshot.getPeriod(); + return true; +} + +void +JsonWriter::doneVisitingSnapshot(const MetricSnapshot&) +{ + assert(_flag == SNAPSHOT_STARTED); + _stream << End() << End(); + _flag = NOT_STARTED; + _period = 0; +} + +void +JsonWriter::pushLegacyDimensionFromSet(const MetricSet& ms) +{ + _dimensionStack.push_back({{ms.getDimensionKey(), ms.getName()}}); +} + +void +JsonWriter::pushAllDimensionsFromSet(const MetricSet& ms) +{ + _dimensionStack.push_back(ms.getTags()); +} + +bool +JsonWriter::visitMetricSet(const MetricSet& set, bool) +{ + if (!set.getDimensionKey().empty()) { + pushLegacyDimensionFromSet(set); + } else { + pushAllDimensionsFromSet(set); + } + return true; +} + +void +JsonWriter::doneVisitingMetricSet(const MetricSet&) +{ + _dimensionStack.pop_back(); +} + +void +JsonWriter::writeCommonPrefix(const Metric& m) +{ + if (_flag == NOT_STARTED) { + _stream << Array(); + _flag = METRICS_WRITTEN; + } + _stream << Object() + << "name" << m.getPath() + << "description" << m.getDescription(); +} + +void +JsonWriter::writeDimensions(const DimensionSet& dimensions) +{ + for (const auto& dimension : dimensions) { + if (!dimension.key.empty() && !dimension.value.empty()) { + _stream << dimension.key << dimension.value; + } + } +} + +void +JsonWriter::writeInheritedDimensions() +{ + for (const auto& dimensions : _dimensionStack) { + writeDimensions(dimensions); + } +} + +void +JsonWriter::writeMetricSpecificDimensions(const Metric& m) +{ + if (isLeafMetric(m)) { + writeDimensions(m.getTags()); + } +} + +void +JsonWriter::writeCommonPostfix(const Metric& m) +{ + _stream << "dimensions" << Object(); + + writeInheritedDimensions(); + writeMetricSpecificDimensions(m); + + _stream << End() << End(); +} + +bool +JsonWriter::visitCountMetric(const AbstractCountMetric& m, bool) +{ + writeCommonPrefix(m); + uint64_t count = m.getLongValue("count"); + _stream << "values" << Object() + << "count" << count; + if (_period > 0) { + uint64_t rate = 1000000 * count / _period; + _stream << "rate" << (rate / 1000000.0); + } + _stream << End(); + writeCommonPostfix(m); + return true; +} + +bool +JsonWriter::visitValueMetric(const AbstractValueMetric& m, bool) +{ + writeCommonPrefix(m); + MetricValueClass::UP values(m.getValues()); + _stream << "values" << Object() + << "average"; + if (values->getLongValue("count") == 0) { + _stream << 0.0; + } else { + _stream << (values->getDoubleValue("total") + / values->getDoubleValue("count")); + } + _stream << "count"; + values->output("count", _stream); + if (_period > 0) { + uint64_t rate = 1000000 * values->getLongValue("count") / _period; + _stream << "rate" << (rate / 1000000.0); + } + _stream << "min"; + values->output("min", _stream); + _stream << "max"; + values->output("max", _stream); + _stream << "last"; + values->output("last", _stream); + _stream << End(); + writeCommonPostfix(m); + return true; +} + +void +JsonWriter::doneVisiting() +{ + if (_flag == METRICS_WRITTEN) { + _stream << End(); + _flag = NOT_STARTED; + } + assert(_flag == NOT_STARTED); +} + +} // metrics diff --git a/metrics/src/vespa/metrics/jsonwriter.h b/metrics/src/vespa/metrics/jsonwriter.h new file mode 100644 index 00000000000..37de3c0c0b1 --- /dev/null +++ b/metrics/src/vespa/metrics/jsonwriter.h @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/metrics/metric.h> +#include <vespa/vespalib/util/jsonstream.h> + +namespace metrics { + +class JsonWriter : public MetricVisitor, public vespalib::JsonStreamTypes { + vespalib::JsonStream& _stream; + enum Flag { NOT_STARTED, SNAPSHOT_STARTED, METRICS_WRITTEN }; + Flag _flag; + // Not a "true" set, but dimensions are guaranteed to be unique for any + // given metric set. + using DimensionSet = Metric::Tags; + + std::vector<DimensionSet> _dimensionStack; + uint64_t _period; + +public: + JsonWriter(vespalib::JsonStream&); + +private: + virtual bool visitSnapshot(const MetricSnapshot&); + virtual void doneVisitingSnapshot(const MetricSnapshot&); + virtual bool visitMetricSet(const MetricSet&, bool autoGenerated); + virtual void doneVisitingMetricSet(const MetricSet&); + virtual bool visitCountMetric(const AbstractCountMetric&, + bool autoGenerated); + virtual bool visitValueMetric(const AbstractValueMetric&, + bool autoGenerated); + virtual void doneVisiting(); + + void checkIfArrayNeedsToBeStarted(); + void writeCommonPrefix(const Metric& m); + void writeCommonPostfix(const Metric& m); + + void pushLegacyDimensionFromSet(const MetricSet&); + void pushAllDimensionsFromSet(const MetricSet&); + void writeDimensions(const DimensionSet&); + void writeInheritedDimensions(); + void writeMetricSpecificDimensions(const Metric&); + + bool isLeafMetric(const Metric& m) const { return !m.isMetricSet(); } +}; + +} + diff --git a/metrics/src/vespa/metrics/loadmetric.h b/metrics/src/vespa/metrics/loadmetric.h new file mode 100644 index 00000000000..dc0762dd161 --- /dev/null +++ b/metrics/src/vespa/metrics/loadmetric.h @@ -0,0 +1,167 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \file loadmetric.h + * \ingroup metrics + * + * \brief Utility class for creating metrics for all load types. + * + * To better see how different load types behave in the system we want to log + * separate metrics for various loadtypes. To make it easy to create and use + * such metrics, this class is a wrapper class that sets up one metric per load + * type. + * + * In order to make it easy to use load metrics, they are templated on the type, + * such that you get the correct type out of operator[]. Load metric needs to + * clone metrics on creation though, so if you want load metrics of a metric set + * you need to properly implement clone() for that set. + */ + +#pragma once + +#include <vespa/metrics/metricset.h> +#include <vespa/metrics/summetric.h> +#include <vespa/vespalib/util/linkedptr.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/stllike/hash_map.h> + +namespace metrics { + +class MetricSet; + +class LoadType { +public: + typedef vespalib::string string; + LoadType(uint32_t id, const string& name) : _id(id), _name(name) {} + + uint32_t getId() const { return _id; } + const string& getName() const { return _name; } + + string toString() const { + return vespalib::make_string("%s(%u)", _name.c_str(), _id); + } +private: + uint32_t _id; + string _name; +}; + +typedef std::vector<LoadType> LoadTypeSet; + +template<typename MetricType> +class LoadMetric : public metrics::MetricSet { + std::vector<metrics::Metric::LP> _ownerList; + typedef vespalib::LinkedPtr<MetricType> MetricTypeLP; + vespalib::hash_map<uint32_t, MetricTypeLP> _metrics; + metrics::SumMetric<MetricType> _sum; + +public: + /** + * Create a load metric using the given metric as a template to how they + * shuold look. They will get prefix names based on load types existing. + */ + LoadMetric(const LoadTypeSet& loadTypes, const MetricType& metric, + metrics::MetricSet* owner = 0) + : MetricSet(metric.getName(), "", metric.getDescription(), owner), + _metrics(), + _sum("sum", "loadsum sum", "Sum of all load metrics", this) + { + _metrics.resize(loadTypes.size()); + // Currently, we only set tags and description on the metric set + // itself, to cut down on size of output when downloading metrics, + // and since matching tags of parent is just as good as matching + // them specifically. + setTags(metric.getTags()); + Tags noTags; + for (uint32_t i=0; i<loadTypes.size(); ++i) { + MetricTypeLP copy( + dynamic_cast<MetricType*>( + metric.clone(_ownerList, CLONE, 0, false))); + assert(copy.get()); + copy->setName(loadTypes[i].getName()); + copy->setTags(noTags); + _metrics[loadTypes[i].getId()] = copy; + registerMetric(*copy); + _sum.addMetricToSum(*copy); + } + metrics::trim(_ownerList); + } + + /** + * A load metric implements a copy constructor and a clone functions that + * clone content, and resetting name/tags/description, just so metric + * implementors can implement clone() by using regular construction and + * then assign the values to the new set. (Without screwing up copying as + * the load metric alters this data in supplied metric) + */ + LoadMetric(const LoadMetric<MetricType>& other, metrics::MetricSet* owner) + : MetricSet(other.getName(), "", other.getDescription(), owner), + _metrics(), + _sum("sum", "loadsum sum", "Sum of all load metrics", this) + { + _metrics.resize(2 * other._metrics.size()); + setTags(other.getTags()); + Tags noTags; + for (typename vespalib::hash_map<uint32_t, MetricTypeLP>::const_iterator + it = other._metrics.begin(); it != other._metrics.end(); ++it) + { + MetricTypeLP copy(dynamic_cast<MetricType*>( + it->second->clone(_ownerList, CLONE, 0, false))); + assert(copy.get()); + copy->setName(it->second->getName()); + copy->setTags(noTags); + _metrics[it->first] = copy; + registerMetric(*copy); + _sum.addMetricToSum(*copy); + } + metrics::trim(_ownerList); + } + + virtual Metric* clone(std::vector<Metric::LP>& ownerList, + CopyType copyType, MetricSet* owner, + bool includeUnused = false) const + { + if (copyType == INACTIVE) { + return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused); + } + return new LoadMetric<MetricType>(*this, owner); + } + + MetricType& operator[](const LoadType& type) { return getMetric(type); } + const MetricType& operator[](const LoadType& type) const + { return const_cast<LoadMetric<MetricType>*>(this)->getMetric(type); } + MetricType& getMetric(const LoadType& type) { + MetricType* metric; + + typename vespalib::hash_map<uint32_t, MetricTypeLP>::iterator it( + _metrics.find(type.getId())); + if (it == _metrics.end()) { + it = _metrics.find(0); + assert(it != _metrics.end()); // Default should always exist + metric = it->second.get(); + assert(metric); + } else { + metric = it->second.get(); + assert(metric); + } + + return *metric; + } + + virtual void addMemoryUsage(metrics::MemoryConsumption& mc) const { + ++mc._loadMetricCount; + mc._loadMetricMeta += sizeof(metrics::Metric::LP) * _ownerList.size(); + for (typename vespalib::hash_map<uint32_t, MetricTypeLP>::const_iterator + it = _metrics.begin(); it != _metrics.end(); ++it) + { + mc._loadMetricMeta += sizeof(uint32_t) + sizeof(MetricTypeLP); + } + _sum.addMemoryUsage(mc); + mc._loadMetricMeta += sizeof(LoadMetric<MetricType>) + - sizeof(metrics::MetricSet) + - sizeof(metrics::SumMetric<MetricType>); + metrics::MetricSet::addMemoryUsage(mc); + } + +}; + +} // documentapi + diff --git a/metrics/src/vespa/metrics/memoryconsumption.h b/metrics/src/vespa/metrics/memoryconsumption.h new file mode 100644 index 00000000000..a96ad4d3e43 --- /dev/null +++ b/metrics/src/vespa/metrics/memoryconsumption.h @@ -0,0 +1,192 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class metrics::MemoryConsumption + * \ingroup metrics + * + * \brief An instance of this class describes memory usage for a set of metrics. + * + * Typically, you ask the memory manager for memory consumption of all metrics. + * An instance of this class is created and sent through all metrics to track + * memory consumption. Tracking may be a bit expensive, so this shouldn't be + * checked too often. Primary use is to detect what parts actually use the most + * memory. Secondary use would be to add it as a metric periodically updated. + * + * The memory consumption object keeps track of various groups of memory users, + * such as to give a good overview of where the memory is used. + */ + +#pragma once + +#include <vespa/vespalib/util/printable.h> +#include <vespa/vespalib/stllike/hash_set.h> +#include <sstream> + +namespace metrics { + +struct MemoryConsumption : public vespalib::Printable { + typedef std::unique_ptr<MemoryConsumption> UP; + + uint32_t _consumerCount; + uint32_t _consumerId; + uint32_t _consumerIdUnique; + uint32_t _consumerMetricsInTotal; + uint32_t _consumerMetricIds; + uint32_t _consumerMetricIdsUnique; + uint32_t _consumerMeta; + + uint32_t _snapshotSetCount; + uint32_t _snapshotSetMeta; + + uint32_t _nameHash; + uint32_t _nameHashStrings; + uint32_t _nameHashUnique; + + uint32_t _snapshotCount; + uint32_t _snapshotName; + uint32_t _snapshotNameUnique; + uint32_t _snapshotMeta; + + uint32_t _metricCount; + uint32_t _metricMeta; + uint32_t _metricName; + uint32_t _metricNameUnique; + uint32_t _metricPath; + uint32_t _metricPathUnique; + uint32_t _metricDescription; + uint32_t _metricDescriptionUnique; + uint32_t _metricTagCount; + uint32_t _metricTags; + uint32_t _metricTagsUnique; + + uint32_t _metricSetCount; + uint32_t _metricSetMeta; + uint32_t _metricSetOrder; + + uint32_t _countMetricCount; + uint32_t _countMetricValues; + uint32_t _countMetricMeta; + + uint32_t _valueMetricCount; + uint32_t _valueMetricValues; + uint32_t _valueMetricMeta; + + uint32_t _sumMetricCount; + uint32_t _sumMetricMeta; + uint32_t _sumMetricParentPath; + uint32_t _sumMetricParentPathUnique; + + uint32_t _loadMetricCount; + uint32_t _loadMetricMeta; + + uint32_t _totalStringCount; + + vespalib::hash_set<const void*> _seenStrings; + std::vector<std::pair<std::string, uint32_t> > _snapShotUsage; + + MemoryConsumption() { + memset(&_consumerCount, 0, + reinterpret_cast<size_t>(&_seenStrings) + - reinterpret_cast<size_t>(&_consumerCount)); + _seenStrings.resize(1000); + } + + /** Get memory usage of a string that is not included when doing sizeof */ + uint32_t getStringMemoryUsage(const std::string& s, uint32_t& uniqueCount) { + ++_totalStringCount; + const char* internalString = s.c_str(); + if (_seenStrings.find(internalString) != _seenStrings.end()) { + return 0; + } + ++uniqueCount; + _seenStrings.insert(internalString); + return s.capacity(); + } + + void addSnapShotUsage(const std::string& name, uint32_t usage) { + _snapShotUsage.push_back(std::pair<std::string, uint32_t>(name, usage)); + } + + uint32_t getTotalMemoryUsage() const { + return _consumerId + _consumerMetricIds + _consumerMeta + + _snapshotSetMeta + _snapshotName + _snapshotMeta + _metricMeta + + _metricName + _metricPath + _metricDescription + + _metricTags + _metricSetMeta + _nameHash + _nameHashStrings + + _metricSetOrder + _countMetricValues + _countMetricMeta + + _valueMetricValues + _valueMetricMeta + _sumMetricMeta + + _sumMetricParentPath + _loadMetricMeta; + } + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const + { + (void) verbose; + std::string newl = "\n" + indent + " "; + out << "MemoryConsumption(" + << newl << "Total memory used: " << bval(getTotalMemoryUsage()) + << newl << "Consumer count: " << _consumerCount + << newl << "Consumer ids: " << bval(_consumerId) + << newl << "Consumer metric count: " << _consumerMetricsInTotal + << newl << "Consumer metric ids: " << bval(_consumerMetricIds) + << newl << "Consumer meta: " << bval(_consumerMeta) + << newl << "Name hash: " << bval(_nameHash) + << newl << "Name hash strings: " << bval(_nameHashStrings) + << newl << "Snapshot set count: " << _snapshotSetCount + << newl << "Snapshot set meta: " << bval(_snapshotSetMeta) + << newl << "Snapshot count: " << _snapshotCount + << newl << "Snapshot name: " << bval(_snapshotName) + << newl << "Snapshot meta: " << bval(_snapshotMeta) + << newl << "Metric count: " << _metricCount + << newl << "Metric meta: " << bval(_metricMeta) + << newl << "Metric names: " << bval(_metricName) + << newl << "Metric paths: " << bval(_metricPath) + << newl << "Metric descriptions: " << bval(_metricDescription) + << newl << "Metric tag count: " << _metricTagCount + << newl << "Metric tags: " << bval(_metricTags) + << newl << "Metric set count: " << _metricSetCount + << newl << "Metric set meta: " << bval(_metricSetMeta) + << newl << "Metric set order list: " << bval(_metricSetOrder) + << newl << "Count metric count: " << _countMetricCount + << newl << "Count metric values: " << bval(_countMetricValues) + << newl << "Count metric meta: " << bval(_countMetricMeta) + << newl << "Value metric count: " << _valueMetricCount + << newl << "Value metric values: " << bval(_valueMetricValues) + << newl << "Value metric meta: " << bval(_valueMetricMeta) + << newl << "Sum metric count: " << _sumMetricCount + << newl << "Sum metric meta: " << bval(_sumMetricMeta) + << newl << "Sum metric parent path: " << bval(_sumMetricParentPath) + << newl << "Load metric count: " << _loadMetricCount + << newl << "Load metric meta: " << bval(_loadMetricMeta) + << newl << "Unique string count: " << _seenStrings.size() + << newl << "Strings stored: " << _totalStringCount + << newl << "Unique consumer ids: " << _consumerIdUnique + << newl << "Unique cons metric ids: " << _consumerMetricIdsUnique + << newl << "Unique snapshot names: " << _snapshotNameUnique + << newl << "Unique metric names: " << _metricNameUnique + << newl << "Unique metric paths: " << _metricPathUnique + << newl << "Unique metric descs: " << _metricDescriptionUnique + << newl << "Unique metric tags: " << _metricTagsUnique + << newl << "Unique sum metric paths: " << _sumMetricParentPathUnique + << newl << "Unique name hash strings: " << _nameHashUnique; + for (uint32_t i=0; i<_snapShotUsage.size(); ++i) { + out << newl << "Snapshot " << _snapShotUsage[i].first << ": " + << bval(_snapShotUsage[i].second); + } + out << "\n" << indent << ")"; + } + + static std::string bval(uint32_t bytes) { + std::ostringstream ost; + if (bytes < 10 * 1024) { + ost << bytes << " B"; + } else if (bytes < 10 * 1024 * 1024) { + ost << (bytes / 1024) << " kB"; + } else { + ost << (bytes / (1024 * 1024)) << " MB"; + } + return ost.str(); + } + +}; + +} // metrics + diff --git a/metrics/src/vespa/metrics/metric.cpp b/metrics/src/vespa/metrics/metric.cpp new file mode 100644 index 00000000000..f4ad47871e6 --- /dev/null +++ b/metrics/src/vespa/metrics/metric.cpp @@ -0,0 +1,234 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/metrics/metric.h> + +#include <vespa/metrics/countmetric.h> +#include <vespa/metrics/valuemetric.h> +#include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/metrics/metricset.h> + +#include <algorithm> +#include <iterator> + +namespace metrics { + +bool +MetricVisitor::visitCountMetric(const AbstractCountMetric& m, + bool autoGenerated) +{ + return visitMetric(m, autoGenerated); +} + +bool +MetricVisitor::visitValueMetric(const AbstractValueMetric& m, + bool autoGenerated) +{ + return visitMetric(m, autoGenerated); +} + +bool +MetricVisitor::visitMetric(const Metric&, bool) +{ + throw vespalib::IllegalStateException( + "visitMetric called with default implementation. You should either " + "override specific visit functions or this catchall function.", + VESPA_STRLOC); +} + +namespace { + Metric::Tags legacyTagStringToKeyedTags(const std::string& tagStr) { + vespalib::StringTokenizer tokenizer(tagStr, " \t\r\f"); + Metric::Tags tags; + std::transform(tokenizer.getTokens().begin(), + tokenizer.getTokens().end(), + std::back_inserter(tags), + [](const std::string& s) { return Tag(s, ""); }); + return tags; + } + std::string namePattern = "[a-zA-Z][_a-zA-Z0-9]*"; +} + +vespalib::Regexp Metric::_namePattern(namePattern); + +Metric::Metric(const String& name, + const String& tags, + const String& description, + MetricSet* owner) + : _name(name), + _description(description), + _tags(legacyTagStringToKeyedTags(tags)), + _owner(nullptr) // Set later by registry +{ + verifyConstructionParameters(); + assignMangledNameWithDimensions(); + registerWithOwnerIfRequired(owner); +} + +Metric::Metric(const String& name, + Tags dimensions, + const String& description, + MetricSet* owner) + : _name(name), + _description(description), + _tags(std::move(dimensions)), + _owner(nullptr) +{ + verifyConstructionParameters(); + assignMangledNameWithDimensions(); + registerWithOwnerIfRequired(owner); +} + + +Metric::Metric(const Metric& other, MetricSet* owner) + : Printable(other), + _name(other._name), + _description(other._description), + _tags(other._tags), + _owner(nullptr) +{ + assignMangledNameWithDimensions(); + registerWithOwnerIfRequired(owner); +} + +bool +Metric::tagsSpecifyAtLeastOneDimension(const Tags& tags) const +{ + auto hasNonEmptyTagValue = [](const Tag& t) { return !t.value.empty(); }; + return std::any_of(tags.begin(), tags.end(), hasNonEmptyTagValue); +} + +void +Metric::assignMangledNameWithDimensions() +{ + if (!tagsSpecifyAtLeastOneDimension(_tags)) { + return; + } + sortTagsInDeterministicOrder(); + _mangledName = createMangledNameWithDimensions(); +} + +void +Metric::sortTagsInDeterministicOrder() +{ + std::sort(_tags.begin(), _tags.end(), [](const Tag& a, const Tag& b) { + return a.key < b.key; + }); +} + +std::string +Metric::createMangledNameWithDimensions() const +{ + vespalib::asciistream s; + s << _name << '{'; + const size_t sz = _tags.size(); + for (size_t i = 0; i < sz; ++i) { + const Tag& dimension(_tags[i]); + if (dimension.value.empty()) { + continue; + } + if (i != 0) { + s << ','; + } + s << dimension.key << ':' << dimension.value; + } + s << '}'; + return s.str(); +} + +void +Metric::verifyConstructionParameters() +{ + if (_name.size() == 0) { + throw vespalib::IllegalArgumentException( + "Metric cannot have empty name", VESPA_STRLOC); + } + if (!_namePattern.match(_name)) { + throw vespalib::IllegalArgumentException( + "Illegal metric name '" + _name + "'. Names must match pattern " + + namePattern, VESPA_STRLOC); + } +} + +void +Metric::registerWithOwnerIfRequired(MetricSet* owner) +{ + if (owner) { + owner->registerMetric(*this); + } +} + +const MetricSet* +Metric::getRoot() const +{ + return (_owner == 0 ? (isMetricSet() ? static_cast<const MetricSet*>(this) + : 0) + : _owner->getRoot()); +} + +vespalib::string +Metric::getPath() const +{ + if (_owner == 0 || _owner->_owner == 0) { + return _name; + } else { + return _owner->getPath() + "." + _name; + } +} + +std::vector<Metric::String> +Metric::getPathVector() const +{ + std::vector<String> result; + result.push_back(_name); + const MetricSet* owner(_owner); + while (owner != 0) { + result.push_back(owner->_name); + owner = owner->_owner; + } + std::reverse(result.begin(), result.end()); + return result; +} + +bool +Metric::hasTag(const String& tag) const +{ + return std::find_if(_tags.begin(), _tags.end(), [&](const Tag& t) { + return t.key == tag; + }) != _tags.end(); +} + +void +Metric::addMemoryUsage(MemoryConsumption& mc) const +{ + ++mc._metricCount; + mc._metricName += mc.getStringMemoryUsage(_name, mc._metricNameUnique); + mc._metricDescription += mc.getStringMemoryUsage( + _description, mc._metricDescriptionUnique); + mc._metricTagCount += _tags.size(); + // XXX figure out what we actually want to report from tags here... + // XXX we don't care about unique strings since they don't matter anymore. + //mc._metricTags += mc.getStringMemoryUsage(_tags, mc._metricTagsUnique); + mc._metricMeta += sizeof(Metric); +} + +void +Metric::updateNames(NameHash& hash) const +{ + Metric& m(const_cast<Metric&>(*this)); + hash.updateName(m._name); + hash.updateName(m._description); + // Tags use vespalib::string which isn't refcounted under the hood and + // use small string optimizations, meaning the implicit ref sharing hack + // won't work for them anyway. +} + +void +Metric::printDebug(std::ostream& out, const std::string& indent) const +{ + (void) indent; + out << "name=" << _name << ", instance=" << ((const void*) this) + << ", owner=" << ((const void*) _owner); +} + +} // metrics diff --git a/metrics/src/vespa/metrics/metric.h b/metrics/src/vespa/metrics/metric.h new file mode 100644 index 00000000000..7c42d187f30 --- /dev/null +++ b/metrics/src/vespa/metrics/metric.h @@ -0,0 +1,319 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/printable.h> +#include <map> +#include <vespa/metrics/config-metricsmanager.h> +#include <vespa/metrics/memoryconsumption.h> +#include <vespa/metrics/namehash.h> +#include <string> +#include <vespa/vespalib/objects/cloneable.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/linkedptr.h> +#include <vespa/vespalib/util/jsonwriter.h> +#include <vespa/vespalib/util/regexp.h> +#include <vespa/vespalib/text/stringtokenizer.h> + +namespace metrics { + +class AbstractCountMetric; +class AbstractValueMetric; +class Metric; +class MetricSet; +class MetricSnapshot; +class XmlWriterMetricVisitor; + +template<typename T> +void trim(std::vector<T>& v) { + if (v.size() < v.capacity()) { + std::vector<T> copy(v); + copy.swap(v); + } + assert(v.size() == v.capacity()); +} + +/** Implement class to visit metrics. */ +struct MetricVisitor { + virtual ~MetricVisitor() {} + + /** + * Visit a snapshot. Return true to visit content of the snapshot + */ + virtual bool visitSnapshot(const MetricSnapshot&) { return true; } + virtual void doneVisitingSnapshot(const MetricSnapshot&) {} + + /** + * Visit a metric set. + * + * @param autoGenerated True for metric sets that are generated on the + * fly such as in sum metrics. + * @return True if you want to visit the content of this metric set. + */ + virtual bool visitMetricSet(const MetricSet&, bool autoGenerated) + { (void) autoGenerated; return true; } + + /** + * Callback visitors can use if they need to know the tree traversal of + * metric sets. This function is not called if visitMetricSet returned + * false. + */ + virtual void doneVisitingMetricSet(const MetricSet&) {} + + /** + * Visit a primitive metric within an accepted metric set. + * + * @return True if you want to continue visiting, false to abort. + */ + virtual bool visitCountMetric(const AbstractCountMetric& m, + bool autoGenerated); + /** + * Visit a primitive metric within an accepted metric set. + * + * @return True if you want to continue visiting, false to abort. + */ + virtual bool visitValueMetric(const AbstractValueMetric& m, + bool autoGenerated); + + /** + * Visit function for visiting primitive metrics in one function. Only + * called if not overriding specific primitive metric functions. + */ + virtual bool visitMetric(const Metric&, bool autoGenerated); + + /** + * To avoid having to destruct the visitor in order to get any post + * processing done you can use this hook. (Only called by functions knowing + * they are top level, so you have to call this manually if visiting metrics + * directly, and not through manager) + */ + virtual void doneVisiting() {} +}; + +/** + * A tag is a simple key-value mapping associated with a metric. Tags are used + * for either constraining the set of metrics reported to a consumer (e.g. "do + * not include metrics tagged as 'partofsum'") or to represent dimensions for + * metric sets or individual metrics. In the former case, the key represents + * the tag and the value remains empty. In the latter (dimension) case, the key + * is the dimension name and the value is the dimension value. + */ +struct Tag +{ + vespalib::string key; + vespalib::string value; + + Tag(vespalib::stringref k, vespalib::stringref v) + : key(k), value(v) + { + } +}; + +class Metric : public vespalib::Printable +{ +public: + using String = std::string; + using stringref = vespalib::stringref; + using UP = std::unique_ptr<Metric>; + using LP = vespalib::LinkedPtr<Metric>; + using SP = std::shared_ptr<Metric>; + using Tags = std::vector<Tag>; + + static vespalib::Regexp _namePattern; + + Metric(const String& name, const String& tags, + const String& description, MetricSet* owner = 0); + + Metric(const String& name, + Tags dimensions, + const String& description, + MetricSet* owner = 0); + + Metric(const Metric& other, MetricSet* owner); + virtual ~Metric() {} + + const String& getName() const { return _name; } + /** + * Get mangled name iff the metric contains any dimensions, otherwise + * the original metric name is returned. + */ + const String& getMangledName() const { + return (_mangledName.empty() ? _name : _mangledName); + } + vespalib::string getPath() const; + std::vector<String> getPathVector() const; + const String& getDescription() const { return _description; } + const Tags& getTags() const { return _tags; } + /** Return whether there exists a tag with a key equal to 'tag' */ + bool hasTag(const String& tag) const; + + enum CopyType { CLONE, INACTIVE }; + /** + * The clone function will clone metrics to an identical subtree of + * metrics. Clone is primarily used for load metrics that wants to clone + * a template metric for each loadtype. But it should work generically. + * + * @param type If set to inactive, sum metrics will evaluate to primitives + * and metrics can save memory by knowing no updates are coming. + * @param includeUnused When creating snapshots we do not want to include + * unused metrics, but while generating sum metric sum in active + * metrics we want to. This has no affect if type is CLONE. + */ + virtual Metric* clone(std::vector<Metric::LP>& ownerList, + CopyType type, MetricSet* owner, + bool includeUnused = false) const = 0; + + /** + * Utility function for assigning values from one metric of identical type + * to this metric. For simplicity sake it does a const cast and calls + * addToSnapshot, which should not alter source if reset is false. This can + * not be used to copy between active metrics and inactive copies. + */ + virtual Metric* assignValues(const Metric& m) { + std::vector<Metric::LP> ownerList; + const_cast<Metric&>(m).addToSnapshot(*this, ownerList); + // As this should only be called among active metrics, all metrics + // should exist and owner list should thus always end up empty. + assert(ownerList.empty()); + return this; + } + + /** Reset all metric values. */ + virtual void reset() = 0; + + virtual bool logFromTotalMetrics() const { return false; } + /** Implement to make metric able to log event. */ + virtual bool logEvent(const String& fullName) const = 0; + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const { + print(out, verbose, indent, 0); + } + virtual void print(std::ostream&, bool verbose, const std::string& indent, + uint64_t secondsPassed) const = 0; + + /** + * Most metrics report numbers of some kind. To be able to report numbers + * without having code to handle each possible metric type, these functions + * exist to extract raw data to present easily. + * @param id The part of the metric to extract. For instance, an average + * metric have average, + */ + virtual int64_t getLongValue(const stringref & id) const = 0; + virtual double getDoubleValue(const stringref & id) const = 0; + + /** + * When snapshotting we need to be able to add data from one set of metrics + * to another set of metrics taken at another time. MetricSet doesn't know + * the type of the metrics it contains, so we need a generic function for + * doing this. This function assumes metric given as input is of the exact + * same type as the one it is called on for simplicity. This is true when + * adding to snapshots as they have been created with clone and is thus + * always exactly equal. + * + * @param m Metric of exact same type as this one. (Will core if wrong) + * @param ownerList In case snapshot doesn't contain given metric, it can + * create them and add them to ownerlist. + */ + virtual void addToSnapshot(Metric& m, + std::vector<Metric::LP>& ownerList) const = 0; + + /** + * For sum metrics to work with metric sets, metric sets need operator+=. + * To implement this, we need a function to add any metric type together. + * This is different from adding to snapshot. When adding to snapshots we + * add different time periods to the same metric, but when adding parts + * together we add different metrics for the same time. For instance, an + * average metric of queuesize, should just add new values to create new + * average when adding to snapshot, but when adding parts, the averages + * themselves should be added together. + * + * @param m Metric of exact same type as this one. (Will core if wrong) + */ + virtual void addToPart(Metric& m) const = 0; + + virtual bool visit(MetricVisitor& visitor, + bool tagAsAutoGenerated = false) const = 0; + + /** Used by sum metric to alter name of cloned metric for sum. */ + void setName(const String& name) { _name = name; } + /** Used by sum metric to alter description of cloned metric for sum. */ + void setDescription(const String& d) { _description = d; } + /** Used by sum metric to alter tag of cloned metric for sum. */ + void setTags(Tags tags) { + _tags = std::move(tags); + } + + /** Set whether metrics have ever been set. */ + virtual bool used() const = 0; + + /** Returns true if this metric is registered in a metric set. */ + bool isRegistered() const { return (_owner != 0); } + + const MetricSet* getOwner() const { return _owner; } + const MetricSet* getRoot() const; + + /** Used by metric set in register functions. */ + void setRegistered(MetricSet* owner) { _owner = owner; } + + virtual void addMemoryUsage(MemoryConsumption&) const; + + /** + * Update names using the given name hash, to utilize ref counting. + * + * NOTE: + * This is a hack that only works on GCC until they decide to finally break + * ABI compatibility and remove that particular multicore-hostile feature + * of their std::string implementation. If we want proper string ref + * counting, all strings should be replaced with explicit string handles + * and should only be created via a shared factory. + */ + virtual void updateNames(NameHash&) const; + + /** Print debug information of the metric tree. */ + virtual void printDebug(std::ostream&, const std::string& indent="") const; + + virtual bool isMetricSet() const { return false; } + +private: + + /** + * A Tag instance is only considered to be a valid dimension if it contains + * a value (i.e. not just a key as with legacy tags). + */ + bool tagsSpecifyAtLeastOneDimension(const Tags&) const; + + /** + * Iff _tags contains at least one valid dimension, update _mangledName to + * be a string of the form 'metricname{key_0=value_0,key_1=value_1}' for + * each pair of valid dimension key/values. _mangledName is left untouched + * if no such valid dimension exists. + */ + void assignMangledNameWithDimensions(); + + /** + * Sort _tags so that its entries are order in increasing lexicographic + * order. Dimensions/tags must be in deterministic order for duplicate + * detection based on name to work correctly. + */ + void sortTagsInDeterministicOrder(); + + std::string createMangledNameWithDimensions() const; + + void verifyConstructionParameters(); + /** + * Registers with metric set iff owner is not nullptr. This will implicitly + * set _owner via a metric set callback to Metric::setRegistered(). + */ + void registerWithOwnerIfRequired(MetricSet* owner); + +protected: + String _name; + String _mangledName; + String _description; + std::vector<Tag> _tags; + MetricSet* _owner; + +}; + +} // metrics + diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp new file mode 100644 index 00000000000..b915786afcf --- /dev/null +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -0,0 +1,968 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/metrics/metricmanager.h> + +#include <vespa/config/print/ostreamconfigwriter.h> +#include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/metrics/countmetric.h> +#include <vespa/metrics/valuemetric.h> +#include <vespa/metrics/metricset.h> +#include <vespa/metrics/summetric.h> +#include <vespa/metrics/jsonwriter.h> +#include <vespa/metrics/textwriter.h> +#include <vespa/metrics/xmlwriter.h> +#include <vespa/log/log.h> +#include <stack> + +LOG_SETUP(".metrics.manager"); + +namespace metrics { + +typedef MetricsmanagerConfig Config; + +void +MetricManager::ConsumerSpec::print(std::ostream& out, bool verbose, + const std::string& indent) const +{ + (void) verbose; + out << "ConsumerSpec("; + std::set<Metric::String> sortedMetrics; + for (vespalib::hash_set<Metric::String>::const_iterator it + = includedMetrics.begin(); it != includedMetrics.end(); ++it) + { + sortedMetrics.insert(*it); + } + for (auto s : sortedMetrics) { + out << "\n" << indent << " " << s; + } + out << ")"; +} + +void +MetricManager::ConsumerSpec::addMemoryUsage(MemoryConsumption& mc) const +{ + mc._consumerMetricsInTotal += includedMetrics.size(); + for (vespalib::hash_set<Metric::String>::const_iterator it + = includedMetrics.begin(); it != includedMetrics.end(); ++it) + { + mc._consumerMetricIds += mc.getStringMemoryUsage( + *it, mc._consumerMetricIdsUnique) + + sizeof(Metric::String); + } +} + +MetricManager::MetricManager(std::unique_ptr<Timer> timer) + : _activeMetrics("Active metrics showing updates since last snapshot"), + _configSubscriber(), + _configHandle(), + _config(), + _consumerConfig(), + _logPeriod(5 * 60, 0), + _snapshots(), + _totalMetrics(MetricSnapshot::SP(new MetricSnapshot( + "Empty metrics before init", 0, _activeMetrics.getMetrics(), + false))), + _timer(std::move(timer)), + _lastProcessedTime(0), + _forceEventLogging(false), + _snapshotUnsetMetrics(false), + _consumerConfigChanged(false), + _metricManagerMetrics("metricmanager", "", + "Metrics for the metric manager upkeep tasks"), + _periodicHookLatency("periodichooklatency", "", + "Time in ms used to update a single periodic hook", + &_metricManagerMetrics), + _snapshotHookLatency("snapshothooklatency", "", + "Time in ms used to update a single snapshot hook", + &_metricManagerMetrics), + _resetLatency("resetlatency", "", + "Time in ms used to reset all metrics.", &_metricManagerMetrics), + _snapshotLatency("snapshotlatency", "", + "Time in ms used to take a snapshot", + &_metricManagerMetrics), + _sleepTimes("sleeptime", "", "Time in ms worker thread is sleeping", + &_metricManagerMetrics) +{ + registerMetric(getMetricLock(), _metricManagerMetrics); +} + +MetricManager::~MetricManager() +{ + stop(); +} + +void +MetricManager::stop() +{ + Runnable::stop(); + { + vespalib::MonitorGuard sync(_waiter); + sync.signal(); + } + join(); +} + +void +MetricManager::addMetricUpdateHook(UpdateHook& hook, uint32_t period) +{ + hook._period = period; + vespalib::MonitorGuard sync(_waiter); + // If we've already initialized manager, log period has been set. + // In this case. Call first time after period + hook._nextCall = (_logPeriod.second == 0 ? 0 : _timer->getTime() + period); + if (period == 0) { + for (UpdateHook * sHook : _snapshotUpdateHooks) { + if (sHook == &hook) { + LOG(warning, "Update hook already registered"); + return; + } + } + _snapshotUpdateHooks.push_back(&hook); + } else { + for (UpdateHook * pHook : _periodicUpdateHooks) { + if (pHook == &hook) { + LOG(warning, "Update hook already registered"); + return; + } + } + _periodicUpdateHooks.push_back(&hook); + } +} + +void +MetricManager::removeMetricUpdateHook(UpdateHook& hook) +{ + vespalib::MonitorGuard sync(_waiter); + if (hook._period == 0) { + for (auto it = _snapshotUpdateHooks.begin(); it != _snapshotUpdateHooks.end(); it++) { + if (*it == &hook) { + _snapshotUpdateHooks.erase(it); + return; + } + } + } else { + for (auto it = _periodicUpdateHooks.begin(); it != _periodicUpdateHooks.end(); it++) { + if (*it == &hook) { + _periodicUpdateHooks.erase(it); + return; + } + } + } + LOG(warning, "Update hook not registered"); +} + +void +MetricManager::init(const config::ConfigUri & uri, FastOS_ThreadPool& pool, + bool startThread) +{ + if (_configHandle.get()) { + throw vespalib::IllegalStateException( + "The metric manager have already been initialized. " + "It can only be initialized once.", VESPA_STRLOC); + } + LOG(debug, "Initializing metric manager."); + _configSubscriber.reset(new config::ConfigSubscriber(uri.getContext())); + _configHandle = _configSubscriber->subscribe<Config>(uri.getConfigId()); + _configSubscriber->nextConfig(); + configure(getMetricLock(), _configHandle->getConfig()); + LOG(debug, "Starting worker thread, waiting for first " + "iteration to complete."); + if (startThread) { + Runnable::start(pool); + // Wait for first iteration to have completed, such that it is safe + // to access snapshots afterwards. + vespalib::MonitorGuard sync(_waiter); + while (_lastProcessedTime == 0) { + sync.wait(1); + } + } else { + _configSubscriber.reset(0); + } + LOG(debug, "Metric manager completed initialization."); +} + +namespace { + + struct Path { + vespalib::StringTokenizer _path; + + Path(const vespalib::stringref & fullpath) : _path(fullpath, ".") { } + + vespalib::string toString() const { + vespalib::asciistream ost; + ost << _path[0]; + for (uint32_t i=1; i<path().size(); ++i) ost << "." << _path[i]; + return ost.str(); + } + + bool matchesPattern(const Path& p) const { + if (path().size() != p.path().size()) { + return false; + } + for (uint32_t i=0; i<p.path().size(); ++i) { + if (p._path[i] == "*") continue; + if (_path[i] != p._path[i]) { + return false; + } + } + return true; + } + const vespalib::StringTokenizer::TokenList& path() const { + return _path.getTokens(); + } + }; + + struct ConsumerMetricBuilder : public MetricVisitor { + const Config::Consumer& _consumer; + MetricManager::ConsumerSpec _matchedMetrics; + // Keep a stack of matches to facilitate tree traversal. + struct Result { + bool tagAdded; + bool tagRemoved; + bool nameAdded; + bool nameRemoved; + uint32_t metricCount; + + Result() : tagAdded(false), tagRemoved(false), + nameAdded(false), nameRemoved(false), metricCount(0) {} + }; + std::list<Result> result; + + ConsumerMetricBuilder(const Config::Consumer& c) + : _consumer(c), _matchedMetrics() + { + LOG(spam, "Adding metrics for consumer %s", c.name.c_str()); + } + + bool tagAdded(const Metric& metric) { + for (const auto& s : _consumer.tags) { + if ((s == "*" && !metric.getTags().empty()) + || metric.hasTag(s)) + { + return true; + } + } + return false; + } + bool tagRemoved(const Metric& metric) { + for (const auto& s : _consumer.removedtags) { + if ((s == "*" && !metric.getTags().empty()) + || metric.hasTag(s)) + { + return true; + } + } + return false; + } + bool nameAdded(const Metric& metric) { + Metric::String path(metric.getPath()); + Path mpath(path); + for (const auto& s : _consumer.addedmetrics) { + if (mpath.matchesPattern(Path(s))) { + return true; + } + } + return false; + } + bool nameRemoved(const Metric& metric) { + Metric::String path(metric.getPath()); + Path mpath(path); + for (const auto& s : _consumer.removedmetrics) { + if (mpath.matchesPattern(Path(s))) { + return true; + } + } + return false; + } + + virtual bool visitMetricSet(const MetricSet& metricSet, + bool autoGenerated) + { + (void) autoGenerated; + result.push_back(Result()); + // If current metric match anything, use that. + // Otherwise, copy from parent + if (nameRemoved(metricSet)) { + result.back().nameRemoved = true; + } else if (nameAdded(metricSet)) { + result.back().nameAdded = true; + } else if (tagRemoved(metricSet)) { + result.back().tagRemoved = true; + } else if (tagAdded(metricSet)) { + result.back().tagAdded = true; + } else if (result.size() > 1) { + result.back() = *++result.rbegin(); + result.back().metricCount = 0; + } + return true; + } + virtual void doneVisitingMetricSet(const MetricSet& metricSet) { + if (result.back().metricCount > 0 && result.size() != 1) { + LOG(spam, "Adding metricset %s", metricSet.getPath().c_str()); + _matchedMetrics.includedMetrics.insert(metricSet.getPath()); + } + result.pop_back(); + } + bool visitMetric(const Metric& metric, bool autoGenerated) { + (void) autoGenerated; + if (result.back().nameRemoved || nameRemoved(metric)) return true; + bool nAdded = (result.back().nameAdded || nameAdded(metric)); + if (!nAdded && (result.back().tagRemoved || tagRemoved(metric))) { + return true; + } + if (nAdded || result.back().tagAdded || tagAdded(metric)) { + _matchedMetrics.includedMetrics.insert(metric.getPath()); + LOG(spam, "Adding metric %s", metric.getPath().c_str()); + for (Result & r : result) { + ++r.metricCount; + } + } + return true; + } + + }; + +} + +// This function may be called by clients or worker thread. Thus, worker monitor +// might have already been grabbed +void +MetricManager::checkMetricsAltered(const MetricLockGuard & guard) +{ + if (_activeMetrics.getMetrics().isRegistrationAltered() + || _consumerConfigChanged) + { + handleMetricsAltered(guard); + } +} + +// When calling this function, the metric lock is already taken. The thread +// monitor lock might be taken. +void +MetricManager::handleMetricsAltered(const MetricLockGuard &) +{ + if (_config.get() == NULL) { + LOG(info, "_config is NULL -> very odd indeed."); + return; + } + if (_consumerConfig.empty()) { + LOG(debug, "Setting up consumers for the first time."); + } else { + LOG(info, "Metrics registration changes detected. Handling changes."); + } + std::map<Metric::String, ConsumerSpec::SP> configMap; + _activeMetrics.getMetrics().clearRegistrationAltered(); + LOG(debug, "Calculating new consumer config"); + for (auto consumer : _config->consumer) { + ConsumerMetricBuilder consumerMetricBuilder(consumer); + _activeMetrics.getMetrics().visit(consumerMetricBuilder); + configMap[consumer.name] = ConsumerSpec::SP( + new ConsumerSpec(consumerMetricBuilder._matchedMetrics)); + } + LOG(debug, "Recreating snapshots to include altered metrics"); + _activeMetrics.updateNames(_nameHash); + _totalMetrics->recreateSnapshot(_activeMetrics.getMetrics(), + _snapshotUnsetMetrics); + _totalMetrics->updateNames(_nameHash); + for (uint32_t i=0; i<_snapshots.size(); ++i) { + _snapshots[i]->recreateSnapshot(_activeMetrics.getMetrics(), + _snapshotUnsetMetrics); + _snapshots[i]->updateNames(_nameHash); + } + LOG(debug, "Setting new consumer config. Clearing dirty flag"); + _consumerConfig.swap(configMap); + _consumerConfigChanged = false; + LOG(debug, "Unified %u of %u strings configuring %" PRIu64 " consumers.", + _nameHash.getUnifiedStringCount(), + _nameHash.getCheckedStringCount(), + _config->consumer.size()); + _nameHash.resetCounts(); +} + +namespace { + bool setSnapshotName(std::ostream& out, const char* name, + uint32_t length, uint32_t period) + { + if (length % period != 0) return false; + out << (length / period) << ' ' << name; + if (length / period != 1) out << "s"; + return true; + } +} + +std::vector<MetricManager::SnapSpec> +MetricManager::createSnapshotPeriods(const Config& config) +{ + std::vector<SnapSpec> result; + try{ + for (uint32_t i=0; i<config.snapshot.periods.size(); ++i) { + uint32_t length = config.snapshot.periods[i]; + if (length < 1) throw vespalib::IllegalStateException( + "Snapshot periods must be positive numbers", + VESPA_STRLOC); + std::ostringstream name; + if (setSnapshotName(name, "week", length, 60 * 60 * 24 * 7)) { + } else if (setSnapshotName(name, "day", length, 60 * 60 * 24)) { + } else if (setSnapshotName(name, "hour", length, 60 * 60)) { + } else if (setSnapshotName(name, "minute", length, 60)) { + } else { + name << length << " seconds"; + } + result.push_back(SnapSpec(length, name.str())); + } + for (uint32_t i=1; i<result.size(); ++i) { + if (result[i].first % result[i-1].first != 0) { + std::ostringstream ost; + ost << "Period " << result[i].first + << " is not a multiplum of period " + << result[i-1].first << " which is needs to be."; + throw vespalib::IllegalStateException( + ost.str(), VESPA_STRLOC); + } + } + } catch (vespalib::Exception& e) { + LOG(warning, "Invalid snapshot periods specified. Using defaults: %s", + e.getMessage().c_str()); + result.clear(); + } + if (result.empty()) { + result.push_back(SnapSpec(60 * 5, "5 minute")); + result.push_back(SnapSpec(60 * 60, "1 hour")); + result.push_back(SnapSpec(60 * 60 * 24, "1 day")); + result.push_back(SnapSpec(60 * 60 * 24 * 7, "1 week")); + } + return result; +} + +void +MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> config) +{ + assert(config.get() != 0); + if (LOG_WOULD_LOG(debug)) { + std::ostringstream ost; + config::OstreamConfigWriter w(ost); + w.write(*config); + LOG(debug, "Received new config for metric manager: %s", + ost.str().c_str()); + } + if (_snapshots.empty()) { + LOG(debug, "Initializing snapshots as this is first configure call"); + std::vector<SnapSpec> snapshotPeriods(createSnapshotPeriods(*config)); + + // Set up snapshots only first time. We don't allow live reconfig + // of snapshot periods. + time_t currentTime(_timer->getTime()); + _activeMetrics.setFromTime(currentTime); + uint32_t count = 1; + for (uint32_t i = 0; i< snapshotPeriods.size(); ++i) + { + uint32_t nextCount = 1; + if (i + 1 < snapshotPeriods.size()) { + nextCount = snapshotPeriods[i + 1].first + / snapshotPeriods[i].first; + if (snapshotPeriods[i + 1].first + % snapshotPeriods[i].first != 0) + { + throw vespalib::IllegalStateException( + "Snapshot periods must be multiplum of each other", + VESPA_STRLOC); + } + } + _snapshots.push_back(MetricSnapshotSet::SP(new MetricSnapshotSet( + snapshotPeriods[i].second, snapshotPeriods[i].first, count, + _activeMetrics.getMetrics(), _snapshotUnsetMetrics))); + count = nextCount; + } + // Add all time snapshot. + _totalMetrics = MetricSnapshot::SP(new MetricSnapshot( + "All time snapshot", 0, _activeMetrics.getMetrics(), + _snapshotUnsetMetrics)); + _totalMetrics->reset(currentTime); + } + if (_config.get() == 0 + || _config->consumer.size() != config->consumer.size()) + { + _consumerConfigChanged = true; + } else { + for (uint32_t i=0; i<_config->consumer.size(); ++i) { + if (_config->consumer[i] != config->consumer[i]) { + _consumerConfigChanged = true; + break; + } + } + } + if (_consumerConfigChanged) { + LOG(debug, "Consumer config changed. Tagging consumer config dirty."); + } + _config = std::move(config); + +} + +MetricManager::ConsumerSpec::SP +MetricManager::getConsumerSpec(const MetricLockGuard &, const Metric::String& consumer) const +{ + auto it(_consumerConfig.find(consumer)); + return (it != _consumerConfig.end() ? it->second : ConsumerSpec::SP()); +} + +//#define VERIFY_ALL_METRICS_VISITED 1 + +namespace { + + struct ConsumerMetricVisitor : public MetricVisitor { + const MetricManager::ConsumerSpec& _metricsToMatch; + MetricVisitor& _client; +#ifdef VERIFY_ALL_METRICS_VISITED + std::set<Metric::String> _visitedMetrics; +#endif + + ConsumerMetricVisitor(const MetricManager::ConsumerSpec& spec, + MetricVisitor& clientVisitor) + : _metricsToMatch(spec), _client(clientVisitor) {} + + virtual bool visitMetricSet(const MetricSet& metricSet, + bool autoGenerated) + { + if (metricSet.isTopSet()) return true; + return (_metricsToMatch.contains(metricSet) + && _client.visitMetricSet(metricSet, autoGenerated)); + } + virtual void doneVisitingMetricSet(const MetricSet& metricSet) { + if (!metricSet.isTopSet()) { +#ifdef VERIFY_ALL_METRICS_VISITED + _visitedMetrics.insert(metricSet.getPath()); +#endif + _client.doneVisitingMetricSet(metricSet); + } + } + virtual bool visitCountMetric(const AbstractCountMetric& metric, + bool autoGenerated) + { + if (_metricsToMatch.contains(metric)) { +#ifdef VERIFY_ALL_METRICS_VISITED + _visitedMetrics.insert(metric.getPath()); +#endif + return _client.visitCountMetric(metric, autoGenerated); + } + return true; + } + virtual bool visitValueMetric(const AbstractValueMetric& metric, + bool autoGenerated) + { + if (_metricsToMatch.contains(metric)) { +#ifdef VERIFY_ALL_METRICS_VISITED + _visitedMetrics.insert(metric.getPath()); +#endif + return _client.visitValueMetric(metric, autoGenerated); + } + return true; + } + }; + +} + +void +MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapshot, MetricVisitor& visitor, + const std::string& consumer) const +{ + if (visitor.visitSnapshot(snapshot)) { + if (consumer == "") { + snapshot.getMetrics().visit(visitor); + } else { + ConsumerSpec::SP consumerSpec(getConsumerSpec(guard, consumer)); + if (consumerSpec.get()) { + ConsumerMetricVisitor consumerVis(*consumerSpec, visitor); + snapshot.getMetrics().visit(consumerVis); +#ifdef VERIFY_ALL_METRICS_VISITED + for (auto metric = consumerSpec->includedMetrics) { + if (consumerVis._visitedMetrics.find(metric) + == consumerVis._visitedMetrics.end()) + { + LOG(debug, "Failed to find metric %s to be visited.", metric.c_str()); + } + } +#endif + } else { + LOGBP(debug, + "Requested metrics for non-defined consumer '%s'.", + consumer.c_str()); + } + } + visitor.doneVisitingSnapshot(snapshot); + } + visitor.doneVisiting(); +} + +std::vector<uint32_t> +MetricManager::getSnapshotPeriods(const MetricLockGuard& l) const +{ + assertMetricLockLocked(l); + std::vector<uint32_t> result(_snapshots.size()); + for (uint32_t i=0; i<_snapshots.size(); ++i) { + result[i] = _snapshots[i]->getPeriod(); + } + return result; +} + +// Client should have grabbed metrics lock before doing this +const MetricSnapshot& +MetricManager::getMetricSnapshot(const MetricLockGuard& l, + uint32_t period, bool getInProgressSet) const +{ + assertMetricLockLocked(l); + for (uint32_t i=0; i<_snapshots.size(); ++i) { + if (_snapshots[i]->getPeriod() == period) { + if (_snapshots[i]->getCount() == 1 && getInProgressSet) { + throw vespalib::IllegalStateException( + "No temporary snapshot for set " + + _snapshots[i]->getName(), VESPA_STRLOC); + } + return _snapshots[i]->getSnapshot(getInProgressSet); + } + } + std::ostringstream ost; + ost << "No snapshot for period of length " << period << " exist."; + throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); +} + +// Client should have grabbed metrics lock before doing this +const MetricSnapshotSet& +MetricManager::getMetricSnapshotSet(const MetricLockGuard& l, + uint32_t period) const +{ + assertMetricLockLocked(l); + for (uint32_t i=0; i<_snapshots.size(); ++i) { + if (_snapshots[i]->getPeriod() == period) { + return *_snapshots[i]; + } + } + std::ostringstream ost; + ost << "No snapshot set for period of length " << period << " exist."; + throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); +} + +void +MetricManager::timeChangedNotification() const +{ + vespalib::MonitorGuard sync(_waiter); + sync.broadcast(); +} + +void +MetricManager::updateMetrics(bool includeSnapshotOnlyHooks) +{ + LOG(debug, "Calling metric update hooks%s.", + includeSnapshotOnlyHooks ? ", including snapshot hooks" : ""); + // Ensure we're not in the way of the background thread + vespalib::MonitorGuard sync(_waiter); + LOG(debug, "Giving %" PRIu64 " periodic update hooks.", + _periodicUpdateHooks.size()); + updatePeriodicMetrics(sync, 0, true); + if (includeSnapshotOnlyHooks) { + LOG(debug, "Giving %" PRIu64 " snapshot update hooks.", + _snapshotUpdateHooks.size()); + updateSnapshotMetrics(sync); + } +} + +// When this is called, the thread monitor lock has already been grabbed +time_t +MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updateTime, bool outOfSchedule) +{ + time_t nextUpdateTime = std::numeric_limits<time_t>::max(); + time_t preTime = _timer->getTimeInMilliSecs(); + for (auto hook : _periodicUpdateHooks) { + if (hook->_nextCall <= updateTime) { + hook->updateMetrics(guard); + if (hook->_nextCall + hook->_period < updateTime) { + if (hook->_nextCall != 0) { + LOG(debug, "Updated hook %s at time %" PRIu64 ", but next " + "run in %u seconds have already passed as time" + " is %" PRIu64 ". Bumping next call to current " + "time + period.", + hook->_name, hook->_nextCall, hook->_period, updateTime); + } + hook->_nextCall = updateTime + hook->_period; + } else { + hook->_nextCall += hook->_period; + } + time_t postTime = _timer->getTimeInMilliSecs(); + _periodicHookLatency.addValue(postTime - preTime); + preTime = postTime; + } else if (outOfSchedule) { + hook->updateMetrics(guard); + time_t postTime = _timer->getTimeInMilliSecs(); + _periodicHookLatency.addValue(postTime - preTime); + preTime = postTime; + } + nextUpdateTime = std::min(nextUpdateTime, hook->_nextCall); + } + return nextUpdateTime; +} + +// When this is called, the thread monitor lock has already been grabbed. +void +MetricManager::updateSnapshotMetrics(const MetricLockGuard & guard) +{ + time_t preTime = _timer->getTimeInMilliSecs(); + for (auto it = _snapshotUpdateHooks.begin(); it != _snapshotUpdateHooks.end(); ++it) { + (**it).updateMetrics(guard); + time_t postTime = _timer->getTimeInMilliSecs(); + _snapshotHookLatency.addValue(postTime - preTime); + preTime = postTime; + } +} + +void +MetricManager::forceEventLogging() +{ + LOG(debug, "Forcing event logging to happen."); + // Ensure background thread is not in a current cycle during change. + vespalib::MonitorGuard sync(_waiter); + _forceEventLogging = true; + sync.signal(); +} + +void +MetricManager::reset(time_t currentTime) +{ + time_t preTime = _timer->getTimeInMilliSecs(); + // Resetting implies visiting metrics, which needs to grab metric lock + // to avoid conflict with adding/removal of metrics + vespalib::LockGuard waiterLock(_waiter); + _activeMetrics.reset(currentTime); + for (uint32_t i=0; i<_snapshots.size(); ++i) { + _snapshots[i]->reset(currentTime); + } + _totalMetrics->reset(currentTime); + time_t postTime = _timer->getTimeInMilliSecs(); + _resetLatency.addValue(postTime - preTime); +} + +namespace { + + struct LogMetricVisitor : public MetricVisitor { + bool _total; + + LogMetricVisitor(bool totalVals) : _total(totalVals) {} + + bool visitMetric(const Metric& metric, bool autoGenerated) { + (void) autoGenerated; + if (metric.logFromTotalMetrics() ^ _total) return true; + Metric::String logName = metric.getPath(); + std::replace(logName.begin(), logName.end(), '.', '_'); + if (!metric.logEvent(logName)) { + LOG(spam, "Not logging event from metric %s.%s as no values " + "have been added to metric.", + metric.getPath().c_str(), metric.getName().c_str()); + } + return true; + } + }; + +} // End of anonymous namespace + +void +MetricManager::run() +{ + while (!stopping()) { + time_t currentTime = _timer->getTime(); + vespalib::MonitorGuard sync(_waiter); + time_t next = tick(sync, currentTime); + if (currentTime < next) { + sync.wait((next - currentTime) * 1000); + _sleepTimes.addValue((next - currentTime) * 1000); + } else { + _sleepTimes.addValue(0); + } + } +} + +time_t +MetricManager::tick(const MetricLockGuard & guard, time_t currentTime) +{ + LOG(spam, "Worker thread starting to process for time %" PRIu64 ".", + currentTime); + bool firstIteration = (_logPeriod.second == 0); + // For a slow system to still be doing metrics tasks each n'th + // second, rather than each n'th + time to do something seconds, + // we constantly add next time to do something from the last timer. + // For that to work, we need to initialize timers on first iteration + // to set them to current time. + time_t nextWorkTime; + if (firstIteration) { + // Setting next log period to now, such that we log metrics + // straight away + _logPeriod.second = currentTime; + for (uint32_t i=0; i<_snapshots.size(); ++i) { + _snapshots[i]->setFromTime(currentTime); + } + for (auto hook : _periodicUpdateHooks) { + hook->_nextCall = currentTime; + } + // Ensure correct time for first snapshot + _snapshots[0]->getSnapshot().setToTime(currentTime); + } + + // Check for new config and reconfigure + if (_configSubscriber.get() && _configSubscriber->nextConfig(0)) { + configure(guard, _configHandle->getConfig()); + } + + // If metrics have changed since last time we did a snapshot, + // work that out before taking the snapshot, such that new + // metric can be included + checkMetricsAltered(guard); + + // Set next work time to the time we want to take next snapshot. + nextWorkTime = _snapshots[0]->getToTime() + _snapshots[0]->getPeriod(); + time_t nextUpdateHookTime; + if (nextWorkTime <= currentTime || _forceEventLogging) { + // If taking a new snapshot or logging, force calls to all + // update hooks. + LOG(debug, "%s. Calling update hooks.", nextWorkTime <= currentTime + ? "Time to do snapshot" : "Out of sequence event logging"); + nextUpdateHookTime = updatePeriodicMetrics(guard, currentTime, true); + updateSnapshotMetrics(guard); + } else { + // If not taking a new snapshot. Only give update hooks to + // periodic hooks wanting it. + nextUpdateHookTime = updatePeriodicMetrics(guard, currentTime, false); + } + // Do snapshotting if it is time + if (nextWorkTime <= currentTime) takeSnapshots(guard, nextWorkTime); + // Log if it is time + if (_logPeriod.second <= currentTime || _forceEventLogging) { + LogMetricVisitor totalVisitor(true); + LogMetricVisitor fiveMinVisitor(false); + if (_logPeriod.second <= currentTime) { + LOG(debug, "Logging total metrics."); + visit(guard, *_totalMetrics, totalVisitor, "log"); + visit(guard, _snapshots[0]->getSnapshot(), fiveMinVisitor , "log"); + if (_logPeriod.second + _logPeriod.first < currentTime) { + uint64_t next = _snapshots[0]->getFromTime() + + _logPeriod.first; + LOG(warning, "Logged events at time %" PRIu64 " for time %" + PRIu64 ". Since this is more than a period %u " + "in the past, next run has been set to next " + "snapshot time at %" PRIu64 ".", + currentTime, _logPeriod.second, _logPeriod.first, next); + _logPeriod.second = next; + } else { + _logPeriod.second += _logPeriod.first; + } + } else { + LOG(debug, "Logging total metrics out of sequence."); + metrics::MetricSnapshot snapshot( + "Total out of sequence metrics from start until " + "current time", 0, _totalMetrics->getMetrics(), + _snapshotUnsetMetrics); + _activeMetrics.addToSnapshot(snapshot, false, currentTime); + snapshot.setFromTime(_totalMetrics->getFromTime()); + visit(guard, snapshot, totalVisitor, "log"); + visit(guard, snapshot, fiveMinVisitor, "log"); + } + _forceEventLogging = false; + } + _lastProcessedTime = (nextWorkTime <= currentTime ? nextWorkTime + : currentTime); + LOG(spam, "Worker thread done with processing for time %" PRIu64 ".", + _lastProcessedTime); + time_t next = std::min( + _logPeriod.second, + _snapshots[0]->getPeriod() + _snapshots[0]->getToTime()); + next = std::min(next, nextUpdateHookTime); + return next; +} + +void +MetricManager::takeSnapshots(const MetricLockGuard &, time_t timeToProcess) +{ + // If not time to do dump data from active snapshot yet, nothing to do + if (!_snapshots[0]->timeForAnotherSnapshot(timeToProcess)) { + LOG(spam, "Not time to process snapshot %s at time %" PRIu64 ". Current " + "first period (%u) snapshot goes from %" PRIu64 " to %" PRIu64, + _snapshots[0]->getName().c_str(), timeToProcess, + _snapshots[0]->getPeriod(), _snapshots[0]->getFromTime(), + _snapshots[0]->getToTime()); + return; + } + time_t preTime = _timer->getTimeInMilliSecs(); + LOG(debug, "Updating %s snapshot and total metrics at time %" PRIu64 ".", + _snapshots[0]->getName().c_str(), timeToProcess); + MetricSnapshot& firstTarget(_snapshots[0]->getNextTarget()); + firstTarget.reset(_activeMetrics.getFromTime()); + _activeMetrics.addToSnapshot(firstTarget, false, timeToProcess); + _activeMetrics.addToSnapshot(*_totalMetrics, false, timeToProcess); + _activeMetrics.reset(timeToProcess); + LOG(debug, "After snapshotting, " + "active metrics goes from %" PRIu64 " to %" PRIu64", " + "and 5 minute metrics goes from %" PRIu64 " to %" PRIu64".", + _activeMetrics.getFromTime(), _activeMetrics.getToTime(), + firstTarget.getFromTime(), firstTarget.getToTime()); + + // Update later snapshots if it is time for it + for (uint32_t i=1; i<_snapshots.size(); ++i) { + LOG(debug, "Adding data from last snapshot to building snapshot of " + "next period snapshot %s.", + _snapshots[i]->getName().c_str()); + MetricSnapshot& target(_snapshots[i]->getNextTarget()); + _snapshots[i-1]->getSnapshot().addToSnapshot( + target, false, timeToProcess); + target.setToTime(timeToProcess); + if (!_snapshots[i]->haveCompletedNewPeriod(timeToProcess)) { + LOG(debug, "Not time to roll snapshot %s yet. %u of %u snapshot " + "taken at time %" PRIu64 ", and period of %u is not up " + "yet as we're currently processing for time %" PRIu64 ".", + _snapshots[i]->getName().c_str(), + _snapshots[i]->getBuilderCount(), + _snapshots[i]->getCount(), + _snapshots[i]->getBuilderCount() * _snapshots[i]->getPeriod() + + _snapshots[i]->getFromTime(), + _snapshots[i]->getPeriod(), + timeToProcess); + break; + } else { + LOG(debug, "Rolled snapshot %s at time %" PRIu64 ".", + _snapshots[i]->getName().c_str(), + timeToProcess); + } + } + time_t postTime = _timer->getTimeInMilliSecs(); + _snapshotLatency.addValue(postTime - preTime); +} + +MemoryConsumption::UP +MetricManager::getMemoryConsumption(const MetricLockGuard & guard) const +{ + (void) guard; + MemoryConsumption::UP mc(new MemoryConsumption); + mc->_consumerCount += _consumerConfig.size(); + mc->_consumerMeta += (sizeof(ConsumerSpec::SP) + sizeof(ConsumerSpec)) + * _consumerConfig.size(); + for (auto it = _consumerConfig.begin(); it != _consumerConfig.end(); ++it) { + mc->_consumerId += mc->getStringMemoryUsage( + it->first, mc->_consumerIdUnique) + + sizeof(Metric::String); + it->second->addMemoryUsage(*mc); + } + uint32_t preTotal = mc->getTotalMemoryUsage(); + _activeMetrics.addMemoryUsage(*mc); + uint32_t postTotal = mc->getTotalMemoryUsage(); + mc->addSnapShotUsage("active", postTotal - preTotal); + preTotal = postTotal; + for (uint32_t i=0; i<_snapshots.size(); ++i) { + _snapshots[i]->addMemoryUsage(*mc); + postTotal = mc->getTotalMemoryUsage(); + mc->addSnapShotUsage(_snapshots[i]->getName(), postTotal - preTotal); + preTotal = postTotal; + } + _totalMetrics->addMemoryUsage(*mc); + postTotal = mc->getTotalMemoryUsage(); + mc->addSnapShotUsage("total", postTotal - preTotal); + _nameHash.addMemoryUsage(*mc); + return mc; +} + +} // metrics diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h new file mode 100644 index 00000000000..467e97469c1 --- /dev/null +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -0,0 +1,322 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class metrics::MetricManager + * + * A metrics-enabled application should have a single MetricManager. + * You can register a number of MetricSets in the MetricManager. Each + * metric in the metrics sets can be used by zero or more consumers, + * configurable using readConfig(). + * + * The consumers get their data by calling the getSnapshot() method, + * which gives them a snapshot of all the current metrics which are + * configured for the given name. + * + * Locking strategy: + * + * There are three locks in this class: + * + * Config lock: + * - This protects the class on config changes. It protects the _config and + * _consumerConfig members. + * + * Thread monitor (waiter): + * - This lock is kept by the worker thread while it is doing a work cycle, + * and it uses this monitor to sleep. It is used to make shutdown quick by + * interrupting thread, and to let functions called by clients be able to + * do a change while the worker thread is idle. + * - The log period is protected by the thread monitor. + * - The update hooks is protected by the thread monitor. + * + * Metric lock: + * - The metric log protects the active metric set when adding or removing + * metrics. Clients need to grab this lock before altering active metrics. + * The metric manager needs to grab this lock everytime it visits active + * metrics. + * - The metric log protects the snapshots. The snapshot writer is the metric + * worker thread and will grab the lock while editing them. Readers that + * aren't the worker thread itself must grab lock to be sure. + * + * If multiple locks is taken, the allowed locking order is: + * 1. Thread monitor. + * 2. Metric lock. + * 3. Config lock. + */ +#pragma once + +#include <string> +#include <boost/weak_ptr.hpp> +#include <vespa/vespalib/util/document_runnable.h> +#include <vespa/metrics/config-metricsmanager.h> +#include <vespa/metrics/metricset.h> +#include <vespa/metrics/metricsnapshot.h> +#include <vespa/metrics/namehash.h> +#include <vespa/metrics/valuemetric.h> +#include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/util/sync.h> +#include <map> +#include <list> +#include <vespa/vespalib/util/jsonwriter.h> +#include <vespa/config/config.h> + +namespace metrics { + +typedef vespalib::MonitorGuard MetricLockGuard; + +class MetricManager : private document::Runnable +{ +public: + class UpdateHook { + const char* _name; + time_t _nextCall; + uint32_t _period; + friend class MetricManager; + + public: + typedef vespalib::LinkedPtr<UpdateHook> LP; + using MetricLockGuard = metrics::MetricLockGuard; + + UpdateHook(const char* name) : _name(name), _nextCall(0), _period(0) {} + virtual ~UpdateHook() {} + virtual void updateMetrics(const MetricLockGuard & guard) = 0; + const char* getName() const { return _name; } + }; + + struct Timer { + virtual ~Timer() {} + virtual time_t getTime() const { return time(0); } + virtual time_t getTimeInMilliSecs() const { return getTime() * 1000; } + }; + + /** + * Spec saved from config. If metricSetChildren has content, metric pointed + * to is a metric set. + */ + struct ConsumerSpec : public vespalib::Printable { + typedef std::shared_ptr<ConsumerSpec> SP; + + vespalib::hash_set<Metric::String> includedMetrics; + ConsumerSpec() : includedMetrics() {} + + bool contains(const Metric& m) const { + return (includedMetrics.find(m.getPath()) != includedMetrics.end()); + } + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + + void addMemoryUsage(MemoryConsumption&) const; + }; + +private: + MetricSnapshot _activeMetrics; + std::unique_ptr<config::ConfigSubscriber> _configSubscriber; + config::ConfigHandle<MetricsmanagerConfig>::UP _configHandle; + std::unique_ptr<MetricsmanagerConfig> _config; + std::map<Metric::String, ConsumerSpec::SP> _consumerConfig; + std::list<UpdateHook*> _periodicUpdateHooks; + std::list<UpdateHook*> _snapshotUpdateHooks; + vespalib::Monitor _waiter; + typedef std::pair<uint32_t, time_t> PeriodTimePair; + PeriodTimePair _logPeriod; + std::vector<MetricSnapshotSet::SP> _snapshots; + MetricSnapshot::SP _totalMetrics; + std::unique_ptr<Timer> _timer; + time_t _lastProcessedTime; + bool _forceEventLogging; + // Should be added to config, but wont now due to problems with + // upgrading + bool _snapshotUnsetMetrics; + bool _consumerConfigChanged; + + MetricSet _metricManagerMetrics; + LongAverageMetric _periodicHookLatency; + LongAverageMetric _snapshotHookLatency; + LongAverageMetric _resetLatency; + LongAverageMetric _snapshotLatency; + LongAverageMetric _sleepTimes; + + // Name hash trying to ensure string ref counting works as well as can be + // expected + NameHash _nameHash; + +public: + MetricManager(std::unique_ptr<Timer> timer = std::unique_ptr<Timer>(new Timer)); + ~MetricManager(); + + void stop(); + + void snapshotUnsetMetrics(bool doIt) { _snapshotUnsetMetrics = doIt; } + + /** + * Add a metric update hook. This will always be called prior to + * snapshotting and metric logging, to make the metrics the best as they can + * be at those occasions. + * + * @param period Period in seconds for how often callback should be called. + * The default value of 0, means only before snapshotting or + * logging, while another value will give callbacks each + * period seconds. Expensive metrics to calculate will + * typically only want to do it before snapshotting, while + * inexpensive metrics might want to log their value every 5 + * seconds or so. Any value of period >= the smallest snapshot + * time will behave identically as if period is set to 0. + */ + void addMetricUpdateHook(UpdateHook&, uint32_t period = 0); + + /** Remove a metric update hook so it won't get any more updates. */ + void removeMetricUpdateHook(UpdateHook&); + + /** + * Force a metric update for all update hooks. Useful if you want to ensure + * nice values before reporting something. + * This function can not be called from an update hook callback. + */ + void updateMetrics(bool includeSnapshotOnlyHooks = false); + + /** + * Force event logging to happen now. + * This function can not be called from an update hook callback. + */ + void forceEventLogging(); + + /** + * Register a new metric to be included in the active metric set. You need + * to have grabbed the metric lock in order to do this. (You also need to + * grab that lock if you alter registration of already registered metric + * set.) This function can not be called from an update hook callback. + */ + void registerMetric(const MetricLockGuard& l, Metric& m) { + assertMetricLockLocked(l); + _activeMetrics.getMetrics().registerMetric(m); + } + + /** + * Unregister a metric from the active metric set. You need to have grabbed + * the metric lock in order to do this. (You also need to grab that lock + * if you alter registration of already registered metric set.) + * This function can not be called from an update hook callback. + */ + void unregisterMetric(const MetricLockGuard& l, Metric& m) { + assertMetricLockLocked(l); + _activeMetrics.getMetrics().unregisterMetric(m); + } + + /** + * Reset all metrics including all snapshots. + * This function can not be called from an update hook callback. + */ + void reset(time_t currentTime); + + /** + * Read configuration. Before reading config, all metrics should be set + * up first. By doing this, the metrics manager can optimize reporting + * of consumers. readConfig() will start a config subscription. It should + * not be called multiple times. + */ + void init(const config::ConfigUri & uri, FastOS_ThreadPool&, + bool startThread = true); + + /** + * Visit a given snapshot for a given consumer. (Empty consumer name means + * all metrics). This function can be used for various printing by using + * various writer visitors in the metrics module, or your own. + */ + void visit(const MetricLockGuard & guard, const MetricSnapshot&, + MetricVisitor&, const std::string& consumer) const; + + /** + * The metric lock protects against changes in metric structure. After + * metric manager init, you need to take this lock if you want to add or + * remove metrics from registered metric sets, to avoid that happening at + * the same time as metrics are being visited. Also, when accessing + * snapshots, you need to have this lock to prevent metric manager to alter + * snapshots while you are accessing them. + */ + MetricLockGuard getMetricLock() const { + MetricLockGuard m(_waiter); + return m; + } + + /** While accessing the active metrics you should have the metric lock. */ + MetricSnapshot& getActiveMetrics(const MetricLockGuard& l) { + assertMetricLockLocked(l); + return _activeMetrics; + } + const MetricSnapshot& getActiveMetrics(const MetricLockGuard& l) const { + assertMetricLockLocked(l); + return _activeMetrics; + } + + /** While accessing the total metrics you should have the metric lock. */ + const MetricSnapshot& getTotalMetricSnapshot(const MetricLockGuard& l) const + { + assertMetricLockLocked(l); + return *_totalMetrics; + } + /** While accessing snapshots you should have the metric lock. */ + const MetricSnapshot& getMetricSnapshot( + const MetricLockGuard&, + uint32_t period, bool getInProgressSet = false) const; + const MetricSnapshotSet& getMetricSnapshotSet( + const MetricLockGuard&, uint32_t period) const; + bool hasTemporarySnapshot(const MetricLockGuard& l, uint32_t period) const + { return getMetricSnapshotSet(l, period).hasTemporarySnapshot(); } + + std::vector<uint32_t> getSnapshotPeriods(const MetricLockGuard& l) const; + + ConsumerSpec::SP getConsumerSpec(const MetricLockGuard & guard, const Metric::String& consumer) const; + + /** + * If you add or remove metrics from the active metric sets, normally, + * snapshots will be recreated next snapshot period. However, if you want + * to see the effects of such changes in status pages ahead of that, you + * can call this function in order to check whether snapshots needs to be + * regenerated and regenerate them if needed. + */ + void checkMetricsAltered(const MetricLockGuard &); + + /** Used by unit tests to verify that we have processed for a given time. */ + time_t getLastProcessedTime() const { return _lastProcessedTime; } + + /** Used by unit tests to wake waiters after altering time. */ + void timeChangedNotification() const; + + MemoryConsumption::UP getMemoryConsumption(const MetricLockGuard & guard) const; + +private: + void takeSnapshots(const MetricLockGuard &, time_t timeToProcess); + + friend class MetricManagerTest; + friend class SnapshotTest; + + void configure(const MetricLockGuard & guard, std::unique_ptr<MetricsmanagerConfig> conf); + void run() override; + time_t tick(const MetricLockGuard & guard, time_t currentTime); + /** + * Utility function for updating periodic metrics. + * + * @param updateTime Update metrics timed to update at this time. + * @param outOfSchedule Force calls to all hooks. Don't screw up normal + * schedule though. If not time to update yet, update + * without adjusting schedule for next update. + * @return Time of next hook to be called in the future. + */ + time_t updatePeriodicMetrics(const MetricLockGuard & guard, time_t updateTime, bool outOfSchedule); + void updateSnapshotMetrics(const MetricLockGuard & guard); + + void handleMetricsAltered(const MetricLockGuard &); + + typedef std::pair<uint32_t, std::string> SnapSpec; + static std::vector<SnapSpec> createSnapshotPeriods( + const MetricsmanagerConfig& config); + void assertMetricLockLocked(const MetricLockGuard& g) const { + if (!g.monitors(_waiter)) { + throw vespalib::IllegalArgumentException( + "Given lock does not lock the metric lock.", VESPA_STRLOC); + } + } + +}; + +} // metrics + diff --git a/metrics/src/vespa/metrics/metrics.h b/metrics/src/vespa/metrics/metrics.h new file mode 100644 index 00000000000..4b21dfb8579 --- /dev/null +++ b/metrics/src/vespa/metrics/metrics.h @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * File to include if you want to get all the most used metrics classes + * included. + */ +#pragma once + +#include <vespa/metrics/metric.h> +#include <vespa/metrics/countmetric.h> +#include <vespa/metrics/valuemetric.h> +#include <vespa/metrics/loadmetric.h> +#include <vespa/metrics/summetric.h> +#include <vespa/metrics/metricset.h> +#include <vespa/metrics/metricmanager.h> +#include <vespa/metrics/metricsnapshot.h> +#include <vespa/metrics/metrictimer.h> + diff --git a/metrics/src/vespa/metrics/metricset.cpp b/metrics/src/vespa/metrics/metricset.cpp new file mode 100644 index 00000000000..cbddccfa787 --- /dev/null +++ b/metrics/src/vespa/metrics/metricset.cpp @@ -0,0 +1,366 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/metrics/metricset.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/util/exceptions.h> +#include <list> + +#include <vespa/log/log.h> + +LOG_SETUP(".metrics.metricsset"); + +namespace metrics { + +MetricSet::MetricSet(const String& name, const String& tags, + const String& description, MetricSet* owner, + const std::string& dimensionKey) + : Metric(name, tags, description, owner), + _metricOrder(), + _registrationAltered(false), + _dimensionKey(dimensionKey) +{ +} + +MetricSet::MetricSet(const String& name, Tags dimensions, + const String& description, MetricSet* owner) + : Metric(name, std::move(dimensions), description, owner), + _metricOrder(), + _registrationAltered(false), + _dimensionKey() +{ +} + +MetricSet::MetricSet(const MetricSet& other, + std::vector<Metric::LP>& ownerList, + CopyType copyType, + MetricSet* owner, + bool includeUnused) + : Metric(other, owner), + _metricOrder(), + _registrationAltered(false), + _dimensionKey(other._dimensionKey) +{ + if (copyType == INACTIVE && owner == NULL && includeUnused) { + _dimensionKey = ""; + } + + for (const Metric* metric : other._metricOrder) { + if (copyType != INACTIVE || includeUnused || metric->used()) { + Metric* m = metric->clone(ownerList, copyType, this, includeUnused); + ownerList.push_back(Metric::LP(m)); + } + } +} + +const Metric* +MetricSet::getMetricInternal(const String& name) const +{ + for (const Metric* metric : _metricOrder) { + if (metric->getMangledName() == name) { + return metric; + } + } + return 0; +} + +const Metric* +MetricSet::getMetric(const String& name) const +{ + std::string::size_type pos = name.find('.'); + if (pos == std::string::npos) { + return getMetricInternal(name); + } else { + std::string child(name.substr(0, pos)); + std::string rest(name.substr(pos+1)); + const Metric* m(getMetricInternal(child)); + if (m == 0) return 0; + if (!m->isMetricSet()) { + throw vespalib::IllegalStateException( + "Metric " + child + " is not a metric set. Cannot retrieve " + "metric at path " + name + " within metric " + getPath(), + VESPA_STRLOC); + } + return static_cast<const MetricSet*>(m)->getMetric(rest); + } +} + +namespace { + struct MetricSetFinder : public MetricVisitor { + std::list<MetricSet*> _metricSets; + + virtual bool visitMetricSet(const MetricSet& set, bool autoGenerated) + { + if (autoGenerated) return false; + _metricSets.push_back(const_cast<MetricSet*>(&set)); + return true; + } + virtual bool visitMetric(const Metric&, bool autoGenerated) + { + (void) autoGenerated; + return true; + } + }; +} + +void +MetricSet::clearRegistrationAltered() +{ + MetricSetFinder finder; + visit(finder); + for (MetricSet* metricSet : finder._metricSets) { + metricSet->_registrationAltered = false; + } +} + +void +MetricSet::tagRegistrationAltered() +{ + _registrationAltered = true; + if (_owner != 0) { + _owner->tagRegistrationAltered(); + } +} + +void +MetricSet::registerMetric(Metric& metric) +{ + if (metric.isRegistered()) { + throw vespalib::IllegalStateException( + "Metric " + metric.getMangledName() + + " is already registered in a metric set. " + "Cannot register it twice.", VESPA_STRLOC); + } + const Metric* existing(getMetricInternal(metric.getMangledName())); + if (existing != 0) { + throw vespalib::IllegalStateException( + "A metric named " + metric.getMangledName() + + " is already registered in metric set " + getPath(), + VESPA_STRLOC); + } + _metricOrder.push_back(&metric); + metric.setRegistered(this); + tagRegistrationAltered(); + if (metric.isMetricSet()) { + static_cast<MetricSet &>(metric)._owner = this; + } + LOG(spam, "Registered metric%s %s in metric set %s.", + metric.isMetricSet() ? "set" : "", + metric.getMangledName().c_str(), + getPath().c_str()); +} + +void +MetricSet::unregisterMetric(Metric& metric) +{ + // In case of abrubt shutdowns, don't die hard on attempts to unregister + // non-registered metrics. Just warn and ignore. + const Metric* existing(getMetricInternal(metric.getMangledName())); + if (existing == 0) { + LOG(warning, "Attempt to unregister metric %s in metric set %s, where " + "it wasn't registered to begin with.", + metric.getMangledName().c_str(), + getPath().c_str()); + return; + } + bool found = false; + for (std::vector<Metric*>::iterator it = _metricOrder.begin(); + it != _metricOrder.end(); ++it) + { + if (*it == &metric) { + _metricOrder.erase(it); + found = true; + break; + } + } + assert(found); // We check above for existence. + (void) found; + metric.setRegistered(NULL); + tagRegistrationAltered(); + if (metric.isMetricSet()) { + static_cast<MetricSet &>(metric)._owner = this; + } + LOG(spam, "Unregistered metric%s %s from metric set %s.", + metric.isMetricSet() ? "set" : "", + metric.getMangledName().c_str(), + getPath().c_str()); +} + +namespace { + typedef vespalib::stringref TmpString; + class StringMetric { + public: + StringMetric(const TmpString & s, Metric * m) : first(s), second(m) { } + bool operator == (const StringMetric & b) const { return first == b.first; } + bool operator == (const TmpString & b) const { return first == b; } + bool operator <(const StringMetric & b) const { return first < b.first; } + bool operator <(const TmpString & b) const { return first < b; } + TmpString first; + Metric * second; + }; + bool operator < (const TmpString & a, const StringMetric & b) { return a < b.first; } + + typedef std::vector<StringMetric> SortedVector; + + void createMetricMap(SortedVector& metricMap, + const std::vector<Metric*>& orderedList) + { + metricMap.reserve(orderedList.size()); + for (Metric* metric : orderedList) { + metricMap.push_back(StringMetric(metric->getMangledName(), metric)); + } + std::sort(metricMap.begin(), metricMap.end()); + } +} + +void +MetricSet::addTo(Metric& other, std::vector<Metric::LP>* ownerList) const +{ + bool mustAdd = (ownerList == 0); + MetricSet& o(static_cast<MetricSet&>(other)); + SortedVector map1, map2; + createMetricMap(map1, _metricOrder); + createMetricMap(map2, o._metricOrder); + SortedVector::iterator source(map1.begin()); + SortedVector::iterator target(map2.begin()); + typedef vespalib::hash_map<TmpString, Metric*> HashMap; + HashMap newMetrics; + while (source != map1.end()) { + if (target == map2.end() || source->first < target->first) { + // Source missing in snapshot to add to. Lets create and add. + if (!mustAdd && source->second->used()) { + Metric::LP copy(source->second->clone( + *ownerList, INACTIVE, &o)); + ownerList->push_back(copy); + newMetrics[source->first] = copy.get(); + } + ++source; + } else if (source->first == target->first) { + if (mustAdd) { + source->second->addToPart(*target->second); + } else { + source->second->addToSnapshot(*target->second, *ownerList); + } + ++source; + ++target; + } else { + ++target; + } + } + // If we added metrics, reorder target order list to equal source + if (!newMetrics.empty()) { + std::vector<Metric*> newOrder; + newOrder.reserve(o._metricOrder.size() + newMetrics.size()); + for (const Metric* metric : _metricOrder) { + TmpString v(metric->getMangledName()); + target = std::lower_bound(map2.begin(), map2.end(), v); + if ((target != map2.end()) && (target->first == v)) { + newOrder.push_back(target->second); + } else { + HashMap::iterator starget = newMetrics.find(v); + if (starget != newMetrics.end()) { + newOrder.push_back(starget->second); + } + } + } + // If target had unique metrics, add them at the end + for (Metric* metric : o._metricOrder) { + TmpString v(metric->getMangledName()); + if ( ! std::binary_search(map1.begin(), map1.end(), v) ) { + LOG(debug, "Metric %s exist in one snapshot but not other." + "Order will be messed up. Adding target unique " + "metrics to end.", + metric->getPath().c_str()); + newOrder.push_back(metric); + } + } + o._metricOrder.swap(newOrder); + } +} + +bool +MetricSet::logEvent(const String& fullName) const +{ + (void) fullName; + throw vespalib::IllegalStateException( + "logEvent() cannot be called on metrics set.", VESPA_STRLOC); +} + +void +MetricSet::reset() +{ + for (Metric* metric : _metricOrder) { + metric->reset(); + } +} + +bool +MetricSet::visit(MetricVisitor& visitor, bool tagAsAutoGenerated) const +{ + if (!visitor.visitMetricSet(*this, tagAsAutoGenerated)) return true; + for (const Metric* metric : _metricOrder) { + if (!metric->visit(visitor, tagAsAutoGenerated)) { + break; + } + } + visitor.doneVisitingMetricSet(*this); + return true; +} + +void +MetricSet::print(std::ostream& out, bool verbose, + const std::string& indent, uint64_t secondsPassed) const +{ + out << _name << ":"; + for (const Metric* metric : _metricOrder) { + out << "\n" << indent << " "; + metric->print(out, verbose, indent + " ", secondsPassed); + } +} + +bool +MetricSet::used() const +{ + for (const Metric* metric : _metricOrder) { + if (metric->used()) return true; + } + return false; +} + +void +MetricSet::addMemoryUsage(MemoryConsumption& mc) const +{ + Metric::addMemoryUsage(mc); + ++mc._metricSetCount; + mc._metricSetMeta += sizeof(MetricSet) - sizeof(Metric); + mc._metricSetOrder += _metricOrder.size() * 3 * sizeof(void*); + for (const Metric* metric : _metricOrder) { + metric->addMemoryUsage(mc); + } +} + +void +MetricSet::updateNames(NameHash& hash) const +{ + Metric::updateNames(hash); + for (const Metric* metric : _metricOrder) { + metric->updateNames(hash); + } +} + +void +MetricSet::printDebug(std::ostream& out, const std::string& indent) const +{ + out << "set "; + Metric::printDebug(out, indent); + if (_registrationAltered) out << ", regAltered"; + out << " {"; + for (const Metric* metric : _metricOrder) { + out << "\n" << indent << " "; + metric->printDebug(out, indent + " "); + } + out << "}"; +} + +} // metrics + diff --git a/metrics/src/vespa/metrics/metricset.h b/metrics/src/vespa/metrics/metricset.h new file mode 100644 index 00000000000..257bb699690 --- /dev/null +++ b/metrics/src/vespa/metrics/metricset.h @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class metrics::MetricSet + * \ingroup metrics + * + * \brief Class containing a set of metrics. + * + * This is a class to bundle related metrics. Note that the metricsset is + * itself a metric so this generates a tree where metricssets are leaf nodes. + */ +#pragma once + +#include <boost/weak_ptr.hpp> +#include <map> +#include <vespa/metrics/metric.h> + +namespace metrics { + +class MetricSet : public Metric +{ + std::vector<Metric*> _metricOrder; // Keep added order for reporting + bool _registrationAltered; // Set to true if metrics have been + // registered/unregistered since last time + // it was reset + std::string _dimensionKey; // If this metric is part of a dimension (yamas-speak), + // the key of the dimension should be set here. + // If so, the name of the metric is used as dimension value. + +public: + typedef std::unique_ptr<MetricSet> UP; + typedef std::shared_ptr<MetricSet> SP; + + MetricSet(const String& name, const String& tags, + const String& description, MetricSet* owner = 0, + const std::string& dimensionKey = ""); + + MetricSet(const String& name, Tags dimensions, + const String& description, MetricSet* owner = 0); + + MetricSet(const MetricSet&, std::vector<Metric::LP>& ownerList, + CopyType, MetricSet* owner = 0, bool includeUnused = false); + virtual ~MetricSet() {} + + // If no path, this metric is not registered within another + bool isTopSet() const { return _owner == 0; } + + /** + * Returns true if registration has been altered since it was last + * cleared. Used by the metric manager to know when it needs to recalculate + * which consumers will see what. + */ + bool isRegistrationAltered() const { return _registrationAltered; } + /** Clear all registration altered flags. */ + void clearRegistrationAltered(); + + void registerMetric(Metric& m); + void unregisterMetric(Metric& m); + + virtual Metric* clone(std::vector<Metric::LP>& ownerList, + CopyType type, MetricSet* owner, + bool includeUnused = false) const + { return new MetricSet(*this, ownerList, type, owner, includeUnused); } + + void reset(); + + virtual bool visit(MetricVisitor&, bool tagAsAutoGenerated = false) const; + + virtual bool logEvent(const String& fullName) const; + + virtual void print(std::ostream&, bool verbose, + const std::string& indent, uint64_t secondsPassed) const; + + // These should never be called on metrics set. + virtual int64_t getLongValue(const stringref & id) const + { (void) id; assert(false); return 0; } + virtual double getDoubleValue(const stringref & id) const + { (void) id; assert(false); return 0; } + + const Metric* getMetric(const String& name) const; + Metric* getMetric(const String& name) { + return const_cast<Metric*>( + const_cast<const MetricSet*>(this)->getMetric(name)); + } + + virtual void addToSnapshot(Metric& m, std::vector<Metric::LP>& o) const + { addTo(m, &o); } + + const std::vector<Metric*>& getRegisteredMetrics() const + { return _metricOrder; } + + virtual bool used() const; + + virtual void addMemoryUsage(MemoryConsumption&) const; + + /** Update names using the given name hash, to utilize ref counting. */ + virtual void updateNames(NameHash&) const; + + virtual void printDebug(std::ostream&, const std::string& indent="") const; + + virtual bool isMetricSet() const { return true; } + + virtual void addToPart(Metric& m) const + { addTo(m, 0); } + + /** + * Returns the key of the dimension this metric is part of (if any). + */ + const std::string& getDimensionKey() const { return _dimensionKey; } + +private: + // Do not generate default copy constructor or assignment operator + // These would screw up metric registering + MetricSet(const MetricSet&); + MetricSet& operator=(const MetricSet&); + + void tagRegistrationAltered(); + const Metric* getMetricInternal(const String& name) const; + + virtual void addTo(Metric&, std::vector<Metric::LP>* ownerList) const; +}; + +} // metrics + diff --git a/metrics/src/vespa/metrics/metricsmanager.def b/metrics/src/vespa/metrics/metricsmanager.def new file mode 100644 index 00000000000..fdf6d1b78e5 --- /dev/null +++ b/metrics/src/vespa/metrics/metricsmanager.def @@ -0,0 +1,64 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=4 +namespace=metrics + +# If any snapshot periods is set, these override all the default ones. +# The array sets the snapshot periods in number of seconds. Note that when +# sorted in rising time length order, it is required that the snapshot length of +# the next snapshot is a multiplum of the previous snapshot. +## restart flag was added automatically and needs to be verified. +snapshot.periods[] int restart + +# The name of the consumer that should include the given metrics specified. +## restart flag was added automatically and needs to be verified. +consumer[].name string restart + +# Include metrics that match the given tags. +# +# A tag specification either adds a single tag by given the full tag name, or +# any tag, by specifying an asterix as wildcard ('*'). An asterix will not match +# metrics without any tags set, but you can match these by specifying an empty +# string. +# +# A metric set matched by tags, will also include every metric contained in it +# regardless of tags. +## restart flag was added automatically and needs to be verified. +consumer[].tags[] string restart + +# Do not include metrics that match the specific tags. This has higher +# presedence than the adding of tags, but lower presedence than name patterns. +# +# Removed tags are specified as added tags, with a small exception. Removing +# wildcard or untagged metrics does not make sense, as metrics are not added by +# default. If you don not want to include these metrics, just not specify them +# as tags to be added. Thus, these are not allowed in remove tags spec. +# +# A metric set matched by remove tag will remove all metrics below that tag. +## restart flag was added automatically and needs to be verified. +consumer[].removedtags[] string restart + +# Include all metrics that matches the names given here. +# Metrics added here have presedence above tag specifications. +# +# Names are specified as a complete dot separated path to metric. Asterix can +# be used to match any part. An asterix will only match a complete name without +# a dot. So if you have an average metric called 'queuesize' in your top level +# metric set called 'myapp', you can add this metric with the spec +# 'myapp.queuesize'. You can also add non-default average metric entries by +# specifying which, such as 'myapp.queuesize.last' or 'myapp.queuesize.max'. +# Patterns can be used, to for instance specify '*.queuesize', 'myapp.*', '*.*', +# 'myapp.*.max' or similar. +# +# A metric set added with specific name will add all metrics within it. +## restart flag was added automatically and needs to be verified. +consumer[].addedmetrics[] string restart + +# Do not include metrics with the below names. +# This has highest presedence. Metrics removed like this will override any +# specification otherwise that would include them. +# +# The remove metrics are specified exactly the same as the added metrics. +# +# A metric set removed with specific name will remove all metrics within it. +## restart flag was added automatically and needs to be verified. +consumer[].removedmetrics[] string restart diff --git a/metrics/src/vespa/metrics/metricsnapshot.cpp b/metrics/src/vespa/metrics/metricsnapshot.cpp new file mode 100644 index 00000000000..afb4aa305d4 --- /dev/null +++ b/metrics/src/vespa/metrics/metricsnapshot.cpp @@ -0,0 +1,164 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/metrics/metricsnapshot.h> + +#include <vespa/log/log.h> +#include <vespa/metrics/metricmanager.h> + +LOG_SETUP(".metrics.snapshot"); + +namespace metrics { + +MetricSnapshot::MetricSnapshot(const Metric::String& name) + : _name(name), + _period(0), + _fromTime(0), + _toTime(0), + _snapshot(new MetricSet("top", "", "")), + _metrics() +{ +} + +MetricSnapshot::MetricSnapshot( + const Metric::String& name, uint32_t period, const MetricSet& source, + bool copyUnset) + : _name(name), + _period(period), + _fromTime(0), + _toTime(0), + _snapshot(), + _metrics() +{ + Metric* m = source.clone(_metrics, Metric::INACTIVE, 0, copyUnset); + assert(m->isMetricSet()); + _snapshot.reset(static_cast<MetricSet*>(m)); + trim(_metrics); +} + +void +MetricSnapshot::reset(time_t currentTime) +{ + _fromTime = currentTime; + _toTime = 0; + _snapshot->reset(); +} + +void +MetricSnapshot::recreateSnapshot(const MetricSet& metrics, bool copyUnset) +{ + std::vector<Metric::LP> newMetrics; + Metric* m = metrics.clone(newMetrics, Metric::INACTIVE, 0, copyUnset); + assert(m->isMetricSet()); + MetricSet::UP newSnapshot(static_cast<MetricSet*>(m)); + newSnapshot->reset(); + _snapshot->addToSnapshot(*newSnapshot, newMetrics); + _snapshot = std::move(newSnapshot); + _metrics.swap(newMetrics); + trim(_metrics); +} + +void +MetricSnapshot::addMemoryUsage(MemoryConsumption& mc) const +{ + ++mc._snapshotCount; + mc._snapshotName += mc.getStringMemoryUsage(_name, mc._snapshotNameUnique); + mc._snapshotMeta += sizeof(MetricSnapshot) + + _metrics.capacity() * sizeof(Metric::SP); + _snapshot->addMemoryUsage(mc); +} + +MetricSnapshotSet::MetricSnapshotSet( + const Metric::String& name, uint32_t period, + uint32_t count, const MetricSet& source, bool snapshotUnsetMetrics) + : _count(count), + _builderCount(0), + _current(new MetricSnapshot(name, period, source, snapshotUnsetMetrics)), + _building(count == 1 ? 0 : new MetricSnapshot( + name, period, source, snapshotUnsetMetrics)) +{ + _current->reset(0); + if (_building.get()) _building->reset(0); +} + +MetricSnapshot& +MetricSnapshotSet::getNextTarget() +{ + if (_count == 1) return *_current; + return *_building; +} + +bool +MetricSnapshotSet::haveCompletedNewPeriod(time_t newFromTime) +{ + if (_count == 1) { + _current->setToTime(newFromTime); + return true; + } + _building->setToTime(newFromTime); + // If not time to roll yet, just return + if (++_builderCount < _count) return false; + // Building buffer done. Use that as current and reset current. + MetricSnapshot::UP tmp(std::move(_current)); + _current = std::move(_building); + _building = std::move(tmp); + _building->reset(newFromTime); + _builderCount = 0; + return true; +} + +bool +MetricSnapshotSet::timeForAnotherSnapshot(time_t currentTime) { + time_t lastTime = getToTime(); + if (currentTime >= lastTime + getPeriod()) { + if (currentTime >= lastTime + 2 * getPeriod()) { + LOG(warning, "Metric snapshot set %s was asked if it was time for " + "another snapshot, a whole period beyond when it " + "should have been done (Last update was at time %" + PRIu64 ", current time is %" PRIu64" and period is %u). " + "Clearing data and updating time to current time.", + getName().c_str(), lastTime, currentTime, getPeriod()); + reset(currentTime); + } + return true; + } + return false; +} + +void +MetricSnapshotSet::reset(time_t currentTime) { + if (_count != 1) _building->reset(currentTime); + _current->reset(currentTime); + _builderCount = 0; +} + +void +MetricSnapshotSet::recreateSnapshot(const MetricSet& metrics, bool copyUnset) +{ + if (_count != 1) _building->recreateSnapshot(metrics, copyUnset); + _current->recreateSnapshot(metrics, copyUnset); +} + +void +MetricSnapshotSet::addMemoryUsage(MemoryConsumption& mc) const +{ + ++mc._snapshotSetCount; + mc._snapshotSetMeta += sizeof(MetricSnapshotSet); + if (_count != 1) _building->addMemoryUsage(mc); + _current->addMemoryUsage(mc); +} + +void +MetricSnapshotSet::updateNames(NameHash& hash) const +{ + if (_count != 1) _building->updateNames(hash); + _current->updateNames(hash); +} + +void +MetricSnapshotSet::setFromTime(time_t fromTime) +{ + if (_count != 1) _building->setFromTime(fromTime); + _current->setFromTime(fromTime); +} + +} // metrics diff --git a/metrics/src/vespa/metrics/metricsnapshot.h b/metrics/src/vespa/metrics/metricsnapshot.h new file mode 100644 index 00000000000..910d97906de --- /dev/null +++ b/metrics/src/vespa/metrics/metricsnapshot.h @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * \class metrics::MetricSnapshot + * \ingroup metrics + * + * \brief Represents a snapshot of a metric set. + */ +#pragma once + +#include <boost/weak_ptr.hpp> +#include <map> +#include <vespa/metrics/metric.h> +#include <vespa/metrics/metricset.h> + +namespace metrics { + +class MetricManager; + +class MetricSnapshot +{ + Metric::String _name; + // Period length of this snapshot + uint32_t _period; + // Time this snapshot was last updated. + time_t _fromTime; + // If set to 0, use _fromTime + _period. + time_t _toTime; + // Keeps the metrics set view of the snapshot + MetricSet::UP _snapshot; + // Snapshots must own their own metrics + mutable std::vector<Metric::LP> _metrics; + +public: + typedef std::unique_ptr<MetricSnapshot> UP; + typedef std::shared_ptr<MetricSnapshot> SP; + + /** Create a fresh empty top level snapshot. */ + MetricSnapshot(const Metric::String& name); + /** Create a snapshot of another metric source. */ + MetricSnapshot(const Metric::String& name, uint32_t period, + const MetricSet& source, bool copyUnset); + virtual ~MetricSnapshot() {} + + void addToSnapshot(MetricSnapshot& other, bool reset_, time_t currentTime) { + _snapshot->addToSnapshot(other.getMetrics(), other._metrics); + if (reset_) reset(currentTime); + other._toTime = currentTime; + } + void addToSnapshot(MetricSnapshot& other, time_t currentTime) const { + _snapshot->addToSnapshot(other.getMetrics(), other._metrics); + other._toTime = currentTime; + } + void setFromTime(time_t fromTime) { _fromTime = fromTime; } + void setToTime(time_t toTime) { _toTime = toTime; } + + const Metric::String& getName() const { return _name; } + uint32_t getPeriod() const { return _period; } + time_t getFromTime() const { return _fromTime; } + time_t getToTime() const { return _toTime; } + time_t getLength() const + { return (_toTime != 0 ? _toTime - _fromTime : _fromTime + _period); } + const MetricSet& getMetrics() const { return *_snapshot; } + MetricSet& getMetrics() { return *_snapshot; } + void reset(time_t currentTime); + /** + * Recreate snapshot by cloning given metric set and then add the data + * from the old one. New metrics have been added. + */ + void recreateSnapshot(const MetricSet& metrics, bool copyUnset); + + void addMemoryUsage(MemoryConsumption&) const; + void updateNames(NameHash& hash) const { _snapshot->updateNames(hash); } +}; + +class MetricSnapshotSet { + uint32_t _count; // Number of times we need to add to building period + // before we have a full time window. + uint32_t _builderCount; // Number of times we've currently added to the + // building instance. + MetricSnapshot::UP _current; // The last full period + MetricSnapshot::UP _building; // The building period +public: + typedef std::shared_ptr<MetricSnapshotSet> SP; + + MetricSnapshotSet(const Metric::String& name, uint32_t period, + uint32_t count, const MetricSet& source, + bool snapshotUnsetMetrics); + + const Metric::String& getName() const { return _current->getName(); } + uint32_t getPeriod() const { return _current->getPeriod(); } + time_t getFromTime() const { return _current->getFromTime(); } + time_t getToTime() const { return _current->getToTime(); } + uint32_t getCount() const { return _count; } + uint32_t getBuilderCount() const { return _builderCount; } + bool hasTemporarySnapshot() const { return (_building.get() != 0); } + MetricSnapshot& getSnapshot(bool temporary = false) + { return *((temporary && _count > 1) ? _building : _current); } + const MetricSnapshot& getSnapshot(bool temporary = false) const + { return *((temporary && _count > 1) ? _building : _current); } + MetricSnapshot& getNextTarget(); + bool timeForAnotherSnapshot(time_t currentTime); + bool haveCompletedNewPeriod(time_t newFromTime); + void reset(time_t currentTime); + /** + * Recreate snapshot by cloning given metric set and then add the data + * from the old one. New metrics have been added. + */ + void recreateSnapshot(const MetricSet& metrics, bool copyUnset); + void addMemoryUsage(MemoryConsumption&) const; + void updateNames(NameHash& hash) const; + void setFromTime(time_t fromTime); +}; + +} // metrics + diff --git a/metrics/src/vespa/metrics/metrictimer.cpp b/metrics/src/vespa/metrics/metrictimer.cpp new file mode 100644 index 00000000000..dfaa982d809 --- /dev/null +++ b/metrics/src/vespa/metrics/metrictimer.cpp @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/metrics/metrictimer.h> + +namespace metrics { + +MetricTimer::MetricTimer() +{ + // Amusingly enough, steady_clock was not actually steady by default on + // GCC < 4.8.1, so add a bit of compile-time paranoia just to make sure. + static_assert(std::chrono::steady_clock::is_steady, + "Old/broken STL implementation; steady_clock not steady"); + _startTime = std::chrono::steady_clock::now(); +} + +} // metrics diff --git a/metrics/src/vespa/metrics/metrictimer.h b/metrics/src/vespa/metrics/metrictimer.h new file mode 100644 index 00000000000..7228daced34 --- /dev/null +++ b/metrics/src/vespa/metrics/metrictimer.h @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class MetricTimer + * @ingruop metrics + * + * @brief Used to add time values to metrics. + */ + +#pragma once + +#include <vespa/fastos/fastos.h> +#include <vespa/metrics/valuemetric.h> +#include <chrono> + +namespace metrics { + +class MetricTimer +{ +public: + MetricTimer(); + + /** + * Adds ms passed since this timer was constructed to given value metric. + * Returns that value as well. + * + * Uses a steady (monotonic) clock internally so value should never + * underflow or be affected by system clock changes. + */ + template<typename AvgVal, typename TotVal, bool SumOnAdd> + uint64_t stop(ValueMetric<AvgVal, TotVal, SumOnAdd>& metric) { + const auto delta = std::chrono::steady_clock::now() - _startTime; + const uint64_t deltaMs( + std::chrono::duration_cast<std::chrono::milliseconds>(delta) + .count()); + metric.addValue(deltaMs); + return deltaMs; + } + +private: + std::chrono::time_point<std::chrono::steady_clock> _startTime; +}; + +} // metrics + diff --git a/metrics/src/vespa/metrics/metricvalueset.h b/metrics/src/vespa/metrics/metricvalueset.h new file mode 100644 index 00000000000..98bc6160833 --- /dev/null +++ b/metrics/src/vespa/metrics/metricvalueset.h @@ -0,0 +1,182 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class MetricValueSet + * \ingroup metrics + * + * \brief Utility for doing lockless metric updates and reads. + * + * We don't want to use regular locking while updating metrics due to overhead. + * We use this class to make metric updates as safe as possible without + * requiring locks. + * + * It keeps the set of values a metric wants to set. Thus it is templated on + * the class keeping the values. All that is required of this class is that it + * has an empty constructor and a copy constructor. + * + * The locking works, by keeping a set of values, with an active pointer into + * the value vector. Assuming only one thread calls setValues(), it can update + * the active pointer safely. We assume updating the active pointer is a + * non-interruptable operations, such that other threads will see either the new + * or the old value correctly. This should be the case on our platforms. + * + * Due to the reset functionality, it is possible to miss out on a metrics + * added during the reset, but this is very unlikely. For that to happen, when + * someone sets the reset flag, the writer thread must be in setValues(), + * having already passed the check for the reset flag, but not finished setting + * the values yet. + */ +#pragma once + +#include <sstream> +#include <vector> +#include <atomic> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/jsonstream.h> + +namespace metrics { + +struct MetricValueClass { + typedef std::unique_ptr<MetricValueClass> UP; + virtual ~MetricValueClass() {} + + virtual double getDoubleValue(const vespalib::stringref & id) const = 0; + virtual uint64_t getLongValue(const vespalib::stringref & id) const = 0; + virtual void output(const std::string& id, std::ostream&) const = 0; + virtual void output(const std::string& id, vespalib::JsonStream&) const = 0; + std::string toString(const std::string& id) { + std::ostringstream ost; + output(id, ost); + return ost.str(); + } +}; + +template<typename ValueClass> +class MetricValueSet { + using AtomicValues = typename ValueClass::AtomicImpl; + std::vector<AtomicValues> _values; + std::atomic<uint32_t> _activeValueIndex; + std::atomic<uint32_t> _flags; + + enum Flag { RESET = 1 }; + bool isReset() const { return hasFlag(RESET); } + + void validateCorrectValueSuperClass(const MetricValueClass&) {} + +public: + MetricValueSet(uint32_t copyCount = 3) + : _values(copyCount), + _activeValueIndex(0), + _flags(0) + { + } + + MetricValueSet(const MetricValueSet& other, uint32_t copyCount = 3) + : _values(copyCount), + _activeValueIndex(0), + _flags(other._flags.load(std::memory_order_relaxed)) + { + setValues(other.getValues()); + } + + MetricValueSet& operator=(const MetricValueSet& other) + { + setValues(other.getValues()); + return *this; + } + + + /** Get the current values. */ + ValueClass getValues() const { + ValueClass v{}; + if (!isReset()) { + // Must load with acquire to match release store in setValues. + // Note that despite being atomic on _individual fields_, this + // does not guarantee reading a consistent snapshot _across_ + // fields for any given metric. + const size_t readIndex( + _activeValueIndex.load(std::memory_order_acquire)); + v.relaxedLoadFrom(_values[readIndex]); + } + return v; + } + + /** + * Get the current values from the metric. This function should not be + * called in parallel. Only call it from a single thread or use external + * locking. If it returns false, it means the metric have just been reset. + * In which case, redo getValues(), apply the update again, and call + * setValues() again. + */ + bool setValues(const ValueClass& values) { + validateCorrectValueSuperClass(values); + // Only setter-thread can write _activeValueIndex, so relaxed + // load suffices. + uint32_t nextIndex = (_activeValueIndex.load(std::memory_order_relaxed) + + 1) % _values.size(); + // Reset flag is loaded/stored with relaxed semantics since it does not + // carry data dependencies. _activeValueIndex has a dependency on + // _values, however, so we must ensure that stores are published + // and loads acquired. + if (isReset()) { + removeFlag(RESET); + ValueClass resetValues{}; + resetValues.relaxedStoreInto(_values[nextIndex]); + _activeValueIndex.store(nextIndex, std::memory_order_release); + return false; + } else { + values.relaxedStoreInto(_values[nextIndex]); + _activeValueIndex.store(nextIndex, std::memory_order_release); + return true; + } + } + + /** + * Retrieve and reset in a single operation, to minimize chance of + * alteration in the process. + */ + ValueClass getValuesAndReset() { + ValueClass result(getValues()); + setFlag(RESET); + return result; + } + + void reset() { + setFlag(RESET); + } + + std::string toString() { + std::ostringstream ost; + ost << "MetricValueSet(reset=" << (isReset() ? "true" : "false") + << ", active " << _activeValueIndex; + ost << "\n empty: " << ValueClass().toString(); + for (uint32_t i=0; i<_values.size(); ++i) { + ost << "\n " << _values[i].toString(); + } + ost << "\n)"; + return ost.str(); + } + + uint32_t getMemoryUsageAllocatedInternally() const { + return _values.capacity() * sizeof(ValueClass); + } + + uint32_t size() const { return _values.size(); } + + bool hasFlag(uint32_t flags) const { + return ((_flags.load(std::memory_order_relaxed) & flags) != 0); + } + void setFlag(uint32_t flags) { + _flags.store(_flags.load(std::memory_order_relaxed) | flags, + std::memory_order_relaxed); + } + void removeFlag(uint32_t flags) { + _flags.store(_flags.load(std::memory_order_relaxed) & ~flags, + std::memory_order_relaxed); + } + uint32_t getFlags() const { + return _flags.load(std::memory_order_relaxed); + } +}; + +} // metrics + diff --git a/metrics/src/vespa/metrics/namehash.h b/metrics/src/vespa/metrics/namehash.h new file mode 100644 index 00000000000..9a281e39111 --- /dev/null +++ b/metrics/src/vespa/metrics/namehash.h @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class metrics::NameHash + * \ingroup metrics + * + * \brief Simple class to enable string reference counting to work better. + * + * When creating metrics, it is easy to use const char references from code, + * for instance having a for loop setting up metrics for each thread, this will + * not actually generate ref counted strings, but rather unique strings. + * + * Also, with ref counted strings, it is easy to screw it up if you access the + * string in a way requiring copy. + * + * This class is used to just keep a set of strings, and having a class for + * users to input their strings and get the "master" string with that content. + * + * Metrics use this after having registered metrics, to ensure we dont keep more + * copies of non-unique strings than needed. + */ +#pragma once + +#include <boost/utility.hpp> +#include <vespa/metrics/memoryconsumption.h> +#include <vespa/vespalib/stllike/hash_set.h> + +namespace metrics { + +class NameHash : private boost::noncopyable { + vespalib::hash_set<std::string> _hash; + uint32_t _unifiedCounter; + uint32_t _checkedCounter; + +public: + NameHash() : _unifiedCounter(0), _checkedCounter(0) {} + + void updateName(std::string& name) { + ++_checkedCounter; + vespalib::hash_set<std::string>::const_iterator it(_hash.find(name)); + if (it != _hash.end()) { + if (name.c_str() != it->c_str()) { + name = *it; + ++_unifiedCounter; + } + } else { + _hash.insert(name); + } + } + + uint32_t getUnifiedStringCount() const { return _unifiedCounter; } + uint32_t getCheckedStringCount() const { return _checkedCounter; } + void resetCounts() { _unifiedCounter = 0; _checkedCounter = 0; } + void addMemoryUsage(MemoryConsumption& mc) const { + mc._nameHash += sizeof(NameHash) + + _hash.getMemoryConsumption() + - sizeof(vespalib::hash_set<std::string>); + for (vespalib::hash_set<std::string>::const_iterator it(_hash.begin()); + it != _hash.end(); ++it) + { + mc._nameHashStrings + += mc.getStringMemoryUsage(*it, mc._nameHashUnique); + } + } +}; + + +} // metrics + diff --git a/metrics/src/vespa/metrics/printutils.h b/metrics/src/vespa/metrics/printutils.h new file mode 100644 index 00000000000..42f640e83d2 --- /dev/null +++ b/metrics/src/vespa/metrics/printutils.h @@ -0,0 +1,418 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * This file contains utility functions to help print out metric snapshots in + * a user friendly way. It defines value types, functions for retrieving and + * doing algorithmics with the values, and printing them in an HTML table. + * + * This is used by storage to print HTML metrics report for its status page. + */ + +#pragma once + +#include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/metrics/metricmanager.h> + +namespace metrics { +namespace printutils { + +typedef std::pair<int64_t, bool> LongValue; +typedef std::pair<double, bool> DoubleValue; + +struct MetricSource { + typedef Metric::String String; + + const MetricSnapshot& _snapshot; + String _metricsPrefix; + // If no map is supplied, this map will own the data in the metrics + // accessed map. + std::map<String, Metric::SP> _metricsAccessedOwner; + std::map<String, Metric::SP>& _metricsAccessed; + + MetricSource(const MetricSnapshot& s, + const String& metricsPrefix, + std::map<String, Metric::SP>* metricsAccessed = 0) + : _snapshot(s), + _metricsPrefix(metricsPrefix), + _metricsAccessedOwner(), + _metricsAccessed(metricsAccessed == 0 ? _metricsAccessedOwner + : *metricsAccessed) + { + } + + String createAbsoluteMetricName(const String& name) const { + String prefix = _metricsPrefix; + String addition = name; + while (addition.find("../") == 0) { + String::size_type pos1 = prefix.rfind('.'); + if (pos1 == String::npos) + throw vespalib::IllegalArgumentException( + "Cannot go back anymore in path " + prefix, + VESPA_STRLOC); + prefix = prefix.substr(0, pos1); + addition = addition.substr(3); + } + return (prefix.empty() ? addition : prefix + "." + addition); + } + + struct SourceMetricVisitor : public metrics::MetricVisitor { + String _stringPath; + vespalib::StringTokenizer::TokenList _path; + int32_t _pathIndex; + Metric::UP _resultMetric; + bool _prefixMatch; + std::vector<String> _prefixMatches; + + SourceMetricVisitor(const String& path, bool prefixMatch) + : _stringPath(path), + _path(vespalib::StringTokenizer(path, ".").getTokens()), + _pathIndex(-1), + _resultMetric(), + _prefixMatch(prefixMatch), + _prefixMatches() + { + } + + void checkForPrefixMatch(const Metric& metric) { + if (metric.getName().size() >= _path[_pathIndex].size()) { + if (metric.getName().find(_path[_pathIndex]) == 0) { + _prefixMatches.push_back(metric.getName()); + } + } + } + + bool visitMetricSet(const MetricSet& set, bool) { + if (_pathIndex == -1) { + _pathIndex = 0; + return true; + } + if (_prefixMatch + && static_cast<size_t>(_pathIndex + 1) == _path.size()) + { + for (std::vector<Metric*>::const_iterator it + = set.getRegisteredMetrics().begin(); + it != set.getRegisteredMetrics().end(); ++it) + { + checkForPrefixMatch(**it); + } + return false; + } + if (set.getName() != _path[_pathIndex]) return false; + if (static_cast<size_t>(++_pathIndex) >= _path.size()) { + throw vespalib::IllegalArgumentException( + "Path " + _stringPath + " points to a metric set. " + "Only primitive metrics can be retrieved.", + VESPA_STRLOC); + } + return true; + } + void doneVisitingMetricSet(const MetricSet&) { --_pathIndex; } + bool visitMetric(const Metric& metric, bool) { + if (_prefixMatch) { + checkForPrefixMatch(metric); + } + if (_path[_pathIndex] != metric.getName()) { + return true; + } + if (_prefixMatch) { + throw vespalib::IllegalArgumentException( + "Cannot find existing entries with prefix " + + _stringPath + " since element " + metric.getName() + + " is not a metric set", VESPA_STRLOC); + } + if (static_cast<size_t>(_pathIndex + 1) < _path.size()) { + throw vespalib::IllegalArgumentException( + "Path " + _stringPath + " cannot exist since element " + + _path[_pathIndex] + " is not a metric set: " + + metric.toString(), + VESPA_STRLOC); + } + std::vector<Metric::LP> ownerList; + _resultMetric.reset(metric.clone(ownerList, Metric::INACTIVE, 0)); + if (!ownerList.empty()) { + throw vespalib::IllegalArgumentException( + "Metric " + metric.getName() + " added entries to " + "owners list when cloning. This should not happen " + "for primitive metrics.", VESPA_STRLOC); + } + return false; + } + + }; + + const Metric* getMetric(const String& name) { + String path = createAbsoluteMetricName(name); + std::map<String, Metric::SP>::const_iterator it( + _metricsAccessed.find(path)); + if (it != _metricsAccessed.end()) { + return it->second.get(); + } + SourceMetricVisitor visitor(path, false); + _snapshot.getMetrics().visit(visitor); + if (visitor._resultMetric.get() == 0) { + throw vespalib::IllegalArgumentException( + "Metric " + path + " was not found.", VESPA_STRLOC); + } + Metric::SP metric(visitor._resultMetric.release()); + _metricsAccessed[path] = metric; + return metric.get(); + } + + std::vector<String> + getPathsMatchingPrefix(const String& prefix) const + { + String path = createAbsoluteMetricName(prefix); + SourceMetricVisitor visitor(path, true); + _snapshot.getMetrics().visit(visitor); + return visitor._prefixMatches; + } +}; + +// Addition functions. Ensure that if floating point value is used, +// result ends up as floating point too. +LongValue operator+(LongValue addend1, LongValue addend2) +{ + return LongValue(addend1.first + addend2.first, + addend1.second && addend2.second); +} + +template<typename ValueType1, typename ValueType2> +DoubleValue operator+(std::pair<ValueType1, bool> addend1, + std::pair<ValueType2, bool> addend2) +{ + return DoubleValue(addend1.first + addend2.first, + addend1.second && addend2.second); +} + +// Subtraction functions. Ensure that if floating point value is used, +// result ends up as floating point too. +LongValue operator-(LongValue minuend, LongValue subtrahend) +{ + return LongValue(minuend.first - subtrahend.first, + minuend.second && subtrahend.second); +} + +template<typename ValueType1, typename ValueType2> +DoubleValue operator-(std::pair<ValueType1, bool> minuend, + std::pair<ValueType2, bool> subtrahend) +{ + return DoubleValue(minuend.first - subtrahend.first, + minuend.second && subtrahend.second); +} + +// Multiplication functions. Ensure that if floating point value is used, +// result ends up as floating point too. + +LongValue operator*(LongValue factor1, LongValue factor2) +{ + return std::pair<int64_t, bool>(factor1.first * factor2.first, + factor1.second && factor2.second); +} + +template<typename ValueType1, typename ValueType2> +DoubleValue operator*(std::pair<ValueType1, bool> factor1, + std::pair<ValueType2, bool> factor2) +{ + return std::pair<double, bool>(factor1.first * factor2.first, + factor1.second && factor2.second); +} + +// Division functions. Ensure that if floating point value is used, +// result ends up as floating point too. + +LongValue operator/(LongValue dividend, LongValue divisor) +{ + if (dividend.first == 0) return LongValue( + 0, dividend.second && divisor.second); + if (divisor.first == 0) return LongValue( + std::numeric_limits<int64_t>().max(), + dividend.second && divisor.second); + return LongValue(dividend.first / divisor.first, + dividend.second && divisor.second); +} + +template<typename ValueType1, typename ValueType2> +DoubleValue operator/(std::pair<ValueType1, bool> dividend, + std::pair<ValueType2, bool> divisor) +{ + // In case divisor is integer, we will core if we attempt to divide + // with it. + if (dividend.first == 0) return DoubleValue( + 0, dividend.second && divisor.second); + if (divisor.first == 0) return DoubleValue( + std::numeric_limits<double>().infinity(), + dividend.second && divisor.second); + return DoubleValue( + dividend.first / static_cast<double>(divisor.first), + dividend.second && divisor.second); +} + +// Min/Max functions +template<typename ValueType> +std::pair<ValueType, bool> getMin(std::pair<ValueType, bool> val1, + std::pair<ValueType, bool> val2) +{ + if (!val1.second) return val2; + if (!val2.second) return val1; + return std::pair<ValueType, bool>(std::min(val1.first, val2.first), true); +} + +template<typename ValueType> +std::pair<ValueType, bool> getMax(std::pair<ValueType, bool> val1, + std::pair<ValueType, bool> val2) +{ + if (!val1.second) return val2; + if (!val2.second) return val1; + return std::pair<ValueType, bool>(std::max(val1.first, val2.first), true); +} + +// Wrapper types for primitives. Would be nice to allow primitives directly, but +// using a wrapper we can get by with less operator overloads above. + +template<typename ValueType> +struct VW : public std::pair<ValueType, bool> { + VW(ValueType val) : std::pair<ValueType, bool>(val, true) {} +}; + +typedef VW<int64_t> LVW; +typedef VW<double> DVW; + + +/** Get metric with given name from source. Set bool true if existing. */ +LongValue getLongMetric(const std::string& name, MetricSource& source) +{ + std::string::size_type pos = name.rfind('.'); + const Metric* metric = (pos == std::string::npos + ? 0 : source.getMetric(name.substr(0, pos))); + try{ + return LongValue(metric == 0 + ? 0 : metric->getLongValue(name.substr(pos+1)), metric != 0); + } catch (vespalib::IllegalArgumentException& e) { + return LongValue(0, false); + } +} + +/** Get metric with given name from source. Set bool true if existing. */ +DoubleValue getDoubleMetric(const std::string& name, MetricSource& source) +{ + std::string::size_type pos = name.rfind('.'); + const Metric* metric = (pos == std::string::npos + ? 0 : source.getMetric(name.substr(0, pos))); + try{ + return DoubleValue(metric == 0 + ? 0.0 : metric->getDoubleValue(name.substr(pos+1)), metric != 0); + } catch (vespalib::IllegalArgumentException& e) { + return DoubleValue(0, false); + } +} + +std::string getValueString(LongValue value, const char* format = "%'lld") +{ + if (!value.second) return "na"; + std::vector<char> buffer(30); + snprintf(&buffer[0], 30, format, value.first); + return std::string(&buffer[0]); +} + +std::string getValueString(DoubleValue value, const char* format = "%'f") +{ + if (!value.second) return "na"; + std::vector<char> buffer(30); + snprintf(&buffer[0], 30, format, value.first); + return std::string(&buffer[0]); +} + +template<typename ValueType> +std::string getByteValueString(std::pair<ValueType, bool> val) +{ + static const int64_t k = (1ul << 10); + static const int64_t m = (1ul << 20); + static const int64_t g = (1ul << 30); + + if (!val.second) return "na"; + std::pair<int64_t, bool> value( + static_cast<int64_t>(val.first), val.second); + if (value.first < 64 * k) { + return getValueString(value, "%'llu B"); + } + if (value.first < 64 * m) { + value.first /= k; + return getValueString(value, "%'llu kB"); + } + if (value.first < 64 * g) { + value.first /= m; + return getValueString(value, "%'llu MB"); + } + value.first /= g; + return getValueString(value, "%'llu GB"); +} + +struct HttpTable { + std::string title; + std::string topLeftText; + std::vector<std::string> colNames; + std::vector<std::string> rowNames; + struct Cell { + bool set; + std::string value; + + Cell() : set(false), value() {} + + void operator=(const std::string& val) { value = val; set = true; } + }; + struct Row { + std::vector<Cell> cells; + Cell& operator[](uint32_t i) { + if (i >= cells.size()) cells.resize(i + 1); + return cells[i]; + } + }; + std::vector<Row> cells; + + HttpTable(const std::string& title_, const std::string& topLeftText_) + : title(title_), topLeftText(topLeftText_) {} + + Row& operator[](uint32_t i) { + if (i >= cells.size()) cells.resize(i + 1); + return cells[i]; + } + + void fillInEmptyHoles() { + if (rowNames.size() < cells.size()) rowNames.resize(cells.size()); + if (rowNames.size() > cells.size()) cells.resize(rowNames.size()); + for (uint32_t i=0; i<cells.size(); ++i) { + if (colNames.size() < cells[i].cells.size()) + colNames.resize(cells[i].cells.size()); + if (colNames.size() > cells[i].cells.size()) + cells[i].cells.resize(colNames.size()); + } + } + + void print(std::ostream& out) { + out << "<h3>" << title << "</h3>\n"; + out << "<table border=\"1\">\n"; + fillInEmptyHoles(); + for (uint32_t i=0; i<=rowNames.size(); ++i) { + if (i == 0) { + out << "<tr><th>" << topLeftText << "</th>"; + for (uint32_t j=0; j<colNames.size(); ++j) { + out << "<th>" << colNames[j] << "</th>"; + } + out << "</tr>\n"; + } else { + out << "<tr><td>" << rowNames[i - 1] << "</td>"; + for (uint32_t j=0; j<colNames.size(); ++j) { + out << "<td align=\"right\">" + << (cells[i - 1][j].set ? cells[i - 1][j].value : "-") + << "</td>"; + } + out << "</tr>\n"; + } + } + out << "</table>\n"; + } +}; + +} // printutils +} // metrics + diff --git a/metrics/src/vespa/metrics/state_api_adapter.cpp b/metrics/src/vespa/metrics/state_api_adapter.cpp new file mode 100644 index 00000000000..606f57a87fe --- /dev/null +++ b/metrics/src/vespa/metrics/state_api_adapter.cpp @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "state_api_adapter.h" +#include "jsonwriter.h" + +namespace metrics { + +vespalib::string +StateApiAdapter::getMetrics(const vespalib::string &consumer) +{ + metrics::MetricLockGuard guard(_manager.getMetricLock()); + std::vector<uint32_t> periods = _manager.getSnapshotPeriods(guard); + if (periods.empty()) { + return ""; // no configuration yet + } + const metrics::MetricSnapshot &snapshot( + _manager.getMetricSnapshot(guard, periods[0])); + vespalib::asciistream json; + vespalib::JsonStream stream(json); + metrics::JsonWriter metricJsonWriter(stream); + _manager.visit(guard, snapshot, metricJsonWriter, consumer); + stream.finalize(); + return json.str(); +} + +vespalib::string +StateApiAdapter::getTotalMetrics(const vespalib::string &consumer) +{ + _manager.updateMetrics(true); + metrics::MetricLockGuard guard(_manager.getMetricLock()); + _manager.checkMetricsAltered(guard); + time_t currentTime = time(0); + std::unique_ptr<metrics::MetricSnapshot> generated( + new metrics::MetricSnapshot( + "Total metrics from start until current time", 0, + _manager.getTotalMetricSnapshot(guard).getMetrics(), + true)); + _manager.getActiveMetrics(guard) + .addToSnapshot(*generated, false, currentTime); + generated->setFromTime( + _manager.getTotalMetricSnapshot(guard).getFromTime()); + const metrics::MetricSnapshot &snapshot = *generated; + vespalib::asciistream json; + vespalib::JsonStream stream(json); + metrics::JsonWriter metricJsonWriter(stream); + _manager.visit(guard, snapshot, metricJsonWriter, consumer); + stream.finalize(); + return json.str(); +} + +} // namespace metrics diff --git a/metrics/src/vespa/metrics/state_api_adapter.h b/metrics/src/vespa/metrics/state_api_adapter.h new file mode 100644 index 00000000000..dd041a3c37c --- /dev/null +++ b/metrics/src/vespa/metrics/state_api_adapter.h @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/net/metrics_producer.h> +#include "metricmanager.h" + +namespace metrics { + +/** + * This is an adapter class that implements the metrics producer + * interface defined by the state api implementation in vespalib by + * extracting metrics in json format from a metric manager. + **/ +class StateApiAdapter : public vespalib::MetricsProducer +{ +private: + MetricManager &_manager; + +public: + StateApiAdapter(MetricManager &manager) : _manager(manager) {} + + vespalib::string getMetrics(const vespalib::string &consumer) override; + vespalib::string getTotalMetrics(const vespalib::string &consumer) override; +}; + + +} // namespace metrics + diff --git a/metrics/src/vespa/metrics/summetric.h b/metrics/src/vespa/metrics/summetric.h new file mode 100644 index 00000000000..6f1c92e5b4f --- /dev/null +++ b/metrics/src/vespa/metrics/summetric.h @@ -0,0 +1,422 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class metrics::CounterMetric + * @ingroup metrics + * + * @brief Counts a value that only moves upwards. + * + * NB! If you have a MetricSet subclass you want to create a sum for, use + * MetricSet itself as the template argument. Otherwise you'll need to override + * clone(...) in order to make it return the correct type for your + * implementation. + */ + +#pragma once + +#include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/metrics/metric.h> +#include <vespa/metrics/metricset.h> + +namespace metrics { + +template<typename AddendMetric> +class SumMetric : public Metric +{ +public: + class StartValue + { + private: + std::vector<Metric::LP> _startValueChildren; + std::unique_ptr<AddendMetric> _startValue; + + public: + typedef std::shared_ptr<StartValue> SP; + StartValue(const AddendMetric &metric) + : _startValueChildren(), + _startValue(metric.clone(_startValueChildren, CLONE, 0, false)) {} + const AddendMetric &getStartValue() const { return *_startValue; } + }; + +private: + typename StartValue::SP _startValue; + std::vector<const AddendMetric*> _metricsToSum; + +public: + SumMetric(const String& name, const String& tags, + const String& description, MetricSet* owner = 0); + SumMetric(const SumMetric<AddendMetric>& other, + std::vector<Metric::LP>& ownerList, MetricSet* owner = 0); + + virtual Metric* clone( + std::vector<Metric::LP>&, CopyType, MetricSet* owner, + bool includeUnused = false) const; + + /** + * If you want to support sums of collections of metrics that may + * be empty, you must supply a start value for the sum operation + * by calling this function. + **/ + void setStartValue(const AddendMetric &metric) { _startValue.reset(new StartValue(metric)); } + typename StartValue::SP getStartValue() const { return _startValue; } + + void addMetricToSum(const AddendMetric&); + void removeMetricFromSum(const AddendMetric&); + + virtual bool logEvent(const String& fullName) const; + + virtual void print(std::ostream&, bool verbose, + const std::string& indent, uint64_t secondsPassed) const; + + virtual int64_t getLongValue(const stringref & id) const; + virtual double getDoubleValue(const stringref & id) const; + + virtual void reset() {} + + virtual bool visit(MetricVisitor& visitor, + bool tagAsAutoGenerated = false) const; + + virtual bool used() const; + + virtual void addMemoryUsage(MemoryConsumption&) const; + + virtual void printDebug(std::ostream&, const std::string& indent="") const; + + virtual void addToPart(metrics::Metric&) const; + virtual void addToSnapshot(metrics::Metric&, + std::vector<Metric::LP>&) const; + +private: + friend class MetricManagerTest; + std::pair<std::vector<Metric::LP>, Metric::LP> generateSum() const; + + virtual void addTo(metrics::Metric&, + std::vector<Metric::LP>* ownerList) const; + bool isAddendType(const Metric* m) const; +}; + +template<typename AddendMetric> +bool +SumMetric<AddendMetric>::visit(MetricVisitor& visitor, + bool tagAsAutoGenerated) const +{ + (void) tagAsAutoGenerated; + if (_metricsToSum.empty()) return true; + std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum()); + return sum.second->visit(visitor, true); +} + +template<typename AddendMetric> +SumMetric<AddendMetric>::SumMetric(const String& name, const String& tags, + const String& description, MetricSet* owner) + : Metric(name, tags, description, owner), + _startValue(), + _metricsToSum() +{ +} + +template<typename AddendMetric> +SumMetric<AddendMetric>::SumMetric(const SumMetric<AddendMetric>& other, + std::vector<Metric::LP>& ownerList, + MetricSet* owner) + : Metric(other, owner), + _startValue(other._startValue), + _metricsToSum() +{ + (void) ownerList; + if (other._owner == 0) { + throw vespalib::IllegalStateException( + "Cannot copy a sum metric not registered in a metric set, as " + "we need to use parent to detect new metrics to point to.", + VESPA_STRLOC); + } + if (owner == 0) { + throw vespalib::IllegalStateException( + "Cannot copy a sum metric directly. One needs to at least " + "include metric set above it in order to include metrics " + "summed.", VESPA_STRLOC); + } + std::vector<String> parentPath(other._owner->getPathVector()); + _metricsToSum.reserve(other._metricsToSum.size()); + for (typename std::vector<const AddendMetric*>::const_iterator it + = other._metricsToSum.begin(); + it != other._metricsToSum.end(); ++it) + { + std::vector<String> addendPath((**it).getPathVector()); + MetricSet* newAddendParent = owner; + for (uint32_t i=parentPath.size(), n=addendPath.size() - 1; i<n; ++i) { + Metric* child = newAddendParent->getMetric(addendPath[i]); + if (child == 0) { + throw vespalib::IllegalStateException( + "Metric " + addendPath[i] + " in metric set " + + newAddendParent->getPath() + " was expected to " + "exist. This sounds like a bug.", VESPA_STRLOC); + } + if (!child->isMetricSet()) { + throw vespalib::IllegalStateException( + "Metric " + addendPath[i] + " in metric set " + + newAddendParent->getPath() + " was expected to be a " + "metric set. This sounds like a bug.", VESPA_STRLOC); + } + newAddendParent = static_cast<MetricSet*>(child); + } + Metric* child = newAddendParent->getMetric( + addendPath[addendPath.size() - 1]); + if (child == 0) { + throw vespalib::IllegalStateException( + "Metric " + addendPath[addendPath.size() - 1] + " in " + "metric set " + newAddendParent->getPath() + " was " + "expected to exist. This sounds like a bug.", VESPA_STRLOC); + } + AddendMetric* am(dynamic_cast<AddendMetric*>(child)); + if (am == 0) { + throw vespalib::IllegalStateException( + "Metric " + child->getPath() + " is of wrong type for sum " + + other.getPath() + ". This sounds like a bug.", + VESPA_STRLOC); + } + _metricsToSum.push_back(am); + } +} + +template<typename AddendMetric> +Metric* +SumMetric<AddendMetric>::clone(std::vector<Metric::LP>& ownerList, + CopyType copyType, MetricSet* owner, + bool includeUnused) const +{ + (void) includeUnused; + if (_metricsToSum.empty() && _startValue.get() == 0) { + throw vespalib::IllegalStateException( + "Attempted to clone sum metric without any children or start value. " + "This is currently illegal, to avoid needing to be able to " + "construct a metric of appropriate type without having a " + "template. (Hard to know how to construct any metric.", + VESPA_STRLOC); + } + if (copyType == CLONE) { + return new SumMetric<AddendMetric>(*this, ownerList, owner); + } + // Else we're generating an inactive copy by evaluating sum + typename std::vector<const AddendMetric*>::const_iterator it( + _metricsToSum.begin()); + // Clone start value or first child and use as accumulator + // As the metric cloned will have wrong info, we have to wait to + // register it in parent until we have fixed that. + Metric *m = 0; + if (_startValue.get() != 0) { + m = _startValue->getStartValue().clone(ownerList, INACTIVE, 0, true); + } else { + m = (**it).clone(ownerList, INACTIVE, 0, true); + ++it; + } + m->setName(getName()); + m->setDescription(getDescription()); + m->setTags(getTags()); + if (owner != 0) owner->registerMetric(*m); + // Add the others to the metric cloned. + for (; it != _metricsToSum.end(); ++it) { + (**it).addToPart(*m); + } + return m; +} + +template<typename AddendMetric> +void +SumMetric<AddendMetric>::addToPart(metrics::Metric& m) const +{ + std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum()); + sum.second->addToPart(m); +} + +template<typename AddendMetric> +void +SumMetric<AddendMetric>::addToSnapshot( + metrics::Metric& m, std::vector<Metric::LP>& ownerList) const +{ + if (isAddendType(&m)) { + // If the type to add to is an addend metric, it is part of an inactive + // copy we need to add data to. + std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum()); + sum.second->addToSnapshot(m, ownerList); + } +} + +template<typename AddendMetric> +void +SumMetric<AddendMetric>::addTo(metrics::Metric& m, + std::vector<Metric::LP>* ownerList) const +{ + if (ownerList == 0) { + std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum()); + sum.second->addToPart(m); + } else { + if (isAddendType(&m)) { + // If the type to add to is an addend metric, it is part of an + // inactive copy we need to add data to. + std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum()); + sum.second->addToSnapshot(m, *ownerList); + } + } +} + +template<typename AddendMetric> +void +SumMetric<AddendMetric>::addMetricToSum(const AddendMetric& metric) +{ + if (_owner == 0) { + throw vespalib::IllegalStateException( + "Sum metric needs to be registered in a parent metric set " + "prior to adding metrics to sum.", VESPA_STRLOC); + } + std::vector<String> sumParentPath(_owner->getPathVector()); + std::vector<String> addedPath(metric.getPathVector()); + bool error = false; + if (addedPath.size() <= sumParentPath.size()) { + error = true; + } else for (uint32_t i=0; i<sumParentPath.size(); ++i) { + if (sumParentPath[i] != addedPath[i]) { + error = true; + break; + } + } + if (error) { + throw vespalib::IllegalStateException( + "Metric added to sum is required to be a child of the sum's " + "direct parent metric set. (Need not be a direct child) " + "Metric set " + metric.getPath() + " is not a child of " + + _owner->getPath(), VESPA_STRLOC); + } + std::vector<const AddendMetric*> metrics(_metricsToSum.size() + 1); + for (uint32_t i=0; i<_metricsToSum.size(); ++i) { + metrics[i] = _metricsToSum[i]; + } + metrics[metrics.size() - 1] = &metric; + _metricsToSum.swap(metrics); + // Ensure we don't use extra memory + assert(_metricsToSum.capacity() == _metricsToSum.size()); +} + +template<typename AddendMetric> +void +SumMetric<AddendMetric>::removeMetricFromSum(const AddendMetric &metric) +{ + _metricsToSum.erase(remove(_metricsToSum.begin(), _metricsToSum.end(), + &metric)); +} + +template<typename AddendMetric> +std::pair<std::vector<Metric::LP>, Metric::LP> +SumMetric<AddendMetric>::generateSum() const +{ + std::pair<std::vector<Metric::LP>, Metric::LP> retVal; + Metric* m = clone(retVal.first, INACTIVE, 0, true); + m->setRegistered(_owner); + retVal.second.reset(m); + return retVal; +} + +template<typename AddendMetric> +int64_t +SumMetric<AddendMetric>::getLongValue(const stringref & id) const +{ + std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum()); + if (sum.second.get() == 0) return 0; + return sum.second->getLongValue(id); +} + +template<typename AddendMetric> +double +SumMetric<AddendMetric>::getDoubleValue(const stringref & id) const +{ + std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum()); + if (sum.second.get() == 0) return 0.0; + return sum.second->getDoubleValue(id); +} + +template<typename AddendMetric> +bool +SumMetric<AddendMetric>::logEvent(const String & fullName) const +{ + std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum()); + if (sum.second.get() == 0) return false; + return sum.second->logEvent(fullName); +} + +template<typename AddendMetric> +void +SumMetric<AddendMetric>::print(std::ostream& out, bool verbose, + const std::string& indent, + uint64_t secondsPassed) const +{ + std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum()); + if (sum.second.get() == 0) return; + sum.second->print(out, verbose, indent, secondsPassed); +} + +template<typename AddendMetric> +bool +SumMetric<AddendMetric>::used() const +{ + for(typename std::vector<const AddendMetric*>::const_iterator it( + _metricsToSum.begin()); it != _metricsToSum.end(); ++it) + { + if ((**it).used()) return true; + } + return false; +} + +template<typename AddendMetric> +void +SumMetric<AddendMetric>::addMemoryUsage(MemoryConsumption& mc) const +{ + ++mc._sumMetricCount; + mc._sumMetricMeta += sizeof(SumMetric<AddendMetric>) - sizeof(Metric) + + _metricsToSum.capacity() * sizeof(Metric*); + Metric::addMemoryUsage(mc); +} + +template<typename AddendMetric> +void +SumMetric<AddendMetric>::printDebug(std::ostream& out, + const std::string& indent) const +{ + out << "sum "; + Metric::printDebug(out, indent); + out << " {"; + for(typename std::vector<const AddendMetric*>::const_iterator it( + _metricsToSum.begin()); it != _metricsToSum.end(); ++it) + { + out << "\n" << indent << " "; + (**it).printDebug(out, indent + " "); + } + out << "}"; +} + +template<typename AddendMetric> +bool +SumMetric<AddendMetric>::isAddendType(const Metric* m) const +{ + // If metric to addend it a metric set, we can only check if target also is + // a metric set, as other type will be lost when going inactive. Is there a + // way to do this without using an actual instance? + if (_metricsToSum.empty() && _startValue.get() == 0) { + throw vespalib::IllegalStateException( + "Attempted to verify addend type for sum metric without any " + "children or start value.", VESPA_STRLOC); + } + const Metric* wantedType; + if (!_metricsToSum.empty()) { + wantedType = _metricsToSum[0]; + } else { + wantedType = &_startValue->getStartValue(); + } + if (wantedType->isMetricSet()) { + return (m->isMetricSet()); + } else { + return (dynamic_cast<const AddendMetric*>(m) != 0); + } +} + +} // metrics + diff --git a/metrics/src/vespa/metrics/textwriter.cpp b/metrics/src/vespa/metrics/textwriter.cpp new file mode 100644 index 00000000000..0b5cb9a287b --- /dev/null +++ b/metrics/src/vespa/metrics/textwriter.cpp @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/metrics/textwriter.h> + +#include <vespa/metrics/countmetric.h> +#include <vespa/metrics/metricset.h> +#include <vespa/metrics/metricsnapshot.h> +#include <vespa/metrics/valuemetric.h> + +namespace metrics { + +TextWriter::TextWriter(std::ostream& out, uint32_t period, + const std::string& regex, bool verbose) + : _period(period), _out(out), _regex(regex), _verbose(verbose) +{ +} + +bool +TextWriter::visitSnapshot(const MetricSnapshot& snapshot) +{ + _out << "snapshot \"" << snapshot.getName() << "\" from " + << snapshot.getFromTime() << " to " << snapshot.getToTime() + << " period " << snapshot.getPeriod(); + return true; +} + +void +TextWriter::doneVisitingSnapshot(const MetricSnapshot&) +{ +} + +bool +TextWriter::visitMetricSet(const MetricSet& set, bool) +{ + _path.push_back(set.getMangledName()); + return true; +} +void +TextWriter::doneVisitingMetricSet(const MetricSet&) { + _path.pop_back(); +} + +bool +TextWriter::writeCommon(const Metric& metric) +{ + std::ostringstream path; + for (uint32_t i=0; i<_path.size(); ++i) { + path << _path[i] << "."; + } + std::string mypath(path.str()); + path << metric.getMangledName(); + if (_regex.match(path.str())) { + if (metric.used() || _verbose) { + _out << "\n" << mypath; + return true; + } + } + return false; +} + +bool +TextWriter::visitCountMetric(const AbstractCountMetric& m, bool) +{ + if (writeCommon(m)) { + if (_verbose || m.used()) { + MetricValueClass::UP values(m.getValues()); + _out << m.getMangledName() + << (m.sumOnAdd() ? " count=" : " value="); + values->output("count", _out); + } + } + return true; +} + +bool +TextWriter::visitValueMetric(const AbstractValueMetric& m, bool) +{ + if (writeCommon(m)) { + m.print(_out, _verbose, " ", _period); + } + return true; +} + +} // metrics diff --git a/metrics/src/vespa/metrics/textwriter.h b/metrics/src/vespa/metrics/textwriter.h new file mode 100644 index 00000000000..dadeb70a7fc --- /dev/null +++ b/metrics/src/vespa/metrics/textwriter.h @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/regexp.h> +#include <vespa/metrics/metric.h> + +namespace metrics { + +class TextWriter : public MetricVisitor { + uint32_t _period; + std::ostream& _out; + std::vector<std::string> _path; + vespalib::Regexp _regex; + bool _verbose; + +public: + TextWriter(std::ostream& out, uint32_t period, + const std::string& regex, bool verbose); + + virtual bool visitSnapshot(const MetricSnapshot&); + virtual void doneVisitingSnapshot(const MetricSnapshot&); + virtual bool visitMetricSet(const MetricSet& set, bool); + virtual void doneVisitingMetricSet(const MetricSet&); + virtual bool visitCountMetric(const AbstractCountMetric&, + bool autoGenerated); + virtual bool visitValueMetric(const AbstractValueMetric&, + bool autoGenerated); + +private: + bool writeCommon(const Metric& m); +}; + +} + diff --git a/metrics/src/vespa/metrics/valuemetric.cpp b/metrics/src/vespa/metrics/valuemetric.cpp new file mode 100644 index 00000000000..3da48885054 --- /dev/null +++ b/metrics/src/vespa/metrics/valuemetric.cpp @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/metrics/valuemetric.h> +#include <atomic> + +#include <vespa/log/log.h> + +LOG_SETUP(".metrics.metric.value"); + +namespace metrics { + +namespace { + +std::atomic<bool> hasWarned {false}; + +} + +void +AbstractValueMetric::logWarning(const char* msg) const +{ + LOG(warning, "%s", msg); +} + +void +AbstractValueMetric::sendLogEvent(Metric::String name, double value) const +{ + EV_VALUE(name.c_str(), value); +} + +void +AbstractValueMetric::logNonFiniteValueWarning() const +{ + if (hasWarned.exchange(true, std::memory_order_relaxed) == false) { + LOG(warning, + "Metric '%s' attempted updated with a value that is NaN or " + "Infinity; update ignored! No further warnings will be printed for " + "such updates on any metrics, but they can be observed with debug " + "logging enabled on component 'metrics.metric.value'.", + getPath().c_str()); + } else { + LOG(debug, + "Metric '%s' attempted updated with a value that is NaN/Infinity; " + "update ignored!", + getPath().c_str()); + } +} + +} // metrics diff --git a/metrics/src/vespa/metrics/valuemetric.h b/metrics/src/vespa/metrics/valuemetric.h new file mode 100644 index 00000000000..9c9d750a1a8 --- /dev/null +++ b/metrics/src/vespa/metrics/valuemetric.h @@ -0,0 +1,532 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class metrics::ValueMetric + * @ingroup metrics + * + * @brief Creates a metric measuring any value. + * + * A value metric have the following properties: + * - Logs the average as a value event. (It is not strictly increasing) + * - When summing average metrics together, the sum becomes the average of + * all values added to both. + */ + +#pragma once + +#include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/util/exceptions.h> +#include <sstream> +#include <type_traits> +#include <cmath> +#include <vespa/metrics/metric.h> +#include <vespa/metrics/metricvalueset.h> +#include <vespa/vespalib/util/jsonstream.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <atomic> + +namespace metrics { + +struct AbstractValueMetric : public Metric { + virtual bool visit(MetricVisitor& visitor, + bool tagAsAutoGenerated = false) const + { + return visitor.visitValueMetric(*this, tagAsAutoGenerated); + } + virtual MetricValueClass::UP getValues() const = 0; + virtual bool inUse(const MetricValueClass& v) const = 0; + virtual bool summedAverage() const = 0; + +protected: + AbstractValueMetric(const String& name, const String& tags, + const String& description, MetricSet* owner) + : Metric(name, tags, description, owner) {} + + AbstractValueMetric(const String& name, Tags dimensions, + const String& description, MetricSet* owner) + : Metric(name, std::move(dimensions), description, owner) {} + + AbstractValueMetric(const AbstractValueMetric& other, MetricSet* owner) + : Metric(other, owner) {} + + void logWarning(const char* msg) const; + void logNonFiniteValueWarning() const; + void sendLogEvent(Metric::String name, double value) const; +}; + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +class ValueMetric + : public AbstractValueMetric +{ + typedef Metric::String String; // Redefine so template finds it easy + + struct Values : MetricValueClass { + uint32_t _count; + AvgVal _min, _max, _last; + TotVal _total; + + struct AtomicImpl { + std::atomic<uint32_t> _count {0}; + std::atomic<AvgVal> _min {std::numeric_limits<AvgVal>::max()}; + std::atomic<AvgVal> _max {std::numeric_limits<AvgVal>::min()}; + std::atomic<AvgVal> _last {0}; + std::atomic<TotVal> _total {0}; + }; + + Values() + : _count(0), + _min(std::numeric_limits<AvgVal>::max()), + _max(std::numeric_limits<AvgVal>::min()), + _last(0), + _total(0) + { + // numeric_limits min() returns smallest positive number + // for signed floating point types. Haven't found a way to + // get minimum negative value, so using -1 * positive for those. + if (std::numeric_limits<AvgVal>::is_signed + && !std::numeric_limits<AvgVal>::is_integer) + { + _max = -1 * std::numeric_limits<AvgVal>::max(); + } + } + + void relaxedStoreInto(AtomicImpl& target) const noexcept { + target._count.store(_count, std::memory_order_relaxed); + target._min.store(_min, std::memory_order_relaxed); + target._max.store(_max, std::memory_order_relaxed); + target._last.store(_last, std::memory_order_relaxed); + target._total.store(_total, std::memory_order_relaxed); + } + + void relaxedLoadFrom(const AtomicImpl& source) noexcept { + _count = source._count.load(std::memory_order_relaxed); + _min = source._min.load(std::memory_order_relaxed); + _max = source._max.load(std::memory_order_relaxed); + _last = source._last.load(std::memory_order_relaxed); + _total = source._total.load(std::memory_order_relaxed); + } + + template<typename T> + T getValue(const stringref & id) const { + if (id == "last") return static_cast<T>(_last); + if (id == "count") return static_cast<T>(_count); + if (id == "total") return static_cast<T>(_total); + if (id == "min") return static_cast<T>(_count > 0 ? _min : 0); + if (id == "max") return static_cast<T>( _count > 0 ? _max : 0); + throw vespalib::IllegalArgumentException( + "No value " + vespalib::string(id) + " in value metric.", VESPA_STRLOC); + } + + double getDoubleValue(const stringref & id) const { + return getValue<double>(id); + } + uint64_t getLongValue(const stringref & id) const { + return getValue<uint64_t>(id); + } + void output(const std::string& id, std::ostream& out) const { + if (id == "last") { out << _last; return; } + if (id == "count") { out << _count; return; } + if (id == "total") { out << _total; return; } + if (id == "min") { out << (_count > 0 ? _min : 0); return; } + if (id == "max") { out << (_count > 0 ? _max : 0); return; } + throw vespalib::IllegalArgumentException( + "No value " + id + " in value metric.", VESPA_STRLOC); + } + void output(const std::string& id, vespalib::JsonStream& stream) const { + if (id == "last") { stream << _last; return; } + if (id == "count") { stream << _count; return; } + if (id == "total") { stream << _total; return; } + if (id == "min") { stream << (_count > 0 ? _min : 0); return; } + if (id == "max") { stream << (_count > 0 ? _max : 0); return; } + throw vespalib::IllegalArgumentException( + "No value " + id + " in value metric.", VESPA_STRLOC); + } + friend std::ostream & operator << (std::ostream & os, const Values & v) { + os << "count=" << v._count; + os << ", total=" << v._total; + return os; + } + }; + + MetricValueSet<Values> _values; + + enum Flag { SUMMED_AVERAGE = 2, UNSET_ON_ZERO_VALUE = 4, LOG_IF_UNSET = 8 }; + + virtual bool summedAverage() const + { return _values.hasFlag(SUMMED_AVERAGE); } + bool unsetOnZeroValue() const + { return _values.hasFlag(UNSET_ON_ZERO_VALUE); } + bool logIfUnset() const { return _values.hasFlag(LOG_IF_UNSET); } + + void add(const Values& values, bool sumOnAdd); + void dec(const Values& values); + + void addValueWithCount(AvgVal avg, TotVal tot, uint32_t count); + + // Finite number (not infinity/NaN) check using type trait tag dispatch. + // 2nd param is instance of std::true_type iff AvgVal is floating point. + bool checkFinite(AvgVal v, std::true_type) { + if (!std::isfinite(v)) { + logNonFiniteValueWarning(); + return false; + } + return true; + } + bool checkFinite(AvgVal, std::false_type) { return true; } + +public: + ValueMetric(const String& name, const String& tags, + const String& description, MetricSet* owner = 0); + + ValueMetric(const ValueMetric<AvgVal, TotVal, SumOnAdd>&, + CopyType, MetricSet* owner); + + ValueMetric(const String& name, Tags dimensions, + const String& description, MetricSet* owner = 0); + + virtual MetricValueClass::UP getValues() const + { return MetricValueClass::UP(new Values(_values.getValues())); } + + void unsetOnZeroValue() { _values.setFlag(UNSET_ON_ZERO_VALUE); } + void logOnlyIfSet() { _values.removeFlag(LOG_IF_UNSET); } + + virtual ValueMetric<AvgVal, TotVal, SumOnAdd>* clone( + std::vector<Metric::LP>&, CopyType type, MetricSet* owner, + bool /*includeUnused*/) const + { return new ValueMetric<AvgVal,TotVal,SumOnAdd>(*this, type, owner); } + + ValueMetric & operator+=(const ValueMetric &); + + friend ValueMetric operator+(const ValueMetric & a, const ValueMetric & b) { + ValueMetric t(a); t += b; return t; + } + + void addAvgValueWithCount(AvgVal avg, uint32_t count) + { if (count) { addValueWithCount(avg, avg * count, count); } } + void addTotalValueWithCount(TotVal tot, uint32_t count) + { if (count) { addValueWithCount(tot / count, tot, count); } } + void addValueBatch(AvgVal avg, uint32_t count) { + addAvgValueWithCount(avg, count); + } + virtual void addValue(AvgVal avg) { addAvgValueWithCount(avg, 1); } + virtual void set(AvgVal avg) { addValue(avg); } + virtual void inc(AvgVal val = 1); + virtual void dec(AvgVal val = 1); + + double getAverage() const; + AvgVal getMinimum() const { return _values.getValues()._min; } + AvgVal getMaximum() const { return _values.getValues()._max; } + AvgVal getCount() const { return _values.getValues()._count; } + AvgVal getLast() const { return _values.getValues()._last; } + + void reset() { _values.reset(); } + + virtual bool logEvent(const String& fullName) const; + + virtual void print(std::ostream&, bool verbose, + const std::string& indent, uint64_t secondsPassed) const; + + virtual int64_t getLongValue(const stringref & id) const; + virtual double getDoubleValue(const stringref & id) const; + + + virtual bool inUse(const MetricValueClass& v) const { + const Values& values(static_cast<const Values&>(v)); + return (values._total != 0 + || (values._count != 0 && !unsetOnZeroValue())); + } + virtual bool used() const { + return inUse(_values.getValues()); + } + + virtual void addMemoryUsage(MemoryConsumption&) const; + + virtual void printDebug(std::ostream&, const std::string& indent) const; + + virtual void addToPart(Metric&) const; + virtual void addToSnapshot(Metric&, std::vector<Metric::LP>&) const; +}; + +typedef ValueMetric<double, double, true> DoubleValueMetric; +typedef ValueMetric<double, double, false> DoubleAverageMetric; +typedef ValueMetric<int64_t, int64_t, true> LongValueMetric; +typedef ValueMetric<int64_t, int64_t, false> LongAverageMetric; + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +ValueMetric<AvgVal, TotVal, SumOnAdd>::ValueMetric( + const String& name, const String& tags, + const String& description, MetricSet* owner) + : AbstractValueMetric(name, tags, description, owner), + _values() +{ + _values.setFlag(LOG_IF_UNSET); +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +ValueMetric<AvgVal, TotVal, SumOnAdd>::ValueMetric( + const String& name, const Tags dimensions, + const String& description, MetricSet* owner) + : AbstractValueMetric(name, std::move(dimensions), description, owner), + _values() +{ + _values.setFlag(LOG_IF_UNSET); +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +ValueMetric<AvgVal, TotVal, SumOnAdd>::ValueMetric( + const ValueMetric<AvgVal, TotVal, SumOnAdd>& other, + CopyType copyType, MetricSet* owner) + : AbstractValueMetric(other, owner), + _values(other._values, copyType == CLONE ? other._values.size() : 1) +{ +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +void ValueMetric<AvgVal, TotVal, SumOnAdd>::inc(AvgVal incVal) +{ + if (!checkFinite(incVal, std::is_floating_point<AvgVal>())) { + return; + } + Values values; + do { + values = _values.getValues(); + AvgVal val = values._last + incVal; + ++values._count; + values._total += val; + if (val < values._min) values._min = val; + if (val > values._max) values._max = val; + values._last = val; + } while (!_values.setValues(values)); +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +void ValueMetric<AvgVal, TotVal, SumOnAdd>::dec(AvgVal decVal) +{ + if (!checkFinite(decVal, std::is_floating_point<AvgVal>())) { + return; + } + Values values; + do { + values = _values.getValues(); + AvgVal val = values._last - decVal; + ++values._count; + values._total += val; + if (val < values._min) values._min = val; + if (val > values._max) values._max = val; + values._last = val; + } while (!_values.setValues(values)); +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +void +ValueMetric<AvgVal, TotVal, SumOnAdd>::addToSnapshot( + Metric& other, std::vector<Metric::LP>&) const +{ + ValueMetric<AvgVal, TotVal, SumOnAdd>& o( + reinterpret_cast<ValueMetric<AvgVal, TotVal, SumOnAdd>&>(other)); + if (_values.getValues()._count == 0) return; // Don't add if not set + o.add(_values.getValues(), false); +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +void +ValueMetric<AvgVal, TotVal, SumOnAdd>::addToPart(Metric& other) const +{ + ValueMetric<AvgVal, TotVal, SumOnAdd>& o( + reinterpret_cast<ValueMetric<AvgVal, TotVal, SumOnAdd>&>( + other)); + o.add(_values.getValues(), SumOnAdd); +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +void +ValueMetric<AvgVal, TotVal, SumOnAdd>::add(const Values& values2, bool sumOnAdd) +{ + bool overflow; + Values values; + do { + values = _values.getValues(); + overflow = values._count > values._count + values2._count + || (values2._total >= 0 + ? values._total > values._total + values2._total + : values._total < values._total + values2._total); + if (values._count == 0) { + values = values2; + } else if (values2._count == 0) { + // Do nothing + } else if (sumOnAdd) { + double totalAverage + = static_cast<double>(values._total) / values._count + + static_cast<double>(values2._total) / values2._count; + values._count += values2._count; + values._total = static_cast<TotVal>(totalAverage * values._count); + values._last += values2._last; + _values.setFlag(SUMMED_AVERAGE); + } else { + values._count += values2._count; + values._total += values2._total; + values._last = values2._last; + } + if (values._min > values2._min) values._min = values2._min; + if (values._max < values2._max) values._max = values2._max; + } while (!_values.setValues(values)); + if (overflow) { + std::ostringstream ost; + ost << "ValueMetric " << getPath() << " overflowed. Resetting it. Old value = ("; + ost << values << "), Incoming = (" << values2 << ")"; + logWarning(ost.str().c_str()); + _values.reset(); + } +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +void +ValueMetric<AvgVal, TotVal, SumOnAdd>::dec(const Values& values2) +{ + bool underflow; + Values values; + do { + values = _values.getValues(); + underflow = values._count < values._count - values2._count + || values._total < values._total - values2._total; + values._count -= values2._count; + values._total -= values2._total; + } while (!_values.setValues(values)); + if (underflow) { + _values.reset(); + std::ostringstream ost; + ost << "ValueMetric " << getPath() << " underflowed. Resetting it."; + logWarning(ost.str().c_str()); + } +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +void ValueMetric<AvgVal, TotVal, SumOnAdd>::addValueWithCount( + AvgVal avg, TotVal tot, uint32_t count) +{ + if (!checkFinite(avg, std::is_floating_point<AvgVal>())) { + return; + } + Values values; + do { + values = _values.getValues(); + values._count += count; + values._total += tot; + if (avg < values._min) values._min = avg; + if (avg > values._max) values._max = avg; + values._last = avg; + } while (!_values.setValues(values)); +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +ValueMetric<AvgVal, TotVal, SumOnAdd>& +ValueMetric<AvgVal, TotVal, SumOnAdd>::operator+=( + const ValueMetric<AvgVal, TotVal, SumOnAdd>& other) +{ + add(other._values.getValues(), SumOnAdd); + return *this; +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +double +ValueMetric<AvgVal, TotVal, SumOnAdd>::getAverage() const +{ + Values values(_values.getValues()); + if (values._count == 0) return 0; + return static_cast<double>(values._total) / values._count; +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +bool +ValueMetric<AvgVal, TotVal, SumOnAdd>::logEvent(const String& fullName) const +{ + Values values(_values.getValues()); + if (!logIfUnset() && !inUse(values)) return false; + sendLogEvent(fullName, SumOnAdd + ? static_cast<double>(values._last) + : static_cast<double>(values._total) / values._count); + return true; +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +void +ValueMetric<AvgVal, TotVal, SumOnAdd>::print( + std::ostream& out, bool verbose, const std::string& indent, + uint64_t secondsPassed) const +{ + (void) indent; + (void) secondsPassed; + Values values(_values.getValues()); + if (!inUse(values) && !verbose) return; + out << this->_name << " average=" << (values._count == 0 + ? 0 : static_cast<double>(values._total) / values._count) + << " last=" << values._last; + if (!summedAverage()) { + if (values._count > 0) { + out << " min=" << values._min << " max=" << values._max; + } + out << " count=" << values._count << " total=" << values._total; + } +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +int64_t +ValueMetric<AvgVal, TotVal, SumOnAdd>::getLongValue(const stringref & id) const +{ + Values values(_values.getValues()); + if (id == "last" || (SumOnAdd && id == "value")) + return static_cast<int64_t>(values._last); + if (id == "average" || (!SumOnAdd && id == "value")) + return static_cast<int64_t>(getAverage()); + if (id == "count") return static_cast<int64_t>(values._count); + if (id == "total") return static_cast<int64_t>(values._total); + if (id == "min") return static_cast<int64_t>( + values._count > 0 ? values._min : 0); + if (id == "max") return static_cast<int64_t>( + values._count > 0 ? values._max : 0); + throw vespalib::IllegalArgumentException( + "No value " + id + " in average metric.", VESPA_STRLOC); +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +double +ValueMetric<AvgVal, TotVal, SumOnAdd>::getDoubleValue(const stringref & id) const +{ + Values values(_values.getValues()); + if (id == "last" || (SumOnAdd && id == "value")) + return static_cast<double>(values._last); + if (id == "average" || (!SumOnAdd && id == "value")) + return getAverage(); + if (id == "count") return static_cast<double>(values._count); + if (id == "total") return static_cast<double>(values._total); + if (id == "min") return static_cast<double>( + values._count > 0 ? values._min : 0); + if (id == "max") return static_cast<double>( + values._count > 0 ? values._max : 0); + throw vespalib::IllegalArgumentException( + "No value " + vespalib::string(id) + " in average metric.", VESPA_STRLOC); +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +void +ValueMetric<AvgVal, TotVal, SumOnAdd>::addMemoryUsage( + MemoryConsumption& mc) const +{ + ++mc._valueMetricCount; + mc._valueMetricValues += _values.getMemoryUsageAllocatedInternally(); + mc._valueMetricMeta += sizeof(ValueMetric<AvgVal, TotVal, SumOnAdd>) + - sizeof(Metric); + Metric::addMemoryUsage(mc); +} + +template<typename AvgVal, typename TotVal, bool SumOnAdd> +void +ValueMetric<AvgVal, TotVal, SumOnAdd>::printDebug( + std::ostream& out, const std::string& indent) const +{ + Values values(_values.getValues()); + out << "value=" << values._last << " "; + Metric::printDebug(out, indent); +} + +} // metrics + diff --git a/metrics/src/vespa/metrics/xmlwriter.cpp b/metrics/src/vespa/metrics/xmlwriter.cpp new file mode 100644 index 00000000000..01c509030ff --- /dev/null +++ b/metrics/src/vespa/metrics/xmlwriter.cpp @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/metrics/xmlwriter.h> + +#include <vespa/metrics/countmetric.h> +#include <vespa/metrics/metricset.h> +#include <vespa/metrics/metricsnapshot.h> +#include <vespa/metrics/valuemetric.h> + +namespace metrics { + +XmlWriter::XmlWriter(vespalib::xml::XmlOutputStream& xos, + uint32_t period, int verbosity) + : _period(period), _xos(xos), _verbosity(verbosity) {} + +bool +XmlWriter::visitSnapshot(const MetricSnapshot& snapshot) +{ + using namespace vespalib::xml; + _xos << XmlTag("snapshot") << XmlAttribute("name", snapshot.getName()) + << XmlAttribute("from", snapshot.getFromTime()) + << XmlAttribute("to", snapshot.getToTime()) + << XmlAttribute("period", snapshot.getPeriod()); + return true; +} + +void +XmlWriter::doneVisitingSnapshot(const MetricSnapshot&) +{ + using namespace vespalib::xml; + _xos << XmlEndTag(); +} + +bool +XmlWriter::visitMetricSet(const MetricSet& set, bool) +{ + using namespace vespalib::xml; + if (set.used() || _verbosity >= 2) { + _xos << XmlTag(set.getName(), CONVERT_ILLEGAL_CHARACTERS); + printCommonXmlParts(set); + return true; + } + return false; +} +void +XmlWriter::doneVisitingMetricSet(const MetricSet&) { + using namespace vespalib::xml; + _xos << XmlEndTag(); +} + +bool +XmlWriter::visitCountMetric(const AbstractCountMetric& metric, bool) +{ + MetricValueClass::UP values(metric.getValues()); + if (!metric.inUse(*values) && _verbosity < 2) return true; + using namespace vespalib::xml; + std::ostringstream ost; + _xos << XmlTag(metric.getName(), CONVERT_ILLEGAL_CHARACTERS) + << XmlAttribute(metric.sumOnAdd() + ? "count" : "value", values->toString("count")); + printCommonXmlParts(metric); + _xos << XmlEndTag(); + return true; +} + +bool +XmlWriter::visitValueMetric(const AbstractValueMetric& metric, bool) +{ + MetricValueClass::UP values(metric.getValues()); + if (!metric.inUse(*values) && _verbosity < 2) return true; + using namespace vespalib::xml; + _xos << XmlTag(metric.getName(), CONVERT_ILLEGAL_CHARACTERS) + << XmlAttribute("average", values->getLongValue("count") == 0 + ? 0 : values->getDoubleValue("total") + / values->getDoubleValue("count")) + << XmlAttribute("last", values->toString("last")); + if (!metric.summedAverage()) { + if (values->getLongValue("count") > 0) { + _xos << XmlAttribute("min", values->toString("min")) + << XmlAttribute("max", values->toString("max")); + } + _xos << XmlAttribute("count", values->getLongValue("count")); + if (_verbosity >= 2) { + _xos << XmlAttribute("total", values->toString("total")); + } + } + printCommonXmlParts(metric); + _xos << XmlEndTag(); + return true; +} + +void +XmlWriter::printCommonXmlParts(const Metric& metric) const +{ + using namespace vespalib::xml; + const Metric::Tags& tags(metric.getTags()); + if (_verbosity >= 3 && tags.size() > 0) { + std::ostringstream ost; + // XXX print tag values as well + ost << tags[0].key; + for (uint32_t i=1; i<tags.size(); ++i) { + ost << "," << tags[i].key; + } + _xos << XmlAttribute("tags", ost.str()); + } + if (_verbosity >= 1 && !metric.getDescription().empty()) { + _xos << XmlAttribute("description", metric.getDescription()); + } +} + +} // metrics diff --git a/metrics/src/vespa/metrics/xmlwriter.h b/metrics/src/vespa/metrics/xmlwriter.h new file mode 100644 index 00000000000..c55ed82f093 --- /dev/null +++ b/metrics/src/vespa/metrics/xmlwriter.h @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/metrics/metric.h> +#include <vespa/vespalib/util/xmlserializable.h> + +namespace metrics { + +class XmlWriter : public MetricVisitor { + uint32_t _period; + vespalib::xml::XmlOutputStream& _xos; + int _verbosity; + +public: + XmlWriter(vespalib::xml::XmlOutputStream& xos, + uint32_t period, int verbosity); + + virtual bool visitSnapshot(const MetricSnapshot&); + virtual void doneVisitingSnapshot(const MetricSnapshot&); + virtual bool visitMetricSet(const MetricSet& set, bool); + virtual void doneVisitingMetricSet(const MetricSet&); + virtual bool visitCountMetric(const AbstractCountMetric&, + bool autoGenerated); + virtual bool visitValueMetric(const AbstractValueMetric&, + bool autoGenerated); + +private: + void printCommonXmlParts(const Metric& metric) const; +}; + +} + |