summaryrefslogtreecommitdiffstats
path: root/metrics
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
committerJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
commit72231250ed81e10d66bfe70701e64fa5fe50f712 (patch)
tree2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /metrics
Publish
Diffstat (limited to 'metrics')
-rw-r--r--metrics/.gitignore5
-rw-r--r--metrics/CMakeLists.txt16
-rw-r--r--metrics/OWNERS1
-rw-r--r--metrics/pom.xml155
-rw-r--r--metrics/src/.gitignore4
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/ConsumerSpec.java32
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/CountMetric.java222
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/DoubleValue.java128
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/EventLogger.java7
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/JoinBehavior.java11
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/LongValue.java143
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/Metric.java256
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/MetricManager.java704
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/MetricSet.java239
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/MetricSnapshot.java119
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/MetricSnapshotSet.java116
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java32
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/SimpleMetricSet.java25
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/SumMetric.java238
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/Timer.java10
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/ValueMetric.java261
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/VespaLogEventLogger.java17
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/util/HasCopy.java7
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/util/MetricValueKeeper.java21
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/util/MetricValueSet.java141
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/util/SimpleMetricValueKeeper.java38
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/util/ThreadLocalDirectoryValueKeeper.java76
-rw-r--r--metrics/src/main/java/com/yahoo/metrics/util/ValueType.java11
-rw-r--r--metrics/src/main/metrics-with-dependencies.xml19
-rw-r--r--metrics/src/test/java/com/yahoo/metrics/AveragedDoubleValueMetric.java18
-rw-r--r--metrics/src/test/java/com/yahoo/metrics/AveragedLongValueMetric.java18
-rw-r--r--metrics/src/test/java/com/yahoo/metrics/CountMetricTest.java43
-rw-r--r--metrics/src/test/java/com/yahoo/metrics/DummyTimer.java11
-rw-r--r--metrics/src/test/java/com/yahoo/metrics/MetricManagerTest.java192
-rw-r--r--metrics/src/test/java/com/yahoo/metrics/MetricSetTest.java338
-rw-r--r--metrics/src/test/java/com/yahoo/metrics/SumMetricTest.java153
-rw-r--r--metrics/src/test/java/com/yahoo/metrics/SummedDoubleValueMetric.java16
-rw-r--r--metrics/src/test/java/com/yahoo/metrics/SummedLongValueMetric.java17
-rw-r--r--metrics/src/test/java/com/yahoo/metrics/ValueMetricTest.java182
-rw-r--r--metrics/src/tests/.gitignore7
-rw-r--r--metrics/src/tests/CMakeLists.txt18
-rw-r--r--metrics/src/tests/countmetrictest.cpp54
-rw-r--r--metrics/src/tests/loadmetrictest.cpp138
-rw-r--r--metrics/src/tests/metricmanagertest.cpp1280
-rw-r--r--metrics/src/tests/metricsettest.cpp157
-rw-r--r--metrics/src/tests/metrictest.cpp112
-rw-r--r--metrics/src/tests/snapshottest.cpp354
-rw-r--r--metrics/src/tests/stresstest.cpp144
-rw-r--r--metrics/src/tests/summetrictest.cpp150
-rw-r--r--metrics/src/tests/testrunner.cpp15
-rw-r--r--metrics/src/tests/valuemetrictest.cpp281
-rw-r--r--metrics/src/vespa/metrics/.gitignore10
-rw-r--r--metrics/src/vespa/metrics/CMakeLists.txt19
-rw-r--r--metrics/src/vespa/metrics/countmetric.cpp25
-rw-r--r--metrics/src/vespa/metrics/countmetric.h357
-rw-r--r--metrics/src/vespa/metrics/jsonwriter.cpp179
-rw-r--r--metrics/src/vespa/metrics/jsonwriter.h48
-rw-r--r--metrics/src/vespa/metrics/loadmetric.h167
-rw-r--r--metrics/src/vespa/metrics/memoryconsumption.h192
-rw-r--r--metrics/src/vespa/metrics/metric.cpp234
-rw-r--r--metrics/src/vespa/metrics/metric.h319
-rw-r--r--metrics/src/vespa/metrics/metricmanager.cpp968
-rw-r--r--metrics/src/vespa/metrics/metricmanager.h322
-rw-r--r--metrics/src/vespa/metrics/metrics.h17
-rw-r--r--metrics/src/vespa/metrics/metricset.cpp366
-rw-r--r--metrics/src/vespa/metrics/metricset.h123
-rw-r--r--metrics/src/vespa/metrics/metricsmanager.def64
-rw-r--r--metrics/src/vespa/metrics/metricsnapshot.cpp164
-rw-r--r--metrics/src/vespa/metrics/metricsnapshot.h116
-rw-r--r--metrics/src/vespa/metrics/metrictimer.cpp16
-rw-r--r--metrics/src/vespa/metrics/metrictimer.h44
-rw-r--r--metrics/src/vespa/metrics/metricvalueset.h182
-rw-r--r--metrics/src/vespa/metrics/namehash.h68
-rw-r--r--metrics/src/vespa/metrics/printutils.h418
-rw-r--r--metrics/src/vespa/metrics/state_api_adapter.cpp50
-rw-r--r--metrics/src/vespa/metrics/state_api_adapter.h29
-rw-r--r--metrics/src/vespa/metrics/summetric.h422
-rw-r--r--metrics/src/vespa/metrics/textwriter.cpp84
-rw-r--r--metrics/src/vespa/metrics/textwriter.h35
-rw-r--r--metrics/src/vespa/metrics/valuemetric.cpp49
-rw-r--r--metrics/src/vespa/metrics/valuemetric.h532
-rw-r--r--metrics/src/vespa/metrics/xmlwriter.cpp111
-rw-r--r--metrics/src/vespa/metrics/xmlwriter.h33
83 files changed, 12515 insertions, 0 deletions
diff --git a/metrics/.gitignore b/metrics/.gitignore
new file mode 100644
index 00000000000..cb87b051cf2
--- /dev/null
+++ b/metrics/.gitignore
@@ -0,0 +1,5 @@
+/libexec
+/target
+/pom.xml.build
+Makefile
+Testing
diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt
new file mode 100644
index 00000000000..03da6f2ee64
--- /dev/null
+++ b/metrics/CMakeLists.txt
@@ -0,0 +1,16 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_define_module(
+ DEPENDS
+ fastos
+ vespalog
+ vespalib
+ staging_vespalib
+ config_cloudconfig
+
+ TEST_EXTERNAL_DEPENDS
+ cppunit
+
+ TESTS
+ src/tests
+ src/vespa/metrics
+)
diff --git a/metrics/OWNERS b/metrics/OWNERS
new file mode 100644
index 00000000000..dbcff24b338
--- /dev/null
+++ b/metrics/OWNERS
@@ -0,0 +1 @@
+vekterli
diff --git a/metrics/pom.xml b/metrics/pom.xml
new file mode 100644
index 00000000000..0aa81dff64b
--- /dev/null
+++ b/metrics/pom.xml
@@ -0,0 +1,155 @@
+<?xml version="1.0"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>parent</artifactId>
+ <version>6-SNAPSHOT</version>
+ <relativePath>../parent/pom.xml</relativePath>
+ </parent>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>metrics</artifactId>
+ <packaging>jar</packaging>
+ <version>6-SNAPSHOT</version>
+ <name>metrics</name>
+ <description></description>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespajlib</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.4</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>jdisc_core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-di</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>container-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <compilerArgs>
+ <arg>-Xlint:rawtypes</arg>
+ <arg>-Xlint:unchecked</arg>
+ <arg>-Xlint:deprecation</arg>
+ </compilerArgs>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>com.yahoo.metrics.foo</mainClass>
+ </manifest>
+ </archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <!-- append to the packaging phase. -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>javacc-maven-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>javacc</id>
+ <goals>
+ <goal>javacc</goal>
+ </goals>
+ <configuration>
+ <lookAhead>1</lookAhead>
+ <isStatic>false</isStatic>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-class-plugin</artifactId>
+ <version>${project.version}</version>
+ <configuration>
+ <defFilesDirectories>
+ src/vespa/metrics,
+ src/main/resources/configdefinitions
+ </defFilesDirectories>
+ </configuration>
+ <executions>
+ <execution>
+ <id>config-gen</id>
+ <goals>
+ <goal>config-gen</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-install-plugin</artifactId>
+ <configuration>
+ <updateReleaseInfo>true</updateReleaseInfo>
+ </configuration>
+ </plugin>
+ <plugin>
+ <!-- Remove when v2.1 is the default
+ - it is required by maven-project-info-reports-plugin v2.2 -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>2.1</version>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/metrics/src/.gitignore b/metrics/src/.gitignore
new file mode 100644
index 00000000000..698739d7bbc
--- /dev/null
+++ b/metrics/src/.gitignore
@@ -0,0 +1,4 @@
+Makefile.ini
+config_command.sh
+project.dsw
+/metrics.mak
diff --git a/metrics/src/main/java/com/yahoo/metrics/ConsumerSpec.java b/metrics/src/main/java/com/yahoo/metrics/ConsumerSpec.java
new file mode 100644
index 00000000000..ec6963bef30
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/ConsumerSpec.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.StringTokenizer;
+
+/**
+ * Spec saved from config. If metricSetChildren has content, metric pointed
+ * to is a metric set.
+ */
+class ConsumerSpec {
+ Set<String> includedMetrics = new HashSet<String>();
+
+ public boolean contains(Metric m) {
+ return includedMetrics.contains(m.getPath());
+ }
+
+ public void register(String path) {
+ StringTokenizer tokenizer = new StringTokenizer(path, ".");
+
+ String total = "";
+
+ while (tokenizer.hasMoreTokens()) {
+ if (!total.isEmpty()) {
+ total += ".";
+ }
+ total += tokenizer.nextToken();
+ includedMetrics.add(total);
+ }
+ }
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/CountMetric.java b/metrics/src/main/java/com/yahoo/metrics/CountMetric.java
new file mode 100644
index 00000000000..409866e4040
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/CountMetric.java
@@ -0,0 +1,222 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import com.yahoo.metrics.util.MetricValueSet;
+import com.yahoo.metrics.util.ValueType;
+import com.yahoo.metrics.util.HasCopy;
+import com.yahoo.text.Utf8String;
+import com.yahoo.text.XMLWriter;
+
+import java.util.Collection;
+import java.util.Locale;
+import java.util.logging.Logger;
+
+/**
+ * A metric that counts something. The value should always be positive.
+ */
+@SuppressWarnings("unchecked")
+public class CountMetric extends Metric {
+
+ public static final int LOG_IF_UNSET = 2;
+
+ private static final Utf8String AVERAGE_CHANGE_PER_SECOND = new Utf8String("average_change_per_second");
+ private static final Utf8String COUNT = new Utf8String("count");
+ private static final Logger log = Logger.getLogger(CountMetric.class.getName());
+ private final MetricValueSet<CountValue> values;
+ private int flags;
+
+ public CountMetric(String name, String tags, String description, MetricSet owner) {
+ super(name, tags, description, owner);
+ values = new MetricValueSet<CountValue>();
+ flags = LOG_IF_UNSET;
+ }
+
+ public CountMetric(CountMetric other, CopyType copyType, MetricSet owner) {
+ super(other, owner);
+ values = new MetricValueSet<CountValue>(other.values, copyType == CopyType.CLONE ? other.values.size() : 1);
+ flags = other.flags;
+ }
+
+ private CountValue getValues() {
+ return values.getValue();
+ }
+
+ public long getValue() {
+ CountValue val = getValues();
+ return (val == null ? 0 : val.value);
+ }
+
+ @SuppressWarnings("UnusedDeclaration")
+ public void logOnlyIfSet() {
+ flags &= LOG_IF_UNSET;
+ }
+
+ public void set(long value) {
+ while (!values.setValue(new CountValue(value))) {
+ // try again
+ }
+ }
+
+ public void inc() {
+ inc(1);
+ }
+
+ public void dec() {
+ dec(1);
+ }
+
+ public void inc(long i) {
+ boolean overflow;
+ CountValue val;
+ do {
+ val = getValues();
+ if (val == null) {
+ val = new CountValue(0);
+ }
+ overflow = (val.value + i < val.value);
+ val.value += i;
+ } while (!values.setValue(val));
+
+ if (overflow) {
+ reset();
+ log.fine("Overflow in metric " + getName() + ". Resetting it.");
+ }
+ }
+
+ public void dec(long i) {
+ boolean underflow;
+ CountValue val;
+ do {
+ val = getValues();
+ if (val == null) {
+ val = new CountValue(0);
+ }
+ underflow = (val.value - i > val.value);
+ val.value -= i;
+ } while (!values.setValue(val));
+
+ if (underflow) {
+ reset();
+ log.fine("Underflow in metric " + getName() + ". Resetting it.");
+ }
+ }
+
+ @Override
+ public void reset() {
+ values.reset();
+ }
+
+ @Override
+ public boolean logFromTotalMetrics() {
+ return true;
+ }
+
+ @Override
+ public void logEvent(EventLogger logger, String fullName) {
+ CountValue val = getValues();
+
+ if ((flags & LOG_IF_UNSET) != 0 || val != null) {
+ logger.count(fullName, val == null ? 0 : val.value);
+ }
+ }
+
+ @Override
+ public void printXml(XMLWriter writer,
+ int secondsPassed,
+ int verbosity)
+ {
+ CountValue valRef = getValues();
+ if (valRef == null && verbosity < 2) {
+ return;
+ }
+ long val = valRef != null ? valRef.value : 0;
+ openXMLTag(writer, verbosity);
+ writer.attribute(COUNT, String.valueOf(val));
+
+ if (secondsPassed > 0) {
+ writer.attribute(AVERAGE_CHANGE_PER_SECOND,
+ String.format(Locale.US, "%.2f", (double)val / secondsPassed));
+ }
+
+ writer.closeTag();
+ }
+
+ // Only one metric in valuemetric, so return it on any id.
+ @Override
+ public long getLongValue(String id) {
+ CountValue val = getValues();
+ return (val == null ? 0 : val.value);
+ }
+
+ @Override
+ public double getDoubleValue(String id) {
+ CountValue val = getValues();
+ return (val == null ? 0 : val.value);
+ }
+
+ @Override
+ public boolean used() {
+ return getValues() != null;
+ }
+
+ @Override
+ public void addToSnapshot(Metric m) {
+ CountValue val = getValues();
+ if (val != null) {
+ ((CountMetric)m).inc(val.value);
+ }
+ }
+
+ @Override
+ public void addToPart(Metric m) {
+ CountValue val = getValues();
+ if (val != null) {
+ ((CountMetric)m).inc(val.value);
+ }
+ }
+
+ @Override
+ public Metric clone(CopyType type, MetricSet owner, boolean includeUnused) {
+ return new CountMetric(this, type, owner);
+ }
+
+ private static class CountValue implements ValueType, HasCopy<CountValue> {
+
+ long value;
+
+ private CountValue(long value) {
+ this.value = value;
+ }
+
+ public CountValue clone() {
+ try {
+ return (CountValue)super.clone();
+ } catch (CloneNotSupportedException e) {
+ return null;
+ }
+ }
+
+ public void add(ValueType other) {
+ value += ((CountValue)other).value;
+ }
+
+ public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior) {
+ CountValue result = new CountValue(0);
+ for (ValueType t : sources) {
+ result.add(t);
+ }
+ if (joinBehavior == JoinBehavior.AVERAGE_ON_JOIN) {
+ result.value /= sources.size();
+ }
+ return result;
+ }
+
+ public String toString() {
+ return Long.toString(value);
+ }
+
+ public CountValue copyObject() {
+ return new CountValue(value);
+ }
+ }
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/DoubleValue.java b/metrics/src/main/java/com/yahoo/metrics/DoubleValue.java
new file mode 100644
index 00000000000..6309e4a9ea2
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/DoubleValue.java
@@ -0,0 +1,128 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import com.yahoo.metrics.util.ValueType;
+import com.yahoo.metrics.util.HasCopy;
+
+import java.util.Collection;
+import java.util.Locale;
+
+public class DoubleValue implements ValueMetric.Value<Double> {
+ private int count;
+ private double min, max, last;
+ private double total;
+
+ public DoubleValue() {
+ count = 0;
+ min = Double.POSITIVE_INFINITY;
+ max = Double.NEGATIVE_INFINITY;
+ last = 0;
+ total = 0;
+ }
+
+ public String toString() {
+ return "(count " + count + ", min " + min + ", max " + max + ", last " + last + ", total " + total + ")";
+ }
+
+ public void add(Double v) {
+ count = count + 1;
+ total = total + v;
+ min = Math.min(min, v);
+ max = Math.max(max, v);
+ last = v;
+ }
+
+ public void join(ValueMetric.Value<Double> v2, boolean createAverageOnJoin) {
+ //StringBuffer sb = new StringBuffer();
+ //sb.append("Adding " + this + " to " + v2);
+ if (createAverageOnJoin) {
+ count += v2.getCount();
+ total += v2.getTotal();
+ last = v2.getLast();
+
+ } else {
+ double totalAverage = getAverage() + v2.getAverage();
+ count += v2.getCount();
+ total = totalAverage * count;
+ last += v2.getLast();
+ }
+ min = Math.min(min, v2.getMin());
+ max = Math.max(max, v2.getMax());
+ //sb.append(" and got " + this);
+ //System.err.println(sb.toString());
+ }
+
+ public void add(ValueType other) {
+ DoubleValue dv = (DoubleValue) other;
+ count = count + dv.count;
+ total = total + dv.total;
+ min = Math.min(min, dv.min);
+ max = Math.max(max, dv.max);
+ last = dv.last;
+ }
+
+ public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior) {
+ DoubleValue result = new DoubleValue();
+ for (ValueType t : sources) {
+ DoubleValue dv = (DoubleValue) t;
+ result.count = result.count + dv.count;
+ result.total = result.total + dv.total;
+ result.min = Math.min(result.min, dv.min);
+ result.max = Math.max(result.max, dv.max);
+ result.last += dv.last;
+ }
+ if (joinBehavior == JoinBehavior.AVERAGE_ON_JOIN) {
+ result.last /= sources.size();
+ } else {
+ result.total *= sources.size();
+ }
+ return result;
+ }
+
+ public boolean overflow(ValueMetric.Value<Double> v2) {
+ if (count > (count + v2.getCount())) {
+ return true;
+ }
+ if (v2.getTotal() > 0 && getTotal() > getTotal() + v2.getTotal()) {
+ return true;
+ }
+ if (v2.getTotal() < 0 && getTotal() < getTotal() + v2.getTotal()) {
+ return true;
+ }
+
+ return false;
+ }
+
+ public int getCount() { return count; }
+ public Double getMin() { return (count > 0) ? min : 0; }
+ public Double getMax() { return (count > 0) ? max : 0; }
+ public Double getLast() { return last; }
+ public Double getTotal() { return total; }
+
+ public Double getAverage() {
+ if (count == 0) {
+ return 0.0;
+ }
+
+ return total / count;
+ }
+
+ public String valueToString(Double val) {
+ if (val == Double.MIN_VALUE || val == Double.MAX_VALUE) {
+ return "0.00";
+ }
+
+ return String.format(Locale.US, "%.2f", val);
+ }
+
+ public DoubleValue clone() {
+ try{
+ return (DoubleValue) super.clone();
+ } catch (CloneNotSupportedException e) { return null; }
+ }
+
+ public ValueMetric.Value<Double> copyObject() {
+ return clone();
+ }
+
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/EventLogger.java b/metrics/src/main/java/com/yahoo/metrics/EventLogger.java
new file mode 100644
index 00000000000..15ebb2ed0ef
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/EventLogger.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+public interface EventLogger {
+ public void value(String name, double value);
+ public void count(String name, long value);
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/JoinBehavior.java b/metrics/src/main/java/com/yahoo/metrics/JoinBehavior.java
new file mode 100644
index 00000000000..eb8f5a540aa
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/JoinBehavior.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+/**
+ * When joining multiple metrics as a result of dimension
+ * removal. Should the result be an average or a sum? As an example,
+ * a latency metric should likely be averaged, while a number of
+ * pending metric should likely be summed. This join behavior property
+ * lets the metric framework know how to remove dimensions.
+ **/
+public enum JoinBehavior { AVERAGE_ON_JOIN, SUM_ON_JOIN }
diff --git a/metrics/src/main/java/com/yahoo/metrics/LongValue.java b/metrics/src/main/java/com/yahoo/metrics/LongValue.java
new file mode 100644
index 00000000000..29a8202f845
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/LongValue.java
@@ -0,0 +1,143 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import com.yahoo.metrics.util.ValueType;
+import com.yahoo.metrics.util.HasCopy;
+
+import java.util.Collection;
+
+/**
+ * @author thomasg
+ */
+public class LongValue
+ implements ValueMetric.Value<Long>
+{
+ private int count;
+ private long min, max, last;
+ private long total;
+
+ public LongValue() {
+ count = 0;
+ min = Long.MAX_VALUE;
+ max = Long.MIN_VALUE;
+ last = 0;
+ total = 0;
+ }
+
+ @Override
+ public void add(Long v) {
+ LongValue val = this;
+ val.count = count + 1;
+ val.total = total + v;
+ val.min = Math.min(min, v);
+ val.max = Math.max(max, v);
+ val.last = v;
+ }
+
+ @Override
+ public void join(ValueMetric.Value<Long> v2, boolean createAverageOnJoin) {
+ LongValue value = this;
+
+ if (createAverageOnJoin) {
+ value.count = count + v2.getCount();
+ value.total = total + v2.getTotal();
+ value.last = v2.getLast();
+ } else {
+ double totalAverage = getAverage() + v2.getAverage();
+ value.count = count + v2.getCount();
+ value.total = (long) (totalAverage * value.count); // Total is "wrong" I guess.
+ value.last = last + v2.getLast();
+ }
+
+ value.min = Math.min(min, v2.getMin());
+ value.max = Math.max(max, v2.getMax());
+ }
+
+ @Override
+ public void add(ValueType other) {
+ LongValue dv = (LongValue) other;
+ count = count + dv.count;
+ total = total + dv.total;
+ min = Math.min(min, dv.min);
+ max = Math.max(max, dv.max);
+ last = dv.last;
+ }
+
+ @Override
+ public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior) {
+ LongValue result = new LongValue();
+ for (ValueType t : sources) {
+ LongValue dv = (LongValue) t;
+ result.count = result.count + dv.count;
+ result.total = result.total + dv.total;
+ result.min = Math.min(result.min, dv.min);
+ result.max = Math.max(result.max, dv.max);
+ result.last += dv.last;
+ }
+ if (joinBehavior == JoinBehavior.AVERAGE_ON_JOIN) {
+ result.last /= sources.size();
+ } else {
+ result.total *= sources.size();
+ }
+ return result;
+ }
+
+ @Override
+ public boolean overflow(ValueMetric.Value<Long> v2) {
+ if (count > (count + v2.getCount())) {
+ return true;
+ }
+ if (v2.getTotal() > 0 && getTotal() > getTotal() + v2.getTotal()) {
+ return true;
+ }
+ if (v2.getTotal() < 0 && getTotal() < getTotal() + v2.getTotal()) {
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public int getCount() { return count; }
+
+ @Override
+ public Long getMin() { return (count > 0) ? min : 0; }
+
+ @Override
+ public Long getMax() { return (count > 0) ? max : 0; }
+
+ @Override
+ public Long getLast() { return last; }
+
+ @Override
+ public Long getTotal() { return total; }
+
+ @Override
+ public Double getAverage() {
+ if (count == 0) {
+ return 0.0;
+ }
+ return ((double) total) / count;
+ }
+
+ @Override
+ public String valueToString(Long val) {
+ if (val == Long.MIN_VALUE || val == Long.MAX_VALUE) {
+ return valueToString((long)0);
+ }
+
+ return val.toString();
+ }
+
+ @Override
+ public LongValue clone() {
+ try{
+ return (LongValue) super.clone();
+ } catch (CloneNotSupportedException e) { return null; }
+ }
+
+ @Override
+ public ValueMetric.Value<Long> copyObject() {
+ return clone();
+ }
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/Metric.java b/metrics/src/main/java/com/yahoo/metrics/Metric.java
new file mode 100644
index 00000000000..0ef7dfbcd0a
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/Metric.java
@@ -0,0 +1,256 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import com.yahoo.text.XMLWriter;
+import com.yahoo.text.Utf8String;
+
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+public abstract class Metric
+{
+ private String name;
+ private String tags;
+ private String description;
+
+ public String getXMLTag() {
+ return getName();
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public void setTags(String tags) {
+ this.tags = tags;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public String getTags() {
+ return tags;
+ }
+
+ MetricSet owner;
+
+ public Metric(String name, String tags, String description) {
+ this(name, tags, description, null);
+ }
+
+ public Metric(String name, String tags, String description, MetricSet owner) {
+ this.name = name;
+ this.tags = tags;
+ this.description = description;
+
+ if (owner != null) {
+ owner.registerMetric(this);
+ }
+ }
+
+ public Metric(Metric other, MetricSet owner) {
+ this(other.name, other.tags, other.description, owner);
+ }
+
+ public String getName() { return name; }
+
+ public String getPath() {
+ if (owner == null || owner.owner == null) {
+ return getName();
+ }
+
+ return owner.getPath() + "." + getName();
+ }
+
+ public List<String> getPathVector() {
+ List<String> result = new ArrayList<String>();
+ result.add(getName());
+ MetricSet owner = this.owner;
+ while (owner != null) {
+ result.add(0, owner.getName());
+ owner = owner.owner;
+ }
+ return result;
+ }
+
+ public String getDescription() { return description; }
+
+ public String[] getTagVector() {
+ return getTags().split("[ \r\t\f]");
+ }
+
+ /**
+ * Returns true if the given tag exists in this metric's tag list.
+ *
+ * @return true if tag exists in tag list
+ */
+ public boolean hasTag(String tag) {
+ for (String s : getTagVector()) {
+ if (s.equals(tag)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public enum CopyType { CLONE, INACTIVE }
+
+ /**
+ * The clone function will clone metrics to an identical subtree of
+ * metrics. Clone is primarily used for load metrics that wants to clone
+ * a template metric for each loadtype. But it should work generically.
+ *
+ * @param type If set to inactive, sum metrics will evaluate to primitives
+ * and metrics can save memory by knowing no updates are coming.
+ * @param includeUnused When creating snapshots we do not want to include
+ * unused metrics, but while generating sum metric sum in active
+ * metrics we want to. This has no affect if type is CLONE.
+ */
+ public abstract Metric clone(CopyType type, MetricSet owner, boolean includeUnused);
+
+ /**
+ * Utility function for assigning values from one metric of identical type
+ * to this metric. For simplicity sake it does a const cast and calls
+ * addToSnapshot, which should not alter source if reset is false. This can
+ * not be used to copy between active metrics and inactive copies.
+ *
+ * @return Returns itself.
+ */
+ public Metric assignValues(Metric m) {
+ m.addToSnapshot(this);
+ // As this should only be called among active metrics, all metrics
+ // should exist and owner list should thus always end up empty.
+ return this;
+ }
+
+ /** Reset all metric values. */
+ public abstract void reset();
+
+ public boolean logFromTotalMetrics() { return false; }
+
+ /** Implement to make metric able to log event.
+ *
+ * @param logger An event logger to use for logging.
+ * @param fullName The name to use for the event.
+ */
+ public abstract void logEvent(EventLogger logger, String fullName);
+
+ public static final Utf8String TAG_NAME = new Utf8String("name");
+ public static final Utf8String TAG_TAGS = new Utf8String("tags");
+ public static final Utf8String TAG_DESC = new Utf8String("description");
+
+ void openXMLTag(XMLWriter writer, int verbosity) {
+ String[] tags = getTagVector();
+
+ writer.openTag(getXMLTag());
+
+ if (getXMLTag() != getName()) {
+ writer.attribute(TAG_NAME, getName());
+ }
+
+ if (verbosity >= 3 && tags.length > 0) {
+ String tagStr = "";
+ for (String tag : tags) {
+ if (!tagStr.isEmpty()) {
+ tagStr = ",";
+ }
+ tagStr += tag;
+ }
+
+ writer.attribute(TAG_TAGS, tagStr);
+ }
+
+ if (verbosity >= 1 && !getDescription().isEmpty()) {
+ writer.attribute(TAG_DESC, getDescription());
+ }
+ }
+
+ /**
+ * The verbosity says how much to print.
+ * At verbosity level 0, only the most critical parts are printed.
+ * At verbosity level 1, descriptions are added.
+ * At verbosity level 2, metrics without data is added.
+ * At verbosity level 3, tags are included too.
+ */
+ public abstract void printXml(XMLWriter writer,
+ int secondsPassed,
+ int verbosity);
+
+ public String toXml(int secondsPassed, int verbosity) {
+ StringWriter writer = new StringWriter();
+ printXml(new XMLWriter(writer), secondsPassed, verbosity);
+ return writer.toString();
+ }
+
+ /**
+ * Most metrics report numbers of some kind. To be able to report numbers
+ * without having code to handle each possible metric type, these functions
+ * exist to extract raw data to present easily.
+ * @param id The part of the metric to extract. For instance, an average
+ * metric have average,
+ */
+ public abstract long getLongValue(String id);
+ public abstract double getDoubleValue(String id);
+
+ /**
+ * When snapshotting we need to be able to join data from one set of metrics
+ * to another set of metrics taken at another time. MetricSet doesn't know
+ * the type of the metrics it contains, so we need a generic function for
+ * doing this. This function assumes metric given as input is of the exact
+ * same type as the one it is called on for simplicity. This is true when
+ * adding to snapshots as they have been created with clone and is thus
+ * always exactly equal.
+ *
+ * @param m Metric of exact same type as this one. (Will core if wrong)
+ */
+ abstract void addToSnapshot(Metric m);
+
+ /**
+ * For sum metrics to work with metric sets, metric sets need operator+=.
+ * To implement this, we need a function to join any metric type together.
+ * This is different from adding to snapshot. When adding to snapshots we
+ * join different time periods to the same metric, but when adding parts
+ * together we join different metrics for the same time. For instance, an
+ * average metric of queuesize, should just join new values to create new
+ * average when adding to snapshot, but when adding parts, the averages
+ * themselves should be added together.
+ *
+ * @param m Metric of exact same type as this one. (Will core if wrong)
+ */
+ abstract void addToPart(Metric m);
+
+ public boolean visit(MetricVisitor visitor, boolean tagAsAutoGenerated) {
+ return visitor.visitPrimitiveMetric(this, tagAsAutoGenerated);
+ }
+
+ /** Set whether metrics have ever been set. */
+ public abstract boolean used();
+
+ /** Returns true if this metric is registered in a metric set. */
+ public boolean isRegistered() { return (owner != null); }
+
+ /**
+ * If this metric is registered with an owner, remove itself from that owner.
+ */
+ public void unregister() {
+ if (isRegistered()) {
+ getOwner().unregisterMetric(this);
+ }
+ }
+
+ public MetricSet getOwner() { return owner; }
+
+ public MetricSet getRoot() {
+ if (owner == null) {
+ if (this instanceof MetricSet) {
+ return (MetricSet)this;
+ } else {
+ return null;
+ }
+ } else {
+ return owner.getRoot();
+ }
+ }
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricManager.java b/metrics/src/main/java/com/yahoo/metrics/MetricManager.java
new file mode 100644
index 00000000000..b44205660de
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/MetricManager.java
@@ -0,0 +1,704 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.text.XMLWriter;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.logging.Logger;
+
+/**
+ * A metrics-enabled application should have a single MetricManager. You can register a number of MetricSets in the
+ * MetricManager. Each metric in the metrics sets can be used by zero or more consumers, configurable using
+ * readConfig().
+ *
+ * The consumers get their data by calling the getMetrics() method, which gives them a snapshot of all the current
+ * metrics which are configured for the given name.
+ *
+ * Locking strategy:
+ *
+ * There are three locks in this class:
+ *
+ * Config lock: - This protects the class on config changes. It protects the _config and _consumerConfig members.
+ *
+ * Thread monitor (waiter): - This lock is kept by the worker thread while it is doing a work cycle, and it uses this
+ * monitor to sleep. It is used to make shutdown quick by interrupting thread, and to let functions called by clients be
+ * able to do a change while the worker thread is idle. - The log period is protected by the thread monitor. - The
+ * update hooks is protected by the thread monitor.
+ *
+ * Metric lock: - The metric log protects the active metric set when adding or removing metrics. Clients need to grab
+ * this lock before altering active metrics. The metric manager needs to grab this lock everytime it visits active
+ * metrics. - The metric log protects the snapshots. The snapshot writer is the metric worker thread and will grab the
+ * lock while editing them. Readers that aren't the worker thread itself must grab lock to be sure.
+ *
+ * If multiple locks is taken, the allowed locking order is: 1. Thread monitor. 2. Metric lock. 3. Config lock.
+ */
+public class MetricManager implements Runnable {
+
+ private static final int STATE_CREATED = 0;
+ private static final int STATE_RUNNING = 1;
+ private static final int STATE_STOPPED = 2;
+ private static final Logger log = Logger.getLogger(MetricManager.class.getName());
+ private final CountDownLatch termination = new CountDownLatch(1);
+ private final MetricSnapshot activeMetrics = new MetricSnapshot("Active metrics showing updates since " +
+ "last snapshot");
+ private final Map<String, ConsumerSpec> consumerConfig = new HashMap<String, ConsumerSpec>();
+ private final List<UpdateHook> periodicUpdateHooks = new ArrayList<UpdateHook>();
+ private final List<UpdateHook> snapshotUpdateHooks = new ArrayList<UpdateHook>();
+ private final Timer timer;
+ private Pair<Integer, Integer> logPeriod;
+ private List<MetricSnapshotSet> snapshots = new ArrayList<MetricSnapshotSet>();
+ private MetricSnapshot totalMetrics = new MetricSnapshot("Empty metrics before init", 0,
+ activeMetrics.getMetrics(), false);
+ private int state = STATE_CREATED;
+ private int lastProcessedTime = 0;
+ private boolean forceEventLogging = false;
+ private boolean snapshotUnsetMetrics = false; // TODO: add to config
+
+ public MetricManager() {
+ this(new Timer());
+ }
+
+ MetricManager(Timer timer) {
+ this.timer = timer;
+ initializeSnapshots();
+ logPeriod = new Pair<Integer, Integer>(snapshots.get(0).getPeriod(), 0);
+ }
+
+ void initializeSnapshots() {
+ int currentTime = timer.secs();
+
+ List<Pair<Integer, String>> snapshotPeriods = new ArrayList<Pair<Integer, String>>();
+ snapshotPeriods.add(new Pair<Integer, String>(60 * 5, "5 minute"));
+ snapshotPeriods.add(new Pair<Integer, String>(60 * 60, "1 hour"));
+ snapshotPeriods.add(new Pair<Integer, String>(60 * 60 * 24, "1 day"));
+ snapshotPeriods.add(new Pair<Integer, String>(60 * 60 * 24 * 7, "1 week"));
+
+ int count = 1;
+ for (int i = 0; i < snapshotPeriods.size(); ++i) {
+ int nextCount = 1;
+ if (i + 1 < snapshotPeriods.size()) {
+ nextCount = snapshotPeriods.get(i + 1).getFirst()
+ / snapshotPeriods.get(i).getFirst();
+ if (snapshotPeriods.get(i + 1).getFirst() % snapshotPeriods.get(i).getFirst() != 0) {
+ throw new IllegalStateException("Snapshot periods must be multiplum of each other");
+ }
+ }
+ snapshots.add(new MetricSnapshotSet(snapshotPeriods.get(i).getSecond(),
+ snapshotPeriods.get(i).getFirst(),
+ count,
+ activeMetrics.getMetrics(),
+ snapshotUnsetMetrics));
+ count = nextCount;
+ }
+ // Add all time snapshot.
+ totalMetrics = new MetricSnapshot("All time snapshot",
+ 0, activeMetrics.getMetrics(),
+ snapshotUnsetMetrics);
+ totalMetrics.reset(currentTime);
+ }
+
+ public void stop() {
+ synchronized (this) {
+ int prevState = state;
+ state = STATE_STOPPED;
+ if (prevState == STATE_CREATED) {
+ return;
+ }
+ notifyAll();
+ }
+ try {
+ termination.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ @SuppressWarnings("UnusedDeclaration")
+ void setSnapshotUnsetMetrics(boolean value) {
+ snapshotUnsetMetrics = value;
+ }
+
+ /**
+ * Add a metric update hook. This will always be called prior to snapshotting and metric logging, to make the
+ * metrics the best as they can be at those occasions.
+ *
+ * @param hook The hook to add.
+ * @param period Period in seconds for how often callback should be called. The default value of 0, means only
+ * before snapshotting or logging, while another value will give callbacks each period seconds.
+ * Expensive metrics to calculate will typically only want to do it before snapshotting, while
+ * inexpensive metrics might want to log their value every 5 seconds or so. Any value of period &gt;= the
+ * smallest snapshot time will behave identically as if period is set to 0.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public synchronized void addMetricUpdateHook(UpdateHook hook, int period) {
+ hook.period = period;
+
+ // If we've already initialized manager, log period has been set.
+ // In this case. Call first time after period
+ hook.nextCall = (logPeriod.getSecond() == 0 ? 0 : timer.secs() + period);
+ if (period == 0) {
+ if (!snapshotUpdateHooks.contains(hook)) {
+ snapshotUpdateHooks.add(hook);
+ }
+ } else {
+ if (!periodicUpdateHooks.contains(hook)) {
+ periodicUpdateHooks.add(hook);
+ }
+ }
+ }
+
+ @SuppressWarnings("UnusedDeclaration")
+ public synchronized void removeMetricUpdateHook(UpdateHook hook) {
+ if (hook.period == 0) {
+ snapshotUpdateHooks.remove(hook);
+ } else {
+ periodicUpdateHooks.remove(hook);
+ }
+ }
+
+ /**
+ * Force a metric update for all update hooks. Useful if you want to ensure nice values before reporting something.
+ * This function can not be called from an update hook callback.
+ *
+ * @param includeSnapshotOnlyHooks True to also run snapshot hooks.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public synchronized void updateMetrics(boolean includeSnapshotOnlyHooks) {
+ log.fine("Giving " + periodicUpdateHooks.size() + " periodic update hooks.");
+
+ updatePeriodicMetrics(0, true);
+
+ if (includeSnapshotOnlyHooks) {
+ log.fine("Giving " + snapshotUpdateHooks.size() + " snapshot update hooks.");
+ updateSnapshotMetrics();
+ }
+ }
+
+ /**
+ * Force event logging to happen now. This function can not be called from an update hook callback.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void forceEventLogging() {
+ log.fine("Forcing event logging to happen.");
+ // Ensure background thread is not in a current cycle during change.
+
+ synchronized (this) {
+ forceEventLogging = true;
+ this.notifyAll();
+ }
+ }
+
+ /**
+ * Register a new metric to be included in the active metric set. You need to have grabbed the metric lock in order
+ * to do this. (You also need to grab that lock if you alter registration of already registered metric set.) This
+ * function can not be called from an update hook callback.
+ *
+ * @param m The metric to register.
+ */
+ public void registerMetric(Metric m) {
+ activeMetrics.getMetrics().registerMetric(m);
+ }
+
+ /**
+ * Unregister a metric from the active metric set. You need to have grabbed the metric lock in order to do this.
+ * (You also need to grab that lock if you alter registration of already registered metric set.) This function can
+ * not be called from an update hook callback.
+ *
+ * @param m The Metric to unregister.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public void unregisterMetric(Metric m) {
+ activeMetrics.getMetrics().unregisterMetric(m);
+ }
+
+ /**
+ * Reset all metrics including all snapshots. This function can not be called from an update hook callback.
+ *
+ * @param currentTime The current time.
+ */
+ public synchronized void reset(int currentTime) {
+ activeMetrics.reset(currentTime);
+
+ for (MetricSnapshotSet m : snapshots) {
+ m.reset(currentTime);
+ }
+ totalMetrics.reset(currentTime);
+ }
+
+ /**
+ * Read configuration. Before reading config, all metrics should be set up first. By doing this, the metrics manager
+ * can optimize reporting of consumers. readConfig() will start a config subscription. It should not be called
+ * multiple times.
+ */
+/* public synchronized void init(String configId, ThreadPool pool) {
+ log.fine("Initializing metric manager")
+
+ LOG(debug, "Initializing metric manager.");
+ _configSubscription = Config::subscribe(configId, *this);
+ LOG(debug, "Starting worker thread, waiting for first "
+ "iteration to complete.");
+ Runnable::start(pool);
+ // Wait for first iteration to have completed, such that it is safe
+ // to access snapshots afterwards.
+ vespalib::MonitorGuard sync(_waiter);
+ while (_lastProcessedTime == 0) {
+ sync.wait(1);
+ }
+ LOG(debug, "Metric manager completed initialization.");
+}
+
+*/
+
+ class ConsumerMetricVisitor extends MetricVisitor {
+
+ ConsumerSpec metricsToMatch;
+ MetricVisitor clientVisitor;
+
+ ConsumerMetricVisitor(ConsumerSpec spec,
+ MetricVisitor clientVisitor)
+ {
+ metricsToMatch = spec;
+ this.clientVisitor = clientVisitor;
+ log.fine("Consuming metrics: " + spec.includedMetrics);
+ }
+
+ public boolean visitMetricSet(MetricSet metricSet, boolean autoGenerated) {
+ if (metricSet.getOwner() == null) {
+ return true;
+ }
+
+ if (!metricsToMatch.contains(metricSet)) {
+ log.fine("Metric doesn't match " + metricSet.getPath());
+ return false;
+ }
+
+ return clientVisitor.visitMetricSet(metricSet, autoGenerated);
+ }
+
+ public void doneVisitingMetricSet(MetricSet metricSet) {
+ if (metricSet.getOwner() != null) {
+ clientVisitor.doneVisitingMetricSet(metricSet);
+ }
+ }
+
+ public boolean visitPrimitiveMetric(Metric metric, boolean autoGenerated) {
+ if (metricsToMatch.contains(metric)) {
+ return clientVisitor.visitPrimitiveMetric(metric, autoGenerated);
+ } else {
+ log.fine("Metric doesn't match " + metric.getPath());
+ }
+ return true;
+ }
+ }
+
+ public synchronized void visit(MetricSet metrics, MetricVisitor visitor, String consumer) {
+ if (consumer.isEmpty()) {
+ metrics.visit(visitor, false);
+ return;
+ }
+
+ ConsumerSpec spec = getConsumerSpec(consumer);
+
+ if (spec != null) {
+ ConsumerMetricVisitor consumerVis = new ConsumerMetricVisitor(spec, visitor);
+ metrics.visit(consumerVis, false);
+ } else {
+ log.warning("Requested metrics for non-defined consumer " + consumer);
+ }
+ }
+
+ class XmlWriterMetricVisitor extends MetricVisitor {
+
+ int period;
+ XMLWriter writer;
+ int verbosity;
+
+ XmlWriterMetricVisitor(XMLWriter writer, int period, int verbosity) {
+ this.period = period;
+ this.verbosity = verbosity;
+ this.writer = writer;
+ }
+
+ public boolean visitMetricSet(MetricSet set, boolean autoGenerated) {
+ if (set.used() || verbosity >= 2) {
+ set.openXMLTag(writer, verbosity);
+ return true;
+ }
+ return false;
+ }
+
+ public void doneVisitingMetricSet(MetricSet set) {
+ writer.closeTag();
+ }
+
+ public boolean visitPrimitiveMetric(Metric metric, boolean autoGenerated) {
+ metric.printXml(writer, period, verbosity);
+ return true;
+ }
+ }
+
+ void printXml(MetricSet set, XMLWriter writer, int period, String consumer, int verbosity) {
+ visit(set, new XmlWriterMetricVisitor(writer, period, verbosity), consumer);
+ }
+
+ /**
+ * Synchronize over this while the returned object
+ *
+ * @return The MetricSnapshot of all active metrics.
+ */
+ public MetricSnapshot getActiveMetrics() {
+ return activeMetrics;
+ }
+
+ /**
+ * Synchronize over this while the returned object
+ *
+ * @return The MetricSnapshot for the total metric.
+ */
+ public MetricSnapshot getTotalMetricSnapshot() {
+ return totalMetrics;
+ }
+
+ public synchronized List<Integer> getSnapshotPeriods() {
+ List<Integer> retVal = new ArrayList<Integer>();
+
+ for (MetricSnapshotSet m : snapshots) {
+ retVal.add(m.getPeriod());
+ }
+ return retVal;
+ }
+
+ /**
+ * While accessing snapshots you should synchronize over this
+ *
+ * @param period The id of the snapshot period to access.
+ * @param getInProgressSet True to retrieve the snapshot currently being built.
+ * @return The appropriate MetricSnapshot.
+ */
+ MetricSnapshot getMetricSnapshot(int period, boolean getInProgressSet) {
+ return getMetricSnapshotSet(period).getSnapshot(getInProgressSet);
+ }
+
+ MetricSnapshot getMetricSnapshot(int period) {
+ return getMetricSnapshot(period, false);
+ }
+
+ public MetricSnapshotSet getMetricSnapshotSet(int period) {
+ for (MetricSnapshotSet m : snapshots) {
+ if (m.getPeriod() == period) {
+ return m;
+ }
+ }
+
+ throw new IllegalArgumentException("No snapshot for period of length " + period + " exists.");
+ }
+
+ @SuppressWarnings("UnusedDeclaration")
+ public synchronized boolean hasTemporarySnapshot(int period) {
+ return getMetricSnapshotSet(period).hasTemporarySnapshot();
+ }
+
+ public synchronized void addMetricToConsumer(String consumerName, String metricPath) {
+ ConsumerSpec spec = getConsumerSpec(consumerName);
+ if (spec == null) {
+ spec = new ConsumerSpec();
+ consumerConfig.put(consumerName, spec);
+ }
+ spec.register(metricPath);
+ }
+
+ public synchronized ConsumerSpec getConsumerSpec(String consumer) {
+ return consumerConfig.get(consumer);
+ }
+
+ /**
+ * If you join or remove metrics from the active metric sets, normally, snapshots will be recreated next snapshot
+ * period. However, if you want to see the effects of such changes in status pages ahead of that, you can call this
+ * function in order to check whether snapshots needs to be regenerated and regenerate them if needed.
+ */
+ public synchronized void checkMetricsAltered() {
+ if (activeMetrics.getMetrics().isRegistrationAltered()) {
+ handleMetricsAltered();
+ }
+ }
+
+ /**
+ * Used by unit tests to verify that we have processed for a given time.
+ *
+ * @return Returns the timestamp of the previous tick.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public int getLastProcessedTime() {
+ return lastProcessedTime;
+ }
+
+ class LogMetricVisitor extends MetricVisitor {
+
+ boolean total;
+ EventLogger logger;
+
+ LogMetricVisitor(boolean totalVals, EventLogger logger) {
+ total = totalVals;
+ this.logger = logger;
+ }
+
+ public boolean visitPrimitiveMetric(Metric metric, boolean autoGenerated) {
+ if (metric.logFromTotalMetrics() == total) {
+ String logName = metric.getPath().replace('.', '_');
+ metric.logEvent(logger, logName);
+ }
+ return true;
+ }
+ }
+
+ public void logTotal(int currentTime, EventLogger logger) {
+ LogMetricVisitor totalVisitor = new LogMetricVisitor(true, logger);
+ LogMetricVisitor fiveMinVisitor = new LogMetricVisitor(false, logger);
+
+ if (logPeriod.getSecond() <= currentTime) {
+ log.fine("Logging total metrics.");
+ visit(totalMetrics.getMetrics(), totalVisitor, "log");
+ visit(snapshots.get(0).getSnapshot().getMetrics(), fiveMinVisitor, "log");
+ if (logPeriod.getSecond() + logPeriod.getFirst() < currentTime) {
+ logPeriod = new Pair<Integer, Integer>(logPeriod.getFirst(),
+ snapshots.get(0).getFromTime() + logPeriod.getFirst());
+ } else {
+ logPeriod =
+ new Pair<Integer, Integer>(logPeriod.getFirst(), logPeriod.getSecond() + logPeriod.getFirst());
+ }
+ }
+ }
+
+ public void logOutOfSequence(int currentTime, EventLogger logger) {
+ LogMetricVisitor totalVisitor = new LogMetricVisitor(true, logger);
+ LogMetricVisitor fiveMinVisitor = new LogMetricVisitor(false, logger);
+
+ log.fine("Logging total metrics out of sequence.");
+ MetricSnapshot snapshot = new MetricSnapshot(
+ "Total out of sequence metrics from start until current time",
+ 0,
+ totalMetrics.getMetrics(),
+ snapshotUnsetMetrics);
+
+ activeMetrics.addToSnapshot(snapshot, currentTime, false);
+ snapshot.setFromTime(totalMetrics.getFromTime());
+ visit(snapshot.getMetrics(), totalVisitor, "log");
+ visit(snapshot.getMetrics(), fiveMinVisitor, "log");
+ }
+
+ /**
+ * Runs one iteration of the thread activity.
+ *
+ * @param logger An event logger to use for any new events generated.
+ * @return The number of milliseconds to sleep until waking up again
+ */
+ public synchronized int tick(EventLogger logger) {
+ int currentTime = timer.secs();
+
+ log.finest("Worker thread starting to process for time " + currentTime);
+
+ boolean firstIteration = (logPeriod.getSecond() == 0);
+ // For a slow system to still be doing metrics tasks each n'th
+ // second, rather than each n'th + time to do something seconds,
+ // we constantly join next time to do something from the last timer.
+ // For that to work, we need to initialize timers on first iteration
+ // to set them to current time.
+ if (firstIteration) {
+ // Setting next log period to now, such that we log metrics
+ // straight away
+ logPeriod = new Pair<Integer, Integer>(logPeriod.getFirst(), currentTime);
+ for (MetricSnapshotSet m : snapshots) {
+ m.setFromTime(currentTime);
+ }
+ for (UpdateHook h : periodicUpdateHooks) {
+ h.nextCall = currentTime;
+ }
+ }
+
+ // If metrics have changed since last time we did a snapshot,
+ // work that out before taking the snapshot, such that new
+ // metric can be included
+ checkMetricsAltered();
+
+ // Set next work time to the time we want to take next snapshot.
+ int nextWorkTime = snapshots.get(0).getPeriod() + snapshots.get(0).getFromTime();
+
+ int nextUpdateHookTime;
+
+ if (nextWorkTime <= currentTime) {
+ log.fine("Time to do snapshot. Calling update hooks");
+ nextUpdateHookTime = updatePeriodicMetrics(currentTime, true);
+ updateSnapshotMetrics();
+ takeSnapshots(nextWorkTime);
+ } else if (forceEventLogging) {
+ log.fine("Out of sequence event logging. Calling update hooks");
+ nextUpdateHookTime = updatePeriodicMetrics(currentTime, true);
+ updateSnapshotMetrics();
+ } else {
+ // If not taking a new snapshot. Only give update hooks to
+ // periodic hooks wanting it.
+ nextUpdateHookTime = updatePeriodicMetrics(currentTime, false);
+ }
+
+ // Log if it is time
+ if (logPeriod.getSecond() <= currentTime || forceEventLogging) {
+ logTotal(currentTime, logger);
+ } else {
+ logOutOfSequence(currentTime, logger);
+ }
+
+ forceEventLogging = false;
+ lastProcessedTime = (nextWorkTime <= currentTime ? nextWorkTime : currentTime);
+ log.fine("Worker thread done with processing for time " + lastProcessedTime);
+
+ int next = Math.min(logPeriod.getSecond(), snapshots.get(0).getPeriod() + snapshots.get(0).getFromTime());
+ next = Math.min(next, nextUpdateHookTime);
+ if (currentTime < next) {
+ return (next - currentTime) * 1000;
+ }
+
+ return 0;
+ }
+
+ @Override
+ public synchronized void run() {
+ if (state != STATE_CREATED) {
+ throw new IllegalStateException();
+ }
+ try {
+ for (state = STATE_RUNNING; state == STATE_RUNNING; ) {
+ int timeout = tick(new VespaLogEventLogger());
+ if (timeout > 0) {
+ wait(timeout);
+ }
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ termination.countDown();
+ }
+ }
+
+ public synchronized void takeSnapshots(int timeToProcess) {
+ // If not time to do dump data from active snapshot yet, nothing to do
+ if (!snapshots.get(0).timeForAnotherSnapshot(timeToProcess)) {
+ return;
+ }
+
+ log.fine("Updating " + snapshots.get(0).getName() + " snapshot from active metrics");
+ // int fromTime = snapshots.get(0).getSnapshot().getToTime();
+ MetricSnapshot firstTarget = (snapshots.get(0).getNextTarget());
+ firstTarget.reset(timeToProcess);
+ activeMetrics.addToSnapshot(firstTarget, timeToProcess, false);
+ log.fine("Updating total metrics with five minute period of active metrics");
+ activeMetrics.addToSnapshot(totalMetrics, timeToProcess, false);
+ activeMetrics.reset(timeToProcess);
+
+ for (int i = 1; i < snapshots.size(); ++i) {
+ MetricSnapshotSet s = snapshots.get(i);
+
+ log.fine("Adding data from last snapshot to building snapshot of " +
+ "next period snapshot " + s.getName());
+
+ MetricSnapshot target = s.getNextTarget();
+ snapshots.get(i - 1).getSnapshot().addToSnapshot(target, timeToProcess, false);
+ target.setToTime(timeToProcess);
+
+ if (!snapshots.get(i).haveCompletedNewPeriod(timeToProcess)) {
+ log.fine("Not time to roll snapshot " + s.getName() + " yet. " +
+ s.getBuilderCount() + " of " + s.getCount() + " snapshot " +
+ "taken at time" + (s.getBuilderCount() * s.getPeriod() + s.getFromTime()) +
+ ", and period of " + s.getPeriod() + " is not up " +
+ "yet as we're currently processing for time " + timeToProcess);
+ break;
+ } else {
+ log.fine("Rolled snapshot " + s.getName() + " at time " + timeToProcess);
+ }
+ }
+ }
+
+ /**
+ * Utility function for updating periodic metrics.
+ *
+ * @param updateTime Update metrics timed to update at this time.
+ * @param outOfSchedule Force calls to all hooks. Don't screw up normal schedule though. If not time to update yet,
+ * update without adjusting schedule for next update.
+ * @return Time of next hook to be called in the future.
+ */
+ int updatePeriodicMetrics(int updateTime, boolean outOfSchedule) {
+ int nextUpdateTime = Integer.MAX_VALUE;
+ for (UpdateHook h : periodicUpdateHooks) {
+ if (h.nextCall <= updateTime) {
+ h.updateMetrics();
+ if (h.nextCall + h.period < updateTime) {
+ h.nextCall = updateTime + h.period;
+ } else {
+ h.nextCall += h.period;
+ }
+ } else if (outOfSchedule) {
+ h.updateMetrics();
+ }
+ nextUpdateTime = Math.min(nextUpdateTime, h.nextCall);
+ }
+ return nextUpdateTime;
+ }
+
+ void updateSnapshotMetrics() {
+ for (UpdateHook h : snapshotUpdateHooks) {
+ h.updateMetrics();
+ }
+ }
+
+ synchronized void handleMetricsAltered() {
+/* if (consumerConfig.isEmpty()) {
+ log.fine("Setting up consumers for the first time.");
+ } else {
+ log.info("Metrics registration changes detected. Handling changes.");
+ }
+
+ Map<String, ConsumerSpec> configMap = new HashMap<String, ConsumerSpec>();
+ activeMetrics.getMetrics().clearRegistrationAltered();
+
+
+ for (<config::MetricsmanagerConfig::Consumer>::const_iterator it
+ = _config.consumer.begin(); it != _config.consumer.end(); ++it)
+ {
+ ConsumerMetricBuilder consumerMetricBuilder(*it);
+ _activeMetrics.getMetrics().visit(consumerMetricBuilder);
+ configMap[it->name] = ConsumerSpec::SP(
+ new ConsumerSpec(consumerMetricBuilder._matchedMetrics));
+ }
+ _consumerConfig.swap(configMap);
+ */
+ log.fine("Recreating snapshots to include altered metrics");
+ totalMetrics.recreateSnapshot(activeMetrics.getMetrics(), snapshotUnsetMetrics);
+
+ for (MetricSnapshotSet set : snapshots) {
+ set.recreateSnapshot(activeMetrics.getMetrics(), snapshotUnsetMetrics);
+ }
+ }
+
+ abstract class UpdateHook {
+
+ String name;
+ int nextCall;
+ int period;
+
+ public UpdateHook(String name) {
+ this.name = name;
+ nextCall = 0;
+ period = 0;
+ }
+
+ public abstract void updateMetrics();
+
+ public String getName() {
+ return name;
+ }
+ }
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricSet.java b/metrics/src/main/java/com/yahoo/metrics/MetricSet.java
new file mode 100644
index 00000000000..f10834d93cc
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/MetricSet.java
@@ -0,0 +1,239 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import com.yahoo.text.XMLWriter;
+
+import java.util.*;
+import java.util.logging.Logger;
+
+public abstract class MetricSet extends Metric
+{
+ private static Logger log = Logger.getLogger(MetricSet.class.getName());
+
+ List<Metric> metricOrder = new ArrayList<Metric>(); // Keep added order for reporting
+ boolean registrationAltered; // Set to true if metrics have been
+ // registered/unregistered since last time
+ // it was reset
+
+ public MetricSet(String name, String tags, String description, MetricSet owner) {
+ super(name, tags, description, owner);
+ }
+
+ public MetricSet(MetricSet other, CopyType copyType, MetricSet owner, boolean includeUnused) {
+ super(other, owner);
+
+ for (Metric m : other.metricOrder) {
+ if (copyType != CopyType.INACTIVE || includeUnused || m.used()) {
+ m.clone(copyType, this, includeUnused);
+ }
+ }
+ }
+
+ /**
+ * @return Returns true if registration has been altered since it was last
+ * cleared. Used by the metric manager to know when it needs to recalculate
+ * which consumers will see what.
+ */
+ public boolean isRegistrationAltered() { return registrationAltered; }
+
+ /** Clear all registration altered flags. */
+ void clearRegistrationAltered() {
+ visit(new MetricVisitor() {
+ public boolean visitMetricSet(MetricSet set, boolean autoGenerated) {
+ if (autoGenerated) {
+ return false;
+ }
+
+ set.registrationAltered = false;
+ return true;
+ }
+ }, false);
+ }
+
+ public void registerMetric(Metric m) {
+ if (m.isRegistered()) {
+ throw new IllegalStateException("Metric " + m.getName() +
+ " is already registered in a metric set. Cannot register it twice.");
+ }
+
+ if (getMetricInternal(m.getName()) != null) {
+ throw new IllegalStateException("A metric named " + m.getName() + " is already registered "
+ + "in metric set " + getPath());
+ }
+
+ metricOrder.add(m);
+ m.owner = this;
+ tagRegistrationAltered();
+ }
+
+ public void unregisterMetric(Metric m) {
+ // In case of abrubt shutdowns, don't die hard on attempts to unregister
+ // non-registered metrics. Just warn and ignore.
+ if (!metricOrder.remove(m)) {
+ log.warning("Attempt to unregister metric " + m.getName() + " in metric set " + getPath() +
+ ", where it wasn't registered to begin with.");
+ return;
+ }
+
+ m.owner = null;
+ tagRegistrationAltered();
+
+ log.finest("Unregistered metric " + m.getName() + " from metric set " + getPath() + ".");
+ }
+
+ @Override
+ public void reset() {
+ for (Metric m : metricOrder) {
+ m.reset();
+ }
+ }
+
+ @Override
+ public boolean visit(MetricVisitor visitor, boolean tagAsAutoGenerated) {
+ if (!visitor.visitMetricSet(this, tagAsAutoGenerated)) {
+ return true;
+ }
+
+ for (Metric m : metricOrder) {
+ if (!m.visit(visitor, tagAsAutoGenerated)) {
+ break;
+ }
+ }
+
+ visitor.doneVisitingMetricSet(this);
+ return true;
+ }
+
+ @Override
+ public void logEvent(EventLogger logger, String fullName) {
+ throw new IllegalStateException("Can't log event from a MetricsSet: " + fullName);
+ }
+
+ // These should never be called on metrics set.
+ @Override
+ public long getLongValue(String id) {
+ throw new IllegalStateException("Tried to get long from metricset");
+ }
+
+ @Override
+ public double getDoubleValue(String id) {
+ throw new IllegalStateException("Tried to get double from metricset");
+ }
+
+ public Metric getMetric(String name) {
+ int pos = name.indexOf('.');
+ if (pos == -1) {
+ return getMetricInternal(name);
+ } else {
+ String child = name.substring(0, pos);
+ String rest = name.substring(pos + 1);
+
+ Metric m = getMetricInternal(child);
+ if (m == null || !(m instanceof MetricSet)) {
+ return null;
+ } else {
+ return ((MetricSet)m).getMetric(rest);
+ }
+ }
+ }
+
+ private Metric getMetricInternal(String name) {
+ for (Metric m : metricOrder) {
+ if (m.getName().equals(name)) {
+ return m;
+ }
+ }
+ return null;
+ }
+
+ Map<String, Metric> createMetricMap() {
+ Map<String, Metric> map = new TreeMap<String, Metric>();
+
+ for (Metric m : metricOrder) {
+ map.put(m.getName(), m);
+ }
+
+ return map;
+ }
+
+ @Override
+ public void addToSnapshot(Metric snapshotMetric) {
+ MetricSet o = (MetricSet)snapshotMetric;
+
+ Map<String, Metric> map1 = createMetricMap();
+ Set<String> seen = new HashSet<String>();
+
+ // For all the metrics in the other's order, join ours to the snapshot.
+ for (Metric m : o.metricOrder) {
+ Metric myCopy = map1.get(m.getName());
+
+ if (myCopy != null) {
+ seen.add(m.getName());
+ myCopy.addToSnapshot(m);
+ }
+ }
+
+ // For all the remaining metrics, just join them to the other one.
+ for (Metric m : metricOrder) {
+ if (!seen.contains(m.getName())) {
+ m.clone(CopyType.INACTIVE, o, false);
+ }
+ }
+ }
+
+ List<Metric> getRegisteredMetrics()
+ { return metricOrder; }
+
+ @Override
+ public boolean used() {
+ for (Metric m : metricOrder) {
+ if (m.used()) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ @Override
+ public void addToPart(Metric partMetric) {
+ MetricSet o = (MetricSet)partMetric;
+
+ Map<String, Metric> map2 = o.createMetricMap();
+
+ for (Metric m : metricOrder) {
+ Metric other = map2.get(m.getName());
+ if (other != null) {
+ m.addToPart(other);
+ } else {
+ m.clone(CopyType.INACTIVE, o, false);
+ }
+ }
+ }
+
+ private void tagRegistrationAltered() {
+ registrationAltered = true;
+ if (owner != null) {
+ owner.tagRegistrationAltered();
+ }
+ }
+
+ @Override
+ public void printXml(XMLWriter writer,
+ int secondsPassed, int verbosity)
+ {
+ if (!used() && verbosity < 3) {
+ return;
+ }
+
+ openXMLTag(writer, verbosity);
+
+ for (Metric m : metricOrder) {
+ m.printXml(writer, secondsPassed, verbosity);
+ }
+
+ writer.closeTag();
+ }
+
+
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricSnapshot.java b/metrics/src/main/java/com/yahoo/metrics/MetricSnapshot.java
new file mode 100644
index 00000000000..cd02b170b4a
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/MetricSnapshot.java
@@ -0,0 +1,119 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import com.yahoo.text.XMLWriter;
+import com.yahoo.text.Utf8String;
+
+import java.io.StringWriter;
+
+public class MetricSnapshot
+{
+ String name;
+ int period;
+ int fromTime;
+ int toTime;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getPeriod() {
+ return period;
+ }
+
+ public void setPeriod(int period) {
+ this.period = period;
+ }
+
+ public int getFromTime() {
+ return fromTime;
+ }
+
+ public void setFromTime(int fromTime) {
+ this.fromTime = fromTime;
+ }
+
+ public int getToTime() {
+ return toTime;
+ }
+
+ public void setToTime(int toTime) {
+ this.toTime = toTime;
+ }
+
+ public MetricSet getMetrics() {
+ return snapshot;
+ }
+
+ MetricSet snapshot;
+
+ public MetricSnapshot(String name) {
+ this.name = name;
+ this.period = 0;
+ this.fromTime = 0;
+ this.toTime = 0;
+ snapshot = new SimpleMetricSet("metrics", "", "");
+ }
+
+ public MetricSnapshot(String name,
+ int period,
+ MetricSet source,
+ boolean copyUnset) {
+ this(name);
+ this.period = period;
+ snapshot = (MetricSet)source.clone(Metric.CopyType.INACTIVE, null, copyUnset);
+ }
+
+ void reset(int currentTime) {
+ fromTime = currentTime;
+ toTime = 0;
+ snapshot.reset();
+ }
+
+ public void recreateSnapshot(MetricSet source, boolean copyUnset) {
+ MetricSet newSnapshot = (MetricSet)source.clone(Metric.CopyType.INACTIVE, null, copyUnset);
+ newSnapshot.reset();
+ snapshot.addToSnapshot(newSnapshot);
+ snapshot = newSnapshot;
+ }
+
+ public static final Utf8String TAG_NAME = new Utf8String("name");
+ public static final Utf8String TAG_FROM = new Utf8String("from");
+ public static final Utf8String TAG_TO = new Utf8String("to");
+ public static final Utf8String TAG_PERIOD = new Utf8String("period");
+
+ public void printXml(MetricManager man, String consumer, int verbosity, XMLWriter writer) {
+ writer.openTag("snapshot");
+ writer.attribute(TAG_NAME, name);
+ writer.attribute(TAG_FROM, fromTime);
+ writer.attribute(TAG_TO, toTime);
+ writer.attribute(TAG_PERIOD, period);
+
+ for (Metric m : snapshot.getRegisteredMetrics()) {
+ if (m instanceof MetricSet) {
+ man.printXml((MetricSet)m, writer, toTime > fromTime ? (toTime - fromTime) : period, consumer, verbosity);
+ }
+ }
+
+ writer.closeTag();
+ }
+
+ public String toXml(MetricManager man, String consumer, int verbosity) {
+ StringWriter str = new StringWriter();
+ XMLWriter writer = new XMLWriter(str);
+ printXml(man, consumer, verbosity, writer);
+ return str.toString();
+ }
+
+ public void addToSnapshot(MetricSnapshot other, int currentTime, boolean reset) {
+ snapshot.addToSnapshot(other.getMetrics());
+ if (reset) {
+ reset(currentTime);
+ }
+ other.toTime = currentTime;
+ }
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricSnapshotSet.java b/metrics/src/main/java/com/yahoo/metrics/MetricSnapshotSet.java
new file mode 100644
index 00000000000..e65f1db8fb4
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/MetricSnapshotSet.java
@@ -0,0 +1,116 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+/**
+ * Represents two snapshots for the same time period.
+ */
+public class MetricSnapshotSet {
+ int count; // Number of times we need to join to building period
+ // before we have a full time window.
+ int builderCount; // Number of times we've currently added to the
+ // building instance.
+ MetricSnapshot current = null; // The last full period
+ MetricSnapshot building = null; // The building period
+
+ MetricSnapshotSet(String name, int period, int count, MetricSet source, boolean snapshotUnsetMetrics) {
+ this.count = count;
+ this.builderCount = 0;
+ current = new MetricSnapshot(name, period, source, snapshotUnsetMetrics);
+ current.reset(0);
+ if (count != 1) {
+ building = new MetricSnapshot(name, period, source, snapshotUnsetMetrics);
+ building.reset(0);
+ }
+ }
+
+ MetricSnapshot getNextTarget() {
+ if (count == 1) {
+ return current;
+ } else {
+ return building;
+ }
+ }
+
+ public boolean haveCompletedNewPeriod(int newFromTime) {
+ if (count == 1) {
+ current.setToTime(newFromTime);
+ return true;
+ }
+ building.setToTime(newFromTime);
+
+ // If not time to roll yet, just return
+ if (++builderCount < count) return false;
+ // Building buffer done. Use that as current and reset current.
+ MetricSnapshot tmp = current;
+ current = building;
+ building = tmp;
+ building.setFromTime(newFromTime);
+ building.setToTime(0);
+ builderCount = 0;
+ return true;
+ }
+
+ public boolean timeForAnotherSnapshot(int currentTime) {
+ int lastTime = getFromTime() + builderCount * getPeriod();
+ return currentTime >= lastTime + getPeriod();
+ }
+
+ public void reset(int currentTime) {
+ if (count != 1) building.reset(currentTime);
+ current.reset(currentTime);
+ builderCount = 0;
+ }
+
+ public void recreateSnapshot(MetricSet metrics, boolean copyUnset) {
+ if (count != 1) building.recreateSnapshot(metrics, copyUnset);
+ current.recreateSnapshot(metrics, copyUnset);
+ }
+
+ public void setFromTime(int fromTime)
+ {
+ if (count != 1) {
+ building.setFromTime(fromTime);
+ } else {
+ current.setFromTime(fromTime);
+ }
+ }
+
+ public int getPeriod() {
+ return current.getPeriod();
+ }
+
+ public int getFromTime() {
+ return current.getFromTime();
+ }
+
+ public int getCount() {
+ return count;
+ }
+
+ public MetricSnapshot getSnapshot() {
+ return getSnapshot(false);
+ }
+
+ public MetricSnapshot getSnapshot(boolean getBuilding) {
+ if (getBuilding) {
+ if (count == 1) {
+ throw new IllegalStateException("No temporary snapshot for set " + current.name);
+ }
+ return building;
+ }
+
+ return current;
+ }
+
+ public boolean hasTemporarySnapshot() {
+ return count > 1;
+ }
+
+ public String getName() {
+ return current.getName();
+ }
+
+ public int getBuilderCount() {
+ return builderCount;
+ }
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java b/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java
new file mode 100644
index 00000000000..4a9ea225bdd
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+public abstract class MetricVisitor {
+ /**
+ * Visit a metric set.
+ *
+ * @param autoGenerated True for metric sets that are generated on the
+ * fly such as in sum metrics.
+ * @return True if you want to visit the content of this metric set.
+ */
+ public boolean visitMetricSet(MetricSet set, boolean autoGenerated) {
+ return true;
+ }
+
+ /**
+ * Callback visitors can use if they need to know the tree traversal of
+ * metric sets. This function is not called if visitMetricSet returned
+ * false.
+ */
+ public void doneVisitingMetricSet(MetricSet set) {
+ }
+
+ /**
+ * Visit a primitive metric within an accepted metric set.
+ *
+ * @return True if you want to continue visiting, false to abort.
+ */
+ public boolean visitPrimitiveMetric(Metric m, boolean autoGenerated) {
+ return true;
+ }
+} \ No newline at end of file
diff --git a/metrics/src/main/java/com/yahoo/metrics/SimpleMetricSet.java b/metrics/src/main/java/com/yahoo/metrics/SimpleMetricSet.java
new file mode 100644
index 00000000000..2eab3c6635e
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/SimpleMetricSet.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+/**
+ * A final metric set.
+ */
+public final class SimpleMetricSet extends MetricSet {
+
+ public SimpleMetricSet(String name, String tags, String description) {
+ this(name, tags, description, null);
+ }
+
+ public SimpleMetricSet(String name, String tags, String description, MetricSet owner) {
+ super(name, tags, description, owner);
+ }
+
+ public SimpleMetricSet(SimpleMetricSet other, Metric.CopyType copyType, MetricSet owner, boolean includeUnused) {
+ super(other, copyType, owner, includeUnused);
+ }
+
+ @Override
+ public Metric clone(Metric.CopyType type, MetricSet owner, boolean includeUnused)
+ { return new SimpleMetricSet(this, type, owner, includeUnused); }
+
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/SumMetric.java b/metrics/src/main/java/com/yahoo/metrics/SumMetric.java
new file mode 100644
index 00000000000..6335d329836
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/SumMetric.java
@@ -0,0 +1,238 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class metrics::CounterMetric
+ * @ingroup metrics
+ *
+ * @brief Counts a value that only moves upwards.
+ *
+ * NB! If you have a MetricSet subclass you want to create a sum for, use
+ * MetricSet itself as the template argument. Otherwise you'll need to override
+ * clone(...) in order to make it return the correct type for your
+ * implementation.
+ */
+
+package com.yahoo.metrics;
+
+import com.yahoo.text.XMLWriter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class SumMetric extends Metric
+{
+ ArrayList<Metric> metricsToSum = new ArrayList<Metric>();
+
+ public SumMetric(String name, String tags, String description, MetricSet owner) {
+ super(name, tags, description, owner);
+ metricsToSum = new ArrayList<Metric>();
+ }
+
+ public SumMetric(SumMetric other, MetricSet owner) {
+ super(other, owner);
+
+
+ if (other.owner == null) {
+ throw new IllegalStateException(
+ "Cannot copy a sum metric not registered in a metric set, as " +
+ "we need to use parent to detect new metrics to point to.");
+ }
+ if (owner == null) {
+ throw new IllegalStateException(
+ "Cannot copy a sum metric directly. One needs to at least " +
+ "include metric set above it in order to include metrics " +
+ "summed.");
+ }
+
+ metricsToSum.ensureCapacity(other.metricsToSum.size());
+ List<String> parentPath = other.owner.getPathVector();
+
+ for (Metric metric : other.metricsToSum) {
+ List<String> addendPath = metric.getPathVector();
+ MetricSet newAddendParent = owner;
+
+ for (int i = parentPath.size(); i < addendPath.size() - 1; ++i) {
+ String path = addendPath.get(i);
+ Metric child = newAddendParent.getMetric(path);
+ if (child == null) {
+ throw new IllegalStateException(
+ "Metric " + path + " in metric set "
+ + newAddendParent.getPath() + " was expected to " +
+ "exist. This sounds like a bug.");
+ }
+
+ newAddendParent = (MetricSet)child;
+ }
+
+ String path = addendPath.get(addendPath.size() - 1);
+ Metric child = newAddendParent.getMetric(path);
+ if (child == null) {
+ throw new IllegalStateException(
+ "Metric " + path + " in metric set "
+ + newAddendParent.getPath() + " was expected to " +
+ "exist. This sounds like a bug.");
+ }
+
+ metricsToSum.add(child);
+ }
+ }
+
+ @Override
+ public boolean visit(MetricVisitor visitor, boolean tagAsAutoGenerated) {
+ if (metricsToSum.isEmpty()) return true;
+
+ Metric sum = generateSum();
+
+ if (sum == null) {
+ return true;
+ }
+
+ if (sum instanceof MetricSet) {
+ sum.visit(visitor, true);
+ return true;
+ } else {
+ return visitor.visitPrimitiveMetric(sum, true);
+ }
+ }
+
+ @Override
+ public Metric clone(CopyType copyType, MetricSet owner, boolean includeUnused) {
+ if (copyType == CopyType.CLONE) {
+ return new SumMetric(this, owner);
+ }
+
+ Metric sum = null;
+ for (Metric metric : metricsToSum) {
+ if (sum == null) {
+ sum = metric.clone(CopyType.INACTIVE, null, includeUnused);
+ sum.setName(getName());
+ sum.setDescription(getDescription());
+ sum.setTags(getTags());
+
+ if (owner != null) {
+ owner.registerMetric(sum);
+ }
+ } else {
+ metric.addToPart(sum);
+ }
+ }
+
+ return sum;
+ }
+
+ @Override
+ public void addToPart(Metric partMetric) {
+ Metric m = generateSum();
+ if (m != null) {
+ m.addToPart(partMetric);
+ }
+ }
+
+ @Override
+ public void addToSnapshot(Metric snapshotMetric) {
+ Metric m = generateSum();
+ if (m != null) {
+ m.addToSnapshot(snapshotMetric);
+ }
+ }
+
+ public void addMetricToSum(Metric metric) {
+ if (owner == null) {
+ throw new IllegalStateException(
+ "Sum metric needs to be registered in a parent metric set " +
+ "prior to adding metrics to sum.");
+ }
+ if (!metricsToSum.isEmpty() && !(metric.getClass().equals(metricsToSum.get(0).getClass()))) {
+ throw new IllegalStateException(
+ "All metrics in a metric set must be of the same type.");
+ }
+
+ List<String> sumParentPath = owner.getPathVector();
+ List<String> addedPath = metric.getPathVector();
+
+ boolean error = false;
+ if (addedPath.size() <= sumParentPath.size()) {
+ error = true;
+ } else for (int i=0; i<sumParentPath.size(); ++i) {
+ if (!sumParentPath.get(i).equals(addedPath.get(i))) {
+ error = true;
+ break;
+ }
+ }
+ if (error) {
+ throw new IllegalStateException(
+ "Metric added to sum is required to be a child of the sum's " +
+ "direct parent metric set. (Need not be a direct child) " +
+ "Metric set " + metric.getPath() + " is not a child of " +
+ owner.getPath());
+ }
+
+ ArrayList<Metric> metrics = new ArrayList<Metric>(metricsToSum.size() + 1);
+ for (int i = 0; i < metricsToSum.size(); ++i) {
+ metrics.add(metricsToSum.get(i));
+ }
+ metrics.add(metric);
+ metricsToSum = metrics;
+ }
+
+ public void removeMetricFromSum(Metric metric) {
+ metricsToSum.remove(metric);
+ }
+
+ public Metric generateSum() {
+ Metric m = clone(CopyType.INACTIVE, null, true);
+
+ if (m != null) {
+ m.owner = owner;
+ }
+ return m;
+ }
+
+ @Override
+ public long getLongValue(String id) {
+ Metric s = generateSum();
+ if (s == null) {
+ return 0;
+ }
+ return s.getLongValue(id);
+ }
+
+ @Override
+ public double getDoubleValue(String id) {
+ Metric s = generateSum();
+ if (s == null) {
+ return 0.0;
+ }
+ return generateSum().getDoubleValue(id);
+ }
+
+ @Override
+ public void logEvent(EventLogger logger, String fullName) {
+ Metric s = generateSum();
+ if (s != null) {
+ s.logEvent(logger, fullName);
+ }
+ }
+
+ @Override
+ public void printXml(XMLWriter writer, int secondsPassed, int verbosity) {
+ Metric s = generateSum();
+ if (s == null) {
+ openXMLTag(writer, verbosity);
+ writer.closeTag();
+ return;
+ }
+
+ generateSum().printXml(writer, secondsPassed, verbosity);
+ }
+
+ @Override
+ public boolean used() {
+ for (Metric m : metricsToSum) {
+ if (m.used()) return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void reset() {}
+} \ No newline at end of file
diff --git a/metrics/src/main/java/com/yahoo/metrics/Timer.java b/metrics/src/main/java/com/yahoo/metrics/Timer.java
new file mode 100644
index 00000000000..12258ebfeb1
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/Timer.java
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+/**
+* @author thomasg
+*/
+class Timer {
+ int secs() { return (int)(milliSecs() / 1000); }
+ long milliSecs() { return System.currentTimeMillis(); }
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/ValueMetric.java b/metrics/src/main/java/com/yahoo/metrics/ValueMetric.java
new file mode 100644
index 00000000000..42cea298439
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/ValueMetric.java
@@ -0,0 +1,261 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.metrics.util.*;
+import com.yahoo.text.XMLWriter;
+import com.yahoo.text.Utf8String;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.logging.Logger;
+
+/**
+ * A metric that represents the value of something.
+ */
+public class ValueMetric<N extends Number>
+ extends Metric
+{
+ private static Logger log = Logger.getLogger(ValueMetric.class.getName());
+ private static AtomicBoolean hasWarnedOnNonFinite = new AtomicBoolean(false);
+
+ public static interface Value<N extends Number> extends ValueType, HasCopy<Value<N>> {
+ void add(N v);
+ void join(Value<N> v2, boolean createAverageOnJoin);
+ boolean overflow(Value<N> v2);
+
+ int getCount();
+ N getMin();
+ N getMax();
+ N getLast();
+ N getTotal();
+ Double getAverage();
+
+ String valueToString(N value);
+ }
+
+ public boolean hasFlag(int flag) { return (flags & flag) != 0; }
+ public ValueMetric<N> setFlag(int flag) { flags |= flag; return this; }
+
+ MetricValueKeeper<Value<N>> values;
+ int flags = 0;
+
+ static int AVERAGE_METRIC = 1;
+ static int CREATE_AVERAGE_ON_JOIN = 2;
+ static int UNSET_ON_ZERO_VALUE = 4;
+ static int LOG_IF_UNSET = 8;
+
+ boolean isAverageMetric() { return hasFlag(AVERAGE_METRIC); }
+ boolean doCreateAverageOnJoin() { return hasFlag(CREATE_AVERAGE_ON_JOIN); }
+ boolean isUnsetOnZeroValue() { return hasFlag(UNSET_ON_ZERO_VALUE); }
+ boolean doLogIfUnset() { return hasFlag(LOG_IF_UNSET); }
+ JoinBehavior getJoinBehavior() {
+ return hasFlag(CREATE_AVERAGE_ON_JOIN) ? JoinBehavior.AVERAGE_ON_JOIN
+ : JoinBehavior.SUM_ON_JOIN;
+ }
+
+ public ValueMetric<N> averageMetric() { return setFlag(AVERAGE_METRIC); }
+ public ValueMetric<N> createAverageOnJoin() { return setFlag(CREATE_AVERAGE_ON_JOIN); }
+ public ValueMetric<N> unsetOnZeroValue() { return setFlag(UNSET_ON_ZERO_VALUE); }
+ public ValueMetric<N> logIfUnset() { return setFlag(LOG_IF_UNSET); }
+
+ public ValueMetric(String name, String tags, String description, MetricSet owner) {
+ super(name, tags, description, owner);
+ //values = new MetricValueSet();
+ values = new ThreadLocalDirectoryValueKeeper<Value<N>>();
+ }
+
+ public ValueMetric(ValueMetric<N> other, CopyType copyType, MetricSet owner) {
+ super(other, owner);
+ if (copyType == CopyType.INACTIVE || other.values instanceof SimpleMetricValueKeeper) {
+ values = new SimpleMetricValueKeeper<Value<N>>();
+ values.set(other.values.get(getJoinBehavior()));
+ } else {
+ //values = new MetricValueSet((MetricValueSet) other.values, ((MetricValueSet) other.values).size());
+ values = new ThreadLocalDirectoryValueKeeper<Value<N>>(other.values);
+ }
+ this.flags = other.flags;
+ }
+
+ private void logNonFiniteValueWarning() {
+ if (!hasWarnedOnNonFinite.getAndSet(true)) {
+ log.log(LogLevel.WARNING,
+ "Metric '" + getPath() + "' attempted updated with a value that is NaN or " +
+ "Infinity; update ignored! No further warnings will be printed for " +
+ "such updates on any metrics, but they can be observed with debug " +
+ "logging enabled on component " + log.getName());
+ } else if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG,
+ "Metric '" + getPath() + "' attempted updated with a value that is NaN or " +
+ "Infinity; update ignored!");
+ }
+ }
+
+ public void addValue(N v) {
+ if (v == null) throw new NullPointerException("Cannot have null value");
+ if (v instanceof Long) {
+ LongValue lv = new LongValue();
+ lv.add((Long) v);
+ values.add((Value<N>) lv);
+ } else {
+ Double d = (Double)v;
+ if (d.isNaN() || d.isInfinite()) {
+ logNonFiniteValueWarning();
+ return;
+ }
+ DoubleValue dv = new DoubleValue();
+ dv.add(d);
+ values.add((Value<N>) dv);
+ }
+ }
+
+ private Value<N> getValues() {
+ return (Value<N>) values.get(getJoinBehavior());
+ }
+
+ @Override
+ public void addToSnapshot(Metric snapshotMetric) {
+ Value<N> v = getValues();
+ if (v != null) ((ValueMetric<N>)snapshotMetric).join(v, true);
+ }
+
+ @Override
+ void addToPart(Metric m) {
+ Value<N> v = getValues();
+ if (v != null) ((ValueMetric<N>) m).join(v, doCreateAverageOnJoin());
+ }
+
+ public void join(Value<N> v2, boolean createAverageOnJoin) {
+ Value<N> tmpVals = getValues();
+ if (tmpVals == null) {
+ if (v2 instanceof LongValue) {
+ tmpVals = (Value<N>) new LongValue();
+ } else {
+ tmpVals = (Value<N>) new DoubleValue();
+ }
+ }
+ if (tmpVals.overflow(v2)) {
+ this.values.reset();
+ log.fine("Metric " + getPath() + " overflowed, resetting it.");
+ return;
+ }
+ if (tmpVals.getCount() == 0) {
+ tmpVals = v2;
+ } else if (v2.getCount() == 0) {
+ // Do nothing
+ } else {
+ tmpVals.join(v2, createAverageOnJoin);
+ }
+ this.values.set(tmpVals);
+ }
+
+ public static final Utf8String TAG_AVG = new Utf8String("average");
+ public static final Utf8String TAG_LAST = new Utf8String("last");
+ public static final Utf8String TAG_MIN = new Utf8String("min");
+ public static final Utf8String TAG_MAX = new Utf8String("max");
+ public static final Utf8String TAG_CNT = new Utf8String("count");
+ public static final Utf8String TAG_TOT = new Utf8String("total");
+
+ @Override
+ public void printXml(XMLWriter writer,
+ int timePassed,
+ int verbosity)
+ {
+ Value<N> val = getValues();
+ if (!inUse(val) && verbosity < 2) {
+ return;
+ }
+ if (val == null) val = (Value<N>) new LongValue();
+
+ openXMLTag(writer, verbosity);
+ writer.attribute(TAG_AVG, new DoubleValue().valueToString(val.getAverage()));
+ writer.attribute(TAG_LAST, val.valueToString(val.getLast()));
+
+ if (val.getCount() > 0) {
+ writer.attribute(TAG_MIN, val.valueToString(val.getMin()));
+ writer.attribute(TAG_MAX, val.valueToString(val.getMax()));
+ }
+ writer.attribute(TAG_CNT, val.getCount());
+ if (verbosity >= 2) {
+ writer.attribute(TAG_TOT, val.valueToString(val.getTotal()));
+ }
+
+ writer.closeTag();
+ }
+
+ public Number getValue(String id) {
+ Value<N> val = getValues();
+ if (val == null) return 0;
+
+ if (id.equals("last") || (!isAverageMetric() && id.equals("value"))) {
+ return val.getLast();
+ } else if (id.equals("average") || (isAverageMetric() && id.equals("value"))) {
+ return val.getAverage();
+ } else if (id.equals("count")) {
+ return val.getCount();
+ } else if (id.equals("total")) {
+ return val.getTotal();
+ } else if (id.equals("min")) {
+ return val.getMin();
+ } else if (id.equals("max")) {
+ return val.getMax();
+ } else {
+ throw new IllegalArgumentException("No id " + id + " in value metric " + getName());
+ }
+ }
+
+ @Override
+ public long getLongValue(String id) {
+ return getValue(id).longValue();
+ }
+
+ @Override
+ public double getDoubleValue(String id) {
+ return getValue(id).doubleValue();
+ }
+
+ @Override
+ public ValueMetric<N> clone(CopyType type, MetricSet owner, boolean includeUnused) {
+ return new ValueMetric<N>(this, type, owner);
+ }
+
+ @Override
+ public void reset() { values.reset(); }
+
+ @Override
+ public void logEvent(EventLogger logger, String fullName) {
+ Value<N> val = getValues();
+ if (!doLogIfUnset() && !inUse(val)) {
+ return;
+ }
+ logger.value(fullName, val == null ? 0
+ : isAverageMetric() ? val.getAverage().doubleValue()
+ : val.getLast().doubleValue());
+ }
+
+ public boolean inUse(Value<?> value) {
+ return (value != null
+ && (value.getTotal().longValue() != 0
+ || (value.getCount() != 0 && !isUnsetOnZeroValue())));
+ }
+
+ @Override
+ public boolean used() {
+ return inUse(getValues());
+ }
+
+ @Override
+ public String toString() {
+ Value<N> tmpVals = getValues();
+ if (tmpVals == null) {
+ tmpVals = (Value<N>) new LongValue();
+ }
+ return ("count=\"" + tmpVals.getCount() +
+ "\" min=\"" + tmpVals.valueToString(tmpVals.getMin()) +
+ "\" max=\"" + tmpVals.valueToString(tmpVals.getMax()) +
+ "\" last=\""+ tmpVals.valueToString(tmpVals.getLast()) +
+ "\" total=\"" + tmpVals.valueToString(tmpVals.getTotal()) +
+ "\" average=\"" + new DoubleValue().valueToString(tmpVals.getAverage()) +
+ "\"");
+ }
+
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/VespaLogEventLogger.java b/metrics/src/main/java/com/yahoo/metrics/VespaLogEventLogger.java
new file mode 100644
index 00000000000..aa2698c92fb
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/VespaLogEventLogger.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import com.yahoo.log.event.Event;
+
+/**
+ * @author thomasg
+ */
+public class VespaLogEventLogger implements EventLogger {
+ public void value(String name, double value) {
+ Event.value(name, value);
+ }
+
+ public void count(String name, long value) {
+ Event.count(name, value);
+ }
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/util/HasCopy.java b/metrics/src/main/java/com/yahoo/metrics/util/HasCopy.java
new file mode 100644
index 00000000000..497f640e0fe
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/util/HasCopy.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics.util;
+
+public interface HasCopy<T> {
+ public T copyObject();
+}
+
diff --git a/metrics/src/main/java/com/yahoo/metrics/util/MetricValueKeeper.java b/metrics/src/main/java/com/yahoo/metrics/util/MetricValueKeeper.java
new file mode 100644
index 00000000000..65bd5abdd3e
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/util/MetricValueKeeper.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics.util;
+
+import com.yahoo.metrics.JoinBehavior;
+import java.util.Collection;
+
+/**
+ *
+ */
+public interface MetricValueKeeper<Value> {
+ /** Add a value to a given value. Used when metrics are written to by metric writer. */
+ public void add(Value v);
+ /** Set the value to exactly this, forgetting old value. This operation needs only be supported by value keepers used in inactive snapshots. */
+ public void set(Value v);
+ /** Get the value of the metrics written. If multiple values exist, use given join behavior. */
+ public Value get(JoinBehavior joinBehavior);
+ /** Reset the value of the metrics to zero. */
+ public void reset();
+ /** Return copy of current view */
+ public Collection<Value> getDirectoryView();
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/util/MetricValueSet.java b/metrics/src/main/java/com/yahoo/metrics/util/MetricValueSet.java
new file mode 100644
index 00000000000..e04c4b4eb25
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/util/MetricValueSet.java
@@ -0,0 +1,141 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class MetricValueSet
+ * \ingroup metrics
+ *
+ * \brief Utility for doing lockless metric updates and reads.
+ *
+ * We don't want to use regular locking while updating metrics due to overhead.
+ * We use this class to make metric updates as safe as possible without
+ * requiring locks.
+ *
+ * It keeps the set of values a metric wants to set. Thus it is templated on
+ * the class keeping the values. All that is required of this class is that it
+ * has an empty constructor and a copy constructor.
+ *
+ * The locking works, by keeping a set of values, with an active pointer into
+ * the value vector. Assuming only one thread calls setValues(), it can update
+ * the active pointer safely. We assume updating the active pointer is a
+ * non-interruptable operations, such that other threads will see either the new
+ * or the old value correctly. This should be the case on our platforms.
+ *
+ * Due to the reset functionality, it is possible to miss out on a metrics
+ * added during the reset, but this is very unlikely. For that to happen, when
+ * someone sets the reset flag, the writer thread must be in setValues(),
+ * having already passed the check for the reset flag, but not finished setting
+ * the values yet.
+ */
+
+package com.yahoo.metrics.util;
+
+import java.util.Collection;
+import com.yahoo.metrics.JoinBehavior;
+
+public class MetricValueSet<Value extends HasCopy<Value> & ValueType>
+ implements MetricValueKeeper<Value>
+{
+ private final ValueType[] values;
+
+ private void setIndexedValue(int idx, Value v) {
+ values[idx] = v;
+ }
+
+ @SuppressWarnings("unchecked")
+ private Value getIndexedValue(int idx) {
+ return (Value) values[idx];
+ }
+
+
+ private volatile int activeValueIndex;
+ private volatile boolean reset = false;
+
+ public MetricValueSet() {
+ values = new ValueType[3];
+ activeValueIndex = 0;
+ }
+
+ public MetricValueSet(MetricValueSet<Value> other, int copyCount) {
+ values = new ValueType[copyCount];
+ activeValueIndex = 0;
+ setValue(other.getValue());
+ }
+
+ /** Get the current values. */
+ public Value getValue() {
+ if (reset) return null;
+ Value v = getIndexedValue(activeValueIndex);
+ return (v == null ? null : v.copyObject());
+ }
+
+ /**
+ * Get the current values from the metric. This function should not be
+ * called in parallel. Only call it from a single thread or use external
+ * locking. If it returns false, it means the metric have just been reset.
+ * In which case, redo getValues(), apply the update again, and call
+ * setValues() again.
+ */
+ public boolean setValue(Value value) {
+ int nextIndex = (activeValueIndex + 1) % values.length;
+ if (reset) {
+ reset = false;
+ setIndexedValue(nextIndex, null);
+ activeValueIndex = nextIndex;
+ return false;
+ } else {
+ setIndexedValue(nextIndex, value);
+ activeValueIndex = nextIndex;
+ return true;
+ }
+ }
+
+ /**
+ * Retrieve and reset in a single operation, to minimize chance of
+ * alteration in the process.
+ */
+ public Value getValuesAndReset() {
+ Value result = getValue();
+ reset = true;
+ return result;
+ }
+
+ public String toString() {
+ String retVal = "MetricValueSet(reset=" + reset
+ + ", active " + activeValueIndex + "[";
+ for (ValueType n : values) retVal += n + ",";
+ retVal += "])";
+ return retVal;
+ }
+
+ public int size() { return values.length; }
+
+ public void add(Value v) {
+ while (true) {
+ Value current = getValue();
+ if (current != null) {
+ current.add(v);
+ if (setValue(current)) {
+ return;
+ }
+ } else {
+ if (setValue(v)) {
+ return;
+ }
+ }
+ }
+ }
+
+ public void set(Value v) {
+ setValue(v);
+ }
+
+ public Value get(JoinBehavior joinBehavior) {
+ return getValue();
+ }
+
+ public void reset() { reset = true; }
+
+ public Collection<Value> getDirectoryView() {
+ return null;
+ }
+
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/util/SimpleMetricValueKeeper.java b/metrics/src/main/java/com/yahoo/metrics/util/SimpleMetricValueKeeper.java
new file mode 100644
index 00000000000..f3a3e64beb6
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/util/SimpleMetricValueKeeper.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics.util;
+
+import java.util.Collection;
+import com.yahoo.metrics.JoinBehavior;
+
+/**
+ * Simple implementation of the metric value keeper for use with snapshots. Only keeps
+ * one instance of data, does not need to worry about threads, and should have a small
+ * memory footprint.
+ */
+public class SimpleMetricValueKeeper<Value> implements MetricValueKeeper<Value> {
+ private Value value;
+
+ public SimpleMetricValueKeeper() {
+ }
+
+ public void add(Value v) {
+ throw new UnsupportedOperationException("Not supported. This value keeper is not intended to be used for live metrics.");
+ }
+
+ public void set(Value v) {
+ this.value = v;
+ }
+
+ public Value get(JoinBehavior joinBehavior) {
+ return value;
+ }
+
+ public void reset() {
+ value = null;
+ }
+
+ public Collection<Value> getDirectoryView() {
+ return null;
+ }
+
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/util/ThreadLocalDirectoryValueKeeper.java b/metrics/src/main/java/com/yahoo/metrics/util/ThreadLocalDirectoryValueKeeper.java
new file mode 100644
index 00000000000..72dcef377d0
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/util/ThreadLocalDirectoryValueKeeper.java
@@ -0,0 +1,76 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics.util;
+
+import com.yahoo.concurrent.ThreadLocalDirectory;
+import com.yahoo.metrics.DoubleValue;
+import com.yahoo.metrics.LongValue;
+import com.yahoo.metrics.JoinBehavior;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Value keeper class using ThreadLocalDirectory to maintain thread safety from multiple threads writing.
+ */
+public class ThreadLocalDirectoryValueKeeper<Value extends ValueType & HasCopy<Value>>
+ implements MetricValueKeeper<Value>, ThreadLocalDirectory.ObservableUpdater<Value, Value>
+{
+ private final ThreadLocalDirectory<Value, Value> directory = new ThreadLocalDirectory<Value, Value>(this);
+ private List<Value> initialValues = null;
+
+ public ThreadLocalDirectoryValueKeeper() {
+ }
+
+ public ThreadLocalDirectoryValueKeeper(MetricValueKeeper<Value> other) {
+ initialValues = new LinkedList<Value>();
+ for (Value v : other.getDirectoryView()) initialValues.add(v.copyObject());
+ }
+
+ public Value createGenerationInstance(Value value) {
+ return null;
+ }
+
+ public Value update(Value current, Value newValue) {
+ if (current == null) {
+ if (newValue instanceof DoubleValue) {
+ current = (Value) new DoubleValue();
+ } else {
+ current = (Value) new LongValue();
+ }
+ }
+ current.add(newValue);
+ return current;
+ }
+
+ public Value copy(Value value) {
+ return value.copyObject();
+ }
+
+ public void add(Value v) {
+ directory.update(v, directory.getLocalInstance());
+ }
+
+ public void set(Value v) {
+ throw new UnsupportedOperationException("This value keeper is only intended to use with active metrics. Set operation not supported.");
+ }
+
+ public Value get(JoinBehavior joinBehavior) {
+ List<Value> values = directory.view();
+ if (initialValues != null) values.addAll(initialValues);
+ if (values.isEmpty()) {
+ return null;
+ }
+ return (Value) values.get(0).join(Collections.<ValueType>unmodifiableList(values), joinBehavior);
+ }
+
+ public void reset() {
+ directory.fetch();
+ initialValues = null;
+ }
+
+ public Collection<Value> getDirectoryView() {
+ return directory.view();
+ }
+}
diff --git a/metrics/src/main/java/com/yahoo/metrics/util/ValueType.java b/metrics/src/main/java/com/yahoo/metrics/util/ValueType.java
new file mode 100644
index 00000000000..b777e213406
--- /dev/null
+++ b/metrics/src/main/java/com/yahoo/metrics/util/ValueType.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics.util;
+
+import com.yahoo.metrics.JoinBehavior;
+
+import java.util.Collection;
+
+public interface ValueType extends Cloneable {
+ public void add(ValueType other);
+ public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior);
+}
diff --git a/metrics/src/main/metrics-with-dependencies.xml b/metrics/src/main/metrics-with-dependencies.xml
new file mode 100644
index 00000000000..f16f31e0398
--- /dev/null
+++ b/metrics/src/main/metrics-with-dependencies.xml
@@ -0,0 +1,19 @@
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<assembly>
+ <id>jar-with-dependencies</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <unpack>true</unpack>
+ <scope>runtime</scope>
+ </dependencySet>
+ </dependencySets>
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.outputDirectory}</directory>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/metrics/src/test/java/com/yahoo/metrics/AveragedDoubleValueMetric.java b/metrics/src/test/java/com/yahoo/metrics/AveragedDoubleValueMetric.java
new file mode 100644
index 00000000000..933f1ec5fa0
--- /dev/null
+++ b/metrics/src/test/java/com/yahoo/metrics/AveragedDoubleValueMetric.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+class AveragedDoubleValueMetric extends ValueMetric<Double> {
+
+ public AveragedDoubleValueMetric(String name, String tags, String description, MetricSet owner) {
+ super(name, tags, description, owner);
+ averageMetric();
+ createAverageOnJoin();
+ }
+
+ public AveragedDoubleValueMetric(AveragedDoubleValueMetric other, CopyType copyType, MetricSet owner) {
+ super(other, copyType, owner);
+ }
+}
diff --git a/metrics/src/test/java/com/yahoo/metrics/AveragedLongValueMetric.java b/metrics/src/test/java/com/yahoo/metrics/AveragedLongValueMetric.java
new file mode 100644
index 00000000000..a1abfced548
--- /dev/null
+++ b/metrics/src/test/java/com/yahoo/metrics/AveragedLongValueMetric.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+class AveragedLongValueMetric extends ValueMetric<Long> {
+
+ public AveragedLongValueMetric(String name, String tags, String description, MetricSet owner) {
+ super(name, tags, description, owner);
+ averageMetric();
+ createAverageOnJoin();
+ }
+
+ public AveragedLongValueMetric(AveragedLongValueMetric other, CopyType copyType, MetricSet owner) {
+ super(other, copyType, owner);
+ }
+}
diff --git a/metrics/src/test/java/com/yahoo/metrics/CountMetricTest.java b/metrics/src/test/java/com/yahoo/metrics/CountMetricTest.java
new file mode 100644
index 00000000000..ad09e289b0f
--- /dev/null
+++ b/metrics/src/test/java/com/yahoo/metrics/CountMetricTest.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import junit.framework.TestCase;
+
+/**
+ * @author thomasg
+ */
+public class CountMetricTest extends TestCase {
+
+ public void testCountMetric() {
+ CountMetric m = new CountMetric("test", "tag", "description", null);
+ assertEquals(false, m.used());
+ m.set(100);
+ assertEquals(true, m.used());
+ assertEquals(100, m.getValue());
+ m.inc(5);
+ assertEquals(105, m.getValue());
+ m.dec(15);
+ assertEquals(90, m.getValue());
+
+ CountMetric m2 = new CountMetric(m, Metric.CopyType.CLONE, null);
+ m.reset();
+ assertEquals(90, m2.getValue());
+ assertEquals(0, m.getValue());
+
+ CountMetric n = new CountMetric("m2", "", "desc", null);
+ n.set(6);
+
+ n.addToSnapshot(m2);
+ assertEquals(96, m2.getValue());
+ n.addToPart(m);
+ assertEquals(6, m.getValue());
+
+ assertEquals(6, n.getValue());
+
+ assertEquals("<test description=\"description\" count=\"96\"/>\n", m2.toXml(0, 2));
+ assertEquals("<test description=\"description\" count=\"96\" average_change_per_second=\"9.60\"/>\n", m2.toXml(10, 2));
+ assertEquals(96.0, m2.getDoubleValue("value"));
+ assertEquals(96, m2.getLongValue("value"));
+ }
+
+}
diff --git a/metrics/src/test/java/com/yahoo/metrics/DummyTimer.java b/metrics/src/test/java/com/yahoo/metrics/DummyTimer.java
new file mode 100644
index 00000000000..8264d8e00eb
--- /dev/null
+++ b/metrics/src/test/java/com/yahoo/metrics/DummyTimer.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+/**
+ * @author thomasg
+ */
+public class DummyTimer extends Timer {
+ long ms = 0;
+
+ public long milliSecs() { return ms; }
+}
diff --git a/metrics/src/test/java/com/yahoo/metrics/MetricManagerTest.java b/metrics/src/test/java/com/yahoo/metrics/MetricManagerTest.java
new file mode 100644
index 00000000000..dad476d2bdb
--- /dev/null
+++ b/metrics/src/test/java/com/yahoo/metrics/MetricManagerTest.java
@@ -0,0 +1,192 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import org.junit.Test;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author thomasg
+ */
+public class MetricManagerTest {
+
+ @Test
+ public void requireThatManagerCanBeStopped() throws InterruptedException {
+ final MetricManager manager = new MetricManager(new DummyTimer());
+ Thread managerThread = new Thread(manager);
+ managerThread.start();
+ Thread stopThread = new Thread(new Runnable() {
+
+ @Override
+ public void run() {
+ manager.stop();
+ }
+ });
+ stopThread.start();
+ stopThread.join(TimeUnit.SECONDS.toMillis(60));
+ assertFalse(stopThread.isAlive());
+ managerThread.join(TimeUnit.SECONDS.toMillis(60));
+ assertFalse(managerThread.isAlive());
+ }
+
+ @Test
+ public void requireThatManagerCanBeStoppedBeforeStarting() throws InterruptedException {
+ final MetricManager manager = new MetricManager(new DummyTimer());
+ Thread thread = new Thread(new Runnable() {
+
+ @Override
+ public void run() {
+ manager.stop();
+ }
+ });
+ thread.start();
+ thread.join(TimeUnit.SECONDS.toMillis(60));
+ assertFalse(thread.isAlive());
+ }
+
+ @Test
+ public void requireThatManagerCanNotBeStartedAfterItHasBeenStopped() {
+ final MetricManager manager = new MetricManager(new DummyTimer());
+ manager.stop();
+ try {
+ manager.run();
+ fail();
+ } catch (IllegalStateException e) {
+
+ }
+ }
+
+ @Test
+ public void testSimpleLogging() {
+ DummyTimer timer = new DummyTimer();
+ MetricManager manager = new MetricManager(timer);
+ SimpleMetricSet sms = new SimpleMetricSet("foo", "", "", null);
+ manager.registerMetric(sms);
+
+ CountMetric bar = new CountMetric("bar", "", "", sms);
+ bar.inc(3);
+
+ manager.addMetricToConsumer("log", "foo.bar");
+
+ DummyEventLogger logger = new DummyEventLogger();
+ manager.tick(logger);
+ //assertEquals("CNT foo_bar: 0\n", logger.output);
+
+ timer.ms = 300000;
+
+ manager.tick(logger);
+ assertEquals("CNT foo_bar: 0\nCNT foo_bar: 3\n", logger.output.toString());
+ }
+
+ @Test
+ public void testSumLogging() {
+ DummyTimer timer = new DummyTimer();
+ MetricManager manager = new MetricManager(timer);
+ SimpleMetricSet sms = new SimpleMetricSet("foo", "", "", null);
+ manager.registerMetric(sms);
+
+ CountMetric v1 = new CountMetric("v1", "", "", sms);
+ CountMetric v2 = new CountMetric("v2", "", "", sms);
+ CountMetric v3 = new CountMetric("v3", "", "", sms);
+ SumMetric sum = new SumMetric("sum", "", "", sms);
+
+ sum.addMetricToSum(v1);
+ sum.addMetricToSum(v2);
+ sum.addMetricToSum(v3);
+
+ v1.inc(3);
+ v2.inc(1);
+ v3.inc(2);
+
+ manager.addMetricToConsumer("log", "foo.sum");
+
+ DummyEventLogger logger = new DummyEventLogger();
+ manager.tick(logger);
+ assertEquals("CNT foo_sum: 0\n", logger.output.toString());
+
+ timer.ms = 300000;
+
+ manager.tick(logger);
+ assertEquals("CNT foo_sum: 0\nCNT foo_sum: 6\n", logger.output.toString());
+ }
+
+ @Test
+ public void testSumAndSetLogging() {
+ DummyTimer timer = new DummyTimer();
+ timer.ms = 1000000;
+ MetricManager manager = new MetricManager(timer);
+ SimpleMetricSet sms = new SimpleMetricSet("foo", "", "", null);
+ manager.registerMetric(sms);
+
+ SimpleMetricSet v1 = new SimpleMetricSet("v1", "", "", sms);
+ SimpleMetricSet v2 = new SimpleMetricSet("v2", "", "", sms);
+
+ CountMetric v1_a = new CountMetric("a", "", "", v1);
+ SummedLongValueMetric v1_b = new SummedLongValueMetric("b", "", "", v1);
+ AveragedLongValueMetric v1_c = new AveragedLongValueMetric("c", "", "", v1);
+
+ CountMetric v2_a = new CountMetric("a", "", "", v2);
+ SummedLongValueMetric v2_b = new SummedLongValueMetric("b", "", "", v2);
+ AveragedLongValueMetric v2_c = new AveragedLongValueMetric("c", "", "", v2);
+
+ SumMetric sum = new SumMetric("sum", "", "", sms);
+
+ sum.addMetricToSum(v1);
+ sum.addMetricToSum(v2);
+
+ v1_a.inc(3);
+ v1_b.addValue((long)1);
+ v1_c.addValue((long)4);
+ v2_a.inc(2);
+ v2_b.addValue((long)2);
+ v2_c.addValue((long)8);
+
+ manager.addMetricToConsumer("log", "foo.sum.a");
+ manager.addMetricToConsumer("log", "foo.sum.b");
+ manager.addMetricToConsumer("log", "foo.sum.c");
+
+ {
+ DummyEventLogger logger = new DummyEventLogger();
+ assertEquals(300000, manager.tick(logger));
+ assertEquals("CNT foo_sum_a: 0\n", logger.output.toString());
+ }
+
+ timer.ms = 1300000;
+
+ {
+ DummyEventLogger logger = new DummyEventLogger();
+ assertEquals(300000, manager.tick(logger));
+ assertEquals("CNT foo_sum_a: 5\nVAL foo_sum_b: 3.0\nVAL foo_sum_c: 6.0\n", logger.output.toString());
+ }
+
+ v1_a.inc(4);
+ v1_b.addValue((long)2);
+ v1_b.addValue((long)6);
+ v2_b.addValue((long)4);
+
+ timer.ms = 1600000;
+
+ {
+ DummyEventLogger logger = new DummyEventLogger();
+ assertEquals(300000, manager.tick(logger));
+ assertEquals("CNT foo_sum_a: 9\nVAL foo_sum_b: 10.0\n", logger.output.toString());
+ }
+ }
+
+ private static class DummyEventLogger implements EventLogger {
+
+ StringBuilder output = new StringBuilder();
+
+ @Override
+ public void value(String name, double value) {
+ output.append("VAL " + name + ": " + value + "\n");
+ }
+
+ @Override
+ public void count(String name, long value) {
+ output.append("CNT " + name + ": " + value + "\n");
+ }
+ }
+}
diff --git a/metrics/src/test/java/com/yahoo/metrics/MetricSetTest.java b/metrics/src/test/java/com/yahoo/metrics/MetricSetTest.java
new file mode 100644
index 00000000000..c9b85566cea
--- /dev/null
+++ b/metrics/src/test/java/com/yahoo/metrics/MetricSetTest.java
@@ -0,0 +1,338 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import junit.framework.TestCase;
+
+
+public class MetricSetTest extends TestCase {
+
+ class TestMetricVisitor extends MetricVisitor {
+ String output = "";
+ int setsToVisit;
+
+ TestMetricVisitor(int setsToVisit) {
+ this.setsToVisit = setsToVisit;
+ }
+
+ public boolean visitMetricSet(MetricSet set, boolean autoGenerated) {
+ output += "[" + (autoGenerated ? "*" : "") + set.getName() + "]\n";
+ if (setsToVisit > 0) {
+ --setsToVisit;
+ return true;
+ }
+ return false;
+ }
+ public boolean visitPrimitiveMetric(Metric m, boolean autoGenerated) {
+ output += (autoGenerated ? "*" : "") + m.getName() + "\n";
+ return true;
+ }
+ }
+
+ public void testNormalUsage()
+ {
+ // Set up some metrics to test..
+ MetricSet set = new SimpleMetricSet("a", "foo", "");
+ SummedDoubleValueMetric v1 = new SummedDoubleValueMetric("c", "foo", "", set);
+ AveragedLongValueMetric v2 = new AveragedLongValueMetric("b", "", "", set);
+ CountMetric v3 = new CountMetric("d", "bar", "", set);
+ MetricSet set2 = new SimpleMetricSet("e", "bar", "", set);
+ CountMetric v4 = new CountMetric("f", "foo", "", set2);
+
+ // Give them some values
+ v1.addValue(4.2);
+ v2.addValue((long)8);
+ v3.inc();
+ v4.inc(3);
+
+ // Check that we can register through registerMetric function too.
+ CountMetric v5 = new CountMetric("g", "", "", null);
+ set.registerMetric(v5);
+ v5.inc(3);
+ v5.dec();
+
+ // Check that getMetric works, and doesn't return copy.
+ AveragedLongValueMetric v2copy = (AveragedLongValueMetric)set.getMetric("b");
+ assertNotNull(v2copy);
+ v2copy.addValue((long)9);
+
+ assertNull(set.getMetric("nonexisting"));
+ assertNull(set.getMetric("non.existing"));
+
+ // Check that paths are set
+ MetricSet topSet = new SimpleMetricSet("top", "", "", null);
+ topSet.registerMetric(set);
+
+ assertEquals("a", set.getPath());
+ assertEquals("a.c", v1.getPath());
+ assertEquals("a.b", v2.getPath());
+ assertEquals("a.d", v3.getPath());
+ assertEquals("a.e", set2.getPath());
+ assertEquals("a.e.f", v4.getPath());
+ assertEquals("a.g", v5.getPath());
+ assertEquals("a.b", v2copy.getPath());
+
+ // Verify XML output. Should be in register order.
+ assertEquals(
+ "<a>\n\n" +
+ " <c average=\"4.20\" last=\"4.20\" min=\"4.20\" max=\"4.20\" count=\"1\" total=\"4.20\"/>\n\n" +
+ " <b average=\"8.50\" last=\"9\" min=\"8\" max=\"9\" count=\"2\" total=\"17\"/>\n\n" +
+ " <d count=\"1\"/>\n\n" +
+ " <e>\n"+
+ " <f count=\"3\"/>\n" +
+ " </e>\n\n" +
+ " <g count=\"2\"/>\n\n"+
+ "</a>\n", set.toXml(0,2));
+
+ // Verify that visiting works. That you get all metrics if you answer
+ // true to all sets, and that you don't get members of sets you answer
+ // false to get.
+ {
+ TestMetricVisitor visitor = new TestMetricVisitor(2);
+ set.visit(visitor, false);
+ assertEquals("[a]\nc\nb\nd\n[e]\nf\ng\n", visitor.output);
+ }
+ {
+ TestMetricVisitor visitor = new TestMetricVisitor(1);
+ set.visit(visitor, false);
+ assertEquals("[a]\nc\nb\nd\n[e]\ng\n", visitor.output);
+ }
+ {
+ TestMetricVisitor visitor = new TestMetricVisitor(0);
+ set.visit(visitor, false);
+ assertEquals("[a]\n", visitor.output);
+ }
+ }
+
+ class MyTimer extends Timer {
+ long timeNow = 0;
+
+ @Override
+ long milliSecs() { return timeNow; }
+ }
+
+ @SuppressWarnings("rawtypes")
+ public void testSumInSet() {
+ MetricSet ms_a = new SimpleMetricSet("a", "", "", null);
+ MyTimer timer = new MyTimer();
+ timer.timeNow = 1;
+
+ MetricManager manager = new MetricManager(timer);
+ manager.registerMetric(ms_a);
+
+ manager.takeSnapshots(1);
+
+ SumMetric s_a = new SumMetric("sum", "", "", ms_a);
+ MetricSet ms_b = new SimpleMetricSet("b", "", "", ms_a);
+ s_a.addMetricToSum(ms_b);
+ MetricSet ms_c = new SimpleMetricSet("c", "", "", ms_a);
+ s_a.addMetricToSum(ms_c);
+
+ SumMetric s_b_a = new SumMetric("sum", "", "", ms_b);
+ SumMetric s_c_a = new SumMetric("sum", "", "", ms_c);
+
+ AveragedDoubleValueMetric v1_b = new AveragedDoubleValueMetric("v1", "", "", ms_b);
+ s_b_a.addMetricToSum(v1_b);
+ AveragedDoubleValueMetric v3_b = new AveragedDoubleValueMetric("v3", "", "", ms_b);
+ s_b_a.addMetricToSum(v3_b);
+
+ AveragedDoubleValueMetric v1_c = new AveragedDoubleValueMetric("v1", "", "", ms_c);
+ s_c_a.addMetricToSum(v1_c);
+ AveragedDoubleValueMetric v2_c = new AveragedDoubleValueMetric("v2", "", "", ms_c);
+ s_c_a.addMetricToSum(v2_c);
+
+ v1_b.addValue(1.0);
+ v3_b.addValue(2.0);
+ v1_c.addValue(3.0);
+ v2_c.addValue(4.0);
+
+ SimpleMetricSet sms = (SimpleMetricSet)s_a.generateSum();
+ ValueMetric av_sum = (ValueMetric)sms.getMetric("sum");
+ ValueMetric av_1 = (ValueMetric)sms.getMetric("v1");
+ ValueMetric av_2 = (ValueMetric)sms.getMetric("v2");
+ ValueMetric av_3 = (ValueMetric)sms.getMetric("v3");
+
+ assertEquals(10.0, av_sum.getDoubleValue("total"));
+ assertEquals(4, av_sum.getLongValue("count"));
+ assertEquals(4.0, av_1.getDoubleValue("total"));
+ assertEquals(4.0, av_2.getDoubleValue("total"));
+ assertEquals(2.0, av_3.getDoubleValue("total"));
+
+ timer.timeNow = 301;
+ manager.takeSnapshots(301);
+
+ MetricSnapshot snapshot = manager.getMetricSnapshot(300);
+
+ assertEquals("<snapshot name=\"5 minute\" from=\"301\" to=\"301\" period=\"300\">\n" +
+ "\n" +
+ " <a>\n" +
+ " <sum>\n" +
+ " <sum average=\"2.50\" last=\"4.00\" min=\"1.00\" max=\"4.00\" count=\"4\" total=\"10.00\"/>\n" +
+ " <v1 average=\"2.00\" last=\"3.00\" min=\"1.00\" max=\"3.00\" count=\"2\" total=\"4.00\"/>\n" +
+ " <v3 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" +
+ " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" +
+ " </sum>\n" +
+ " <b>\n" +
+ " <sum average=\"1.50\" last=\"2.00\" min=\"1.00\" max=\"2.00\" count=\"2\" total=\"3.00\"/>\n" +
+ " <v1 average=\"1.00\" last=\"1.00\" min=\"1.00\" max=\"1.00\" count=\"1\" total=\"1.00\"/>\n" +
+ " <v3 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" +
+ " </b>\n" +
+ " <c>\n" +
+ " <sum average=\"3.50\" last=\"4.00\" min=\"3.00\" max=\"4.00\" count=\"2\" total=\"7.00\"/>\n" +
+ " <v1 average=\"3.00\" last=\"3.00\" min=\"3.00\" max=\"3.00\" count=\"1\" total=\"3.00\"/>\n" +
+ " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" +
+ " </c>\n" +
+ " </a>\n" +
+ "\n" +
+ "</snapshot>\n", snapshot.toXml(manager, "", 2));
+
+ assertEquals("<snapshot name=\"1 hour\" from=\"0\" to=\"301\" period=\"3600\">\n" +
+ "\n" +
+ " <a>\n" +
+ " <sum>\n" +
+ " <sum average=\"2.50\" last=\"4.00\" min=\"1.00\" max=\"4.00\" count=\"4\" total=\"10.00\"/>\n" +
+ " <v1 average=\"2.00\" last=\"3.00\" min=\"1.00\" max=\"3.00\" count=\"2\" total=\"4.00\"/>\n" +
+ " <v3 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" +
+ " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" +
+ " </sum>\n" +
+ " <b>\n" +
+ " <sum average=\"1.50\" last=\"2.00\" min=\"1.00\" max=\"2.00\" count=\"2\" total=\"3.00\"/>\n" +
+ " <v1 average=\"1.00\" last=\"1.00\" min=\"1.00\" max=\"1.00\" count=\"1\" total=\"1.00\"/>\n" +
+ " <v3 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" +
+ " </b>\n" +
+ " <c>\n" +
+ " <sum average=\"3.50\" last=\"4.00\" min=\"3.00\" max=\"4.00\" count=\"2\" total=\"7.00\"/>\n" +
+ " <v1 average=\"3.00\" last=\"3.00\" min=\"3.00\" max=\"3.00\" count=\"1\" total=\"3.00\"/>\n" +
+ " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" +
+ " </c>\n" +
+ " </a>\n" +
+ "\n" +
+ "</snapshot>\n", manager.getMetricSnapshot(3600, true).toXml(manager, "", 2));
+
+ v1_b.addValue(2.0);
+ v3_b.addValue(3.0);
+ v1_c.addValue(4.0);
+ v2_c.addValue(8.0);
+
+ CountMetric c1_b = new CountMetric("cnt", "", "", ms_b);
+ c1_b.inc(500);
+
+ timer.timeNow = 601;
+ manager.takeSnapshots(601);
+
+ // THIS IS WRONG. See ticket
+ assertEquals("<snapshot name=\"5 minute\" from=\"601\" to=\"601\" period=\"300\">\n" +
+ "\n" +
+ " <a>\n" +
+ " <sum>\n" +
+ " <sum average=\"4.25\" last=\"8.00\" min=\"2.00\" max=\"8.00\" count=\"4\" total=\"17.00\"/>\n" +
+ " <v1 average=\"3.00\" last=\"4.00\" min=\"2.00\" max=\"4.00\" count=\"2\" total=\"6.00\"/>\n" +
+ " <v3 average=\"3.00\" last=\"3.00\" min=\"3.00\" max=\"3.00\" count=\"1\" total=\"3.00\"/>\n" +
+ " <v2 average=\"8.00\" last=\"8.00\" min=\"8.00\" max=\"8.00\" count=\"1\" total=\"8.00\"/>\n" +
+ " <cnt count=\"500\" average_change_per_second=\"1.67\"/>\n" +
+ " </sum>\n" +
+ " <b>\n" +
+ " <sum average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" +
+ " <v1 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" +
+ " <v3 average=\"3.00\" last=\"3.00\" min=\"3.00\" max=\"3.00\" count=\"1\" total=\"3.00\"/>\n" +
+ " <cnt count=\"500\" average_change_per_second=\"1.67\"/>\n" +
+ " </b>\n" +
+ " <c>\n" +
+ " <sum average=\"6.00\" last=\"8.00\" min=\"4.00\" max=\"8.00\" count=\"2\" total=\"12.00\"/>\n" +
+ " <v1 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" +
+ " <v2 average=\"8.00\" last=\"8.00\" min=\"8.00\" max=\"8.00\" count=\"1\" total=\"8.00\"/>\n" +
+ " </c>\n" +
+ " </a>\n" +
+ "\n" +
+ "</snapshot>\n", manager.getMetricSnapshot(300, false).toXml(manager, "", 2));
+
+ assertEquals("<snapshot name=\"1 hour\" from=\"0\" to=\"601\" period=\"3600\">\n" +
+ "\n" +
+ " <a>\n" +
+ " <sum>\n" +
+ " <sum average=\"3.38\" last=\"8.00\" min=\"1.00\" max=\"8.00\" count=\"8\" total=\"27.00\"/>\n" +
+ " <v1 average=\"2.50\" last=\"4.00\" min=\"1.00\" max=\"4.00\" count=\"4\" total=\"10.00\"/>\n" +
+ " <v3 average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" +
+ " <v2 average=\"6.00\" last=\"8.00\" min=\"4.00\" max=\"8.00\" count=\"2\" total=\"12.00\"/>\n" +
+ " <cnt count=\"500\" average_change_per_second=\"0.83\"/>\n" +
+ " </sum>\n" +
+ " <b>\n" +
+ " <sum average=\"2.00\" last=\"3.00\" min=\"1.00\" max=\"3.00\" count=\"4\" total=\"8.00\"/>\n" +
+ " <v1 average=\"1.50\" last=\"2.00\" min=\"1.00\" max=\"2.00\" count=\"2\" total=\"3.00\"/>\n" +
+ " <v3 average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" +
+ " <cnt count=\"500\" average_change_per_second=\"0.83\"/>\n" +
+ " </b>\n" +
+ " <c>\n" +
+ " <sum average=\"4.75\" last=\"8.00\" min=\"3.00\" max=\"8.00\" count=\"4\" total=\"19.00\"/>\n" +
+ " <v1 average=\"3.50\" last=\"4.00\" min=\"3.00\" max=\"4.00\" count=\"2\" total=\"7.00\"/>\n" +
+ " <v2 average=\"6.00\" last=\"8.00\" min=\"4.00\" max=\"8.00\" count=\"2\" total=\"12.00\"/>\n" +
+ " </c>\n" +
+ " </a>\n" +
+ "\n" +
+ "</snapshot>\n", manager.getMetricSnapshot(3600, true).toXml(manager, "", 2));
+
+ AveragedDoubleValueMetric v4_c = new AveragedDoubleValueMetric("v4", "", "", ms_c);
+ s_c_a.addMetricToSum(v4_c);
+
+ v2_c.addValue(4.0);
+ v4_c.addValue(12.0);
+ c1_b.inc(3000);
+
+ timer.timeNow = 901;
+ manager.takeSnapshots(901);
+
+ assertEquals("<snapshot name=\"5 minute\" from=\"901\" to=\"901\" period=\"300\">\n" +
+ "\n" +
+ " <a>\n" +
+ " <sum>\n" +
+ " <sum average=\"8.00\" last=\"12.00\" min=\"4.00\" max=\"12.00\" count=\"2\" total=\"16.00\"/>\n" +
+ " <v1 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" +
+ " <v3 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" +
+ " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" +
+ " <cnt count=\"3000\" average_change_per_second=\"10.00\"/>\n" +
+ " <v4 average=\"12.00\" last=\"12.00\" min=\"12.00\" max=\"12.00\" count=\"1\" total=\"12.00\"/>\n" +
+ " </sum>\n" +
+ " <b>\n" +
+ " <sum average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" +
+ " <v1 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" +
+ " <v3 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" +
+ " <cnt count=\"3000\" average_change_per_second=\"10.00\"/>\n" +
+ " </b>\n" +
+ " <c>\n" +
+ " <sum average=\"8.00\" last=\"12.00\" min=\"4.00\" max=\"12.00\" count=\"2\" total=\"16.00\"/>\n" +
+ " <v1 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" +
+ " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" +
+ " <v4 average=\"12.00\" last=\"12.00\" min=\"12.00\" max=\"12.00\" count=\"1\" total=\"12.00\"/>\n" +
+ " </c>\n" +
+ " </a>\n" +
+ "\n" +
+ "</snapshot>\n", manager.getMetricSnapshot(300, false).toXml(manager, "", 2));
+
+ assertEquals("<snapshot name=\"1 hour\" from=\"0\" to=\"901\" period=\"3600\">\n" +
+ "\n" +
+ " <a>\n" +
+ " <sum>\n" +
+ " <sum average=\"4.30\" last=\"12.00\" min=\"1.00\" max=\"12.00\" count=\"10\" total=\"43.00\"/>\n" +
+ " <v1 average=\"2.50\" last=\"4.00\" min=\"1.00\" max=\"4.00\" count=\"4\" total=\"10.00\"/>\n" +
+ " <v3 average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" +
+ " <v2 average=\"5.33\" last=\"4.00\" min=\"4.00\" max=\"8.00\" count=\"3\" total=\"16.00\"/>\n" +
+ " <cnt count=\"3500\" average_change_per_second=\"3.88\"/>\n" +
+ " <v4 average=\"12.00\" last=\"12.00\" min=\"12.00\" max=\"12.00\" count=\"1\" total=\"12.00\"/>\n" +
+ " </sum>\n" +
+ " <b>\n" +
+ " <sum average=\"2.00\" last=\"3.00\" min=\"1.00\" max=\"3.00\" count=\"4\" total=\"8.00\"/>\n" +
+ " <v1 average=\"1.50\" last=\"2.00\" min=\"1.00\" max=\"2.00\" count=\"2\" total=\"3.00\"/>\n" +
+ " <v3 average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" +
+ " <cnt count=\"3500\" average_change_per_second=\"3.88\"/>\n" +
+ " </b>\n" +
+ " <c>\n" +
+ " <sum average=\"5.83\" last=\"12.00\" min=\"3.00\" max=\"12.00\" count=\"6\" total=\"35.00\"/>\n" +
+ " <v1 average=\"3.50\" last=\"4.00\" min=\"3.00\" max=\"4.00\" count=\"2\" total=\"7.00\"/>\n" +
+ " <v2 average=\"5.33\" last=\"4.00\" min=\"4.00\" max=\"8.00\" count=\"3\" total=\"16.00\"/>\n" +
+ " <v4 average=\"12.00\" last=\"12.00\" min=\"12.00\" max=\"12.00\" count=\"1\" total=\"12.00\"/>\n" +
+ " </c>\n" +
+ " </a>\n" +
+ "\n" +
+ "</snapshot>\n", manager.getMetricSnapshot(3600, true).toXml(manager, "", 2));
+ }
+
+} // metrics
diff --git a/metrics/src/test/java/com/yahoo/metrics/SumMetricTest.java b/metrics/src/test/java/com/yahoo/metrics/SumMetricTest.java
new file mode 100644
index 00000000000..09204a66454
--- /dev/null
+++ b/metrics/src/test/java/com/yahoo/metrics/SumMetricTest.java
@@ -0,0 +1,153 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import junit.framework.TestCase;
+
+/**
+ * @author thomasg
+ */
+public class SumMetricTest extends TestCase {
+ public void testSummedCountMetric() {
+ MetricSet parent = new SimpleMetricSet("parent", "", "");
+ SumMetric sum = new SumMetric("foo", "", "foodesc", parent);
+
+ CountMetric v1 = new CountMetric("aa", "", "", parent);
+ CountMetric v2 = new CountMetric("bb", "", "", parent);
+ CountMetric v3 = new CountMetric("cc", "", "", parent);
+
+ sum.addMetricToSum(v1);
+ sum.addMetricToSum(v2);
+ sum.addMetricToSum(v3);
+
+ // Give them some values
+ v1.inc(3);
+ v2.inc(7);
+ v3.inc(5);
+
+ assertEquals("<foo description=\"foodesc\" count=\"15\"/>\n", sum.toXml(0, 2));
+ assertEquals(15, sum.getLongValue("value"));
+
+ v3.inc(5);
+
+ assertEquals("<foo description=\"foodesc\" count=\"20\"/>\n", sum.toXml(0, 2));
+ assertEquals(20, sum.getLongValue("value"));
+ }
+
+ public void testSummedValueMetric() {
+ MetricSet parent = new SimpleMetricSet("parent", "", "");
+ SumMetric sum = new SumMetric("foo", "", "foodesc", parent);
+
+ SummedDoubleValueMetric v1 = new SummedDoubleValueMetric("aa", "", "", parent);
+ SummedDoubleValueMetric v2 = new SummedDoubleValueMetric("bb", "", "", parent);
+ SummedDoubleValueMetric v3 = new SummedDoubleValueMetric("cc", "", "", parent);
+
+ sum.addMetricToSum(v1);
+ sum.addMetricToSum(v2);
+ sum.addMetricToSum(v3);
+
+ v1.addValue(3.0);
+ v1.addValue(2.0);
+
+ v2.addValue(7.0);
+
+ v3.addValue(5.0);
+ v3.addValue(10.0);
+
+ assertEquals("<foo description=\"foodesc\" average=\"17.00\" last=\"19.00\" min=\"2.00\" max=\"10.00\" count=\"5\" total=\"85.00\"/>\n", sum.toXml(0,2));
+
+ assertEquals(19, sum.getLongValue("value"));
+ assertEquals(2, sum.getLongValue("min"));
+ assertEquals(10, sum.getLongValue("max"));
+ }
+
+
+ public void testAveragedValueMetric() {
+ MetricSet parent = new SimpleMetricSet("parent", "", "");
+ SumMetric sum = new SumMetric("foo", "", "foodesc", parent);
+
+ AveragedDoubleValueMetric v1 = new AveragedDoubleValueMetric("aa", "", "", parent);
+ AveragedDoubleValueMetric v2 = new AveragedDoubleValueMetric("bb", "", "", parent);
+ AveragedDoubleValueMetric v3 = new AveragedDoubleValueMetric("cc", "", "", parent);
+
+ sum.addMetricToSum(v1);
+ sum.addMetricToSum(v2);
+ sum.addMetricToSum(v3);
+
+ v1.addValue(3.0);
+ v1.addValue(2.0);
+
+ v2.addValue(7.0);
+
+ v3.addValue(5.0);
+ v3.addValue(10.0);
+
+ assertEquals("<foo description=\"foodesc\" average=\"5.40\" last=\"10.00\" min=\"2.00\" max=\"10.00\" count=\"5\" total=\"27.00\"/>\n", sum.toXml(0,2));
+
+ assertEquals(5.40, sum.getDoubleValue("value"));
+ assertEquals(2.0, sum.getDoubleValue("min"));
+ assertEquals(10.0, sum.getDoubleValue("max"));
+ }
+
+ public void testMetricSet() {
+ MetricSet parent = new SimpleMetricSet("parent", "", "");
+ SumMetric sum = new SumMetric("foo", "", "bar", parent);
+
+ MetricSet set1 = new SimpleMetricSet("a", "", "", parent);
+ MetricSet set2 = new SimpleMetricSet("b", "", "", parent);
+
+ SummedLongValueMetric v1 = new SummedLongValueMetric("c", "", "", set1);
+ SummedLongValueMetric v2 = new SummedLongValueMetric("c", "", "", set2);
+ CountMetric v3 = new CountMetric("e", "", "", set1);
+ CountMetric v4 = new CountMetric("e", "", "", set2);
+
+ sum.addMetricToSum(set1);
+ sum.addMetricToSum(set2);
+
+ // Give them some values
+ v1.addValue((long)3);
+ v2.addValue((long)7);
+ v3.inc(2);
+ v4.inc();
+
+ // Verify XML output. Should be in register order.
+ assertEquals(
+ "<foo description=\"bar\">\n\n" +
+ " <c average=\"10.00\" last=\"10\" min=\"3\" max=\"7\" count=\"2\" total=\"20\"/>\n\n" +
+ " <e count=\"3\"/>\n\n" +
+ "</foo>\n", sum.toXml(0,2));
+
+ }
+
+
+ public void testRemove()
+ {
+ MetricSet parent = new SimpleMetricSet("parent", "", "");
+ SumMetric sum = new SumMetric("foo", "", "foodesc", parent);
+
+ SummedDoubleValueMetric v1 = new SummedDoubleValueMetric("aa", "", "", parent);
+ SummedDoubleValueMetric v2 = new SummedDoubleValueMetric("bb", "", "", parent);
+ SummedDoubleValueMetric v3 = new SummedDoubleValueMetric("cc", "", "", parent);
+
+ sum.addMetricToSum(v1);
+ sum.addMetricToSum(v2);
+ sum.addMetricToSum(v3);
+
+ v1.addValue(3.0);
+ v1.addValue(2.0);
+
+ v2.addValue(7.0);
+
+ v3.addValue(5.0);
+ v3.addValue(10.0);
+
+ sum.removeMetricFromSum(v1);
+
+ assertEquals("<foo description=\"foodesc\" average=\"14.50\" last=\"17.00\" min=\"5.00\" max=\"10.00\" count=\"3\" total=\"43.50\"/>\n", sum.toXml(0,2));
+ }
+
+ public void testEmpty() {
+ SumMetric sum = new SumMetric("foo", "", "foodesc", null);
+ assertEquals("<foo description=\"foodesc\"/>\n", sum.toXml(0,2));
+ }
+
+}
diff --git a/metrics/src/test/java/com/yahoo/metrics/SummedDoubleValueMetric.java b/metrics/src/test/java/com/yahoo/metrics/SummedDoubleValueMetric.java
new file mode 100644
index 00000000000..a869348b332
--- /dev/null
+++ b/metrics/src/test/java/com/yahoo/metrics/SummedDoubleValueMetric.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+class SummedDoubleValueMetric extends ValueMetric<Double> {
+
+ public SummedDoubleValueMetric(String name, String tags, String description, MetricSet owner) {
+ super(name, tags, description, owner);
+ }
+
+ public SummedDoubleValueMetric(SummedDoubleValueMetric other, CopyType copyType, MetricSet owner) {
+ super(other, copyType, owner);
+ }
+}
diff --git a/metrics/src/test/java/com/yahoo/metrics/SummedLongValueMetric.java b/metrics/src/test/java/com/yahoo/metrics/SummedLongValueMetric.java
new file mode 100644
index 00000000000..d08abe6f242
--- /dev/null
+++ b/metrics/src/test/java/com/yahoo/metrics/SummedLongValueMetric.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+class SummedLongValueMetric extends ValueMetric<Long> {
+
+ public SummedLongValueMetric(String name, String tags, String description, MetricSet owner) {
+ super(name, tags, description, owner);
+ }
+
+ public SummedLongValueMetric(SummedLongValueMetric other, CopyType copyType, MetricSet owner) {
+ super(other, copyType, owner);
+ }
+}
+
diff --git a/metrics/src/test/java/com/yahoo/metrics/ValueMetricTest.java b/metrics/src/test/java/com/yahoo/metrics/ValueMetricTest.java
new file mode 100644
index 00000000000..4758888b0c8
--- /dev/null
+++ b/metrics/src/test/java/com/yahoo/metrics/ValueMetricTest.java
@@ -0,0 +1,182 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.metrics;
+
+import junit.framework.TestCase;
+
+public class ValueMetricTest extends TestCase {
+ public void testAveragedDoubleValueMetric()
+ {
+ AveragedDoubleValueMetric m = new AveragedDoubleValueMetric("test", "tag", "description", null);
+ m.addValue(100.0);
+ assertEquals("count=\"1\" min=\"100.00\" max=\"100.00\" last=\"100.00\" total=\"100.00\" average=\"100.00\"", m.toString());
+ m.addValue(100.0);
+ assertEquals("count=\"2\" min=\"100.00\" max=\"100.00\" last=\"100.00\" total=\"200.00\" average=\"100.00\"", m.toString());
+ m.addValue(40.0);
+ assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m.toString());
+
+ AveragedDoubleValueMetric m2 = new AveragedDoubleValueMetric(m, Metric.CopyType.CLONE, null);
+ assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m.toString());
+ assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m2.toString());
+ m.reset();
+ assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString());
+ assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m2.toString());
+
+ AveragedDoubleValueMetric m3 = new AveragedDoubleValueMetric("test", "tag", "description", null);
+ m3.addValue(200.0);
+ m3.addValue(100.0);
+ m3.addValue(400.0);
+
+ AveragedDoubleValueMetric m4 = new AveragedDoubleValueMetric("test", "tag", "description", null);
+ m3.addValue(50.0);
+ m3.addValue(100.0);
+ m3.addValue(2000.0);
+
+ AveragedDoubleValueMetric sum = new AveragedDoubleValueMetric(m2, Metric.CopyType.INACTIVE, null);
+ m3.addToPart(sum);
+ m4.addToPart(sum);
+ assertEquals("count=\"9\" min=\"40.00\" max=\"2000.00\" last=\"2000.00\" total=\"3090.00\" average=\"343.33\"", sum.toString());
+
+ AveragedDoubleValueMetric snapshot = new AveragedDoubleValueMetric(m2, Metric.CopyType.INACTIVE, null);
+ m3.addToSnapshot(snapshot);
+ m4.addToSnapshot(snapshot);
+ assertEquals("count=\"9\" min=\"40.00\" max=\"2000.00\" last=\"2000.00\" total=\"3090.00\" average=\"343.33\"", snapshot.toString());
+
+ assertEquals("<test description=\"description\" average=\"343.33\" last=\"2000.00\" min=\"40.00\" max=\"2000.00\" count=\"9\" total=\"3090.00\"/>\n", sum.toXml(0, 2));
+
+ assertEquals(80.0, m2.getDoubleValue("value"));
+ assertEquals(80.0, m2.getDoubleValue("average"));
+ assertEquals(40.0, m2.getDoubleValue("min"));
+ assertEquals(100.0, m2.getDoubleValue("max"));
+ assertEquals(40.0, m2.getDoubleValue("last"));
+ assertEquals(3.0, m2.getDoubleValue("count"));
+ assertEquals(240.0, m2.getDoubleValue("total"));
+
+ assertEquals(80, m2.getLongValue("value"));
+ assertEquals(80, m2.getLongValue("average"));
+ assertEquals(40, m2.getLongValue("min"));
+ assertEquals(100, m2.getLongValue("max"));
+ assertEquals(40, m2.getLongValue("last"));
+ assertEquals(3, m2.getLongValue("count"));
+ assertEquals(240, m2.getLongValue("total"));
+ }
+
+ public void testDoubleValueMetricNotUpdatedOnNaN() {
+ AveragedDoubleValueMetric m = new AveragedDoubleValueMetric("test", "tag", "description", null);
+ m.addValue(Double.NaN);
+ assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString());
+ }
+
+ public void testDoubleValueMetricNotUpdatedOnInfinity() {
+ AveragedDoubleValueMetric m = new AveragedDoubleValueMetric("test", "tag", "description", null);
+ m.addValue(Double.POSITIVE_INFINITY);
+ assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString());
+ }
+
+ public void testSummedDoubleValueMetric()
+ {
+ SummedDoubleValueMetric m = new SummedDoubleValueMetric("test", "tag", "description", null);
+ m.addValue(100.0);
+ assertEquals("count=\"1\" min=\"100.00\" max=\"100.00\" last=\"100.00\" total=\"100.00\" average=\"100.00\"", m.toString());
+ m.addValue(100.0);
+ assertEquals("count=\"2\" min=\"100.00\" max=\"100.00\" last=\"100.00\" total=\"200.00\" average=\"100.00\"", m.toString());
+ m.addValue(40.0);
+ assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m.toString());
+
+ SummedDoubleValueMetric m2 = new SummedDoubleValueMetric(m, Metric.CopyType.CLONE, null);
+ assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m.toString());
+ assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m2.toString());
+ m.reset();
+ assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString());
+ assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m2.toString());
+
+ SummedDoubleValueMetric m3 = new SummedDoubleValueMetric("test", "tag", "description", null);
+ m3.addValue(200.0);
+ m3.addValue(100.0);
+ m3.addValue(400.0);
+
+ SummedDoubleValueMetric m4 = new SummedDoubleValueMetric("test", "tag", "description", null);
+ m4.addValue(2000.0);
+
+ SummedDoubleValueMetric sum = new SummedDoubleValueMetric(m2, Metric.CopyType.INACTIVE, null);
+ m3.addToPart(sum);
+ m4.addToPart(sum);
+ assertEquals("count=\"7\" min=\"40.00\" max=\"2000.00\" last=\"2440.00\" total=\"16193.33\" average=\"2313.33\"", sum.toString());
+
+ SummedDoubleValueMetric snapshot = new SummedDoubleValueMetric(m2, Metric.CopyType.INACTIVE, null);
+ m3.addToSnapshot(snapshot);
+ m4.addToSnapshot(snapshot);
+ assertEquals("count=\"7\" min=\"40.00\" max=\"2000.00\" last=\"2000.00\" total=\"2940.00\" average=\"420.00\"", snapshot.toString());
+
+ assertEquals("<test description=\"description\" average=\"2313.33\" last=\"2440.00\" min=\"40.00\" max=\"2000.00\" count=\"7\" total=\"16193.33\"/>\n", sum.toXml(0, 2));
+
+ assertEquals(40.0, m2.getDoubleValue("value"));
+ assertEquals(80.0, m2.getDoubleValue("average"));
+ assertEquals(40.0, m2.getDoubleValue("min"));
+ assertEquals(100.0, m2.getDoubleValue("max"));
+ assertEquals(40.0, m2.getDoubleValue("last"));
+ assertEquals(3.0, m2.getDoubleValue("count"));
+ assertEquals(240.0, m2.getDoubleValue("total"));
+
+ assertEquals(40, m2.getLongValue("value"));
+ assertEquals(80, m2.getLongValue("average"));
+ assertEquals(40, m2.getLongValue("min"));
+ assertEquals(100, m2.getLongValue("max"));
+ assertEquals(40, m2.getLongValue("last"));
+ assertEquals(3, m2.getLongValue("count"));
+ assertEquals(240, m2.getLongValue("total"));
+ }
+
+ public void testAveragedLongValueMetric()
+ {
+ AveragedLongValueMetric m = new AveragedLongValueMetric("test", "tag", "description", null);
+
+ assertEquals(0l, m.getLongValue("max"));
+ assertEquals(0l, m.getLongValue("min"));
+
+ m.addValue((long)100);
+ assertEquals("count=\"1\" min=\"100\" max=\"100\" last=\"100\" total=\"100\" average=\"100.00\"", m.toString());
+ m.addValue((long)100);
+ assertEquals("count=\"2\" min=\"100\" max=\"100\" last=\"100\" total=\"200\" average=\"100.00\"", m.toString());
+ m.addValue((long)40);
+ assertEquals("count=\"3\" min=\"40\" max=\"100\" last=\"40\" total=\"240\" average=\"80.00\"", m.toString());
+
+ AveragedLongValueMetric m2 = new AveragedLongValueMetric(m, Metric.CopyType.CLONE, null);
+ assertEquals("count=\"3\" min=\"40\" max=\"100\" last=\"40\" total=\"240\" average=\"80.00\"", m.toString());
+ assertEquals("count=\"3\" min=\"40\" max=\"100\" last=\"40\" total=\"240\" average=\"80.00\"", m2.toString());
+ m.reset();
+ assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString());
+ assertEquals("count=\"3\" min=\"40\" max=\"100\" last=\"40\" total=\"240\" average=\"80.00\"", m2.toString());
+
+ AveragedLongValueMetric m3 = new AveragedLongValueMetric("test", "tag", "description", null);
+ m3.addValue((long)200);
+ m3.addValue((long)100);
+ m3.addValue((long)400);
+
+ AveragedLongValueMetric m4 = new AveragedLongValueMetric("test", "tag", "description", null);
+ m3.addValue((long)50);
+ m3.addValue((long)100);
+ m3.addValue((long)2000);
+
+ AveragedLongValueMetric sum = new AveragedLongValueMetric(m2, Metric.CopyType.INACTIVE, null);
+ m3.addToPart(sum);
+ m4.addToPart(sum);
+ assertEquals("count=\"9\" min=\"40\" max=\"2000\" last=\"2000\" total=\"3090\" average=\"343.33\"", sum.toString());
+
+ AveragedLongValueMetric snapshot = new AveragedLongValueMetric(m2, Metric.CopyType.INACTIVE, null);
+ m3.addToSnapshot(snapshot);
+ m4.addToSnapshot(snapshot);
+ assertEquals("count=\"9\" min=\"40\" max=\"2000\" last=\"2000\" total=\"3090\" average=\"343.33\"", snapshot.toString());
+
+ assertEquals("<test description=\"description\" average=\"343.33\" last=\"2000\" min=\"40\" max=\"2000\" count=\"9\" total=\"3090\"/>\n", sum.toXml(0, 2));
+
+ assertEquals(80, m2.getLongValue("value"));
+ assertEquals(80, m2.getLongValue("average"));
+ assertEquals(40, m2.getLongValue("min"));
+ assertEquals(100, m2.getLongValue("max"));
+ assertEquals(40, m2.getLongValue("last"));
+ assertEquals(3, m2.getLongValue("count"));
+ assertEquals(240, m2.getLongValue("total"));
+ }
+
+
+}
diff --git a/metrics/src/tests/.gitignore b/metrics/src/tests/.gitignore
new file mode 100644
index 00000000000..453dd9e4ab3
--- /dev/null
+++ b/metrics/src/tests/.gitignore
@@ -0,0 +1,7 @@
+.*.swp
+.depend
+.~
+Makefile
+testrunner
+/test.vlog
+metrics_testrunner_app
diff --git a/metrics/src/tests/CMakeLists.txt b/metrics/src/tests/CMakeLists.txt
new file mode 100644
index 00000000000..b84701e9487
--- /dev/null
+++ b/metrics/src/tests/CMakeLists.txt
@@ -0,0 +1,18 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(metrics_testrunner_app
+ SOURCES
+ testrunner.cpp
+ countmetrictest.cpp
+ valuemetrictest.cpp
+ metricsettest.cpp
+ summetrictest.cpp
+ metricmanagertest.cpp
+ loadmetrictest.cpp
+ snapshottest.cpp
+ stresstest.cpp
+ metrictest.cpp
+ DEPENDS
+ metrics
+ vdstestlib
+)
+vespa_add_test(NAME metrics_testrunner_app COMMAND metrics_testrunner_app)
diff --git a/metrics/src/tests/countmetrictest.cpp b/metrics/src/tests/countmetrictest.cpp
new file mode 100644
index 00000000000..1f39113474b
--- /dev/null
+++ b/metrics/src/tests/countmetrictest.cpp
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/vespalib/objects/floatingpointtype.h>
+#include <vespa/metrics/countmetric.h>
+
+using vespalib::Double;
+
+namespace metrics {
+
+struct CountMetricTest : public CppUnit::TestFixture {
+ void testLongCountMetric();
+
+ CPPUNIT_TEST_SUITE(CountMetricTest);
+ CPPUNIT_TEST(testLongCountMetric);
+ CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(CountMetricTest);
+
+void CountMetricTest::testLongCountMetric()
+{
+ LongCountMetric m("test", "tag", "description");
+ m.set(100);
+ CPPUNIT_ASSERT_EQUAL(uint64_t(100), m.getValue());
+ m.inc(5);
+ CPPUNIT_ASSERT_EQUAL(uint64_t(105), m.getValue());
+ m.dec(15);
+ CPPUNIT_ASSERT_EQUAL(uint64_t(90), m.getValue());
+ LongCountMetric m2(m);
+ CPPUNIT_ASSERT_EQUAL(uint64_t(90), m2.getValue());
+ m.reset();
+ CPPUNIT_ASSERT_EQUAL(uint64_t(0), m.getValue());
+
+ LongCountMetric n("m2", "", "desc");
+ n.set(6);
+ CPPUNIT_ASSERT_EQUAL(uint64_t(6), n.getValue());
+
+ LongCountMetric o = m2 + n;
+ CPPUNIT_ASSERT_EQUAL(uint64_t(96), o.getValue());
+
+ o = m2 - n;
+ CPPUNIT_ASSERT_EQUAL(uint64_t(84), o.getValue());
+
+ std::string expected(
+ "test count=84");
+ CPPUNIT_ASSERT_EQUAL(expected, o.toString());
+
+ CPPUNIT_ASSERT_EQUAL(Double(84), Double(o.getDoubleValue("value")));
+ CPPUNIT_ASSERT_EQUAL(int64_t(84), o.getLongValue("value"));
+}
+
+}
diff --git a/metrics/src/tests/loadmetrictest.cpp b/metrics/src/tests/loadmetrictest.cpp
new file mode 100644
index 00000000000..b0755fa46e9
--- /dev/null
+++ b/metrics/src/tests/loadmetrictest.cpp
@@ -0,0 +1,138 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/metrics/loadmetric.h>
+#include <vespa/metrics/valuemetric.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+
+namespace metrics {
+
+struct LoadTypeSetImpl : public LoadTypeSet {
+ LoadTypeSetImpl() {
+ push_back(LoadType(0, "default"));
+ }
+ LoadTypeSetImpl& add(uint32_t id, const char* name) {
+ push_back(LoadType(id, name));
+ return *this;
+ }
+ const LoadType& operator[](const std::string& name) const {
+ for (uint32_t i=0; i<size(); ++i) {
+ const LoadType& lt(LoadTypeSet::operator[](i));
+ if (lt.getName() == name) return lt;
+ }
+ CPPUNIT_FAIL("No load type with name " + name);
+ return operator[](0); // Should never get here
+ }
+};
+
+struct LoadMetricTest : public CppUnit::TestFixture {
+ void testNormalUsage();
+ void testClone(Metric::CopyType);
+ void testInactiveCopy() { testClone(Metric::INACTIVE); }
+ void testActiveCopy() { testClone(Metric::CLONE); }
+ void testAdding();
+
+ CPPUNIT_TEST_SUITE(LoadMetricTest);
+ CPPUNIT_TEST(testNormalUsage);
+ CPPUNIT_TEST(testActiveCopy);
+ CPPUNIT_TEST(testInactiveCopy);
+ CPPUNIT_TEST(testAdding);
+ CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(LoadMetricTest);
+
+void
+LoadMetricTest::testNormalUsage()
+{
+ LoadTypeSetImpl loadTypes;
+ loadTypes.add(32, "foo").add(1000, "bar");
+ LoadMetric<LongValueMetric> metric(
+ loadTypes, LongValueMetric("put", "", "Put"));
+}
+
+namespace {
+ struct MyMetricSet : public MetricSet {
+ LongAverageMetric metric;
+
+ MyMetricSet(MetricSet* owner = 0)
+ : MetricSet("tick", "", "", owner),
+ metric("tack", "", "", this)
+ {
+ }
+
+ Metric* clone(std::vector<Metric::LP>& ownerList,
+ CopyType copyType,
+ MetricSet* owner,
+ bool includeUnused = false) const
+ {
+ if (copyType != CLONE) {
+ return MetricSet::clone(ownerList, copyType, owner,
+ includeUnused);
+ }
+ return (new MyMetricSet(owner))->assignValues(*this);
+ }
+ };
+}
+
+void
+LoadMetricTest::testClone(Metric::CopyType copyType)
+{
+ LoadTypeSetImpl loadTypes;
+ loadTypes.add(32, "foo").add(1000, "bar");
+ MetricSet top("top", "", "");
+ MyMetricSet myset;
+ LoadMetric<MyMetricSet> metric(loadTypes, myset, &top);
+ metric[loadTypes["foo"]].metric.addValue(5);
+
+ std::vector<Metric::LP> ownerList;
+ MetricSet::UP copy(dynamic_cast<MetricSet*>(
+ top.clone(ownerList, copyType, 0, true)));
+ CPPUNIT_ASSERT(copy.get());
+
+ std::string expected =
+ "top:\n"
+ " tick:\n"
+ " sum:\n"
+ " tack average=5 last=5 min=5 max=5 count=1 total=5\n"
+ " default:\n"
+ " tack average=0 last=0 count=0 total=0\n"
+ " foo:\n"
+ " tack average=5 last=5 min=5 max=5 count=1 total=5\n"
+ " bar:\n"
+ " tack average=0 last=0 count=0 total=0";
+
+ CPPUNIT_ASSERT_EQUAL(expected, std::string(copy->toString(true)));
+}
+
+void
+LoadMetricTest::testAdding()
+{
+ LoadTypeSetImpl loadTypes;
+ loadTypes.add(32, "foo").add(1000, "bar");
+ MetricSet top("top", "", "");
+ MyMetricSet myset;
+ LoadMetric<MyMetricSet> metric(loadTypes, myset, &top);
+ metric[loadTypes["foo"]].metric.addValue(5);
+
+ std::vector<Metric::LP> ownerList;
+ MetricSet::UP copy(dynamic_cast<MetricSet*>(
+ top.clone(ownerList, Metric::INACTIVE, 0, false)));
+ CPPUNIT_ASSERT(copy.get());
+
+ top.reset();
+
+ top.addToSnapshot(*copy, ownerList);
+
+ std::string expected =
+ "top:\n"
+ " tick:\n"
+ " sum:\n"
+ " tack average=5 last=5 min=5 max=5 count=1 total=5\n"
+ " foo:\n"
+ " tack average=5 last=5 min=5 max=5 count=1 total=5";
+
+ CPPUNIT_ASSERT_EQUAL(expected, std::string(copy->toString(true)));
+
+}
+
+} // documentapi
diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp
new file mode 100644
index 00000000000..31471cc40ed
--- /dev/null
+++ b/metrics/src/tests/metricmanagertest.cpp
@@ -0,0 +1,1280 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include <vespa/metrics/metrics.h>
+#include <vespa/metrics/xmlwriter.h>
+#include <vespa/metrics/jsonwriter.h>
+#include <vespa/metrics/textwriter.h>
+#include <vespa/metrics/printutils.h>
+#include <vespa/metrics/state_api_adapter.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+
+#include <fstream>
+#include <sstream>
+
+LOG_SETUP(".test.metricmanager");
+
+namespace metrics {
+
+struct MetricManagerTest : public CppUnit::TestFixture {
+ void testConsumerVisitor();
+ void testSnapshots();
+ void testPrintUtils();
+ void testXmlOutput();
+ void testJsonOutput();
+ void jsonOutputSupportsMultipleDimensions();
+ void jsonOutputCanNestDimensionsFromMultipleMetricSets();
+ void jsonOutputCanHaveMultipleSetsWithSameName();
+ void testTextOutput();
+ void textOutputSupportsDimensions();
+ void testUpdateHooks();
+
+ CPPUNIT_TEST_SUITE(MetricManagerTest);
+ CPPUNIT_TEST(testConsumerVisitor);
+ CPPUNIT_TEST(testSnapshots);
+ CPPUNIT_TEST(testPrintUtils);
+ CPPUNIT_TEST(testXmlOutput);
+ CPPUNIT_TEST(testJsonOutput);
+ CPPUNIT_TEST(jsonOutputSupportsMultipleDimensions);
+ CPPUNIT_TEST(jsonOutputCanNestDimensionsFromMultipleMetricSets);
+ CPPUNIT_TEST(jsonOutputCanHaveMultipleSetsWithSameName);
+ CPPUNIT_TEST(testTextOutput);
+ CPPUNIT_TEST(textOutputSupportsDimensions);
+ CPPUNIT_TEST(testUpdateHooks);
+ CPPUNIT_TEST_SUITE_END();
+
+public:
+ // Ugly indirection hack caused by some tests using friended internals of
+ // MetricManager that aren't accessible to "freestanding" fixtures. So we
+ // get the test to do the necessary poking and prodding for us instead.
+ void takeSnapshots(MetricManager& mm, time_t timeToProcess) {
+ mm.takeSnapshots(mm.getMetricLock(), timeToProcess);
+ }
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(MetricManagerTest);
+
+namespace {
+
+struct SubMetricSet : public MetricSet
+{
+ DoubleValueMetric val1;
+ DoubleValueMetric val2;
+ SumMetric<DoubleValueMetric> valsum;
+
+ SubMetricSet(const Metric::String & name, MetricSet* owner)
+ : MetricSet(name, "sub", "sub desc", owner, "sub"),
+ val1("val1", "tag4 snaptest", "val1 desc", this),
+ val2("val2", "tag5", "val2 desc", this),
+ valsum("valsum", "tag4 snaptest", "valsum desc", this)
+ {
+ valsum.addMetricToSum(val1);
+ valsum.addMetricToSum(val2);
+ }
+};
+
+struct MultiSubMetricSet
+{
+ MetricSet set;
+ LongCountMetric count;
+ SubMetricSet a;
+ SubMetricSet b;
+ SumMetric<MetricSet> sum;
+
+ MultiSubMetricSet(MetricSet* owner)
+ : set("multisub", "multisub", "", owner),
+ count("count", "snaptest", "counter", &set),
+ a("a", &set),
+ b("b", &set),
+ sum("sum", "snaptest", "", &set)
+ {
+ sum.addMetricToSum(a);
+ sum.addMetricToSum(b);
+ }
+
+};
+
+struct TestMetricSet
+{
+ MetricSet set;
+ DoubleValueMetric val1;
+ DoubleValueMetric val2;
+ DoubleValueMetric val3;
+ DoubleValueMetric val4;
+ DoubleValueMetric val5;
+ DoubleValueMetric val6;
+ DoubleValueMetric val7;
+ DoubleValueMetric val8;
+ SubMetricSet val9;
+ MultiSubMetricSet val10;
+
+ TestMetricSet()
+ : set("temp", "test", "desc of test set"),
+ val1("val1", "tag1", "val1 desc", &set),
+ val2("val2", "tag1 tag2", "val2 desc", &set),
+ val3("val3", "tag2 tag3", "val3 desc", &set),
+ val4("val4", "tag3", "val4 desc", &set),
+ val5("val5", "tag2", "val5 desc", &set),
+ val6("val6", "tag4 snaptest", "val6 desc", &set),
+ val7("val7", "", "val7 desc", &set),
+ val8("val8", "tag6", "val8 desc", &set),
+ val9("sub", &set),
+ val10(&set)
+ {
+ }
+
+};
+
+struct MetricNameVisitor : public MetricVisitor {
+ std::ostringstream ost;
+ bool debug;
+
+ MetricNameVisitor(bool debug_ = false) : debug(debug_) {}
+
+ virtual bool visitMetricSet(const MetricSet& metricSet,
+ bool autoGenerated)
+ {
+ if (debug) {
+ ost << "<" << (autoGenerated ? "*" : "")
+ << metricSet.getPath() << ">\n";
+ }
+ return true;
+ }
+ virtual void doneVisitingMetricSet(const MetricSet& metricSet) {
+ if (debug) {
+ ost << "</" << metricSet.getPath() << ">\n";
+ }
+ }
+
+ virtual bool visitMetric(const Metric& m, bool autoGenerated) {
+ ost << (autoGenerated ? "*" : "") << m.getPath() << "\n";
+ return true;
+ }
+
+ std::string toString() const { return ost.str(); }
+};
+
+}
+
+namespace {
+ std::pair<std::string, std::string> getMatchedMetrics(
+ const vespalib::string& config)
+ {
+ FastOS_ThreadPool pool(256 * 1024);
+ TestMetricSet mySet;
+ MetricManager mm;
+ mm.registerMetric(mm.getMetricLock(), mySet.set);
+ mm.init(config, pool);
+ MetricNameVisitor visitor;
+
+ /** Take a copy to verify clone works.
+ std::list<Metric::SP> ownerList;
+ MetricSet::UP copy(dynamic_cast<MetricSet*>(
+ mm.getMetrics().clone(ownerList)));
+ mm.visit(*copy, visitor, "consumer");
+ */
+
+ MetricLockGuard g(mm.getMetricLock());
+ mm.visit(g, mm.getActiveMetrics(g), visitor, "consumer");
+ MetricManager::ConsumerSpec::SP consumerSpec(
+ mm.getConsumerSpec(g, "consumer"));
+ return std::pair<std::string, std::string>(
+ visitor.toString(),
+ consumerSpec.get() ? consumerSpec->toString()
+ : "Non-existing consumer");
+ }
+}
+
+#define ASSERT_CONSUMER_MATCH(name, expected, config) \
+{ \
+ std::pair<std::string, std::string> consumerMatch( \
+ getMatchedMetrics(config)); \
+ CPPUNIT_ASSERT_EQUAL_MSG(name + std::string(": ") + consumerMatch.second, \
+ "\n" + expected, "\n" + consumerMatch.first); \
+}
+
+void MetricManagerTest::testConsumerVisitor()
+{
+ // Add one tag and a name, check that we get all three.
+ ASSERT_CONSUMER_MATCH("testAddTagAndName", std::string(
+ "temp.val1\n"
+ "temp.val2\n"
+ "temp.val4\n"
+ "*temp.multisub.sum.val1\n"
+ ),
+ "raw:"
+ "consumer[1]\n"
+ "consumer[0].name consumer\n"
+ "consumer[0].tags[1]\n"
+ "consumer[0].tags[0] tag1\n"
+ "consumer[0].addedmetrics[2]\n"
+ "consumer[0].addedmetrics[0] temp.val4\n"
+ "consumer[0].addedmetrics[1] temp.multisub.sum.val1\n"
+ );
+ // Add two tags, remove one
+ ASSERT_CONSUMER_MATCH("testAddAndRemoveTag", std::string(
+ "temp.val1\n"
+ "temp.val4\n"
+ ),
+ "raw:"
+ "consumer[1]\n"
+ "consumer[0].name consumer\n"
+ "consumer[0].tags[2]\n"
+ "consumer[0].tags[0] tag1\n"
+ "consumer[0].tags[1] tag3\n"
+ "consumer[0].removedtags[1]\n"
+ "consumer[0].removedtags[0] tag2\n"
+ );
+ // Test simple wildcards
+ ASSERT_CONSUMER_MATCH("testWildCards", std::string(
+ "temp.val1\n"
+ "temp.val2\n"
+ "temp.val3\n"
+ "temp.val4\n"
+ "temp.val5\n"
+ "temp.val6\n"
+ "temp.val7\n"
+ "temp.val8\n"
+ ),
+ "raw:"
+ "consumer[1]\n"
+ "consumer[0].name consumer\n"
+ "consumer[0].addedmetrics[1]\n"
+ "consumer[0].addedmetrics[0] temp.*\n"
+ "consumer[0].removedmetrics[2]\n"
+ "consumer[0].removedmetrics[0] temp.sub.*\n"
+ "consumer[0].removedmetrics[1] temp.multisub.*\n"
+ );
+ // Test more wildcards
+ ASSERT_CONSUMER_MATCH("testWildCards2", std::string(
+ "temp.sub.val1\n"
+ ),
+ "raw:"
+ "consumer[1]\n"
+ "consumer[0].name consumer\n"
+ "consumer[0].addedmetrics[1]\n"
+ "consumer[0].addedmetrics[0] temp.*.val1\n"
+ );
+ // test adding all
+ ASSERT_CONSUMER_MATCH("testAddAll", std::string(
+ "metricmanager.periodichooklatency\n"
+ "metricmanager.snapshothooklatency\n"
+ "metricmanager.resetlatency\n"
+ "metricmanager.snapshotlatency\n"
+ "metricmanager.sleeptime\n"
+ "temp.val1\n"
+ "temp.val2\n"
+ "temp.val3\n"
+ "temp.val4\n"
+ "temp.val5\n"
+ "temp.val6\n"
+ "temp.val7\n"
+ "temp.val8\n"
+ "temp.sub.val1\n"
+ "temp.sub.val2\n"
+ "*temp.sub.valsum\n"
+ "temp.multisub.count\n"
+ "temp.multisub.a.val1\n"
+ "temp.multisub.a.val2\n"
+ "*temp.multisub.a.valsum\n"
+ "temp.multisub.b.val1\n"
+ "temp.multisub.b.val2\n"
+ "*temp.multisub.b.valsum\n"
+ "*temp.multisub.sum.val1\n"
+ "*temp.multisub.sum.val2\n"
+ "*temp.multisub.sum.valsum\n"
+ ),
+ "raw:"
+ "consumer[1]\n"
+ "consumer[0].name consumer\n"
+ "consumer[0].addedmetrics[1]\n"
+ "consumer[0].addedmetrics[0] *\n"
+ );
+ // test adding all using tags
+ ASSERT_CONSUMER_MATCH("testAddAll2", std::string(
+ "temp.val1\n"
+ "temp.val2\n"
+ "temp.val3\n"
+ "temp.val4\n"
+ "temp.val5\n"
+ "temp.val6\n"
+ "temp.val7\n"
+ "temp.val8\n"
+ "temp.sub.val1\n"
+ "temp.sub.val2\n"
+ "*temp.sub.valsum\n"
+ "temp.multisub.count\n"
+ "temp.multisub.a.val1\n"
+ "temp.multisub.a.val2\n"
+ "*temp.multisub.a.valsum\n"
+ "temp.multisub.b.val1\n"
+ "temp.multisub.b.val2\n"
+ "*temp.multisub.b.valsum\n"
+ "*temp.multisub.sum.val1\n"
+ "*temp.multisub.sum.val2\n"
+ "*temp.multisub.sum.valsum\n"
+ ),
+ "raw:"
+ "consumer[1]\n"
+ "consumer[0].name consumer\n"
+ "consumer[0].tags[1]\n"
+ "consumer[0].tags[0] *\n"
+ );
+ // Test that all metrics are added when a metricsset is added by name
+ ASSERT_CONSUMER_MATCH("testSpecifiedSetName", std::string(
+ "temp.sub.val1\n"
+ "temp.sub.val2\n"
+ "*temp.sub.valsum\n"
+ ),
+ "raw:"
+ "consumer[1]\n"
+ "consumer[0].name consumer\n"
+ "consumer[0].addedmetrics[1]\n"
+ "consumer[0].addedmetrics[0] temp.sub\n"
+ );
+ // Test that all metrics are added when a metricsset is added by tag
+ ASSERT_CONSUMER_MATCH("testSpecifiedSetTag", std::string(
+ "temp.sub.val1\n"
+ "temp.sub.val2\n"
+ "*temp.sub.valsum\n"
+ "temp.multisub.a.val1\n"
+ "temp.multisub.a.val2\n"
+ "*temp.multisub.a.valsum\n"
+ "temp.multisub.b.val1\n"
+ "temp.multisub.b.val2\n"
+ "*temp.multisub.b.valsum\n"
+ ),
+ "raw:"
+ "consumer[1]\n"
+ "consumer[0].name consumer\n"
+ "consumer[0].tags[1]\n"
+ "consumer[0].tags[0] sub\n"
+ );
+ // Test that all metrics are added from the set except those with a
+ // certain tag.
+ ASSERT_CONSUMER_MATCH("testSpecifiedSetTagWithExceptionTags", std::string(
+ "temp.val1\n"
+ "temp.val4\n"
+ "temp.val7\n"
+ "temp.val8\n"
+ "temp.sub.val2\n"
+ "temp.multisub.count\n"
+ "temp.multisub.a.val2\n"
+ "temp.multisub.b.val2\n"
+ "*temp.multisub.sum.val2\n"
+ ),
+ "raw:"
+ "consumer[1]\n"
+ "consumer[0].name consumer\n"
+ "consumer[0].tags[1]\n"
+ "consumer[0].tags[0] test\n"
+ "consumer[0].removedtags[2]\n"
+ "consumer[0].removedtags[0] tag2\n"
+ "consumer[0].removedtags[1] tag4\n"
+ );
+};
+
+namespace {
+ struct FakeTimer : public MetricManager::Timer {
+ time_t _time;
+ FakeTimer(time_t startTime = 0) : _time(startTime) {}
+ virtual time_t getTime() const { return _time; }
+ };
+
+ struct BriefValuePrinter : public MetricVisitor {
+ uint32_t count;
+ std::ostringstream ost;
+
+ BriefValuePrinter() : count(0), ost() {}
+
+ virtual bool visitMetric(const Metric& metric, bool) {
+ if (++count > 1) ost << ",";
+ //ost << metric.getPath() << ":";
+ ost << metric.getDoubleValue("value");
+ return true;
+ }
+ };
+
+ bool waitForTimeProcessed(const MetricManager& mm,
+ time_t processtime, uint32_t timeout = 120)
+ {
+ uint32_t lastchance = time(0) + timeout;
+ while (time(0) < lastchance) {
+ if (mm.getLastProcessedTime() >= processtime) return true;
+ mm.timeChangedNotification();
+ FastOS_Thread::Sleep(10);
+ }
+ return false;
+ }
+
+ std::string dumpAllSnapshots(const MetricManager& mm,
+ const std::string& consumer)
+ {
+ std::ostringstream ost;
+ ost << "\n";
+ {
+ MetricLockGuard metricLock(mm.getMetricLock());
+ BriefValuePrinter briefValuePrinter;
+ mm.visit(metricLock, mm.getActiveMetrics(metricLock),
+ briefValuePrinter, consumer);
+ ost << "Current: " << briefValuePrinter.ost.str() << "\n";
+ }
+ {
+ MetricLockGuard metricLock(mm.getMetricLock());
+ BriefValuePrinter briefValuePrinter;
+ mm.visit(metricLock, mm.getTotalMetricSnapshot(metricLock),
+ briefValuePrinter, consumer);
+ ost << "Total: " << briefValuePrinter.ost.str() << "\n";
+ }
+ std::vector<uint32_t> periods;
+ {
+ MetricLockGuard metricLock(mm.getMetricLock());
+ periods = mm.getSnapshotPeriods(metricLock);
+ }
+ for (uint32_t i=0; i<periods.size(); ++i) {
+ MetricLockGuard metricLock(mm.getMetricLock());
+ const MetricSnapshotSet& set(mm.getMetricSnapshotSet(
+ metricLock, periods[i]));
+ ost << set.getName() << "\n";
+ uint32_t count = 0;
+ for (uint32_t j=0; j<2; ++j) {
+ if (set.getCount() == 1 && j == 1) continue;
+ const MetricSnapshot& snap(set.getSnapshot(j == 1));
+ BriefValuePrinter briefValuePrinter;
+ mm.visit(metricLock, snap, briefValuePrinter, consumer);
+ ost << " " << count++ << " " << &snap.getMetrics() << ": "
+ << briefValuePrinter.ost.str() << "\n";
+ }
+ }
+ return ost.str();
+ }
+
+}
+
+#define ASSERT_VALUES(mm, period, expected) \
+{ \
+ MetricLockGuard lockGuard(mm.getMetricLock()); \
+ BriefValuePrinter briefValuePrinter; \
+ if (period == -1) { \
+ mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), \
+ briefValuePrinter, "snapper"); \
+ } else if (period == 0) { \
+ mm.visit(lockGuard, mm.getTotalMetricSnapshot(lockGuard), \
+ briefValuePrinter, "snapper"); \
+ } else { \
+ mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, period), \
+ briefValuePrinter, "snapper"); \
+ } \
+ CPPUNIT_ASSERT_EQUAL_MSG(dumpAllSnapshots(mm, "snapper"), \
+ std::string(expected), briefValuePrinter.ost.str()); \
+}
+
+#define ASSERT_PROCESS_TIME(mm, time) \
+{ \
+ LOG(info, "Waiting for processed time %u.", time); \
+ bool gotToCorrectProgress = waitForTimeProcessed(mm, time); \
+ if (!gotToCorrectProgress) \
+ CPPUNIT_FAIL("Failed to get to processed time within timeout"); \
+}
+
+void MetricManagerTest::testSnapshots()
+{
+ FastOS_ThreadPool pool(256 * 1024);
+ FakeTimer* timer = new FakeTimer(1000);
+ std::unique_ptr<MetricManager::Timer> timerImpl(timer);
+ TestMetricSet mySet;
+ MetricManager mm(std::move(timerImpl));
+ {
+ MetricLockGuard lockGuard(mm.getMetricLock());
+ mm.registerMetric(lockGuard, mySet.set);
+ }
+ mm.init("raw:"
+ "consumer[2]\n"
+ "consumer[0].name snapper\n"
+ "consumer[0].tags[1]\n"
+ "consumer[0].tags[0] snaptest\n"
+ "consumer[1].name log\n"
+ "consumer[1].tags[1]\n"
+ "consumer[1].tags[0] snaptest\n",
+ pool);
+ MetricNameVisitor visitor;
+ {
+ MetricLockGuard lockGuard(mm.getMetricLock());
+ mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), visitor, "snapper");
+ MetricManager::ConsumerSpec::SP consumerSpec(
+ mm.getConsumerSpec(lockGuard, "snapper"));
+ CPPUNIT_ASSERT_EQUAL_MSG(consumerSpec.get() ? consumerSpec->toString()
+ : "Non-existing consumer",
+ std::string("\n"
+ "temp.val6\n"
+ "temp.sub.val1\n"
+ "*temp.sub.valsum\n"
+ "temp.multisub.count\n"
+ "temp.multisub.a.val1\n"
+ "*temp.multisub.a.valsum\n"
+ "temp.multisub.b.val1\n"
+ "*temp.multisub.b.valsum\n"
+ "*temp.multisub.sum.val1\n"
+ "*temp.multisub.sum.val2\n"
+ "*temp.multisub.sum.valsum\n"
+ ),
+ "\n" + visitor.toString());
+ }
+ // Initially, there should be no metrics logged
+ ASSERT_PROCESS_TIME(mm, 1000);
+ ASSERT_VALUES(mm, 5 * 60, "");
+
+ // Adding metrics done in first five minutes.
+ mySet.val6.addValue(2);
+ mySet.val9.val1.addValue(4);
+ mySet.val10.count.inc();
+ mySet.val10.a.val1.addValue(7);
+ mySet.val10.a.val2.addValue(2);
+ mySet.val10.b.val1.addValue(1);
+ timer->_time += 5 * 60;
+ ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60);
+ ASSERT_VALUES(mm, 5 * 60, "2,4,4,1,7,9,1,1,8,2,10");
+ ASSERT_VALUES(mm, 60 * 60, "");
+ ASSERT_VALUES(mm, 0 * 60, "2,4,4,1,7,9,1,1,8,2,10");
+
+ // Adding metrics done in second five minute period. Total should
+ // be updated to account for both
+ mySet.val6.addValue(4);
+ mySet.val9.val1.addValue(5);
+ mySet.val10.count.inc();
+ mySet.val10.a.val1.addValue(8);
+ mySet.val10.a.val2.addValue(3);
+ mySet.val10.b.val1.addValue(2);
+ timer->_time += 5 * 60;
+ ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 2);
+ ASSERT_VALUES(mm, 5 * 60, "4,5,5,1,8,11,2,2,10,3,13");
+ ASSERT_VALUES(mm, 60 * 60, "");
+ ASSERT_VALUES(mm, 0 * 60, "4,5,5,2,8,11,2,2,10,3,13");
+
+ //std::cerr << dumpAllSnapshots(mm, "snapper") << "\n";
+
+ // Adding another five minute period where nothing have happened.
+ // Metric for last 5 minutes should be 0.
+ timer->_time += 5 * 60;
+ ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 3);
+ ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0");
+ ASSERT_VALUES(mm, 60 * 60, "");
+ ASSERT_VALUES(mm, 0 * 60, "4,5,5,2,8,11,2,2,10,3,13");
+
+ // Advancing time to 60 minute period, we should create a proper
+ // 60 minute period timer.
+ mySet.val6.addValue(6);
+ for (uint32_t i=0; i<9; ++i) { // 9 x 5 minutes. Avoid snapshot bumping
+ // due to taking snapshots in the past
+ timer->_time += 5 * 60;
+ ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * (4 + i));
+ }
+ ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0");
+ ASSERT_VALUES(mm, 60 * 60, "6,5,5,2,8,11,2,2,10,3,13");
+ ASSERT_VALUES(mm, 0 * 60, "6,5,5,2,8,11,2,2,10,3,13");
+
+ // Test that reset works
+ mm.reset(1000);
+ ASSERT_VALUES(mm, -1, "0,0,0,0,0,0,0,0,0,0,0");
+ ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0");
+ ASSERT_VALUES(mm, 60 * 60, "0,0,0,0,0,0,0,0,0,0,0");
+ ASSERT_VALUES(mm, 0 * 60, "0,0,0,0,0,0,0,0,0,0,0");
+}
+
+void MetricManagerTest::testPrintUtils()
+{
+ FastOS_ThreadPool pool(256 * 1024);
+ FakeTimer* timer = new FakeTimer(1000);
+ std::unique_ptr<MetricManager::Timer> timerImpl(timer);
+ MetricManager mm(std::move(timerImpl));
+ TestMetricSet mySet;
+ {
+ MetricLockGuard lockGuard(mm.getMetricLock());
+ mm.registerMetric(lockGuard, mySet.set);
+ }
+ // Adding metrics to have some values in them
+ mySet.val6.addValue(2);
+ mySet.val9.val1.addValue(4);
+ mySet.val10.count.inc();
+ mySet.val10.a.val1.addValue(7);
+ mySet.val10.a.val2.addValue(2);
+ mySet.val10.b.val1.addValue(1);
+ // Initialize metric manager to get snapshots created.
+ mm.init("raw:"
+ "consumer[2]\n"
+ "consumer[0].name snapper\n"
+ "consumer[0].tags[1]\n"
+ "consumer[0].tags[0] snaptest\n"
+ "consumer[1].name log\n"
+ "consumer[1].tags[1]\n"
+ "consumer[1].tags[0] snaptest\n",
+ pool);
+ using namespace printutils;
+ MetricLockGuard lockGuard(mm.getMetricLock());
+ MetricSource source(mm.getActiveMetrics(lockGuard),
+ "temp.multisub");
+ CPPUNIT_ASSERT(source.getMetric("count") != 0);
+ CPPUNIT_ASSERT(source.getMetric("a.val1") != 0);
+ CPPUNIT_ASSERT(source.getMetric("a.valsum") != 0);
+ CPPUNIT_ASSERT(source.getMetric("sum.val1") != 0);
+ CPPUNIT_ASSERT(source.getMetric("sum.valsum") != 0);
+
+ std::vector<Metric::String> metrics(
+ source.getPathsMatchingPrefix("a.val"));
+ std::vector<Metric::String> expected;
+ expected.push_back("val1");
+ expected.push_back("val2");
+ expected.push_back("valsum");
+ CPPUNIT_ASSERT_EQUAL(expected, metrics);
+
+ HttpTable table("mytable", "stuff");
+ table.colNames.push_back("values");
+ table.rowNames.push_back("valsum");
+ table[0][0] = getValueString(
+ getLongMetric("a.val1.value", source)
+ + getLongMetric("a.val2.value", source));
+ std::ostringstream ost;
+ table.print(ost);
+ CPPUNIT_ASSERT_EQUAL(std::string(
+ "<h3>mytable</h3>\n"
+ "<table border=\"1\">\n"
+ "<tr><th>stuff</th><th>values</th></tr>\n"
+ "<tr><td>valsum</td><td align=\"right\">9</td></tr>\n"
+ "</table>\n"
+ ), ost.str());
+}
+
+void MetricManagerTest::testXmlOutput()
+{
+ FastOS_ThreadPool pool(256 * 1024);
+ FakeTimer* timer = new FakeTimer(1000);
+ std::unique_ptr<MetricManager::Timer> timerImpl(timer);
+ MetricManager mm(std::move(timerImpl));
+ TestMetricSet mySet;
+ {
+ MetricLockGuard lockGuard(mm.getMetricLock());
+ mm.registerMetric(lockGuard, mySet.set);
+ }
+
+ // Initialize metric manager to get snapshots created.
+ mm.init("raw:"
+ "consumer[2]\n"
+ "consumer[0].name snapper\n"
+ "consumer[0].tags[1]\n"
+ "consumer[0].tags[0] snaptest\n"
+ "consumer[1].name log\n"
+ "consumer[1].tags[1]\n"
+ "consumer[1].tags[0] snaptest\n",
+ pool);
+
+ mm.takeSnapshots(mm.getMetricLock(), 1000);
+
+ // Adding metrics to have some values in them
+ mySet.val6.addValue(2);
+ mySet.val9.val1.addValue(4);
+ mySet.val10.count.inc();
+ mySet.val10.a.val1.addValue(7);
+ mySet.val10.a.val2.addValue(2);
+ mySet.val10.b.val1.addValue(1);
+
+ timer->_time = 1300;
+ mm.takeSnapshots(mm.getMetricLock(), 1300);
+
+ std::string expected(
+ "'<snapshot name=\"5 minute\" from=\"1000\" to=\"1300\" period=\"300\">\n"
+ " <temp>\n"
+ " <val6 average=\"2\" last=\"2\" min=\"2\" max=\"2\" count=\"1\"/>\n"
+ " <sub>\n"
+ " <val1 average=\"4\" last=\"4\" min=\"4\" max=\"4\" count=\"1\"/>\n"
+ " <valsum average=\"4\" last=\"4\" min=\"4\" max=\"4\" count=\"1\"/>\n"
+ " </sub>\n"
+ " <multisub>\n"
+ " <count count=\"1\"/>\n"
+ " <a>\n"
+ " <val1 average=\"7\" last=\"7\" min=\"7\" max=\"7\" count=\"1\"/>\n"
+ " <valsum average=\"9\" last=\"9\"/>\n"
+ " </a>\n"
+ " <b>\n"
+ " <val1 average=\"1\" last=\"1\" min=\"1\" max=\"1\" count=\"1\"/>\n"
+ " <valsum average=\"1\" last=\"1\" min=\"1\" max=\"1\" count=\"1\"/>\n"
+ " </b>\n"
+ " <sum>\n"
+ " <val1 average=\"8\" last=\"8\"/>\n"
+ " <val2 average=\"2\" last=\"2\" min=\"2\" max=\"2\" count=\"1\"/>\n"
+ " <valsum average=\"10\" last=\"10\"/>\n"
+ " </sum>\n"
+ " </multisub>\n"
+ " </temp>\n"
+ "</snapshot>'");
+
+ std::ostringstream ost;
+ vespalib::XmlOutputStream xos(ost, " ");
+ XmlWriter writer(xos, 300, 0);
+ {
+ MetricLockGuard lockGuard(mm.getMetricLock());
+ mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false),
+ writer, "snapper");
+ }
+ std::string actual(ost.str());
+ // Not bothering to match all the nitty gritty details as it will test
+ // more than it needs to. Just be here in order to check on XML output
+ // easily if needed.
+ CPPUNIT_ASSERT_EQUAL(expected, "'" + actual + "'");
+}
+
+void MetricManagerTest::testJsonOutput()
+{
+ FastOS_ThreadPool pool(256 * 1024);
+ FakeTimer* timer = new FakeTimer(1000);
+ std::unique_ptr<MetricManager::Timer> timerImpl(timer);
+ MetricManager mm(std::move(timerImpl));
+ TestMetricSet mySet;
+ {
+ MetricLockGuard lockGuard(mm.getMetricLock());
+ mm.registerMetric(lockGuard, mySet.set);
+ }
+
+ // Initialize metric manager to get snapshots created.
+ mm.init("raw:"
+ "consumer[1]\n"
+ "consumer[0].name snapper\n"
+ "consumer[0].tags[1]\n"
+ "consumer[0].tags[0] snaptest\n",
+ pool);
+
+ mm.takeSnapshots(mm.getMetricLock(), 1000);
+
+ // Adding metrics to have some values in them
+ mySet.val6.addValue(2);
+ mySet.val9.val1.addValue(4);
+ mySet.val10.count.inc();
+ mySet.val10.a.val1.addValue(7);
+ mySet.val10.a.val2.addValue(2);
+ mySet.val10.b.val1.addValue(1);
+
+ timer->_time = 1300;
+ mm.takeSnapshots(mm.getMetricLock(), 1300);
+
+ // Create json output
+ vespalib::asciistream as;
+ vespalib::JsonStream jsonStream(as);
+ JsonWriter writer(jsonStream);
+ {
+ MetricLockGuard lockGuard(mm.getMetricLock());
+ mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false),
+ writer, "snapper");
+ }
+ jsonStream.finalize();
+ std::string jsonData = as.str();
+ // Parse it back
+ using namespace vespalib::slime;
+ vespalib::Slime slime;
+ size_t parsed = JsonFormat::decode(Memory(jsonData), slime);
+ if (jsonData.size() != parsed) {
+ SimpleBuffer buffer;
+ JsonFormat::encode(slime, buffer, false);
+ std::ostringstream ost;
+ ost << "Failed to parse JSON: '\n"
+ << jsonData << "'\n:" << buffer.get().make_string() << "\n";
+ CPPUNIT_ASSERT_EQUAL_MSG(ost.str(), jsonData.size(), parsed);
+ }
+ // Verify some content
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 1000.0,
+ slime.get()["snapshot"]["from"].asDouble());
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 1300.0,
+ slime.get()["snapshot"]["to"].asDouble());
+
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, vespalib::string("temp.val6"),
+ slime.get()["values"][0]["name"].asString().make_string());
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, vespalib::string("val6 desc"),
+ slime.get()["values"][0]["description"].asString().make_string());
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 2.0,
+ slime.get()["values"][0]["values"]["average"].asDouble());
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 1.0,
+ slime.get()["values"][0]["values"]["count"].asDouble());
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 2.0,
+ slime.get()["values"][0]["values"]["min"].asDouble());
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 2.0,
+ slime.get()["values"][0]["values"]["max"].asDouble());
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 2.0,
+ slime.get()["values"][0]["values"]["last"].asDouble());
+
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, vespalib::string("temp.multisub.sum.valsum"),
+ slime.get()["values"][10]["name"].asString().make_string());
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, vespalib::string("valsum desc"),
+ slime.get()["values"][10]["description"].asString().make_string());
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 10.0,
+ slime.get()["values"][10]["values"]["average"].asDouble());
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 3.0,
+ slime.get()["values"][10]["values"]["count"].asDouble());
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 1.0,
+ slime.get()["values"][10]["values"]["min"].asDouble());
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 7.0,
+ slime.get()["values"][10]["values"]["max"].asDouble());
+ CPPUNIT_ASSERT_EQUAL_MSG(jsonData, 10.0,
+ slime.get()["values"][10]["values"]["last"].asDouble());
+
+
+ metrics::StateApiAdapter adapter(mm);
+ vespalib::string normal = adapter.getMetrics("snapper");
+ CPPUNIT_ASSERT_EQUAL(vespalib::string(jsonData), normal);
+ vespalib::string total = adapter.getTotalMetrics("snapper");
+ CPPUNIT_ASSERT(total.size() > 0);
+ CPPUNIT_ASSERT(total != normal);
+}
+
+namespace {
+
+struct MetricSnapshotTestFixture
+{
+ static const size_t DEFAULT_THREAD_STACK_SIZE = 256 * 1024;
+
+ MetricManagerTest& test;
+ FastOS_ThreadPool pool;
+ FakeTimer* timer;
+ MetricManager manager;
+ MetricSet& mset;
+
+ MetricSnapshotTestFixture(MetricManagerTest& callerTest, MetricSet& metricSet)
+ : test(callerTest),
+ pool(DEFAULT_THREAD_STACK_SIZE),
+ timer(new FakeTimer(1000)),
+ manager(std::unique_ptr<MetricManager::Timer>(timer)),
+ mset(metricSet)
+ {
+ {
+ MetricLockGuard lockGuard(manager.getMetricLock());
+ manager.registerMetric(lockGuard, mset);
+ }
+
+ // Initialize metric manager to get snapshots created.
+ manager.init(
+ "raw:"
+ "consumer[1]\n"
+ "consumer[0].name snapper\n"
+ "consumer[0].addedmetrics[1]\n"
+ "consumer[0].addedmetrics[0] *\n",
+ pool);
+
+ test.takeSnapshots(manager, 1000);
+ }
+
+ // Take snapshot of metric values from time 1000 to time 1300
+ void takeSnapshotsOnce() {
+ timer->_time = 1300;
+ test.takeSnapshots(manager, 1300);
+ }
+
+ std::string renderLastSnapshotAsJson() const {
+ vespalib::asciistream as;
+ vespalib::JsonStream jsonStream(as, true);
+ JsonWriter writer(jsonStream);
+ {
+ MetricLockGuard lockGuard(manager.getMetricLock());
+ manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false),
+ writer, "snapper");
+ }
+ jsonStream.finalize();
+ return as.str();
+ }
+
+ std::string renderLastSnapshotAsText(
+ const std::string& matchPattern = ".*") const
+ {
+ std::ostringstream ss;
+ TextWriter writer(ss, 300, matchPattern, true);
+ {
+ MetricLockGuard lockGuard(manager.getMetricLock());
+ manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false),
+ writer, "snapper");
+ }
+ return ss.str();
+ }
+};
+
+class JsonMetricWrapper
+{
+ std::string _jsonText;
+ vespalib::Slime _tree;
+public:
+ JsonMetricWrapper(const std::string& jsonText)
+ : _jsonText(jsonText)
+ {
+ vespalib::slime::JsonFormat::decode(
+ vespalib::slime::Memory(jsonText), _tree);
+ }
+
+ // XXX ideally all these should be const, but is not clear how/if it's
+ // possible to get a const cursor into the underlying tree.
+ vespalib::slime::Cursor& root() { return _tree.get(); }
+
+ vespalib::slime::Cursor& nthMetric(size_t metricIndex) {
+ return root()["values"][metricIndex];
+ }
+
+ size_t nthMetricDimensionCount(size_t metricIndex) {
+ return nthMetric(metricIndex)["dimensions"].children();
+ }
+
+ std::string nthMetricName(size_t metricIndex) {
+ return nthMetric(metricIndex)["name"].asString().make_string();
+ }
+
+ std::string nthMetricDimension(size_t metricIndex, const std::string& key) {
+ return nthMetric(metricIndex)["dimensions"][key]
+ .asString().make_string();
+ }
+
+ // Verify that the nth metric has the given name and the given set of
+ // dimension key-values. Cannot use name alone to check, as multiple metrics
+ // may have same name but different dimensions and output ordering of
+ // metrics is well defined as being that of the insertion order.
+ void verifyDimensions(size_t metricIndex,
+ const std::string& name,
+ const Metric::Tags& dimensions) {
+ // Works to do this outside of main test body because cppunit uses
+ // exceptions for its failures.
+ CPPUNIT_ASSERT_EQUAL_MSG(_jsonText, name, nthMetricName(metricIndex));
+ CPPUNIT_ASSERT_EQUAL_MSG(_jsonText, dimensions.size(),
+ nthMetricDimensionCount(metricIndex));
+ for (auto& dim : dimensions) {
+ CPPUNIT_ASSERT_EQUAL_MSG(_jsonText, std::string(dim.value),
+ nthMetricDimension(metricIndex, dim.key));
+ }
+ }
+};
+
+struct DimensionTestMetricSet : MetricSet
+{
+ DoubleValueMetric val1;
+ LongCountMetric val2;
+
+ DimensionTestMetricSet(MetricSet* owner = nullptr)
+ : MetricSet("temp", {{"foo", "megafoo"}, {"bar", "hyperbar"}},
+ "", owner),
+ val1("val1", "tag1", "val1 desc", this), // Legacy tag; not dimension
+ val2("val2", {{"baz", "superbaz"}}, "val2 desc", this)
+ {
+ }
+};
+
+} // anon ns
+
+void
+MetricManagerTest::jsonOutputSupportsMultipleDimensions()
+{
+ DimensionTestMetricSet mset;
+ MetricSnapshotTestFixture fixture(*this, mset);
+
+ mset.val1.addValue(2);
+ mset.val2.inc();
+
+ fixture.takeSnapshotsOnce();
+ std::string actual = fixture.renderLastSnapshotAsJson();
+ JsonMetricWrapper json(actual);
+
+ json.verifyDimensions(0, "temp.val1",
+ {{"foo", "megafoo"}, {"bar", "hyperbar"}});
+ json.verifyDimensions(1, "temp.val2",
+ {{"foo", "megafoo"}, {"bar", "hyperbar"},
+ {"baz", "superbaz"}});
+}
+
+namespace {
+
+struct NestedDimensionTestMetricSet : MetricSet
+{
+ DimensionTestMetricSet nestedSet;
+
+ NestedDimensionTestMetricSet()
+ : MetricSet("outer", {{"fancy", "stuff"}}, ""),
+ nestedSet(this)
+ {
+ }
+};
+
+} // anon ns
+
+void
+MetricManagerTest::jsonOutputCanNestDimensionsFromMultipleMetricSets()
+{
+ NestedDimensionTestMetricSet mset;
+ MetricSnapshotTestFixture fixture(*this, mset);
+
+ mset.nestedSet.val1.addValue(2);
+ mset.nestedSet.val2.inc();
+
+ fixture.takeSnapshotsOnce();
+ std::string actual = fixture.renderLastSnapshotAsJson();
+ JsonMetricWrapper json(actual);
+
+ json.verifyDimensions(0, "outer.temp.val1",
+ {{"foo", "megafoo"}, {"bar", "hyperbar"},
+ {"fancy", "stuff"}});
+ json.verifyDimensions(1, "outer.temp.val2",
+ {{"foo", "megafoo"}, {"bar", "hyperbar"},
+ {"baz", "superbaz"}, {"fancy", "stuff"}});
+}
+
+namespace {
+
+struct DimensionOverridableTestMetricSet : MetricSet
+{
+ DoubleValueMetric val;
+
+ DimensionOverridableTestMetricSet(const std::string& dimValue,
+ MetricSet* owner = nullptr)
+ : MetricSet("temp", {{"foo", dimValue}}, "", owner),
+ val("val", "", "val desc", this)
+ {
+ }
+};
+
+struct SameNamesTestMetricSet : MetricSet
+{
+ DimensionOverridableTestMetricSet set1;
+ DimensionOverridableTestMetricSet set2;
+
+ SameNamesTestMetricSet()
+ : MetricSet("outer", {{"fancy", "stuff"}}, ""),
+ set1("bar", this),
+ set2("baz", this)
+ {
+ }
+};
+
+} // anon ns
+
+void
+MetricManagerTest::jsonOutputCanHaveMultipleSetsWithSameName()
+{
+ SameNamesTestMetricSet mset;
+ MetricSnapshotTestFixture fixture(*this, mset);
+
+ mset.set1.val.addValue(2);
+ mset.set2.val.addValue(5);
+
+ fixture.takeSnapshotsOnce();
+ std::string actual = fixture.renderLastSnapshotAsJson();
+ JsonMetricWrapper json(actual);
+
+ // Note the identical names. Only difference is the dimensions per set.
+ json.verifyDimensions(0, "outer.temp.val",
+ {{"foo", "bar"}, {"fancy", "stuff"}});
+ json.verifyDimensions(1, "outer.temp.val",
+ {{"foo", "baz"}, {"fancy", "stuff"}});
+}
+
+void MetricManagerTest::testTextOutput()
+{
+ FastOS_ThreadPool pool(256 * 1024);
+ FakeTimer* timer = new FakeTimer(1000);
+ std::unique_ptr<MetricManager::Timer> timerImpl(timer);
+ MetricManager mm(std::move(timerImpl));
+ TestMetricSet mySet;
+ {
+ MetricLockGuard lockGuard(mm.getMetricLock());
+ mm.registerMetric(lockGuard, mySet.set);
+ }
+ // Adding metrics to have some values in them
+ mySet.val6.addValue(2);
+ mySet.val9.val1.addValue(4);
+ mySet.val10.count.inc();
+ mySet.val10.a.val1.addValue(7);
+ mySet.val10.a.val2.addValue(2);
+ mySet.val10.b.val1.addValue(1);
+ // Initialize metric manager to get snapshots created.
+ mm.init("raw:"
+ "consumer[2]\n"
+ "consumer[0].name snapper\n"
+ "consumer[0].tags[1]\n"
+ "consumer[0].tags[0] snaptest\n"
+ "consumer[1].name log\n"
+ "consumer[1].tags[1]\n"
+ "consumer[1].tags[0] snaptest\n",
+ pool);
+ std::string expected(
+ "snapshot \"Active metrics showing updates since last snapshot\" from 1000 to 0 period 0\n"
+ "temp.val6 average=2 last=2 min=2 max=2 count=1 total=2\n"
+ "temp.sub.val1 average=4 last=4 min=4 max=4 count=1 total=4\n"
+ "temp.sub.valsum average=4 last=4 min=4 max=4 count=1 total=4\n"
+ "temp.multisub.count count=1\n"
+ "temp.multisub.a.val1 average=7 last=7 min=7 max=7 count=1 total=7\n"
+ "temp.multisub.a.valsum average=9 last=9\n"
+ "temp.multisub.b.val1 average=1 last=1 min=1 max=1 count=1 total=1\n"
+ "temp.multisub.b.valsum average=1 last=1 min=1 max=1 count=1 total=1\n"
+ "temp.multisub.sum.val1 average=8 last=8\n"
+ "temp.multisub.sum.val2 average=2 last=2 min=2 max=2 count=1 total=2\n"
+ "temp.multisub.sum.valsum average=10 last=10");
+ std::ostringstream ost;
+ TextWriter writer(ost, 300, ".*", true);
+ {
+ MetricLockGuard lockGuard(mm.getMetricLock());
+ mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), writer, "snapper");
+ }
+ std::string actual(ost.str());
+ // Not bothering to match all the nitty gritty details as it will test
+ // more than it needs to. Just be here in order to check on XML output
+ // easily if needed.
+ CPPUNIT_ASSERT_EQUAL(expected, actual);
+}
+
+void
+MetricManagerTest::textOutputSupportsDimensions()
+{
+ NestedDimensionTestMetricSet mset;
+ MetricSnapshotTestFixture fixture(*this, mset);
+
+ mset.nestedSet.val1.addValue(2);
+ mset.nestedSet.val2.inc();
+
+ fixture.takeSnapshotsOnce();
+ std::string actual = fixture.renderLastSnapshotAsText("outer.*temp.*val");
+ std::string expected(
+ "snapshot \"5 minute\" from 1000 to 1300 period 300\n"
+ "outer{fancy:stuff}.temp{bar:hyperbar,foo:megafoo}.val1 "
+ "average=2 last=2 min=2 max=2 count=1 total=2\n"
+ "outer{fancy:stuff}.temp{bar:hyperbar,foo:megafoo}.val2"
+ "{baz:superbaz} count=1");
+ CPPUNIT_ASSERT_EQUAL(expected, actual);
+}
+
+namespace {
+ struct MyUpdateHook : public MetricManager::UpdateHook {
+ std::ostringstream& _output;
+ FakeTimer& _timer;
+
+ MyUpdateHook(std::ostringstream& output, const char* name,
+ FakeTimer& timer)
+ : MetricManager::UpdateHook(name), _output(output), _timer(timer) {}
+
+ void updateMetrics(const MetricLockGuard & ) override {
+ _output << _timer.getTime() << ": " << getName() << " called\n";
+ }
+ };
+}
+
+void MetricManagerTest::testUpdateHooks()
+{
+ std::ostringstream output;
+ FastOS_ThreadPool pool(256 * 1024);
+ FakeTimer* timer = new FakeTimer(1000);
+ std::unique_ptr<MetricManager::Timer> timerImpl(timer);
+ // Add a metric set just so one exist
+ TestMetricSet mySet;
+ MetricManager mm(std::move(timerImpl));
+ {
+ MetricLockGuard lockGuard(mm.getMetricLock());
+ mm.registerMetric(lockGuard, mySet.set);
+ }
+
+ MyUpdateHook preInitShort(output, "BIS", *timer);
+ MyUpdateHook preInitLong(output, "BIL", *timer);
+ MyUpdateHook preInitInfinite(output, "BII", *timer);
+ mm.addMetricUpdateHook(preInitShort, 5);
+ mm.addMetricUpdateHook(preInitLong, 300);
+ mm.addMetricUpdateHook(preInitInfinite, 0);
+
+ // All hooks should get called during initialization
+
+ // Initialize metric manager to get snapshots created.
+ output << "Running init\n";
+ mm.init("raw:"
+ "consumer[2]\n"
+ "consumer[0].name snapper\n"
+ "consumer[0].tags[1]\n"
+ "consumer[0].tags[0] snaptest\n"
+ "consumer[1].name log\n"
+ "consumer[1].tags[1]\n"
+ "consumer[1].tags[0] snaptest\n",
+ pool);
+ output << "Init done\n";
+
+ MyUpdateHook postInitShort(output, "AIS", *timer);
+ MyUpdateHook postInitLong(output, "AIL", *timer);
+ MyUpdateHook postInitInfinite(output, "AII", *timer);
+ mm.addMetricUpdateHook(postInitShort, 5);
+ mm.addMetricUpdateHook(postInitLong, 400);
+ mm.addMetricUpdateHook(postInitInfinite, 0);
+
+ // After 5 seconds the short ones should get another.
+
+ timer->_time = 1006;
+ waitForTimeProcessed(mm, 1006);
+
+ // After 4 more seconds the short ones should get another
+ // since last update was a second late. (Stable periods, process time
+ // should not affect how often they are updated)
+
+ timer->_time = 1010;
+ waitForTimeProcessed(mm, 1010);
+
+ // Bumping considerably ahead, such that next update is in the past,
+ // we should only get one update called in this period.
+
+ timer->_time = 1200;
+ waitForTimeProcessed(mm, 1200);
+
+ // No updates at this time.
+ timer->_time = 1204;
+ waitForTimeProcessed(mm, 1204);
+
+ // Give all hooks an update
+ mm.updateMetrics(true);
+
+ // Last update should not have interfered with periods
+ timer->_time = 1205;
+ waitForTimeProcessed(mm, 1205);
+
+ // Time is just ahead of a snapshot.
+ timer->_time = 1299;
+ waitForTimeProcessed(mm, 1299);
+
+ // At time 1300 we are at a 5 minute snapshot bump
+ // All hooks should thus get an update. The one with matching period
+ // should only get one
+ timer->_time = 1300;
+ waitForTimeProcessed(mm, 1300);
+
+ // The snapshot time currently doesn't count for the metric at period
+ // 400. It will get an event at this time.
+ timer->_time = 1450;
+ waitForTimeProcessed(mm, 1450);
+
+ std::string expected(
+ "Running init\n"
+ "1000: BIS called\n"
+ "1000: BIL called\n"
+ "Init done\n"
+ "1006: BIS called\n"
+ "1006: AIS called\n"
+ "1010: BIS called\n"
+ "1010: AIS called\n"
+ "1200: BIS called\n"
+ "1200: AIS called\n"
+ "1204: BIS called\n"
+ "1204: BIL called\n"
+ "1204: AIS called\n"
+ "1204: AIL called\n"
+ "1204: BII called\n"
+ "1204: AII called\n"
+ "1205: BIS called\n"
+ "1205: AIS called\n"
+ "1299: BIS called\n"
+ "1299: AIS called\n"
+ "1300: BIS called\n"
+ "1300: BIL called\n"
+ "1300: AIS called\n"
+ "1300: AIL called\n"
+ "1300: BII called\n"
+ "1300: AII called\n"
+ "1450: BIS called\n"
+ "1450: AIS called\n"
+ "1450: AIL called\n"
+ );
+ std::string actual(output.str());
+ CPPUNIT_ASSERT_EQUAL(expected, actual);
+}
+
+} // metrics
diff --git a/metrics/src/tests/metricsettest.cpp b/metrics/src/tests/metricsettest.cpp
new file mode 100644
index 00000000000..9ce141feaf3
--- /dev/null
+++ b/metrics/src/tests/metricsettest.cpp
@@ -0,0 +1,157 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/vespalib/objects/floatingpointtype.h>
+#include <vespa/metrics/metrics.h>
+
+namespace metrics {
+
+struct MetricSetTest : public CppUnit::TestFixture {
+ void testNormalUsage();
+ void supportMultipleMetricsWithSameNameDifferentDimensions();
+ void uniqueTargetMetricsAreAddedToMetricSet();
+
+ CPPUNIT_TEST_SUITE(MetricSetTest);
+ CPPUNIT_TEST(testNormalUsage);
+ CPPUNIT_TEST(supportMultipleMetricsWithSameNameDifferentDimensions);
+ CPPUNIT_TEST(uniqueTargetMetricsAreAddedToMetricSet);
+ CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(MetricSetTest);
+
+namespace {
+ struct TestMetricVisitor : public MetricVisitor {
+ std::ostringstream ost;
+ uint32_t setsToVisit;
+
+ TestMetricVisitor(uint32_t setsToVisit_ = 100)
+ : ost(), setsToVisit(setsToVisit_) {}
+
+ virtual bool visitMetricSet(const MetricSet& set, bool autoGenerated) {
+ ost << "[" << (autoGenerated ? "*" : "") << set.getName() << "]\n";
+ if (setsToVisit > 0) {
+ --setsToVisit;
+ return true;
+ }
+ return false;
+ }
+ virtual bool visitMetric(const Metric& m, bool autoGenerated) {
+ ost << (autoGenerated ? "*" : "") << m.getName() << "\n";
+ return true;
+ }
+ };
+}
+
+void
+MetricSetTest::testNormalUsage()
+{
+ // Set up some metrics to test..
+ MetricSet set("a", "foo", "");
+ DoubleValueMetric v1("c", "foo", "", &set);
+ LongAverageMetric v2("b", "", "", &set);
+ LongCountMetric v3("d", "bar", "", &set);
+ MetricSet set2("e", "bar", "", &set);
+ LongCountMetric v4("f", "foo", "", &set2);
+
+ // Give them some values
+ v1.addValue(4.2);
+ v2.addValue(8);
+ v3.inc();
+ v4.inc(3);
+
+ // Check that we can register through registerMetric function too.
+ LongCountMetric v5("g", "", "");
+ set.registerMetric(v5);
+ v5.inc(3);
+ v5.dec();
+
+ // Check that getMetric works, and doesn't return copy.
+ LongAverageMetric* v2copy(
+ dynamic_cast<LongAverageMetric*>(set.getMetric("b")));
+ CPPUNIT_ASSERT(v2copy != 0);
+ v2copy->addValue(9);
+ Metric* nonExistingCopy = set.getMetric("nonexisting");
+ CPPUNIT_ASSERT(nonExistingCopy == 0);
+ nonExistingCopy = set.getMetric("non.existing");
+ CPPUNIT_ASSERT(nonExistingCopy == 0);
+
+ // Check that paths are set
+ MetricSet topSet("top", "", "");
+ topSet.registerMetric(set);
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("a"), set.getPath());
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("a.c"), v1.getPath());
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("a.b"), v2.getPath());
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("a.d"), v3.getPath());
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("a.e"), set2.getPath());
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("a.e.f"), v4.getPath());
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("a.g"), v5.getPath());
+ CPPUNIT_ASSERT_EQUAL(vespalib::string("a.b"), v2copy->getPath());
+
+ // Verify XML output. Should be in register order.
+ std::string expected("'\n"
+ "a:\n"
+ " c average=4.2 last=4.2 min=4.2 max=4.2 count=1 total=4.2\n"
+ " b average=8.5 last=9 min=8 max=9 count=2 total=17\n"
+ " d count=1\n"
+ " e:\n"
+ " f count=3\n"
+ " g count=2'"
+ );
+ CPPUNIT_ASSERT_EQUAL(expected, "'\n" + set.toString() + "'");
+
+ // Verify that visiting works. That you get all metrics if you answer
+ // true to all sets, and that you don't get members of sets you answer
+ // false to get.
+ {
+ TestMetricVisitor visitor(2);
+ set.visit(visitor);
+ expected = "[a]\nc\nb\nd\n[e]\nf\ng\n";
+ CPPUNIT_ASSERT_EQUAL("\n" + expected, "\n" + visitor.ost.str());
+ }
+ {
+ TestMetricVisitor visitor(1);
+ set.visit(visitor);
+ expected = "[a]\nc\nb\nd\n[e]\ng\n";
+ CPPUNIT_ASSERT_EQUAL("\n" + expected, "\n" + visitor.ost.str());
+ }
+ {
+ TestMetricVisitor visitor(0);
+ set.visit(visitor);
+ expected = "[a]\n";
+ CPPUNIT_ASSERT_EQUAL("\n" + expected, "\n" + visitor.ost.str());
+ }
+}
+
+void
+MetricSetTest::supportMultipleMetricsWithSameNameDifferentDimensions()
+{
+ MetricSet set("dimset", {{"foo", "bar"}}, "");
+ DoubleValueMetric v1("stuff", {{"baz", "blarg"}}, "", &set);
+ LongAverageMetric v2("stuff", {{"flarn", "yarn"}}, "", &set);
+
+ CPPUNIT_ASSERT_EQUAL(size_t(2), set.getRegisteredMetrics().size());
+}
+
+void
+MetricSetTest::uniqueTargetMetricsAreAddedToMetricSet()
+{
+ MetricSet set1("a", "foo", "");
+ LongCountMetric v1("wow", "foo", "", &set1);
+ MetricSet set2("e", "bar", "");
+ LongCountMetric v2("doge", "foo", "", &set2);
+
+ // Have to actually assign a value to metrics or they won't be carried over.
+ v1.inc();
+ v2.inc();
+
+ // 'doge' metric in set2 must be preserved even though it does not exist
+ // in set1.
+ std::vector<Metric::LP> ownerList;
+ set1.addToSnapshot(set2, ownerList);
+
+ CPPUNIT_ASSERT(set2.getMetric("wow") != nullptr);
+ CPPUNIT_ASSERT(set2.getMetric("doge") != nullptr);
+}
+
+} // metrics
diff --git a/metrics/src/tests/metrictest.cpp b/metrics/src/tests/metrictest.cpp
new file mode 100644
index 00000000000..67dd608051a
--- /dev/null
+++ b/metrics/src/tests/metrictest.cpp
@@ -0,0 +1,112 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/metrics/valuemetric.h>
+#include <vespa/metrics/countmetric.h>
+
+namespace metrics {
+
+struct MetricTest : public CppUnit::TestFixture
+{
+ template <typename MetricImpl>
+ void testMetricsGetDimensionsAsPartOfMangledNameImpl();
+ template <typename MetricImpl>
+ void testMangledNameMayContainMultipleDimensionsImpl();
+
+ void valueMetricsGetDimensionsAsPartOfMangledName();
+ void countMetricsGetDimensionsAsPartOfMangledName();
+ void valueMetricMangledNameMayContainMultipleDimensions();
+ void countMetricMangledNameMayContainMultipleDimensions();
+ void mangledNameListsDimensionsInLexicographicOrder();
+ void manglingDoesNotChangeOriginalMetricName();
+ void legacyTagsDoNotCreateMangledName();
+
+ CPPUNIT_TEST_SUITE(MetricTest);
+ CPPUNIT_TEST(valueMetricsGetDimensionsAsPartOfMangledName);
+ CPPUNIT_TEST(countMetricsGetDimensionsAsPartOfMangledName);
+ CPPUNIT_TEST(valueMetricMangledNameMayContainMultipleDimensions);
+ CPPUNIT_TEST(countMetricMangledNameMayContainMultipleDimensions);
+ CPPUNIT_TEST(mangledNameListsDimensionsInLexicographicOrder);
+ CPPUNIT_TEST(manglingDoesNotChangeOriginalMetricName);
+ CPPUNIT_TEST(legacyTagsDoNotCreateMangledName);
+ CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(MetricTest);
+
+// Metric subclasses have the same constructor parameters, so we template
+// our way from having to duplicate code. Templated GTest fixtures would be
+// a far more elegant solution.
+template <typename MetricImpl>
+void
+MetricTest::testMetricsGetDimensionsAsPartOfMangledNameImpl()
+{
+ MetricImpl m("test", {{"foo", "bar"}}, "description goes here");
+ CPPUNIT_ASSERT_EQUAL(std::string("test{foo:bar}"), m.getMangledName());
+}
+
+template <typename MetricImpl>
+void
+MetricTest::testMangledNameMayContainMultipleDimensionsImpl()
+{
+ MetricImpl m("test",
+ {{"flarn", "yarn"}, {"foo", "bar"}},
+ "description goes here");
+ CPPUNIT_ASSERT_EQUAL(std::string("test{flarn:yarn,foo:bar}"),
+ m.getMangledName());
+}
+
+void
+MetricTest::valueMetricsGetDimensionsAsPartOfMangledName()
+{
+ testMetricsGetDimensionsAsPartOfMangledNameImpl<LongValueMetric>();
+}
+
+void
+MetricTest::countMetricsGetDimensionsAsPartOfMangledName()
+{
+ testMetricsGetDimensionsAsPartOfMangledNameImpl<LongCountMetric>();
+}
+
+void
+MetricTest::valueMetricMangledNameMayContainMultipleDimensions()
+{
+ testMangledNameMayContainMultipleDimensionsImpl<LongValueMetric>();
+}
+
+void
+MetricTest::countMetricMangledNameMayContainMultipleDimensions()
+{
+ testMangledNameMayContainMultipleDimensionsImpl<LongCountMetric>();
+}
+
+// Assuming the above tests pass, we simplify by not requiring all subclasses
+// to be tested since propagation down to the base class has already been
+// verified.
+void
+MetricTest::mangledNameListsDimensionsInLexicographicOrder()
+{
+ LongValueMetric m("test",
+ {{"xyz", "bar"}, {"abc", "foo"}, {"def", "baz"}},
+ "");
+ CPPUNIT_ASSERT_EQUAL(std::string("test{abc:foo,def:baz,xyz:bar}"),
+ m.getMangledName());
+}
+
+void
+MetricTest::manglingDoesNotChangeOriginalMetricName()
+{
+ LongValueMetric m("test", {{"foo", "bar"}}, "");
+ CPPUNIT_ASSERT_EQUAL(std::string("test"), m.getName());
+}
+
+void
+MetricTest::legacyTagsDoNotCreateMangledName()
+{
+ LongValueMetric m("test", "foo bar", "");
+ CPPUNIT_ASSERT_EQUAL(std::string("test"), m.getName());
+ CPPUNIT_ASSERT_EQUAL(std::string("test"), m.getMangledName());
+}
+
+} // metrics
+
diff --git a/metrics/src/tests/snapshottest.cpp b/metrics/src/tests/snapshottest.cpp
new file mode 100644
index 00000000000..800d9f8465a
--- /dev/null
+++ b/metrics/src/tests/snapshottest.cpp
@@ -0,0 +1,354 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include <vespa/metrics/metrics.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+
+LOG_SETUP(".test.snapshot");
+
+namespace metrics {
+
+struct SnapshotTest : public CppUnit::TestFixture {
+ void testSnapshotTwoDays();
+
+ CPPUNIT_TEST_SUITE(SnapshotTest);
+ CPPUNIT_TEST(testSnapshotTwoDays);
+ CPPUNIT_TEST_SUITE_END();
+
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(SnapshotTest);
+
+namespace {
+
+struct SubSubMetricSet : public MetricSet {
+ const LoadTypeSet& loadTypes;
+ int incVal;
+ LongCountMetric count1;
+ LongCountMetric count2;
+ LoadMetric<LongCountMetric> loadCount;
+ SumMetric<LongCountMetric> countSum;
+ DoubleValueMetric value1;
+ DoubleValueMetric value2;
+ LoadMetric<DoubleValueMetric> loadValue;
+ SumMetric<DoubleValueMetric> valueSum;
+ DoubleAverageMetric average1;
+ DoubleAverageMetric average2;
+ LoadMetric<DoubleAverageMetric> loadAverage;
+ SumMetric<DoubleAverageMetric> averageSum;
+
+ SubSubMetricSet(vespalib::stringref name, const LoadTypeSet& loadTypes_,
+ MetricSet* owner = 0)
+ : MetricSet(name, "", "", owner),
+ loadTypes(loadTypes_),
+ incVal(1),
+ count1("count1", "", "", this),
+ count2("count2", "", "", this),
+ loadCount(loadTypes, LongCountMetric("loadCount", "", ""), this),
+ countSum("countSum", "", "", this),
+ value1("value1", "", "", this),
+ value2("value2", "", "", this),
+ loadValue(loadTypes, DoubleValueMetric("loadValue", "", ""), this),
+ valueSum("valueSum", "", "", this),
+ average1("average1", "", "", this),
+ average2("average2", "", "", this),
+ loadAverage(loadTypes, DoubleAverageMetric("loadAverage", "", ""),
+ this),
+ averageSum("averageSum", "", "", this)
+ {
+ countSum.addMetricToSum(count1);
+ countSum.addMetricToSum(count2);
+ valueSum.addMetricToSum(value1);
+ valueSum.addMetricToSum(value2);
+ averageSum.addMetricToSum(average1);
+ averageSum.addMetricToSum(average2);
+ }
+
+ virtual Metric* clone(std::vector<Metric::LP>& ownerList,
+ CopyType copyType, metrics::MetricSet* owner,
+ bool includeUnused) const
+ {
+ if (copyType == INACTIVE) {
+ return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused);
+ }
+ return (SubSubMetricSet*) (new SubSubMetricSet(
+ getName(), loadTypes, owner))
+ ->assignValues(*this);
+ }
+
+ void incValues() {
+ count1.inc(incVal);
+ count2.inc(incVal);
+ for (uint32_t i=0; i<loadTypes.size(); ++i) {
+ loadCount[loadTypes[i]].inc(incVal);
+ }
+ value1.set(incVal);
+ value2.set(incVal);
+ for (uint32_t i=0; i<loadTypes.size(); ++i) {
+ loadValue[loadTypes[i]].set(incVal);
+ }
+ average1.set(incVal);
+ average2.set(incVal);
+ for (uint32_t i=0; i<loadTypes.size(); ++i) {
+ loadAverage[loadTypes[i]].set(incVal);
+ }
+ }
+};
+
+struct SubMetricSet : public MetricSet {
+ const LoadTypeSet& loadTypes;
+ SubSubMetricSet set1;
+ SubSubMetricSet set2;
+ LoadMetric<SubSubMetricSet> loadSet;
+ SumMetric<SubSubMetricSet> setSum;
+
+ SubMetricSet(vespalib::stringref name, const LoadTypeSet& loadTypes_,
+ MetricSet* owner = 0)
+ : MetricSet(name, "", "", owner),
+ loadTypes(loadTypes_),
+ set1("set1", loadTypes, this),
+ set2("set2", loadTypes, this),
+ loadSet(loadTypes, *std::unique_ptr<SubSubMetricSet>(
+ new SubSubMetricSet("loadSet", loadTypes)), this),
+ setSum("setSum", "", "", this)
+ {
+ setSum.addMetricToSum(set1);
+ setSum.addMetricToSum(set2);
+ }
+
+ virtual Metric* clone(std::vector<Metric::LP>& ownerList,
+ CopyType copyType, metrics::MetricSet* owner,
+ bool includeUnused) const
+ {
+ if (copyType == INACTIVE) {
+ return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused);
+ }
+ return (SubMetricSet*) (new SubMetricSet(getName(), loadTypes, owner))
+ ->assignValues(*this);
+ }
+
+ void incValues() {
+ set1.incValues();
+ set2.incValues();
+ for (uint32_t i=0; i<loadTypes.size(); ++i) {
+ loadSet[loadTypes[i]].incValues();
+ }
+ }
+};
+
+struct TestMetricSet : public MetricSet {
+ const LoadTypeSet& loadTypes;
+ SubMetricSet set1;
+ SubMetricSet set2;
+ LoadMetric<SubMetricSet> loadSet;
+ SumMetric<SubMetricSet> setSum;
+
+ TestMetricSet(vespalib::stringref name, const LoadTypeSet& loadTypes_,
+ MetricSet* owner = 0)
+ : MetricSet(name, "", "", owner),
+ loadTypes(loadTypes_),
+ set1("set1", loadTypes, this),
+ set2("set2", loadTypes, this),
+ loadSet(loadTypes, *std::unique_ptr<SubMetricSet>(
+ new SubMetricSet("loadSet", loadTypes)), this),
+ setSum("setSum", "", "", this)
+ {
+ setSum.addMetricToSum(set1);
+ setSum.addMetricToSum(set2);
+ }
+
+ void incValues() {
+ set1.incValues();
+ set2.incValues();
+ for (uint32_t i=0; i<loadTypes.size(); ++i) {
+ loadSet[loadTypes[i]].incValues();
+ }
+ }
+};
+
+struct FakeTimer : public MetricManager::Timer {
+ uint32_t _timeInSecs;
+
+ FakeTimer() : _timeInSecs(1) {}
+
+ virtual time_t getTime() const { return _timeInSecs; }
+};
+
+} // End of anonymous namespace
+
+#define ASSERT_VALUE(value, snapshot, name) \
+{ \
+ const Metric* _metricValue_((snapshot).getMetrics().getMetric(name)); \
+ if (_metricValue_ == 0) { \
+ CPPUNIT_FAIL("Metric value '" + std::string(name) \
+ + "' not found in snapshot"); \
+ } \
+ CPPUNIT_ASSERT_EQUAL(value, \
+ int32_t(_metricValue_->getLongValue("value"))); \
+}
+
+void SnapshotTest::testSnapshotTwoDays()
+{
+ // Create load types
+ LoadTypeSet loadTypes;
+ loadTypes.push_back(LoadType(1, "foo"));
+ loadTypes.push_back(LoadType(2, "bar"));
+
+ TestMetricSet set("test", loadTypes);
+
+ FakeTimer* timer;
+ FastOS_ThreadPool threadPool(256 * 1024);
+ MetricManager mm(
+ std::unique_ptr<MetricManager::Timer>(timer = new FakeTimer));
+ {
+ MetricLockGuard lockGuard(mm.getMetricLock());
+ mm.registerMetric(lockGuard, set);
+ }
+ mm.init("raw:consumer[1]\n"
+ "consumer[0].name \"log\"", threadPool, false);
+ mm.tick(mm.getMetricLock(), timer->_timeInSecs * 1000);
+
+ for (uint32_t days=0; days<2; ++days) {
+ for (uint32_t hour=0; hour<24; ++hour) {
+ for (uint32_t fiveMin=0; fiveMin<12; ++fiveMin) {
+ set.incValues();
+ timer->_timeInSecs += 5 * 60;
+ mm.tick(mm.getMetricLock(), timer->_timeInSecs * 1000);
+ }
+ }
+ }
+
+ // Print all data. Useful for debugging. It's too much to verify
+ // everything, so test will just verify some parts. Add more parts if you
+ // find a failure.
+ /*
+ std::string regex = ".*";
+ bool verbose = true;
+ std::cout << "\n" << mm.getActiveMetrics().toString(mm, "", regex, verbose)
+ << "\n\n";
+ std::vector<uint32_t> periods(mm.getSnapshotPeriods());
+ for (uint32_t i=0; i<periods.size(); ++i) {
+ const MetricSnapshot& snap(mm.getMetricSnapshot(periods[i]));
+ std::cout << snap.toString(mm, "", regex, verbose) << "\n\n";
+ }
+
+ std::cout << mm.getTotalMetricSnapshot().toString(mm, "", regex, verbose)
+ << "\n";
+ */
+
+ const MetricSnapshot* snap = 0;
+ // active snapshot
+ MetricLockGuard lockGuard(mm.getMetricLock());
+ snap = &mm.getActiveMetrics(lockGuard);
+ ASSERT_VALUE(0, *snap, "test.set1.set1.count1");
+ ASSERT_VALUE(0, *snap, "test.set1.set1.loadCount.foo");
+ ASSERT_VALUE(0, *snap, "test.set1.set1.loadCount.sum");
+ ASSERT_VALUE(0, *snap, "test.set1.set1.countSum");
+ ASSERT_VALUE(0, *snap, "test.set1.loadSet.foo.count1");
+ ASSERT_VALUE(0, *snap, "test.set1.loadSet.foo.countSum");
+/* Current test procedure for fetching values, don't work in active sums of sets
+ ASSERT_VALUE(0, *snap, "test.set1.loadSet.sum.count1");
+ ASSERT_VALUE(0, *snap, "test.set1.loadSet.sum.loadCount.foo");
+ ASSERT_VALUE(0, *snap, "test.set1.loadSet.sum.loadCount.sum");
+ ASSERT_VALUE(0, *snap, "test.set1.loadSet.sum.countSum");
+*/
+
+ // 5 minute snapshot
+ snap = &mm.getMetricSnapshot(lockGuard, 5 * 60);
+ ASSERT_VALUE(1, *snap, "test.set1.set1.count1");
+ ASSERT_VALUE(1, *snap, "test.set1.set1.loadCount.foo");
+ ASSERT_VALUE(2, *snap, "test.set1.set1.loadCount.sum");
+ ASSERT_VALUE(2, *snap, "test.set1.set1.countSum");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.count1");
+ ASSERT_VALUE(2, *snap, "test.set1.loadSet.foo.countSum");
+ ASSERT_VALUE(2, *snap, "test.set1.loadSet.sum.count1");
+ ASSERT_VALUE(2, *snap, "test.set1.loadSet.sum.loadCount.foo");
+ ASSERT_VALUE(4, *snap, "test.set1.loadSet.sum.loadCount.sum");
+ ASSERT_VALUE(4, *snap, "test.set1.loadSet.sum.countSum");
+
+ ASSERT_VALUE(1, *snap, "test.set1.set1.average1");
+ ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.foo");
+ ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.sum");
+ ASSERT_VALUE(1, *snap, "test.set1.set1.averageSum");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.average1");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.averageSum");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.average1");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.foo");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.sum");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.averageSum");
+
+ // 1 hour snapshot
+ snap = &mm.getMetricSnapshot(lockGuard, 60 * 60);
+ ASSERT_VALUE(12, *snap, "test.set1.set1.count1");
+ ASSERT_VALUE(12, *snap, "test.set1.set1.loadCount.foo");
+ ASSERT_VALUE(24, *snap, "test.set1.set1.loadCount.sum");
+ ASSERT_VALUE(24, *snap, "test.set1.set1.countSum");
+ ASSERT_VALUE(12, *snap, "test.set1.loadSet.foo.count1");
+ ASSERT_VALUE(24, *snap, "test.set1.loadSet.foo.countSum");
+ ASSERT_VALUE(24, *snap, "test.set1.loadSet.sum.count1");
+ ASSERT_VALUE(24, *snap, "test.set1.loadSet.sum.loadCount.foo");
+ ASSERT_VALUE(48, *snap, "test.set1.loadSet.sum.loadCount.sum");
+ ASSERT_VALUE(48, *snap, "test.set1.loadSet.sum.countSum");
+
+ ASSERT_VALUE(1, *snap, "test.set1.set1.average1");
+ ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.foo");
+ ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.sum");
+ ASSERT_VALUE(1, *snap, "test.set1.set1.averageSum");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.average1");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.averageSum");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.average1");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.foo");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.sum");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.averageSum");
+
+ // 1 day snapshot
+ snap = &mm.getMetricSnapshot(lockGuard, 24 * 60 * 60);
+ ASSERT_VALUE(288, *snap, "test.set1.set1.count1");
+ ASSERT_VALUE(288, *snap, "test.set1.set1.loadCount.foo");
+ ASSERT_VALUE(576, *snap, "test.set1.set1.loadCount.sum");
+ ASSERT_VALUE(576, *snap, "test.set1.set1.countSum");
+ ASSERT_VALUE(288, *snap, "test.set1.loadSet.foo.count1");
+ ASSERT_VALUE(576, *snap, "test.set1.loadSet.foo.countSum");
+ ASSERT_VALUE(576, *snap, "test.set1.loadSet.sum.count1");
+ ASSERT_VALUE(576, *snap, "test.set1.loadSet.sum.loadCount.foo");
+ ASSERT_VALUE(1152, *snap, "test.set1.loadSet.sum.loadCount.sum");
+ ASSERT_VALUE(1152, *snap, "test.set1.loadSet.sum.countSum");
+
+ ASSERT_VALUE(1, *snap, "test.set1.set1.average1");
+ ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.foo");
+ ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.sum");
+ ASSERT_VALUE(1, *snap, "test.set1.set1.averageSum");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.average1");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.averageSum");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.average1");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.foo");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.sum");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.averageSum");
+
+ // total snapshot (2 days currently, not testing weeks)
+ snap = &mm.getTotalMetricSnapshot(lockGuard);
+ ASSERT_VALUE(576, *snap, "test.set1.set1.count1");
+ ASSERT_VALUE(576, *snap, "test.set1.set1.loadCount.foo");
+ ASSERT_VALUE(1152, *snap, "test.set1.set1.loadCount.sum");
+ ASSERT_VALUE(1152, *snap, "test.set1.set1.countSum");
+ ASSERT_VALUE(576, *snap, "test.set1.loadSet.foo.count1");
+ ASSERT_VALUE(1152, *snap, "test.set1.loadSet.foo.countSum");
+ ASSERT_VALUE(1152, *snap, "test.set1.loadSet.sum.count1");
+ ASSERT_VALUE(1152, *snap, "test.set1.loadSet.sum.loadCount.foo");
+ ASSERT_VALUE(2304, *snap, "test.set1.loadSet.sum.loadCount.sum");
+ ASSERT_VALUE(2304, *snap, "test.set1.loadSet.sum.countSum");
+
+ ASSERT_VALUE(1, *snap, "test.set1.set1.average1");
+ ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.foo");
+ ASSERT_VALUE(1, *snap, "test.set1.set1.loadAverage.sum");
+ ASSERT_VALUE(1, *snap, "test.set1.set1.averageSum");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.average1");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.foo.averageSum");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.average1");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.foo");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.loadAverage.sum");
+ ASSERT_VALUE(1, *snap, "test.set1.loadSet.sum.averageSum");
+}
+
+} // metrics
diff --git a/metrics/src/tests/stresstest.cpp b/metrics/src/tests/stresstest.cpp
new file mode 100644
index 00000000000..f88c3a4944c
--- /dev/null
+++ b/metrics/src/tests/stresstest.cpp
@@ -0,0 +1,144 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include <vespa/metrics/metrics.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+
+LOG_SETUP(".metrics.test.stress");
+
+namespace metrics {
+
+struct StressTest : public CppUnit::TestFixture {
+ void testStress();
+
+ CPPUNIT_TEST_SUITE(StressTest);
+ CPPUNIT_TEST(testStress);
+ CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(StressTest);
+
+namespace {
+ struct InnerMetricSet : public MetricSet {
+ const LoadTypeSet& _loadTypes;
+ LongCountMetric _count;
+ LongAverageMetric _value1;
+ LongAverageMetric _value2;
+ SumMetric<LongAverageMetric> _valueSum;
+ LoadMetric<LongAverageMetric> _load;
+
+ InnerMetricSet(const char* name, const LoadTypeSet& lt,
+ MetricSet* owner = 0)
+ : MetricSet(name, "", "", owner),
+ _loadTypes(lt),
+ _count("count", "", "", this),
+ _value1("value1", "", "", this),
+ _value2("value2", "", "", this),
+ _valueSum("valuesum", "", "", this),
+ _load(lt, LongAverageMetric("load", "", ""), this)
+ {
+ _valueSum.addMetricToSum(_value1);
+ _valueSum.addMetricToSum(_value2);
+ }
+
+ Metric* clone(std::vector<Metric::LP>& ownerList, CopyType copyType,
+ MetricSet* owner, bool includeUnused) const
+ {
+ if (copyType != CLONE) {
+ return MetricSet::clone(ownerList, copyType, owner,
+ includeUnused);
+ }
+ return (new InnerMetricSet(getName().c_str(), _loadTypes, owner))
+ ->assignValues(*this);
+ }
+ };
+ struct OuterMetricSet : public MetricSet {
+ InnerMetricSet _inner1;
+ InnerMetricSet _inner2;
+ SumMetric<InnerMetricSet> _innerSum;
+ InnerMetricSet _tmp;
+ LoadMetric<InnerMetricSet> _load;
+
+ OuterMetricSet(const LoadTypeSet& lt, MetricSet* owner = 0)
+ : MetricSet("outer", "", "", owner),
+ _inner1("inner1", lt, this),
+ _inner2("inner2", lt, this),
+ _innerSum("innersum", "", "", this),
+ _tmp("innertmp", lt, 0),
+ _load(lt, _tmp, this)
+ {
+ _innerSum.addMetricToSum(_inner1);
+ _innerSum.addMetricToSum(_inner2);
+ }
+ };
+ struct Hammer : public document::Runnable {
+ typedef vespalib::LinkedPtr<Hammer> LP;
+
+ OuterMetricSet& _metrics;
+ const LoadTypeSet& _loadTypes;
+ LoadType _nonexistingLoadType;
+
+ Hammer(OuterMetricSet& metrics, const LoadTypeSet& lt,
+ FastOS_ThreadPool& threadPool)
+ : _metrics(metrics), _loadTypes(lt),
+ _nonexistingLoadType(123, "nonexisting")
+ {
+ start(threadPool);
+ }
+ ~Hammer() {
+ stop();
+ join();
+ //std::cerr << "Loadgiver thread joined\n";
+ }
+
+ virtual void run() {
+ uint64_t i = 0;
+ while (running()) {
+ ++i;
+ setMetrics(i, _metrics._inner1);
+ setMetrics(i + 3, _metrics._inner2);
+ const LoadType& loadType(_loadTypes[i % _loadTypes.size()]);
+ setMetrics(i + 5, _metrics._load[loadType]);
+ }
+ }
+
+ void setMetrics(uint64_t val, InnerMetricSet& set) {
+ set._count.inc(val);
+ set._value1.addValue(val);
+ set._value2.addValue(val + 10);
+ set._load[_loadTypes[val % _loadTypes.size()]].addValue(val);
+ }
+ };
+}
+
+
+void
+StressTest::testStress()
+{
+ LoadTypeSet loadTypes;
+ loadTypes.push_back(LoadType(0, "default"));
+ loadTypes.push_back(LoadType(2, "foo"));
+ loadTypes.push_back(LoadType(1, "bar"));
+
+ OuterMetricSet metrics(loadTypes);
+
+ LOG(info, "Starting load givers");
+ FastOS_ThreadPool threadPool(256 * 1024);
+ std::vector<Hammer::LP> hammers;
+ for (uint32_t i=0; i<10; ++i) {
+ hammers.push_back(Hammer::LP(
+ new Hammer(metrics, loadTypes, threadPool)));
+ }
+ LOG(info, "Waiting to let loadgivers hammer a while");
+ FastOS_Thread::Sleep(5 * 1000);
+
+ LOG(info, "Removing loadgivers");
+ hammers.clear();
+
+ LOG(info, "Printing end state");
+ std::ostringstream ost;
+ metrics.print(ost, true, "", 5);
+ // std::cerr << ost.str() << "\n";
+}
+
+} // metrics
diff --git a/metrics/src/tests/summetrictest.cpp b/metrics/src/tests/summetrictest.cpp
new file mode 100644
index 00000000000..fd4d67d7f3b
--- /dev/null
+++ b/metrics/src/tests/summetrictest.cpp
@@ -0,0 +1,150 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/metrics/metrics.h>
+
+namespace metrics {
+
+struct SumMetricTest : public CppUnit::TestFixture {
+ void testLongCountMetric();
+ void testAverageMetric();
+ void testMetricSet();
+ void testRemove();
+ void testStartValue();
+
+ CPPUNIT_TEST_SUITE(SumMetricTest);
+ CPPUNIT_TEST(testLongCountMetric);
+ CPPUNIT_TEST(testAverageMetric);
+ CPPUNIT_TEST(testMetricSet);
+ CPPUNIT_TEST(testRemove);
+ CPPUNIT_TEST(testStartValue);
+ CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(SumMetricTest);
+
+void
+SumMetricTest::testLongCountMetric()
+{
+ MetricSet parent("parent", "", "");
+ SumMetric<LongCountMetric> sum("foo", "", "foodesc", &parent);
+
+ LongCountMetric v1("ff", "", "", &parent);
+ LongCountMetric v2("aa", "", "", &parent);
+
+ sum.addMetricToSum(v1);
+ sum.addMetricToSum(v2);
+
+ // Give them some values
+ v1.inc(3);
+ v2.inc(7);
+
+ // Verify XML output. Should be in register order.
+ std::string expected("foo count=10");
+ CPPUNIT_ASSERT_EQUAL(expected, sum.toString());
+ CPPUNIT_ASSERT_EQUAL(int64_t(10), sum.getLongValue("value"));
+}
+
+void
+SumMetricTest::testAverageMetric() {
+ MetricSet parent("parent", "", "");
+ SumMetric<LongAverageMetric> sum("foo", "", "foodesc", &parent);
+
+ LongAverageMetric v1("ff", "", "", &parent);
+ LongAverageMetric v2("aa", "", "", &parent);
+
+ sum.addMetricToSum(v1);
+ sum.addMetricToSum(v2);
+
+ // Give them some values
+ v1.addValue(3);
+ v2.addValue(7);
+
+ // Verify XML output. Should be in register order.
+ std::string expected("foo average=5 last=7 min=3 max=7 count=2 total=10");
+ CPPUNIT_ASSERT_EQUAL(expected, sum.toString());
+ CPPUNIT_ASSERT_EQUAL(int64_t(5), sum.getLongValue("value"));
+ CPPUNIT_ASSERT_EQUAL(int64_t(3), sum.getLongValue("min"));
+ CPPUNIT_ASSERT_EQUAL(int64_t(7), sum.getLongValue("max"));
+}
+
+void
+SumMetricTest::testMetricSet() {
+ MetricSet parent("parent", "", "");
+ SumMetric<MetricSet> sum("foo", "", "bar", &parent);
+
+ MetricSet set1("a", "", "", &parent);
+ MetricSet set2("b", "", "", &parent);
+ LongValueMetric v1("c", "", "", &set1);
+ LongValueMetric v2("d", "", "", &set2);
+ LongCountMetric v3("e", "", "", &set1);
+ LongCountMetric v4("f", "", "", &set2);
+
+ sum.addMetricToSum(set1);
+ sum.addMetricToSum(set2);
+
+ // Give them some values
+ v1.addValue(3);
+ v2.addValue(7);
+ v3.inc(2);
+ v4.inc();
+
+ // Verify XML output. Should be in register order.
+ std::string expected("'\n"
+ "foo:\n"
+ " c average=3 last=3 min=3 max=3 count=1 total=3\n"
+ " e count=2'"
+ );
+ CPPUNIT_ASSERT_EQUAL(expected, "'\n" + sum.toString() + "'");
+}
+
+void
+SumMetricTest::testRemove()
+{
+ MetricSet parent("parent", "", "");
+ SumMetric<LongCountMetric> sum("foo", "", "foodesc", &parent);
+
+ LongCountMetric v1("ff", "", "", &parent);
+ LongCountMetric v2("aa", "", "", &parent);
+ LongCountMetric v3("zz", "", "", &parent);
+
+ sum.addMetricToSum(v1);
+ sum.addMetricToSum(v2);
+ sum.addMetricToSum(v3);
+
+ // Give them some values
+ v1.inc(3);
+ v2.inc(7);
+ v3.inc(10);
+
+ CPPUNIT_ASSERT_EQUAL(int64_t(20), sum.getLongValue("value"));
+ sum.removeMetricFromSum(v2);
+ CPPUNIT_ASSERT_EQUAL(int64_t(13), sum.getLongValue("value"));
+}
+
+void
+SumMetricTest::testStartValue()
+{
+ MetricSnapshot snapshot("active");
+ SumMetric<LongValueMetric> sum("foo", "", "foodesc",
+ &snapshot.getMetrics());
+ LongValueMetric start("start", "", "", 0);
+ start.set(50);
+ sum.setStartValue(start);
+
+ // without children
+ CPPUNIT_ASSERT_EQUAL(int64_t(50), sum.getLongValue("value"));
+
+ MetricSnapshot copy("copy");
+ copy.recreateSnapshot(snapshot.getMetrics(), true);
+ snapshot.addToSnapshot(copy, 100);
+
+ LongValueMetric value("value", "", "", &snapshot.getMetrics());
+ sum.addMetricToSum(value);
+ value.set(10);
+
+ // with children
+ CPPUNIT_ASSERT_EQUAL(int64_t(60), sum.getLongValue("value"));
+}
+
+} // metrics
diff --git a/metrics/src/tests/testrunner.cpp b/metrics/src/tests/testrunner.cpp
new file mode 100644
index 00000000000..5d8dc8d4c1f
--- /dev/null
+++ b/metrics/src/tests/testrunner.cpp
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <iostream>
+#include <vespa/log/log.h>
+#include <vespa/vdstestlib/cppunit/cppunittestrunner.h>
+
+LOG_SETUP("storagecppunittests");
+
+int
+main(int argc, char **argv)
+{
+ vdstestlib::CppUnitTestRunner testRunner;
+ return testRunner.run(argc, argv);
+}
diff --git a/metrics/src/tests/valuemetrictest.cpp b/metrics/src/tests/valuemetrictest.cpp
new file mode 100644
index 00000000000..afe4221481e
--- /dev/null
+++ b/metrics/src/tests/valuemetrictest.cpp
@@ -0,0 +1,281 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/vdstestlib/cppunit/macros.h>
+#include <vespa/vespalib/objects/floatingpointtype.h>
+#include <vespa/metrics/jsonwriter.h>
+#include <vespa/metrics/metricmanager.h>
+#include <vespa/metrics/valuemetric.h>
+
+using vespalib::Double;
+
+namespace metrics {
+
+struct ValueMetricTest : public CppUnit::TestFixture {
+ void testDoubleValueMetric();
+ void testDoubleValueMetricNotUpdatedOnNaN();
+ void testDoubleValueMetricNotUpdatedOnInfinity();
+ void testLongValueMetric();
+ void testSmallAverage();
+ void testAddValueBatch();
+ void testJson();
+
+ CPPUNIT_TEST_SUITE(ValueMetricTest);
+ CPPUNIT_TEST(testDoubleValueMetric);
+ CPPUNIT_TEST(testDoubleValueMetricNotUpdatedOnNaN);
+ CPPUNIT_TEST(testDoubleValueMetricNotUpdatedOnInfinity);
+ CPPUNIT_TEST(testLongValueMetric);
+ CPPUNIT_TEST(testSmallAverage);
+ CPPUNIT_TEST(testAddValueBatch);
+ CPPUNIT_TEST(testJson);
+ CPPUNIT_TEST_SUITE_END();
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION(ValueMetricTest);
+
+#define ASSERT_AVERAGE(metric, avg, min, max, count, last) \
+ CPPUNIT_ASSERT_EQUAL_MSG("avg", Double(avg), Double(metric.getAverage())); \
+ CPPUNIT_ASSERT_EQUAL_MSG("cnt", Double(count), Double(metric.getCount())); \
+ CPPUNIT_ASSERT_EQUAL_MSG("last", Double(last), Double(metric.getLast())); \
+ if (metric.getCount() > 0) { \
+ CPPUNIT_ASSERT_EQUAL_MSG("min", Double(min), Double(metric.getMinimum())); \
+ CPPUNIT_ASSERT_EQUAL_MSG("max", Double(max), Double(metric.getMaximum())); \
+ }
+
+void ValueMetricTest::testDoubleValueMetric()
+{
+ DoubleValueMetric m("test", "tag", "description");
+ m.addValue(100);
+ ASSERT_AVERAGE(m, 100, 100, 100, 1, 100);
+ m.addValue(100);
+ ASSERT_AVERAGE(m, 100, 100, 100, 2, 100);
+ m.addValue(40);
+ ASSERT_AVERAGE(m, 80, 40, 100, 3, 40);
+ DoubleValueMetric m2(m);
+ ASSERT_AVERAGE(m, 80, 40, 100, 3, 40);
+ m.reset();
+ ASSERT_AVERAGE(m, 0, 0, 0, 0, 0);
+
+ DoubleValueMetric n("m2", "", "desc");
+ n.addValue(60);
+ ASSERT_AVERAGE(n, 60, 60, 60, 1, 60);
+
+ DoubleValueMetric o = m2 + n;
+ ASSERT_AVERAGE(o, 140, 40, 100, 4, 100);
+
+ o = n + m2;
+ ASSERT_AVERAGE(o, 140, 40, 100, 4, 100);
+
+ std::string expected(
+ "test average=80 last=40 min=40 max=100 count=3 total=240");
+ CPPUNIT_ASSERT_EQUAL(expected, m2.toString());
+ expected = "m2 average=140 last=100";
+ CPPUNIT_ASSERT_EQUAL(expected, o.toString());
+
+ CPPUNIT_ASSERT_EQUAL(Double(40), Double(m2.getDoubleValue("value")));
+ CPPUNIT_ASSERT_EQUAL(Double(80), Double(m2.getDoubleValue("average")));
+ CPPUNIT_ASSERT_EQUAL(Double(40), Double(m2.getDoubleValue("min")));
+ CPPUNIT_ASSERT_EQUAL(Double(100), Double(m2.getDoubleValue("max")));
+ CPPUNIT_ASSERT_EQUAL(Double(40), Double(m2.getDoubleValue("last")));
+ CPPUNIT_ASSERT_EQUAL(Double(3), Double(m2.getDoubleValue("count")));
+ CPPUNIT_ASSERT_EQUAL(Double(240), Double(m2.getDoubleValue("total")));
+
+ CPPUNIT_ASSERT_EQUAL(int64_t(40), m2.getLongValue("value"));
+ CPPUNIT_ASSERT_EQUAL(int64_t(80), m2.getLongValue("average"));
+ CPPUNIT_ASSERT_EQUAL(int64_t(40), m2.getLongValue("min"));
+ CPPUNIT_ASSERT_EQUAL(int64_t(100), m2.getLongValue("max"));
+ CPPUNIT_ASSERT_EQUAL(int64_t(40), m2.getLongValue("last"));
+ CPPUNIT_ASSERT_EQUAL(int64_t(3), m2.getLongValue("count"));
+ CPPUNIT_ASSERT_EQUAL(int64_t(240), m2.getLongValue("total"));
+}
+
+void
+ValueMetricTest::testDoubleValueMetricNotUpdatedOnNaN()
+{
+ DoubleValueMetric m("test", "tag", "description");
+ m.addValue(std::numeric_limits<double>::quiet_NaN());
+ CPPUNIT_ASSERT_EQUAL(std::string(), m.toString());
+
+ m.addAvgValueWithCount(std::numeric_limits<double>::quiet_NaN(), 123);
+ CPPUNIT_ASSERT_EQUAL(std::string(), m.toString());
+
+ m.inc(std::numeric_limits<double>::quiet_NaN());
+ CPPUNIT_ASSERT_EQUAL(std::string(), m.toString());
+
+ m.dec(std::numeric_limits<double>::quiet_NaN());
+ CPPUNIT_ASSERT_EQUAL(std::string(), m.toString());
+}
+
+void
+ValueMetricTest::testDoubleValueMetricNotUpdatedOnInfinity()
+{
+ DoubleValueMetric m("test", "tag", "description");
+ m.addValue(std::numeric_limits<double>::infinity());
+ CPPUNIT_ASSERT_EQUAL(std::string(), m.toString());
+
+ m.addAvgValueWithCount(std::numeric_limits<double>::quiet_NaN(), 123);
+ CPPUNIT_ASSERT_EQUAL(std::string(), m.toString());
+
+ m.inc(std::numeric_limits<double>::infinity());
+ CPPUNIT_ASSERT_EQUAL(std::string(), m.toString());
+
+ m.dec(std::numeric_limits<double>::infinity());
+ CPPUNIT_ASSERT_EQUAL(std::string(), m.toString());
+}
+
+void ValueMetricTest::testLongValueMetric()
+{
+ LongValueMetric m("test", "tag", "description");
+ m.addValue(100);
+ ASSERT_AVERAGE(m, 100, 100, 100, 1, 100);
+ m.addValue(100);
+ ASSERT_AVERAGE(m, 100, 100, 100, 2, 100);
+ m.addValue(41);
+ ASSERT_AVERAGE(m, 241.0 / 3, 41, 100, 3, 41);
+ LongValueMetric m2(m);
+ ASSERT_AVERAGE(m, 241.0 / 3, 41, 100, 3, 41);
+ m.reset();
+ ASSERT_AVERAGE(m, 0, 0, 0, 0, 0);
+
+ LongValueMetric n("m2", "", "desc");
+ n.addValue(60);
+ ASSERT_AVERAGE(n, 60, 60, 60, 1, 60);
+
+ LongValueMetric o = m2 + n;
+ ASSERT_AVERAGE(o, 140.25, 41, 100, 4, 101);
+
+ o = n + m2;
+ ASSERT_AVERAGE(o, 140.25, 41, 100, 4, 101);
+
+ std::string expected(
+ "test average=80.3333 last=41 min=41 max=100 count=3 total=241");
+ CPPUNIT_ASSERT_EQUAL(expected, m2.toString());
+ expected = "m2 average=140.25 last=101";
+ CPPUNIT_ASSERT_EQUAL(expected, o.toString());
+
+ CPPUNIT_ASSERT_EQUAL(Double(41), Double(m2.getDoubleValue("value")));
+ CPPUNIT_ASSERT_EQUAL(Double(241.0/3), Double(m2.getDoubleValue("average")));
+ CPPUNIT_ASSERT_EQUAL(Double(41), Double(m2.getDoubleValue("min")));
+ CPPUNIT_ASSERT_EQUAL(Double(100), Double(m2.getDoubleValue("max")));
+ CPPUNIT_ASSERT_EQUAL(Double(41), Double(m2.getDoubleValue("last")));
+ CPPUNIT_ASSERT_EQUAL(Double(3), Double(m2.getDoubleValue("count")));
+ CPPUNIT_ASSERT_EQUAL(Double(241), Double(m2.getDoubleValue("total")));
+
+ CPPUNIT_ASSERT_EQUAL(int64_t(41), m2.getLongValue("value"));
+ CPPUNIT_ASSERT_EQUAL(int64_t(80), m2.getLongValue("average"));
+ CPPUNIT_ASSERT_EQUAL(int64_t(41), m2.getLongValue("min"));
+ CPPUNIT_ASSERT_EQUAL(int64_t(100), m2.getLongValue("max"));
+ CPPUNIT_ASSERT_EQUAL(int64_t(41), m2.getLongValue("last"));
+ CPPUNIT_ASSERT_EQUAL(int64_t(3), m2.getLongValue("count"));
+ CPPUNIT_ASSERT_EQUAL(int64_t(241), m2.getLongValue("total"));
+}
+
+void ValueMetricTest::testSmallAverage()
+{
+ DoubleValueMetric m("test", "tag", "description");
+ m.addValue(0.0001);
+ m.addValue(0.0002);
+ m.addValue(0.0003);
+ std::vector<Metric::LP> ownerList;
+ Metric::UP c(m.clone(ownerList, Metric::INACTIVE, 0, false));
+ std::string expect("test average=0.0002 last=0.0003 min=0.0001 max=0.0003 count=3 total=0.0006");
+ CPPUNIT_ASSERT_EQUAL(expect, m.toString());
+ CPPUNIT_ASSERT_EQUAL(expect, c->toString());
+}
+
+void ValueMetricTest::testAddValueBatch() {
+ DoubleValueMetric m("test", "tag", "description");
+ m.addValueBatch(100, 3);
+ ASSERT_AVERAGE(m, 100, 100, 100, 3, 100);
+ m.addValueBatch(0, 0);
+ ASSERT_AVERAGE(m, 100, 100, 100, 3, 100);
+}
+
+namespace {
+ vespalib::string extractMetricJson(vespalib::stringref s) {
+ vespalib::StringTokenizer st(s, "\n", "");
+ for (uint32_t i = st.size() - 1; i < st.size(); --i) {
+ if (st[i].find("\"name\":\"") != std::string::npos) {
+ vespalib::asciistream as;
+ as << "'\n";
+ for (uint32_t j=i-1; j<st.size() - 2; ++j) {
+ as << st[j].substr(4) << "\n";
+ }
+ as << "'";
+ return as.str();
+ }
+ }
+ throw vespalib::IllegalArgumentException("Didn't find metric");
+ }
+ vespalib::string getJson(MetricManager& mm) {
+ vespalib::asciistream as;
+ vespalib::JsonStream stream(as, true);
+ JsonWriter writer(stream);
+ MetricLockGuard guard(mm.getMetricLock());
+ mm.visit(guard, mm.getActiveMetrics(guard), writer, "");
+ stream.finalize();
+ return as.str();
+ }
+}
+
+void ValueMetricTest::testJson() {
+ MetricManager mm;
+ DoubleValueMetric m("test", "tag", "description");
+ mm.registerMetric(mm.getMetricLock(), m);
+
+ vespalib::string expected("'\n"
+ "{\n"
+ " \"name\":\"test\",\n"
+ " \"description\":\"description\",\n"
+ " \"values\":\n"
+ " {\n"
+ " \"average\":0.0,\n"
+ " \"count\":0,\n"
+ " \"min\":0.0,\n"
+ " \"max\":0.0,\n"
+ " \"last\":0.0\n"
+ " },\n"
+ " \"dimensions\":\n"
+ " {\n"
+ " }\n"
+ "}\n'"
+ );
+ CPPUNIT_ASSERT_EQUAL(expected, extractMetricJson(getJson(mm)));
+ m.addValue(100);
+ expected = "'\n"
+ "{\n"
+ " \"name\":\"test\",\n"
+ " \"description\":\"description\",\n"
+ " \"values\":\n"
+ " {\n"
+ " \"average\":100.0,\n"
+ " \"count\":1,\n"
+ " \"min\":100.0,\n"
+ " \"max\":100.0,\n"
+ " \"last\":100.0\n"
+ " },\n"
+ " \"dimensions\":\n"
+ " {\n"
+ " }\n"
+ "}\n'";
+ CPPUNIT_ASSERT_EQUAL(expected, extractMetricJson(getJson(mm)));
+ m.addValue(500);
+ expected = "'\n"
+ "{\n"
+ " \"name\":\"test\",\n"
+ " \"description\":\"description\",\n"
+ " \"values\":\n"
+ " {\n"
+ " \"average\":300.0,\n"
+ " \"count\":2,\n"
+ " \"min\":100.0,\n"
+ " \"max\":500.0,\n"
+ " \"last\":500.0\n"
+ " },\n"
+ " \"dimensions\":\n"
+ " {\n"
+ " }\n"
+ "}\n'";
+ CPPUNIT_ASSERT_EQUAL(expected, extractMetricJson(getJson(mm)));
+}
+
+} // namespace metrics
diff --git a/metrics/src/vespa/metrics/.gitignore b/metrics/src/vespa/metrics/.gitignore
new file mode 100644
index 00000000000..d6b11486175
--- /dev/null
+++ b/metrics/src/vespa/metrics/.gitignore
@@ -0,0 +1,10 @@
+.depend
+.*.swp
+.~
+*.o
+*.So
+Makefile
+config-metricsmanager.h
+config-metricsmanager.cpp
+
+/libmetrics.so.5.1
diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt
new file mode 100644
index 00000000000..7a1813d7e18
--- /dev/null
+++ b/metrics/src/vespa/metrics/CMakeLists.txt
@@ -0,0 +1,19 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(metrics
+ SOURCES
+ metric.cpp
+ countmetric.cpp
+ valuemetric.cpp
+ metrictimer.cpp
+ metricset.cpp
+ metricmanager.cpp
+ metricsnapshot.cpp
+ jsonwriter.cpp
+ textwriter.cpp
+ xmlwriter.cpp
+ state_api_adapter.cpp
+ INSTALL lib64
+ DEPENDS
+)
+vespa_generate_config(metrics metricsmanager.def)
+install(FILES metricsmanager.def DESTINATION var/db/vespa/config_server/serverdb/classes)
diff --git a/metrics/src/vespa/metrics/countmetric.cpp b/metrics/src/vespa/metrics/countmetric.cpp
new file mode 100644
index 00000000000..d152c09a242
--- /dev/null
+++ b/metrics/src/vespa/metrics/countmetric.cpp
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/metrics/countmetric.h>
+
+#include <vespa/log/log.h>
+
+LOG_SETUP(".metrics.metric.count");
+
+namespace metrics {
+
+void
+AbstractCountMetric::logWarning(const char* msg) const
+{
+ LOG(warning, "%s", msg);
+}
+
+void
+AbstractCountMetric::sendLogCountEvent(
+ Metric::String name, uint64_t value) const
+{
+ EV_COUNT(name.c_str(), value);
+}
+
+} // metrics
diff --git a/metrics/src/vespa/metrics/countmetric.h b/metrics/src/vespa/metrics/countmetric.h
new file mode 100644
index 00000000000..c9eef00f6ea
--- /dev/null
+++ b/metrics/src/vespa/metrics/countmetric.h
@@ -0,0 +1,357 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class CountMetric
+ * \ingroup metrics
+ *
+ * \brief Metric representing a count.
+ *
+ * A counter metric have the following properties:
+ * - It can never decrease, only increase.
+ * - Logs its value as a count event.
+ * - When summing counts, the counts are added together.
+ */
+
+#pragma once
+
+#include <sstream>
+#include <vespa/metrics/metric.h>
+#include <vespa/metrics/metricvalueset.h>
+#include <vespa/vespalib/util/jsonstream.h>
+#include <atomic>
+
+namespace metrics {
+
+struct AbstractCountMetric : public Metric {
+ virtual bool visit(MetricVisitor& visitor,
+ bool tagAsAutoGenerated = false) const
+ {
+ return visitor.visitCountMetric(*this, tagAsAutoGenerated);
+ }
+ virtual MetricValueClass::UP getValues() const = 0;
+ virtual bool sumOnAdd() const = 0;
+ virtual bool inUse(const MetricValueClass& v) const = 0;
+
+protected:
+ AbstractCountMetric(const String& name, const String& tags,
+ const String& description, MetricSet* owner = 0)
+ : Metric(name, tags, description, owner)
+ {
+ }
+
+ AbstractCountMetric(const String& name, Tags dimensions,
+ const String& description, MetricSet* owner = 0)
+ : Metric(name, std::move(dimensions), description, owner)
+ {
+ }
+
+ AbstractCountMetric(const AbstractCountMetric& other, MetricSet* owner)
+ : Metric(other, owner)
+ {
+ }
+
+ void logWarning(const char* msg) const;
+ void sendLogCountEvent(Metric::String name, uint64_t value) const;
+};
+
+template<typename T, bool SumOnAdd>
+class CountMetric : public AbstractCountMetric
+{
+ struct Values : public MetricValueClass {
+ T _value;
+
+ struct AtomicImpl {
+ std::atomic<T> _value {0};
+ };
+
+ void relaxedStoreInto(AtomicImpl& target) const noexcept {
+ target._value.store(_value, std::memory_order_relaxed);
+ }
+
+ void relaxedLoadFrom(const AtomicImpl& source) noexcept {
+ _value = source._value.load(std::memory_order_relaxed);
+ }
+
+ Values() : _value(0) {}
+
+ std::string toString() const {
+ std::ostringstream ost;
+ ost << _value;
+ return ost.str();
+ }
+ double getDoubleValue(const stringref &) const
+ { return static_cast<double>(_value); }
+ uint64_t getLongValue(const stringref &) const
+ { return static_cast<uint64_t>(_value); }
+ void output(const std::string&, std::ostream& out) const
+ { out << _value; }
+ void output(const std::string&, vespalib::JsonStream& stream) const
+ { stream << _value; }
+ bool inUse() const { return (_value != 0); }
+ };
+ MetricValueSet<Values> _values;
+
+ enum Flag { LOG_IF_UNSET = 2 };
+
+ bool logIfUnset() const { return _values.hasFlag(LOG_IF_UNSET); }
+
+public:
+ CountMetric(const String& name, const String& tags,
+ const String& description, MetricSet* owner = 0);
+
+ CountMetric(const String& name, Tags dimensions,
+ const String& description, MetricSet* owner = 0);
+
+ CountMetric(const CountMetric<T, SumOnAdd>& other,
+ CopyType, MetricSet* owner);
+
+ virtual MetricValueClass::UP getValues() const
+ { return MetricValueClass::UP(new Values(_values.getValues())); }
+ void logOnlyIfSet() { _values.removeFlag(LOG_IF_UNSET); }
+
+ void set(T value);
+ void inc(T value = 1);
+ void dec(T value = 1);
+ CountMetric & operator++() { inc(); return *this; }
+ CountMetric & operator--() { dec(); return *this; }
+
+ CountMetric operator++(int) { CountMetric tmp(*this); inc(); return tmp; }
+ CountMetric operator--(int) { CountMetric tmp(*this); inc(); return tmp; }
+
+ CountMetric & operator+=(const CountMetric &);
+ CountMetric & operator-=(const CountMetric &);
+ friend CountMetric operator+(const CountMetric & a, const CountMetric & b) {
+ CountMetric t(a); t += b; return t;
+ }
+ friend CountMetric operator-(const CountMetric & a, const CountMetric & b) {
+ CountMetric t(a); t -= b; return t;
+ }
+
+ virtual CountMetric * clone(
+ std::vector<Metric::LP>&, CopyType type, MetricSet* owner,
+ bool /*includeUnused*/) const
+ { return new CountMetric<T, SumOnAdd>(*this, type, owner); }
+
+ T getValue() const { return _values.getValues()._value; }
+
+ void reset() {
+ _values.reset();
+ }
+
+ virtual bool logFromTotalMetrics() const { return true; }
+ virtual bool logEvent(const String& fullName) const;
+
+ virtual void print(std::ostream&, bool verbose,
+ const std::string& indent, uint64_t secondsPassed) const;
+
+ // Only one metric in valuemetric, so return it on any id.
+ virtual int64_t getLongValue(const stringref & id) const
+ { (void) id; return static_cast<int64_t>(getValue()); }
+ virtual double getDoubleValue(const stringref & id) const
+ { (void) id; return static_cast<double>(getValue()); }
+
+ virtual bool inUse(const MetricValueClass& v) const
+ { return static_cast<const Values&>(v).inUse(); }
+ virtual bool used() const { return _values.getValues().inUse(); }
+ virtual bool sumOnAdd() const { return SumOnAdd; }
+
+ virtual void addMemoryUsage(MemoryConsumption&) const;
+
+ virtual void printDebug(std::ostream&, const std::string& indent) const;
+
+ virtual void addToPart(Metric&) const;
+ virtual void addToSnapshot(Metric&, std::vector<Metric::LP>&) const;
+
+};
+
+typedef CountMetric<uint64_t, true> LongCountMetric;
+
+template <typename T, bool SumOnAdd>
+CountMetric<T, SumOnAdd>::CountMetric(const String& name, const String& tags,
+ const String& desc, MetricSet* owner)
+ : AbstractCountMetric(name, tags, desc, owner),
+ _values()
+{
+ _values.setFlag(LOG_IF_UNSET);
+}
+
+template <typename T, bool SumOnAdd>
+CountMetric<T, SumOnAdd>::CountMetric(const String& name, Tags dimensions,
+ const String& desc, MetricSet* owner)
+ : AbstractCountMetric(name, std::move(dimensions), desc, owner),
+ _values()
+{
+ _values.setFlag(LOG_IF_UNSET);
+}
+
+template <typename T, bool SumOnAdd>
+CountMetric<T, SumOnAdd>::CountMetric(const CountMetric<T, SumOnAdd>& other,
+ CopyType copyType, MetricSet* owner)
+ : AbstractCountMetric(other, owner),
+ _values(other._values, copyType == CLONE ? other._values.size() : 1)
+{
+}
+
+template <typename T, bool SumOnAdd>
+CountMetric<T, SumOnAdd>&
+CountMetric<T, SumOnAdd>::operator+=(const CountMetric<T, SumOnAdd>& other)
+{
+ T otherValues(other.getValue());
+ bool overflow;
+ Values values;
+ do {
+ values = _values.getValues();
+ overflow = (values._value + otherValues < values._value);
+ values._value += otherValues;
+ } while (!_values.setValues(values));
+ if (overflow) {
+ _values.reset();
+ std::ostringstream ost;
+ ost << "Overflow in metric " << getPath() << " op +=. Resetting it.";
+ logWarning(ost.str().c_str());
+ }
+ return *this;
+}
+
+template <typename T, bool SumOnAdd>
+CountMetric<T, SumOnAdd>&
+CountMetric<T, SumOnAdd>::operator-=(const CountMetric<T, SumOnAdd>& other)
+{
+ T otherValues(other.getValue());
+ bool underflow;
+ Values values;
+ do {
+ values = _values.getValues();
+ underflow = (values._value - otherValues > values._value);
+ values._value -= otherValues;
+ } while (!_values.setValues(values));
+ if (underflow) {
+ _values.reset();
+ std::ostringstream ost;
+ ost << "Underflow in metric " << getPath() << " op -=. Resetting it.";
+ logWarning(ost.str().c_str());
+ }
+ return *this;
+}
+
+template <typename T, bool SumOnAdd>
+void
+CountMetric<T, SumOnAdd>::set(T value)
+{
+ Values values;
+ values._value = value;
+ while (!_values.setValues(values)) {}
+}
+
+template <typename T, bool SumOnAdd>
+void
+CountMetric<T, SumOnAdd>::inc(T value)
+{
+ bool overflow;
+ Values values;
+ do {
+ values = _values.getValues();
+ overflow = (values._value + value < values._value);
+ values._value += value;
+ } while (!_values.setValues(values));
+ if (overflow) {
+ _values.reset();
+ std::ostringstream ost;
+ ost << "Overflow in metric " << getPath() << ". Resetting it.";
+ logWarning(ost.str().c_str());
+ }
+}
+
+template <typename T, bool SumOnAdd>
+void
+CountMetric<T, SumOnAdd>::dec(T value)
+{
+ bool underflow;
+ Values values;
+ do {
+ values = _values.getValues();
+ underflow = (values._value - value > values._value);
+ values._value -= value;
+ } while (!_values.setValues(values));
+ if (underflow) {
+ _values.reset();
+ std::ostringstream ost;
+ ost << "Underflow in metric " << getPath() << ". Resetting it.";
+ logWarning(ost.str().c_str());
+ }
+}
+
+template <typename T, bool SumOnAdd>
+void
+CountMetric<T, SumOnAdd>::addToSnapshot(
+ Metric& other, std::vector<Metric::LP>&) const
+{
+ CountMetric<T, SumOnAdd>& o(
+ reinterpret_cast<CountMetric<T, SumOnAdd>&>(other));
+ o.inc(_values.getValues()._value);
+}
+
+template <typename T, bool SumOnAdd>
+void
+CountMetric<T, SumOnAdd>::addToPart(Metric& other) const
+{
+ CountMetric<T, SumOnAdd>& o(
+ reinterpret_cast<CountMetric<T, SumOnAdd>&>(other));
+ if (SumOnAdd) {
+ o.inc(_values.getValues()._value);
+ } else {
+ o.set((_values.getValues()._value + o._values.getValues()._value) / 2);
+ }
+}
+
+template <typename T, bool SumOnAdd>
+bool
+CountMetric<T, SumOnAdd>::logEvent(const String& fullName) const
+{
+ Values values(_values.getValues());
+ if (!logIfUnset() && values._value == 0) return false;
+ sendLogCountEvent(
+ fullName, static_cast<uint64_t>(values._value));
+ return true;
+}
+
+template <typename T, bool SumOnAdd>
+void
+CountMetric<T, SumOnAdd>::print(std::ostream& out, bool verbose,
+ const std::string& indent,
+ uint64_t secondsPassed) const
+{
+ (void) indent;
+ Values values(_values.getValues());
+ if (values._value == 0 && !verbose) return;
+ out << this->_name << (SumOnAdd ? " count=" : " value=") << values._value;
+ if (SumOnAdd) {
+ if (secondsPassed != 0) {
+ double avgDiff = values._value / ((double) secondsPassed);
+ out << " average_change_per_second=" << avgDiff;
+ }
+ }
+}
+
+template <typename T, bool SumOnAdd>
+void
+CountMetric<T, SumOnAdd>::addMemoryUsage(MemoryConsumption& mc) const
+{
+ ++mc._countMetricCount;
+ mc._countMetricValues += _values.getMemoryUsageAllocatedInternally();
+ mc._countMetricMeta += sizeof(CountMetric<T, SumOnAdd>)
+ - sizeof(Metric);
+ Metric::addMemoryUsage(mc);
+}
+
+template <typename T, bool SumOnAdd>
+void
+CountMetric<T, SumOnAdd>::printDebug(std::ostream& out,
+ const std::string& indent) const
+{
+ Values values(_values.getValues());
+ out << "count=" << values._value << " ";
+ Metric::printDebug(out, indent);
+}
+
+} // metrics
+
diff --git a/metrics/src/vespa/metrics/jsonwriter.cpp b/metrics/src/vespa/metrics/jsonwriter.cpp
new file mode 100644
index 00000000000..0847a5a1624
--- /dev/null
+++ b/metrics/src/vespa/metrics/jsonwriter.cpp
@@ -0,0 +1,179 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/metrics/jsonwriter.h>
+
+#include <vespa/metrics/countmetric.h>
+#include <vespa/metrics/valuemetric.h>
+#include <vespa/metrics/metricsnapshot.h>
+#include <vespa/vespalib/util/jsonstream.h>
+
+#include <algorithm>
+#include <iterator>
+
+namespace metrics {
+
+JsonWriter::JsonWriter(vespalib::JsonStream& stream)
+ : _stream(stream),
+ _flag(NOT_STARTED),
+ _dimensionStack(),
+ _period(0)
+{
+}
+
+bool
+JsonWriter::visitSnapshot(const MetricSnapshot& snapshot)
+{
+ _stream << Object()
+ << "snapshot" << Object()
+ << "from" << snapshot.getFromTime()
+ << "to" << snapshot.getToTime()
+ << End()
+ << "values" << Array();
+ _flag = SNAPSHOT_STARTED;
+ _period = snapshot.getPeriod();
+ return true;
+}
+
+void
+JsonWriter::doneVisitingSnapshot(const MetricSnapshot&)
+{
+ assert(_flag == SNAPSHOT_STARTED);
+ _stream << End() << End();
+ _flag = NOT_STARTED;
+ _period = 0;
+}
+
+void
+JsonWriter::pushLegacyDimensionFromSet(const MetricSet& ms)
+{
+ _dimensionStack.push_back({{ms.getDimensionKey(), ms.getName()}});
+}
+
+void
+JsonWriter::pushAllDimensionsFromSet(const MetricSet& ms)
+{
+ _dimensionStack.push_back(ms.getTags());
+}
+
+bool
+JsonWriter::visitMetricSet(const MetricSet& set, bool)
+{
+ if (!set.getDimensionKey().empty()) {
+ pushLegacyDimensionFromSet(set);
+ } else {
+ pushAllDimensionsFromSet(set);
+ }
+ return true;
+}
+
+void
+JsonWriter::doneVisitingMetricSet(const MetricSet&)
+{
+ _dimensionStack.pop_back();
+}
+
+void
+JsonWriter::writeCommonPrefix(const Metric& m)
+{
+ if (_flag == NOT_STARTED) {
+ _stream << Array();
+ _flag = METRICS_WRITTEN;
+ }
+ _stream << Object()
+ << "name" << m.getPath()
+ << "description" << m.getDescription();
+}
+
+void
+JsonWriter::writeDimensions(const DimensionSet& dimensions)
+{
+ for (const auto& dimension : dimensions) {
+ if (!dimension.key.empty() && !dimension.value.empty()) {
+ _stream << dimension.key << dimension.value;
+ }
+ }
+}
+
+void
+JsonWriter::writeInheritedDimensions()
+{
+ for (const auto& dimensions : _dimensionStack) {
+ writeDimensions(dimensions);
+ }
+}
+
+void
+JsonWriter::writeMetricSpecificDimensions(const Metric& m)
+{
+ if (isLeafMetric(m)) {
+ writeDimensions(m.getTags());
+ }
+}
+
+void
+JsonWriter::writeCommonPostfix(const Metric& m)
+{
+ _stream << "dimensions" << Object();
+
+ writeInheritedDimensions();
+ writeMetricSpecificDimensions(m);
+
+ _stream << End() << End();
+}
+
+bool
+JsonWriter::visitCountMetric(const AbstractCountMetric& m, bool)
+{
+ writeCommonPrefix(m);
+ uint64_t count = m.getLongValue("count");
+ _stream << "values" << Object()
+ << "count" << count;
+ if (_period > 0) {
+ uint64_t rate = 1000000 * count / _period;
+ _stream << "rate" << (rate / 1000000.0);
+ }
+ _stream << End();
+ writeCommonPostfix(m);
+ return true;
+}
+
+bool
+JsonWriter::visitValueMetric(const AbstractValueMetric& m, bool)
+{
+ writeCommonPrefix(m);
+ MetricValueClass::UP values(m.getValues());
+ _stream << "values" << Object()
+ << "average";
+ if (values->getLongValue("count") == 0) {
+ _stream << 0.0;
+ } else {
+ _stream << (values->getDoubleValue("total")
+ / values->getDoubleValue("count"));
+ }
+ _stream << "count";
+ values->output("count", _stream);
+ if (_period > 0) {
+ uint64_t rate = 1000000 * values->getLongValue("count") / _period;
+ _stream << "rate" << (rate / 1000000.0);
+ }
+ _stream << "min";
+ values->output("min", _stream);
+ _stream << "max";
+ values->output("max", _stream);
+ _stream << "last";
+ values->output("last", _stream);
+ _stream << End();
+ writeCommonPostfix(m);
+ return true;
+}
+
+void
+JsonWriter::doneVisiting()
+{
+ if (_flag == METRICS_WRITTEN) {
+ _stream << End();
+ _flag = NOT_STARTED;
+ }
+ assert(_flag == NOT_STARTED);
+}
+
+} // metrics
diff --git a/metrics/src/vespa/metrics/jsonwriter.h b/metrics/src/vespa/metrics/jsonwriter.h
new file mode 100644
index 00000000000..37de3c0c0b1
--- /dev/null
+++ b/metrics/src/vespa/metrics/jsonwriter.h
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/metrics/metric.h>
+#include <vespa/vespalib/util/jsonstream.h>
+
+namespace metrics {
+
+class JsonWriter : public MetricVisitor, public vespalib::JsonStreamTypes {
+ vespalib::JsonStream& _stream;
+ enum Flag { NOT_STARTED, SNAPSHOT_STARTED, METRICS_WRITTEN };
+ Flag _flag;
+ // Not a "true" set, but dimensions are guaranteed to be unique for any
+ // given metric set.
+ using DimensionSet = Metric::Tags;
+
+ std::vector<DimensionSet> _dimensionStack;
+ uint64_t _period;
+
+public:
+ JsonWriter(vespalib::JsonStream&);
+
+private:
+ virtual bool visitSnapshot(const MetricSnapshot&);
+ virtual void doneVisitingSnapshot(const MetricSnapshot&);
+ virtual bool visitMetricSet(const MetricSet&, bool autoGenerated);
+ virtual void doneVisitingMetricSet(const MetricSet&);
+ virtual bool visitCountMetric(const AbstractCountMetric&,
+ bool autoGenerated);
+ virtual bool visitValueMetric(const AbstractValueMetric&,
+ bool autoGenerated);
+ virtual void doneVisiting();
+
+ void checkIfArrayNeedsToBeStarted();
+ void writeCommonPrefix(const Metric& m);
+ void writeCommonPostfix(const Metric& m);
+
+ void pushLegacyDimensionFromSet(const MetricSet&);
+ void pushAllDimensionsFromSet(const MetricSet&);
+ void writeDimensions(const DimensionSet&);
+ void writeInheritedDimensions();
+ void writeMetricSpecificDimensions(const Metric&);
+
+ bool isLeafMetric(const Metric& m) const { return !m.isMetricSet(); }
+};
+
+}
+
diff --git a/metrics/src/vespa/metrics/loadmetric.h b/metrics/src/vespa/metrics/loadmetric.h
new file mode 100644
index 00000000000..dc0762dd161
--- /dev/null
+++ b/metrics/src/vespa/metrics/loadmetric.h
@@ -0,0 +1,167 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \file loadmetric.h
+ * \ingroup metrics
+ *
+ * \brief Utility class for creating metrics for all load types.
+ *
+ * To better see how different load types behave in the system we want to log
+ * separate metrics for various loadtypes. To make it easy to create and use
+ * such metrics, this class is a wrapper class that sets up one metric per load
+ * type.
+ *
+ * In order to make it easy to use load metrics, they are templated on the type,
+ * such that you get the correct type out of operator[]. Load metric needs to
+ * clone metrics on creation though, so if you want load metrics of a metric set
+ * you need to properly implement clone() for that set.
+ */
+
+#pragma once
+
+#include <vespa/metrics/metricset.h>
+#include <vespa/metrics/summetric.h>
+#include <vespa/vespalib/util/linkedptr.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/stllike/hash_map.h>
+
+namespace metrics {
+
+class MetricSet;
+
+class LoadType {
+public:
+ typedef vespalib::string string;
+ LoadType(uint32_t id, const string& name) : _id(id), _name(name) {}
+
+ uint32_t getId() const { return _id; }
+ const string& getName() const { return _name; }
+
+ string toString() const {
+ return vespalib::make_string("%s(%u)", _name.c_str(), _id);
+ }
+private:
+ uint32_t _id;
+ string _name;
+};
+
+typedef std::vector<LoadType> LoadTypeSet;
+
+template<typename MetricType>
+class LoadMetric : public metrics::MetricSet {
+ std::vector<metrics::Metric::LP> _ownerList;
+ typedef vespalib::LinkedPtr<MetricType> MetricTypeLP;
+ vespalib::hash_map<uint32_t, MetricTypeLP> _metrics;
+ metrics::SumMetric<MetricType> _sum;
+
+public:
+ /**
+ * Create a load metric using the given metric as a template to how they
+ * shuold look. They will get prefix names based on load types existing.
+ */
+ LoadMetric(const LoadTypeSet& loadTypes, const MetricType& metric,
+ metrics::MetricSet* owner = 0)
+ : MetricSet(metric.getName(), "", metric.getDescription(), owner),
+ _metrics(),
+ _sum("sum", "loadsum sum", "Sum of all load metrics", this)
+ {
+ _metrics.resize(loadTypes.size());
+ // Currently, we only set tags and description on the metric set
+ // itself, to cut down on size of output when downloading metrics,
+ // and since matching tags of parent is just as good as matching
+ // them specifically.
+ setTags(metric.getTags());
+ Tags noTags;
+ for (uint32_t i=0; i<loadTypes.size(); ++i) {
+ MetricTypeLP copy(
+ dynamic_cast<MetricType*>(
+ metric.clone(_ownerList, CLONE, 0, false)));
+ assert(copy.get());
+ copy->setName(loadTypes[i].getName());
+ copy->setTags(noTags);
+ _metrics[loadTypes[i].getId()] = copy;
+ registerMetric(*copy);
+ _sum.addMetricToSum(*copy);
+ }
+ metrics::trim(_ownerList);
+ }
+
+ /**
+ * A load metric implements a copy constructor and a clone functions that
+ * clone content, and resetting name/tags/description, just so metric
+ * implementors can implement clone() by using regular construction and
+ * then assign the values to the new set. (Without screwing up copying as
+ * the load metric alters this data in supplied metric)
+ */
+ LoadMetric(const LoadMetric<MetricType>& other, metrics::MetricSet* owner)
+ : MetricSet(other.getName(), "", other.getDescription(), owner),
+ _metrics(),
+ _sum("sum", "loadsum sum", "Sum of all load metrics", this)
+ {
+ _metrics.resize(2 * other._metrics.size());
+ setTags(other.getTags());
+ Tags noTags;
+ for (typename vespalib::hash_map<uint32_t, MetricTypeLP>::const_iterator
+ it = other._metrics.begin(); it != other._metrics.end(); ++it)
+ {
+ MetricTypeLP copy(dynamic_cast<MetricType*>(
+ it->second->clone(_ownerList, CLONE, 0, false)));
+ assert(copy.get());
+ copy->setName(it->second->getName());
+ copy->setTags(noTags);
+ _metrics[it->first] = copy;
+ registerMetric(*copy);
+ _sum.addMetricToSum(*copy);
+ }
+ metrics::trim(_ownerList);
+ }
+
+ virtual Metric* clone(std::vector<Metric::LP>& ownerList,
+ CopyType copyType, MetricSet* owner,
+ bool includeUnused = false) const
+ {
+ if (copyType == INACTIVE) {
+ return MetricSet::clone(ownerList, INACTIVE, owner, includeUnused);
+ }
+ return new LoadMetric<MetricType>(*this, owner);
+ }
+
+ MetricType& operator[](const LoadType& type) { return getMetric(type); }
+ const MetricType& operator[](const LoadType& type) const
+ { return const_cast<LoadMetric<MetricType>*>(this)->getMetric(type); }
+ MetricType& getMetric(const LoadType& type) {
+ MetricType* metric;
+
+ typename vespalib::hash_map<uint32_t, MetricTypeLP>::iterator it(
+ _metrics.find(type.getId()));
+ if (it == _metrics.end()) {
+ it = _metrics.find(0);
+ assert(it != _metrics.end()); // Default should always exist
+ metric = it->second.get();
+ assert(metric);
+ } else {
+ metric = it->second.get();
+ assert(metric);
+ }
+
+ return *metric;
+ }
+
+ virtual void addMemoryUsage(metrics::MemoryConsumption& mc) const {
+ ++mc._loadMetricCount;
+ mc._loadMetricMeta += sizeof(metrics::Metric::LP) * _ownerList.size();
+ for (typename vespalib::hash_map<uint32_t, MetricTypeLP>::const_iterator
+ it = _metrics.begin(); it != _metrics.end(); ++it)
+ {
+ mc._loadMetricMeta += sizeof(uint32_t) + sizeof(MetricTypeLP);
+ }
+ _sum.addMemoryUsage(mc);
+ mc._loadMetricMeta += sizeof(LoadMetric<MetricType>)
+ - sizeof(metrics::MetricSet)
+ - sizeof(metrics::SumMetric<MetricType>);
+ metrics::MetricSet::addMemoryUsage(mc);
+ }
+
+};
+
+} // documentapi
+
diff --git a/metrics/src/vespa/metrics/memoryconsumption.h b/metrics/src/vespa/metrics/memoryconsumption.h
new file mode 100644
index 00000000000..a96ad4d3e43
--- /dev/null
+++ b/metrics/src/vespa/metrics/memoryconsumption.h
@@ -0,0 +1,192 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class metrics::MemoryConsumption
+ * \ingroup metrics
+ *
+ * \brief An instance of this class describes memory usage for a set of metrics.
+ *
+ * Typically, you ask the memory manager for memory consumption of all metrics.
+ * An instance of this class is created and sent through all metrics to track
+ * memory consumption. Tracking may be a bit expensive, so this shouldn't be
+ * checked too often. Primary use is to detect what parts actually use the most
+ * memory. Secondary use would be to add it as a metric periodically updated.
+ *
+ * The memory consumption object keeps track of various groups of memory users,
+ * such as to give a good overview of where the memory is used.
+ */
+
+#pragma once
+
+#include <vespa/vespalib/util/printable.h>
+#include <vespa/vespalib/stllike/hash_set.h>
+#include <sstream>
+
+namespace metrics {
+
+struct MemoryConsumption : public vespalib::Printable {
+ typedef std::unique_ptr<MemoryConsumption> UP;
+
+ uint32_t _consumerCount;
+ uint32_t _consumerId;
+ uint32_t _consumerIdUnique;
+ uint32_t _consumerMetricsInTotal;
+ uint32_t _consumerMetricIds;
+ uint32_t _consumerMetricIdsUnique;
+ uint32_t _consumerMeta;
+
+ uint32_t _snapshotSetCount;
+ uint32_t _snapshotSetMeta;
+
+ uint32_t _nameHash;
+ uint32_t _nameHashStrings;
+ uint32_t _nameHashUnique;
+
+ uint32_t _snapshotCount;
+ uint32_t _snapshotName;
+ uint32_t _snapshotNameUnique;
+ uint32_t _snapshotMeta;
+
+ uint32_t _metricCount;
+ uint32_t _metricMeta;
+ uint32_t _metricName;
+ uint32_t _metricNameUnique;
+ uint32_t _metricPath;
+ uint32_t _metricPathUnique;
+ uint32_t _metricDescription;
+ uint32_t _metricDescriptionUnique;
+ uint32_t _metricTagCount;
+ uint32_t _metricTags;
+ uint32_t _metricTagsUnique;
+
+ uint32_t _metricSetCount;
+ uint32_t _metricSetMeta;
+ uint32_t _metricSetOrder;
+
+ uint32_t _countMetricCount;
+ uint32_t _countMetricValues;
+ uint32_t _countMetricMeta;
+
+ uint32_t _valueMetricCount;
+ uint32_t _valueMetricValues;
+ uint32_t _valueMetricMeta;
+
+ uint32_t _sumMetricCount;
+ uint32_t _sumMetricMeta;
+ uint32_t _sumMetricParentPath;
+ uint32_t _sumMetricParentPathUnique;
+
+ uint32_t _loadMetricCount;
+ uint32_t _loadMetricMeta;
+
+ uint32_t _totalStringCount;
+
+ vespalib::hash_set<const void*> _seenStrings;
+ std::vector<std::pair<std::string, uint32_t> > _snapShotUsage;
+
+ MemoryConsumption() {
+ memset(&_consumerCount, 0,
+ reinterpret_cast<size_t>(&_seenStrings)
+ - reinterpret_cast<size_t>(&_consumerCount));
+ _seenStrings.resize(1000);
+ }
+
+ /** Get memory usage of a string that is not included when doing sizeof */
+ uint32_t getStringMemoryUsage(const std::string& s, uint32_t& uniqueCount) {
+ ++_totalStringCount;
+ const char* internalString = s.c_str();
+ if (_seenStrings.find(internalString) != _seenStrings.end()) {
+ return 0;
+ }
+ ++uniqueCount;
+ _seenStrings.insert(internalString);
+ return s.capacity();
+ }
+
+ void addSnapShotUsage(const std::string& name, uint32_t usage) {
+ _snapShotUsage.push_back(std::pair<std::string, uint32_t>(name, usage));
+ }
+
+ uint32_t getTotalMemoryUsage() const {
+ return _consumerId + _consumerMetricIds + _consumerMeta
+ + _snapshotSetMeta + _snapshotName + _snapshotMeta + _metricMeta
+ + _metricName + _metricPath + _metricDescription
+ + _metricTags + _metricSetMeta + _nameHash + _nameHashStrings
+ + _metricSetOrder + _countMetricValues + _countMetricMeta
+ + _valueMetricValues + _valueMetricMeta + _sumMetricMeta
+ + _sumMetricParentPath + _loadMetricMeta;
+ }
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+ {
+ (void) verbose;
+ std::string newl = "\n" + indent + " ";
+ out << "MemoryConsumption("
+ << newl << "Total memory used: " << bval(getTotalMemoryUsage())
+ << newl << "Consumer count: " << _consumerCount
+ << newl << "Consumer ids: " << bval(_consumerId)
+ << newl << "Consumer metric count: " << _consumerMetricsInTotal
+ << newl << "Consumer metric ids: " << bval(_consumerMetricIds)
+ << newl << "Consumer meta: " << bval(_consumerMeta)
+ << newl << "Name hash: " << bval(_nameHash)
+ << newl << "Name hash strings: " << bval(_nameHashStrings)
+ << newl << "Snapshot set count: " << _snapshotSetCount
+ << newl << "Snapshot set meta: " << bval(_snapshotSetMeta)
+ << newl << "Snapshot count: " << _snapshotCount
+ << newl << "Snapshot name: " << bval(_snapshotName)
+ << newl << "Snapshot meta: " << bval(_snapshotMeta)
+ << newl << "Metric count: " << _metricCount
+ << newl << "Metric meta: " << bval(_metricMeta)
+ << newl << "Metric names: " << bval(_metricName)
+ << newl << "Metric paths: " << bval(_metricPath)
+ << newl << "Metric descriptions: " << bval(_metricDescription)
+ << newl << "Metric tag count: " << _metricTagCount
+ << newl << "Metric tags: " << bval(_metricTags)
+ << newl << "Metric set count: " << _metricSetCount
+ << newl << "Metric set meta: " << bval(_metricSetMeta)
+ << newl << "Metric set order list: " << bval(_metricSetOrder)
+ << newl << "Count metric count: " << _countMetricCount
+ << newl << "Count metric values: " << bval(_countMetricValues)
+ << newl << "Count metric meta: " << bval(_countMetricMeta)
+ << newl << "Value metric count: " << _valueMetricCount
+ << newl << "Value metric values: " << bval(_valueMetricValues)
+ << newl << "Value metric meta: " << bval(_valueMetricMeta)
+ << newl << "Sum metric count: " << _sumMetricCount
+ << newl << "Sum metric meta: " << bval(_sumMetricMeta)
+ << newl << "Sum metric parent path: " << bval(_sumMetricParentPath)
+ << newl << "Load metric count: " << _loadMetricCount
+ << newl << "Load metric meta: " << bval(_loadMetricMeta)
+ << newl << "Unique string count: " << _seenStrings.size()
+ << newl << "Strings stored: " << _totalStringCount
+ << newl << "Unique consumer ids: " << _consumerIdUnique
+ << newl << "Unique cons metric ids: " << _consumerMetricIdsUnique
+ << newl << "Unique snapshot names: " << _snapshotNameUnique
+ << newl << "Unique metric names: " << _metricNameUnique
+ << newl << "Unique metric paths: " << _metricPathUnique
+ << newl << "Unique metric descs: " << _metricDescriptionUnique
+ << newl << "Unique metric tags: " << _metricTagsUnique
+ << newl << "Unique sum metric paths: " << _sumMetricParentPathUnique
+ << newl << "Unique name hash strings: " << _nameHashUnique;
+ for (uint32_t i=0; i<_snapShotUsage.size(); ++i) {
+ out << newl << "Snapshot " << _snapShotUsage[i].first << ": "
+ << bval(_snapShotUsage[i].second);
+ }
+ out << "\n" << indent << ")";
+ }
+
+ static std::string bval(uint32_t bytes) {
+ std::ostringstream ost;
+ if (bytes < 10 * 1024) {
+ ost << bytes << " B";
+ } else if (bytes < 10 * 1024 * 1024) {
+ ost << (bytes / 1024) << " kB";
+ } else {
+ ost << (bytes / (1024 * 1024)) << " MB";
+ }
+ return ost.str();
+ }
+
+};
+
+} // metrics
+
diff --git a/metrics/src/vespa/metrics/metric.cpp b/metrics/src/vespa/metrics/metric.cpp
new file mode 100644
index 00000000000..f4ad47871e6
--- /dev/null
+++ b/metrics/src/vespa/metrics/metric.cpp
@@ -0,0 +1,234 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/metrics/metric.h>
+
+#include <vespa/metrics/countmetric.h>
+#include <vespa/metrics/valuemetric.h>
+#include <vespa/vespalib/text/stringtokenizer.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/metrics/metricset.h>
+
+#include <algorithm>
+#include <iterator>
+
+namespace metrics {
+
+bool
+MetricVisitor::visitCountMetric(const AbstractCountMetric& m,
+ bool autoGenerated)
+{
+ return visitMetric(m, autoGenerated);
+}
+
+bool
+MetricVisitor::visitValueMetric(const AbstractValueMetric& m,
+ bool autoGenerated)
+{
+ return visitMetric(m, autoGenerated);
+}
+
+bool
+MetricVisitor::visitMetric(const Metric&, bool)
+{
+ throw vespalib::IllegalStateException(
+ "visitMetric called with default implementation. You should either "
+ "override specific visit functions or this catchall function.",
+ VESPA_STRLOC);
+}
+
+namespace {
+ Metric::Tags legacyTagStringToKeyedTags(const std::string& tagStr) {
+ vespalib::StringTokenizer tokenizer(tagStr, " \t\r\f");
+ Metric::Tags tags;
+ std::transform(tokenizer.getTokens().begin(),
+ tokenizer.getTokens().end(),
+ std::back_inserter(tags),
+ [](const std::string& s) { return Tag(s, ""); });
+ return tags;
+ }
+ std::string namePattern = "[a-zA-Z][_a-zA-Z0-9]*";
+}
+
+vespalib::Regexp Metric::_namePattern(namePattern);
+
+Metric::Metric(const String& name,
+ const String& tags,
+ const String& description,
+ MetricSet* owner)
+ : _name(name),
+ _description(description),
+ _tags(legacyTagStringToKeyedTags(tags)),
+ _owner(nullptr) // Set later by registry
+{
+ verifyConstructionParameters();
+ assignMangledNameWithDimensions();
+ registerWithOwnerIfRequired(owner);
+}
+
+Metric::Metric(const String& name,
+ Tags dimensions,
+ const String& description,
+ MetricSet* owner)
+ : _name(name),
+ _description(description),
+ _tags(std::move(dimensions)),
+ _owner(nullptr)
+{
+ verifyConstructionParameters();
+ assignMangledNameWithDimensions();
+ registerWithOwnerIfRequired(owner);
+}
+
+
+Metric::Metric(const Metric& other, MetricSet* owner)
+ : Printable(other),
+ _name(other._name),
+ _description(other._description),
+ _tags(other._tags),
+ _owner(nullptr)
+{
+ assignMangledNameWithDimensions();
+ registerWithOwnerIfRequired(owner);
+}
+
+bool
+Metric::tagsSpecifyAtLeastOneDimension(const Tags& tags) const
+{
+ auto hasNonEmptyTagValue = [](const Tag& t) { return !t.value.empty(); };
+ return std::any_of(tags.begin(), tags.end(), hasNonEmptyTagValue);
+}
+
+void
+Metric::assignMangledNameWithDimensions()
+{
+ if (!tagsSpecifyAtLeastOneDimension(_tags)) {
+ return;
+ }
+ sortTagsInDeterministicOrder();
+ _mangledName = createMangledNameWithDimensions();
+}
+
+void
+Metric::sortTagsInDeterministicOrder()
+{
+ std::sort(_tags.begin(), _tags.end(), [](const Tag& a, const Tag& b) {
+ return a.key < b.key;
+ });
+}
+
+std::string
+Metric::createMangledNameWithDimensions() const
+{
+ vespalib::asciistream s;
+ s << _name << '{';
+ const size_t sz = _tags.size();
+ for (size_t i = 0; i < sz; ++i) {
+ const Tag& dimension(_tags[i]);
+ if (dimension.value.empty()) {
+ continue;
+ }
+ if (i != 0) {
+ s << ',';
+ }
+ s << dimension.key << ':' << dimension.value;
+ }
+ s << '}';
+ return s.str();
+}
+
+void
+Metric::verifyConstructionParameters()
+{
+ if (_name.size() == 0) {
+ throw vespalib::IllegalArgumentException(
+ "Metric cannot have empty name", VESPA_STRLOC);
+ }
+ if (!_namePattern.match(_name)) {
+ throw vespalib::IllegalArgumentException(
+ "Illegal metric name '" + _name + "'. Names must match pattern "
+ + namePattern, VESPA_STRLOC);
+ }
+}
+
+void
+Metric::registerWithOwnerIfRequired(MetricSet* owner)
+{
+ if (owner) {
+ owner->registerMetric(*this);
+ }
+}
+
+const MetricSet*
+Metric::getRoot() const
+{
+ return (_owner == 0 ? (isMetricSet() ? static_cast<const MetricSet*>(this)
+ : 0)
+ : _owner->getRoot());
+}
+
+vespalib::string
+Metric::getPath() const
+{
+ if (_owner == 0 || _owner->_owner == 0) {
+ return _name;
+ } else {
+ return _owner->getPath() + "." + _name;
+ }
+}
+
+std::vector<Metric::String>
+Metric::getPathVector() const
+{
+ std::vector<String> result;
+ result.push_back(_name);
+ const MetricSet* owner(_owner);
+ while (owner != 0) {
+ result.push_back(owner->_name);
+ owner = owner->_owner;
+ }
+ std::reverse(result.begin(), result.end());
+ return result;
+}
+
+bool
+Metric::hasTag(const String& tag) const
+{
+ return std::find_if(_tags.begin(), _tags.end(), [&](const Tag& t) {
+ return t.key == tag;
+ }) != _tags.end();
+}
+
+void
+Metric::addMemoryUsage(MemoryConsumption& mc) const
+{
+ ++mc._metricCount;
+ mc._metricName += mc.getStringMemoryUsage(_name, mc._metricNameUnique);
+ mc._metricDescription += mc.getStringMemoryUsage(
+ _description, mc._metricDescriptionUnique);
+ mc._metricTagCount += _tags.size();
+ // XXX figure out what we actually want to report from tags here...
+ // XXX we don't care about unique strings since they don't matter anymore.
+ //mc._metricTags += mc.getStringMemoryUsage(_tags, mc._metricTagsUnique);
+ mc._metricMeta += sizeof(Metric);
+}
+
+void
+Metric::updateNames(NameHash& hash) const
+{
+ Metric& m(const_cast<Metric&>(*this));
+ hash.updateName(m._name);
+ hash.updateName(m._description);
+ // Tags use vespalib::string which isn't refcounted under the hood and
+ // use small string optimizations, meaning the implicit ref sharing hack
+ // won't work for them anyway.
+}
+
+void
+Metric::printDebug(std::ostream& out, const std::string& indent) const
+{
+ (void) indent;
+ out << "name=" << _name << ", instance=" << ((const void*) this)
+ << ", owner=" << ((const void*) _owner);
+}
+
+} // metrics
diff --git a/metrics/src/vespa/metrics/metric.h b/metrics/src/vespa/metrics/metric.h
new file mode 100644
index 00000000000..7c42d187f30
--- /dev/null
+++ b/metrics/src/vespa/metrics/metric.h
@@ -0,0 +1,319 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vespalib/util/printable.h>
+#include <map>
+#include <vespa/metrics/config-metricsmanager.h>
+#include <vespa/metrics/memoryconsumption.h>
+#include <vespa/metrics/namehash.h>
+#include <string>
+#include <vespa/vespalib/objects/cloneable.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/linkedptr.h>
+#include <vespa/vespalib/util/jsonwriter.h>
+#include <vespa/vespalib/util/regexp.h>
+#include <vespa/vespalib/text/stringtokenizer.h>
+
+namespace metrics {
+
+class AbstractCountMetric;
+class AbstractValueMetric;
+class Metric;
+class MetricSet;
+class MetricSnapshot;
+class XmlWriterMetricVisitor;
+
+template<typename T>
+void trim(std::vector<T>& v) {
+ if (v.size() < v.capacity()) {
+ std::vector<T> copy(v);
+ copy.swap(v);
+ }
+ assert(v.size() == v.capacity());
+}
+
+/** Implement class to visit metrics. */
+struct MetricVisitor {
+ virtual ~MetricVisitor() {}
+
+ /**
+ * Visit a snapshot. Return true to visit content of the snapshot
+ */
+ virtual bool visitSnapshot(const MetricSnapshot&) { return true; }
+ virtual void doneVisitingSnapshot(const MetricSnapshot&) {}
+
+ /**
+ * Visit a metric set.
+ *
+ * @param autoGenerated True for metric sets that are generated on the
+ * fly such as in sum metrics.
+ * @return True if you want to visit the content of this metric set.
+ */
+ virtual bool visitMetricSet(const MetricSet&, bool autoGenerated)
+ { (void) autoGenerated; return true; }
+
+ /**
+ * Callback visitors can use if they need to know the tree traversal of
+ * metric sets. This function is not called if visitMetricSet returned
+ * false.
+ */
+ virtual void doneVisitingMetricSet(const MetricSet&) {}
+
+ /**
+ * Visit a primitive metric within an accepted metric set.
+ *
+ * @return True if you want to continue visiting, false to abort.
+ */
+ virtual bool visitCountMetric(const AbstractCountMetric& m,
+ bool autoGenerated);
+ /**
+ * Visit a primitive metric within an accepted metric set.
+ *
+ * @return True if you want to continue visiting, false to abort.
+ */
+ virtual bool visitValueMetric(const AbstractValueMetric& m,
+ bool autoGenerated);
+
+ /**
+ * Visit function for visiting primitive metrics in one function. Only
+ * called if not overriding specific primitive metric functions.
+ */
+ virtual bool visitMetric(const Metric&, bool autoGenerated);
+
+ /**
+ * To avoid having to destruct the visitor in order to get any post
+ * processing done you can use this hook. (Only called by functions knowing
+ * they are top level, so you have to call this manually if visiting metrics
+ * directly, and not through manager)
+ */
+ virtual void doneVisiting() {}
+};
+
+/**
+ * A tag is a simple key-value mapping associated with a metric. Tags are used
+ * for either constraining the set of metrics reported to a consumer (e.g. "do
+ * not include metrics tagged as 'partofsum'") or to represent dimensions for
+ * metric sets or individual metrics. In the former case, the key represents
+ * the tag and the value remains empty. In the latter (dimension) case, the key
+ * is the dimension name and the value is the dimension value.
+ */
+struct Tag
+{
+ vespalib::string key;
+ vespalib::string value;
+
+ Tag(vespalib::stringref k, vespalib::stringref v)
+ : key(k), value(v)
+ {
+ }
+};
+
+class Metric : public vespalib::Printable
+{
+public:
+ using String = std::string;
+ using stringref = vespalib::stringref;
+ using UP = std::unique_ptr<Metric>;
+ using LP = vespalib::LinkedPtr<Metric>;
+ using SP = std::shared_ptr<Metric>;
+ using Tags = std::vector<Tag>;
+
+ static vespalib::Regexp _namePattern;
+
+ Metric(const String& name, const String& tags,
+ const String& description, MetricSet* owner = 0);
+
+ Metric(const String& name,
+ Tags dimensions,
+ const String& description,
+ MetricSet* owner = 0);
+
+ Metric(const Metric& other, MetricSet* owner);
+ virtual ~Metric() {}
+
+ const String& getName() const { return _name; }
+ /**
+ * Get mangled name iff the metric contains any dimensions, otherwise
+ * the original metric name is returned.
+ */
+ const String& getMangledName() const {
+ return (_mangledName.empty() ? _name : _mangledName);
+ }
+ vespalib::string getPath() const;
+ std::vector<String> getPathVector() const;
+ const String& getDescription() const { return _description; }
+ const Tags& getTags() const { return _tags; }
+ /** Return whether there exists a tag with a key equal to 'tag' */
+ bool hasTag(const String& tag) const;
+
+ enum CopyType { CLONE, INACTIVE };
+ /**
+ * The clone function will clone metrics to an identical subtree of
+ * metrics. Clone is primarily used for load metrics that wants to clone
+ * a template metric for each loadtype. But it should work generically.
+ *
+ * @param type If set to inactive, sum metrics will evaluate to primitives
+ * and metrics can save memory by knowing no updates are coming.
+ * @param includeUnused When creating snapshots we do not want to include
+ * unused metrics, but while generating sum metric sum in active
+ * metrics we want to. This has no affect if type is CLONE.
+ */
+ virtual Metric* clone(std::vector<Metric::LP>& ownerList,
+ CopyType type, MetricSet* owner,
+ bool includeUnused = false) const = 0;
+
+ /**
+ * Utility function for assigning values from one metric of identical type
+ * to this metric. For simplicity sake it does a const cast and calls
+ * addToSnapshot, which should not alter source if reset is false. This can
+ * not be used to copy between active metrics and inactive copies.
+ */
+ virtual Metric* assignValues(const Metric& m) {
+ std::vector<Metric::LP> ownerList;
+ const_cast<Metric&>(m).addToSnapshot(*this, ownerList);
+ // As this should only be called among active metrics, all metrics
+ // should exist and owner list should thus always end up empty.
+ assert(ownerList.empty());
+ return this;
+ }
+
+ /** Reset all metric values. */
+ virtual void reset() = 0;
+
+ virtual bool logFromTotalMetrics() const { return false; }
+ /** Implement to make metric able to log event. */
+ virtual bool logEvent(const String& fullName) const = 0;
+
+ virtual void print(std::ostream& out, bool verbose,
+ const std::string& indent) const {
+ print(out, verbose, indent, 0);
+ }
+ virtual void print(std::ostream&, bool verbose, const std::string& indent,
+ uint64_t secondsPassed) const = 0;
+
+ /**
+ * Most metrics report numbers of some kind. To be able to report numbers
+ * without having code to handle each possible metric type, these functions
+ * exist to extract raw data to present easily.
+ * @param id The part of the metric to extract. For instance, an average
+ * metric have average,
+ */
+ virtual int64_t getLongValue(const stringref & id) const = 0;
+ virtual double getDoubleValue(const stringref & id) const = 0;
+
+ /**
+ * When snapshotting we need to be able to add data from one set of metrics
+ * to another set of metrics taken at another time. MetricSet doesn't know
+ * the type of the metrics it contains, so we need a generic function for
+ * doing this. This function assumes metric given as input is of the exact
+ * same type as the one it is called on for simplicity. This is true when
+ * adding to snapshots as they have been created with clone and is thus
+ * always exactly equal.
+ *
+ * @param m Metric of exact same type as this one. (Will core if wrong)
+ * @param ownerList In case snapshot doesn't contain given metric, it can
+ * create them and add them to ownerlist.
+ */
+ virtual void addToSnapshot(Metric& m,
+ std::vector<Metric::LP>& ownerList) const = 0;
+
+ /**
+ * For sum metrics to work with metric sets, metric sets need operator+=.
+ * To implement this, we need a function to add any metric type together.
+ * This is different from adding to snapshot. When adding to snapshots we
+ * add different time periods to the same metric, but when adding parts
+ * together we add different metrics for the same time. For instance, an
+ * average metric of queuesize, should just add new values to create new
+ * average when adding to snapshot, but when adding parts, the averages
+ * themselves should be added together.
+ *
+ * @param m Metric of exact same type as this one. (Will core if wrong)
+ */
+ virtual void addToPart(Metric& m) const = 0;
+
+ virtual bool visit(MetricVisitor& visitor,
+ bool tagAsAutoGenerated = false) const = 0;
+
+ /** Used by sum metric to alter name of cloned metric for sum. */
+ void setName(const String& name) { _name = name; }
+ /** Used by sum metric to alter description of cloned metric for sum. */
+ void setDescription(const String& d) { _description = d; }
+ /** Used by sum metric to alter tag of cloned metric for sum. */
+ void setTags(Tags tags) {
+ _tags = std::move(tags);
+ }
+
+ /** Set whether metrics have ever been set. */
+ virtual bool used() const = 0;
+
+ /** Returns true if this metric is registered in a metric set. */
+ bool isRegistered() const { return (_owner != 0); }
+
+ const MetricSet* getOwner() const { return _owner; }
+ const MetricSet* getRoot() const;
+
+ /** Used by metric set in register functions. */
+ void setRegistered(MetricSet* owner) { _owner = owner; }
+
+ virtual void addMemoryUsage(MemoryConsumption&) const;
+
+ /**
+ * Update names using the given name hash, to utilize ref counting.
+ *
+ * NOTE:
+ * This is a hack that only works on GCC until they decide to finally break
+ * ABI compatibility and remove that particular multicore-hostile feature
+ * of their std::string implementation. If we want proper string ref
+ * counting, all strings should be replaced with explicit string handles
+ * and should only be created via a shared factory.
+ */
+ virtual void updateNames(NameHash&) const;
+
+ /** Print debug information of the metric tree. */
+ virtual void printDebug(std::ostream&, const std::string& indent="") const;
+
+ virtual bool isMetricSet() const { return false; }
+
+private:
+
+ /**
+ * A Tag instance is only considered to be a valid dimension if it contains
+ * a value (i.e. not just a key as with legacy tags).
+ */
+ bool tagsSpecifyAtLeastOneDimension(const Tags&) const;
+
+ /**
+ * Iff _tags contains at least one valid dimension, update _mangledName to
+ * be a string of the form 'metricname{key_0=value_0,key_1=value_1}' for
+ * each pair of valid dimension key/values. _mangledName is left untouched
+ * if no such valid dimension exists.
+ */
+ void assignMangledNameWithDimensions();
+
+ /**
+ * Sort _tags so that its entries are order in increasing lexicographic
+ * order. Dimensions/tags must be in deterministic order for duplicate
+ * detection based on name to work correctly.
+ */
+ void sortTagsInDeterministicOrder();
+
+ std::string createMangledNameWithDimensions() const;
+
+ void verifyConstructionParameters();
+ /**
+ * Registers with metric set iff owner is not nullptr. This will implicitly
+ * set _owner via a metric set callback to Metric::setRegistered().
+ */
+ void registerWithOwnerIfRequired(MetricSet* owner);
+
+protected:
+ String _name;
+ String _mangledName;
+ String _description;
+ std::vector<Tag> _tags;
+ MetricSet* _owner;
+
+};
+
+} // metrics
+
diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp
new file mode 100644
index 00000000000..b915786afcf
--- /dev/null
+++ b/metrics/src/vespa/metrics/metricmanager.cpp
@@ -0,0 +1,968 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/metrics/metricmanager.h>
+
+#include <vespa/config/print/ostreamconfigwriter.h>
+#include <vespa/vespalib/text/stringtokenizer.h>
+#include <vespa/metrics/countmetric.h>
+#include <vespa/metrics/valuemetric.h>
+#include <vespa/metrics/metricset.h>
+#include <vespa/metrics/summetric.h>
+#include <vespa/metrics/jsonwriter.h>
+#include <vespa/metrics/textwriter.h>
+#include <vespa/metrics/xmlwriter.h>
+#include <vespa/log/log.h>
+#include <stack>
+
+LOG_SETUP(".metrics.manager");
+
+namespace metrics {
+
+typedef MetricsmanagerConfig Config;
+
+void
+MetricManager::ConsumerSpec::print(std::ostream& out, bool verbose,
+ const std::string& indent) const
+{
+ (void) verbose;
+ out << "ConsumerSpec(";
+ std::set<Metric::String> sortedMetrics;
+ for (vespalib::hash_set<Metric::String>::const_iterator it
+ = includedMetrics.begin(); it != includedMetrics.end(); ++it)
+ {
+ sortedMetrics.insert(*it);
+ }
+ for (auto s : sortedMetrics) {
+ out << "\n" << indent << " " << s;
+ }
+ out << ")";
+}
+
+void
+MetricManager::ConsumerSpec::addMemoryUsage(MemoryConsumption& mc) const
+{
+ mc._consumerMetricsInTotal += includedMetrics.size();
+ for (vespalib::hash_set<Metric::String>::const_iterator it
+ = includedMetrics.begin(); it != includedMetrics.end(); ++it)
+ {
+ mc._consumerMetricIds += mc.getStringMemoryUsage(
+ *it, mc._consumerMetricIdsUnique)
+ + sizeof(Metric::String);
+ }
+}
+
+MetricManager::MetricManager(std::unique_ptr<Timer> timer)
+ : _activeMetrics("Active metrics showing updates since last snapshot"),
+ _configSubscriber(),
+ _configHandle(),
+ _config(),
+ _consumerConfig(),
+ _logPeriod(5 * 60, 0),
+ _snapshots(),
+ _totalMetrics(MetricSnapshot::SP(new MetricSnapshot(
+ "Empty metrics before init", 0, _activeMetrics.getMetrics(),
+ false))),
+ _timer(std::move(timer)),
+ _lastProcessedTime(0),
+ _forceEventLogging(false),
+ _snapshotUnsetMetrics(false),
+ _consumerConfigChanged(false),
+ _metricManagerMetrics("metricmanager", "",
+ "Metrics for the metric manager upkeep tasks"),
+ _periodicHookLatency("periodichooklatency", "",
+ "Time in ms used to update a single periodic hook",
+ &_metricManagerMetrics),
+ _snapshotHookLatency("snapshothooklatency", "",
+ "Time in ms used to update a single snapshot hook",
+ &_metricManagerMetrics),
+ _resetLatency("resetlatency", "",
+ "Time in ms used to reset all metrics.", &_metricManagerMetrics),
+ _snapshotLatency("snapshotlatency", "",
+ "Time in ms used to take a snapshot",
+ &_metricManagerMetrics),
+ _sleepTimes("sleeptime", "", "Time in ms worker thread is sleeping",
+ &_metricManagerMetrics)
+{
+ registerMetric(getMetricLock(), _metricManagerMetrics);
+}
+
+MetricManager::~MetricManager()
+{
+ stop();
+}
+
+void
+MetricManager::stop()
+{
+ Runnable::stop();
+ {
+ vespalib::MonitorGuard sync(_waiter);
+ sync.signal();
+ }
+ join();
+}
+
+void
+MetricManager::addMetricUpdateHook(UpdateHook& hook, uint32_t period)
+{
+ hook._period = period;
+ vespalib::MonitorGuard sync(_waiter);
+ // If we've already initialized manager, log period has been set.
+ // In this case. Call first time after period
+ hook._nextCall = (_logPeriod.second == 0 ? 0 : _timer->getTime() + period);
+ if (period == 0) {
+ for (UpdateHook * sHook : _snapshotUpdateHooks) {
+ if (sHook == &hook) {
+ LOG(warning, "Update hook already registered");
+ return;
+ }
+ }
+ _snapshotUpdateHooks.push_back(&hook);
+ } else {
+ for (UpdateHook * pHook : _periodicUpdateHooks) {
+ if (pHook == &hook) {
+ LOG(warning, "Update hook already registered");
+ return;
+ }
+ }
+ _periodicUpdateHooks.push_back(&hook);
+ }
+}
+
+void
+MetricManager::removeMetricUpdateHook(UpdateHook& hook)
+{
+ vespalib::MonitorGuard sync(_waiter);
+ if (hook._period == 0) {
+ for (auto it = _snapshotUpdateHooks.begin(); it != _snapshotUpdateHooks.end(); it++) {
+ if (*it == &hook) {
+ _snapshotUpdateHooks.erase(it);
+ return;
+ }
+ }
+ } else {
+ for (auto it = _periodicUpdateHooks.begin(); it != _periodicUpdateHooks.end(); it++) {
+ if (*it == &hook) {
+ _periodicUpdateHooks.erase(it);
+ return;
+ }
+ }
+ }
+ LOG(warning, "Update hook not registered");
+}
+
+void
+MetricManager::init(const config::ConfigUri & uri, FastOS_ThreadPool& pool,
+ bool startThread)
+{
+ if (_configHandle.get()) {
+ throw vespalib::IllegalStateException(
+ "The metric manager have already been initialized. "
+ "It can only be initialized once.", VESPA_STRLOC);
+ }
+ LOG(debug, "Initializing metric manager.");
+ _configSubscriber.reset(new config::ConfigSubscriber(uri.getContext()));
+ _configHandle = _configSubscriber->subscribe<Config>(uri.getConfigId());
+ _configSubscriber->nextConfig();
+ configure(getMetricLock(), _configHandle->getConfig());
+ LOG(debug, "Starting worker thread, waiting for first "
+ "iteration to complete.");
+ if (startThread) {
+ Runnable::start(pool);
+ // Wait for first iteration to have completed, such that it is safe
+ // to access snapshots afterwards.
+ vespalib::MonitorGuard sync(_waiter);
+ while (_lastProcessedTime == 0) {
+ sync.wait(1);
+ }
+ } else {
+ _configSubscriber.reset(0);
+ }
+ LOG(debug, "Metric manager completed initialization.");
+}
+
+namespace {
+
+ struct Path {
+ vespalib::StringTokenizer _path;
+
+ Path(const vespalib::stringref & fullpath) : _path(fullpath, ".") { }
+
+ vespalib::string toString() const {
+ vespalib::asciistream ost;
+ ost << _path[0];
+ for (uint32_t i=1; i<path().size(); ++i) ost << "." << _path[i];
+ return ost.str();
+ }
+
+ bool matchesPattern(const Path& p) const {
+ if (path().size() != p.path().size()) {
+ return false;
+ }
+ for (uint32_t i=0; i<p.path().size(); ++i) {
+ if (p._path[i] == "*") continue;
+ if (_path[i] != p._path[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+ const vespalib::StringTokenizer::TokenList& path() const {
+ return _path.getTokens();
+ }
+ };
+
+ struct ConsumerMetricBuilder : public MetricVisitor {
+ const Config::Consumer& _consumer;
+ MetricManager::ConsumerSpec _matchedMetrics;
+ // Keep a stack of matches to facilitate tree traversal.
+ struct Result {
+ bool tagAdded;
+ bool tagRemoved;
+ bool nameAdded;
+ bool nameRemoved;
+ uint32_t metricCount;
+
+ Result() : tagAdded(false), tagRemoved(false),
+ nameAdded(false), nameRemoved(false), metricCount(0) {}
+ };
+ std::list<Result> result;
+
+ ConsumerMetricBuilder(const Config::Consumer& c)
+ : _consumer(c), _matchedMetrics()
+ {
+ LOG(spam, "Adding metrics for consumer %s", c.name.c_str());
+ }
+
+ bool tagAdded(const Metric& metric) {
+ for (const auto& s : _consumer.tags) {
+ if ((s == "*" && !metric.getTags().empty())
+ || metric.hasTag(s))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+ bool tagRemoved(const Metric& metric) {
+ for (const auto& s : _consumer.removedtags) {
+ if ((s == "*" && !metric.getTags().empty())
+ || metric.hasTag(s))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+ bool nameAdded(const Metric& metric) {
+ Metric::String path(metric.getPath());
+ Path mpath(path);
+ for (const auto& s : _consumer.addedmetrics) {
+ if (mpath.matchesPattern(Path(s))) {
+ return true;
+ }
+ }
+ return false;
+ }
+ bool nameRemoved(const Metric& metric) {
+ Metric::String path(metric.getPath());
+ Path mpath(path);
+ for (const auto& s : _consumer.removedmetrics) {
+ if (mpath.matchesPattern(Path(s))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ virtual bool visitMetricSet(const MetricSet& metricSet,
+ bool autoGenerated)
+ {
+ (void) autoGenerated;
+ result.push_back(Result());
+ // If current metric match anything, use that.
+ // Otherwise, copy from parent
+ if (nameRemoved(metricSet)) {
+ result.back().nameRemoved = true;
+ } else if (nameAdded(metricSet)) {
+ result.back().nameAdded = true;
+ } else if (tagRemoved(metricSet)) {
+ result.back().tagRemoved = true;
+ } else if (tagAdded(metricSet)) {
+ result.back().tagAdded = true;
+ } else if (result.size() > 1) {
+ result.back() = *++result.rbegin();
+ result.back().metricCount = 0;
+ }
+ return true;
+ }
+ virtual void doneVisitingMetricSet(const MetricSet& metricSet) {
+ if (result.back().metricCount > 0 && result.size() != 1) {
+ LOG(spam, "Adding metricset %s", metricSet.getPath().c_str());
+ _matchedMetrics.includedMetrics.insert(metricSet.getPath());
+ }
+ result.pop_back();
+ }
+ bool visitMetric(const Metric& metric, bool autoGenerated) {
+ (void) autoGenerated;
+ if (result.back().nameRemoved || nameRemoved(metric)) return true;
+ bool nAdded = (result.back().nameAdded || nameAdded(metric));
+ if (!nAdded && (result.back().tagRemoved || tagRemoved(metric))) {
+ return true;
+ }
+ if (nAdded || result.back().tagAdded || tagAdded(metric)) {
+ _matchedMetrics.includedMetrics.insert(metric.getPath());
+ LOG(spam, "Adding metric %s", metric.getPath().c_str());
+ for (Result & r : result) {
+ ++r.metricCount;
+ }
+ }
+ return true;
+ }
+
+ };
+
+}
+
+// This function may be called by clients or worker thread. Thus, worker monitor
+// might have already been grabbed
+void
+MetricManager::checkMetricsAltered(const MetricLockGuard & guard)
+{
+ if (_activeMetrics.getMetrics().isRegistrationAltered()
+ || _consumerConfigChanged)
+ {
+ handleMetricsAltered(guard);
+ }
+}
+
+// When calling this function, the metric lock is already taken. The thread
+// monitor lock might be taken.
+void
+MetricManager::handleMetricsAltered(const MetricLockGuard &)
+{
+ if (_config.get() == NULL) {
+ LOG(info, "_config is NULL -> very odd indeed.");
+ return;
+ }
+ if (_consumerConfig.empty()) {
+ LOG(debug, "Setting up consumers for the first time.");
+ } else {
+ LOG(info, "Metrics registration changes detected. Handling changes.");
+ }
+ std::map<Metric::String, ConsumerSpec::SP> configMap;
+ _activeMetrics.getMetrics().clearRegistrationAltered();
+ LOG(debug, "Calculating new consumer config");
+ for (auto consumer : _config->consumer) {
+ ConsumerMetricBuilder consumerMetricBuilder(consumer);
+ _activeMetrics.getMetrics().visit(consumerMetricBuilder);
+ configMap[consumer.name] = ConsumerSpec::SP(
+ new ConsumerSpec(consumerMetricBuilder._matchedMetrics));
+ }
+ LOG(debug, "Recreating snapshots to include altered metrics");
+ _activeMetrics.updateNames(_nameHash);
+ _totalMetrics->recreateSnapshot(_activeMetrics.getMetrics(),
+ _snapshotUnsetMetrics);
+ _totalMetrics->updateNames(_nameHash);
+ for (uint32_t i=0; i<_snapshots.size(); ++i) {
+ _snapshots[i]->recreateSnapshot(_activeMetrics.getMetrics(),
+ _snapshotUnsetMetrics);
+ _snapshots[i]->updateNames(_nameHash);
+ }
+ LOG(debug, "Setting new consumer config. Clearing dirty flag");
+ _consumerConfig.swap(configMap);
+ _consumerConfigChanged = false;
+ LOG(debug, "Unified %u of %u strings configuring %" PRIu64 " consumers.",
+ _nameHash.getUnifiedStringCount(),
+ _nameHash.getCheckedStringCount(),
+ _config->consumer.size());
+ _nameHash.resetCounts();
+}
+
+namespace {
+ bool setSnapshotName(std::ostream& out, const char* name,
+ uint32_t length, uint32_t period)
+ {
+ if (length % period != 0) return false;
+ out << (length / period) << ' ' << name;
+ if (length / period != 1) out << "s";
+ return true;
+ }
+}
+
+std::vector<MetricManager::SnapSpec>
+MetricManager::createSnapshotPeriods(const Config& config)
+{
+ std::vector<SnapSpec> result;
+ try{
+ for (uint32_t i=0; i<config.snapshot.periods.size(); ++i) {
+ uint32_t length = config.snapshot.periods[i];
+ if (length < 1) throw vespalib::IllegalStateException(
+ "Snapshot periods must be positive numbers",
+ VESPA_STRLOC);
+ std::ostringstream name;
+ if (setSnapshotName(name, "week", length, 60 * 60 * 24 * 7)) {
+ } else if (setSnapshotName(name, "day", length, 60 * 60 * 24)) {
+ } else if (setSnapshotName(name, "hour", length, 60 * 60)) {
+ } else if (setSnapshotName(name, "minute", length, 60)) {
+ } else {
+ name << length << " seconds";
+ }
+ result.push_back(SnapSpec(length, name.str()));
+ }
+ for (uint32_t i=1; i<result.size(); ++i) {
+ if (result[i].first % result[i-1].first != 0) {
+ std::ostringstream ost;
+ ost << "Period " << result[i].first
+ << " is not a multiplum of period "
+ << result[i-1].first << " which is needs to be.";
+ throw vespalib::IllegalStateException(
+ ost.str(), VESPA_STRLOC);
+ }
+ }
+ } catch (vespalib::Exception& e) {
+ LOG(warning, "Invalid snapshot periods specified. Using defaults: %s",
+ e.getMessage().c_str());
+ result.clear();
+ }
+ if (result.empty()) {
+ result.push_back(SnapSpec(60 * 5, "5 minute"));
+ result.push_back(SnapSpec(60 * 60, "1 hour"));
+ result.push_back(SnapSpec(60 * 60 * 24, "1 day"));
+ result.push_back(SnapSpec(60 * 60 * 24 * 7, "1 week"));
+ }
+ return result;
+}
+
+void
+MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> config)
+{
+ assert(config.get() != 0);
+ if (LOG_WOULD_LOG(debug)) {
+ std::ostringstream ost;
+ config::OstreamConfigWriter w(ost);
+ w.write(*config);
+ LOG(debug, "Received new config for metric manager: %s",
+ ost.str().c_str());
+ }
+ if (_snapshots.empty()) {
+ LOG(debug, "Initializing snapshots as this is first configure call");
+ std::vector<SnapSpec> snapshotPeriods(createSnapshotPeriods(*config));
+
+ // Set up snapshots only first time. We don't allow live reconfig
+ // of snapshot periods.
+ time_t currentTime(_timer->getTime());
+ _activeMetrics.setFromTime(currentTime);
+ uint32_t count = 1;
+ for (uint32_t i = 0; i< snapshotPeriods.size(); ++i)
+ {
+ uint32_t nextCount = 1;
+ if (i + 1 < snapshotPeriods.size()) {
+ nextCount = snapshotPeriods[i + 1].first
+ / snapshotPeriods[i].first;
+ if (snapshotPeriods[i + 1].first
+ % snapshotPeriods[i].first != 0)
+ {
+ throw vespalib::IllegalStateException(
+ "Snapshot periods must be multiplum of each other",
+ VESPA_STRLOC);
+ }
+ }
+ _snapshots.push_back(MetricSnapshotSet::SP(new MetricSnapshotSet(
+ snapshotPeriods[i].second, snapshotPeriods[i].first, count,
+ _activeMetrics.getMetrics(), _snapshotUnsetMetrics)));
+ count = nextCount;
+ }
+ // Add all time snapshot.
+ _totalMetrics = MetricSnapshot::SP(new MetricSnapshot(
+ "All time snapshot", 0, _activeMetrics.getMetrics(),
+ _snapshotUnsetMetrics));
+ _totalMetrics->reset(currentTime);
+ }
+ if (_config.get() == 0
+ || _config->consumer.size() != config->consumer.size())
+ {
+ _consumerConfigChanged = true;
+ } else {
+ for (uint32_t i=0; i<_config->consumer.size(); ++i) {
+ if (_config->consumer[i] != config->consumer[i]) {
+ _consumerConfigChanged = true;
+ break;
+ }
+ }
+ }
+ if (_consumerConfigChanged) {
+ LOG(debug, "Consumer config changed. Tagging consumer config dirty.");
+ }
+ _config = std::move(config);
+
+}
+
+MetricManager::ConsumerSpec::SP
+MetricManager::getConsumerSpec(const MetricLockGuard &, const Metric::String& consumer) const
+{
+ auto it(_consumerConfig.find(consumer));
+ return (it != _consumerConfig.end() ? it->second : ConsumerSpec::SP());
+}
+
+//#define VERIFY_ALL_METRICS_VISITED 1
+
+namespace {
+
+ struct ConsumerMetricVisitor : public MetricVisitor {
+ const MetricManager::ConsumerSpec& _metricsToMatch;
+ MetricVisitor& _client;
+#ifdef VERIFY_ALL_METRICS_VISITED
+ std::set<Metric::String> _visitedMetrics;
+#endif
+
+ ConsumerMetricVisitor(const MetricManager::ConsumerSpec& spec,
+ MetricVisitor& clientVisitor)
+ : _metricsToMatch(spec), _client(clientVisitor) {}
+
+ virtual bool visitMetricSet(const MetricSet& metricSet,
+ bool autoGenerated)
+ {
+ if (metricSet.isTopSet()) return true;
+ return (_metricsToMatch.contains(metricSet)
+ && _client.visitMetricSet(metricSet, autoGenerated));
+ }
+ virtual void doneVisitingMetricSet(const MetricSet& metricSet) {
+ if (!metricSet.isTopSet()) {
+#ifdef VERIFY_ALL_METRICS_VISITED
+ _visitedMetrics.insert(metricSet.getPath());
+#endif
+ _client.doneVisitingMetricSet(metricSet);
+ }
+ }
+ virtual bool visitCountMetric(const AbstractCountMetric& metric,
+ bool autoGenerated)
+ {
+ if (_metricsToMatch.contains(metric)) {
+#ifdef VERIFY_ALL_METRICS_VISITED
+ _visitedMetrics.insert(metric.getPath());
+#endif
+ return _client.visitCountMetric(metric, autoGenerated);
+ }
+ return true;
+ }
+ virtual bool visitValueMetric(const AbstractValueMetric& metric,
+ bool autoGenerated)
+ {
+ if (_metricsToMatch.contains(metric)) {
+#ifdef VERIFY_ALL_METRICS_VISITED
+ _visitedMetrics.insert(metric.getPath());
+#endif
+ return _client.visitValueMetric(metric, autoGenerated);
+ }
+ return true;
+ }
+ };
+
+}
+
+void
+MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapshot, MetricVisitor& visitor,
+ const std::string& consumer) const
+{
+ if (visitor.visitSnapshot(snapshot)) {
+ if (consumer == "") {
+ snapshot.getMetrics().visit(visitor);
+ } else {
+ ConsumerSpec::SP consumerSpec(getConsumerSpec(guard, consumer));
+ if (consumerSpec.get()) {
+ ConsumerMetricVisitor consumerVis(*consumerSpec, visitor);
+ snapshot.getMetrics().visit(consumerVis);
+#ifdef VERIFY_ALL_METRICS_VISITED
+ for (auto metric = consumerSpec->includedMetrics) {
+ if (consumerVis._visitedMetrics.find(metric)
+ == consumerVis._visitedMetrics.end())
+ {
+ LOG(debug, "Failed to find metric %s to be visited.", metric.c_str());
+ }
+ }
+#endif
+ } else {
+ LOGBP(debug,
+ "Requested metrics for non-defined consumer '%s'.",
+ consumer.c_str());
+ }
+ }
+ visitor.doneVisitingSnapshot(snapshot);
+ }
+ visitor.doneVisiting();
+}
+
+std::vector<uint32_t>
+MetricManager::getSnapshotPeriods(const MetricLockGuard& l) const
+{
+ assertMetricLockLocked(l);
+ std::vector<uint32_t> result(_snapshots.size());
+ for (uint32_t i=0; i<_snapshots.size(); ++i) {
+ result[i] = _snapshots[i]->getPeriod();
+ }
+ return result;
+}
+
+// Client should have grabbed metrics lock before doing this
+const MetricSnapshot&
+MetricManager::getMetricSnapshot(const MetricLockGuard& l,
+ uint32_t period, bool getInProgressSet) const
+{
+ assertMetricLockLocked(l);
+ for (uint32_t i=0; i<_snapshots.size(); ++i) {
+ if (_snapshots[i]->getPeriod() == period) {
+ if (_snapshots[i]->getCount() == 1 && getInProgressSet) {
+ throw vespalib::IllegalStateException(
+ "No temporary snapshot for set "
+ + _snapshots[i]->getName(), VESPA_STRLOC);
+ }
+ return _snapshots[i]->getSnapshot(getInProgressSet);
+ }
+ }
+ std::ostringstream ost;
+ ost << "No snapshot for period of length " << period << " exist.";
+ throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC);
+}
+
+// Client should have grabbed metrics lock before doing this
+const MetricSnapshotSet&
+MetricManager::getMetricSnapshotSet(const MetricLockGuard& l,
+ uint32_t period) const
+{
+ assertMetricLockLocked(l);
+ for (uint32_t i=0; i<_snapshots.size(); ++i) {
+ if (_snapshots[i]->getPeriod() == period) {
+ return *_snapshots[i];
+ }
+ }
+ std::ostringstream ost;
+ ost << "No snapshot set for period of length " << period << " exist.";
+ throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC);
+}
+
+void
+MetricManager::timeChangedNotification() const
+{
+ vespalib::MonitorGuard sync(_waiter);
+ sync.broadcast();
+}
+
+void
+MetricManager::updateMetrics(bool includeSnapshotOnlyHooks)
+{
+ LOG(debug, "Calling metric update hooks%s.",
+ includeSnapshotOnlyHooks ? ", including snapshot hooks" : "");
+ // Ensure we're not in the way of the background thread
+ vespalib::MonitorGuard sync(_waiter);
+ LOG(debug, "Giving %" PRIu64 " periodic update hooks.",
+ _periodicUpdateHooks.size());
+ updatePeriodicMetrics(sync, 0, true);
+ if (includeSnapshotOnlyHooks) {
+ LOG(debug, "Giving %" PRIu64 " snapshot update hooks.",
+ _snapshotUpdateHooks.size());
+ updateSnapshotMetrics(sync);
+ }
+}
+
+// When this is called, the thread monitor lock has already been grabbed
+time_t
+MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updateTime, bool outOfSchedule)
+{
+ time_t nextUpdateTime = std::numeric_limits<time_t>::max();
+ time_t preTime = _timer->getTimeInMilliSecs();
+ for (auto hook : _periodicUpdateHooks) {
+ if (hook->_nextCall <= updateTime) {
+ hook->updateMetrics(guard);
+ if (hook->_nextCall + hook->_period < updateTime) {
+ if (hook->_nextCall != 0) {
+ LOG(debug, "Updated hook %s at time %" PRIu64 ", but next "
+ "run in %u seconds have already passed as time"
+ " is %" PRIu64 ". Bumping next call to current "
+ "time + period.",
+ hook->_name, hook->_nextCall, hook->_period, updateTime);
+ }
+ hook->_nextCall = updateTime + hook->_period;
+ } else {
+ hook->_nextCall += hook->_period;
+ }
+ time_t postTime = _timer->getTimeInMilliSecs();
+ _periodicHookLatency.addValue(postTime - preTime);
+ preTime = postTime;
+ } else if (outOfSchedule) {
+ hook->updateMetrics(guard);
+ time_t postTime = _timer->getTimeInMilliSecs();
+ _periodicHookLatency.addValue(postTime - preTime);
+ preTime = postTime;
+ }
+ nextUpdateTime = std::min(nextUpdateTime, hook->_nextCall);
+ }
+ return nextUpdateTime;
+}
+
+// When this is called, the thread monitor lock has already been grabbed.
+void
+MetricManager::updateSnapshotMetrics(const MetricLockGuard & guard)
+{
+ time_t preTime = _timer->getTimeInMilliSecs();
+ for (auto it = _snapshotUpdateHooks.begin(); it != _snapshotUpdateHooks.end(); ++it) {
+ (**it).updateMetrics(guard);
+ time_t postTime = _timer->getTimeInMilliSecs();
+ _snapshotHookLatency.addValue(postTime - preTime);
+ preTime = postTime;
+ }
+}
+
+void
+MetricManager::forceEventLogging()
+{
+ LOG(debug, "Forcing event logging to happen.");
+ // Ensure background thread is not in a current cycle during change.
+ vespalib::MonitorGuard sync(_waiter);
+ _forceEventLogging = true;
+ sync.signal();
+}
+
+void
+MetricManager::reset(time_t currentTime)
+{
+ time_t preTime = _timer->getTimeInMilliSecs();
+ // Resetting implies visiting metrics, which needs to grab metric lock
+ // to avoid conflict with adding/removal of metrics
+ vespalib::LockGuard waiterLock(_waiter);
+ _activeMetrics.reset(currentTime);
+ for (uint32_t i=0; i<_snapshots.size(); ++i) {
+ _snapshots[i]->reset(currentTime);
+ }
+ _totalMetrics->reset(currentTime);
+ time_t postTime = _timer->getTimeInMilliSecs();
+ _resetLatency.addValue(postTime - preTime);
+}
+
+namespace {
+
+ struct LogMetricVisitor : public MetricVisitor {
+ bool _total;
+
+ LogMetricVisitor(bool totalVals) : _total(totalVals) {}
+
+ bool visitMetric(const Metric& metric, bool autoGenerated) {
+ (void) autoGenerated;
+ if (metric.logFromTotalMetrics() ^ _total) return true;
+ Metric::String logName = metric.getPath();
+ std::replace(logName.begin(), logName.end(), '.', '_');
+ if (!metric.logEvent(logName)) {
+ LOG(spam, "Not logging event from metric %s.%s as no values "
+ "have been added to metric.",
+ metric.getPath().c_str(), metric.getName().c_str());
+ }
+ return true;
+ }
+ };
+
+} // End of anonymous namespace
+
+void
+MetricManager::run()
+{
+ while (!stopping()) {
+ time_t currentTime = _timer->getTime();
+ vespalib::MonitorGuard sync(_waiter);
+ time_t next = tick(sync, currentTime);
+ if (currentTime < next) {
+ sync.wait((next - currentTime) * 1000);
+ _sleepTimes.addValue((next - currentTime) * 1000);
+ } else {
+ _sleepTimes.addValue(0);
+ }
+ }
+}
+
+time_t
+MetricManager::tick(const MetricLockGuard & guard, time_t currentTime)
+{
+ LOG(spam, "Worker thread starting to process for time %" PRIu64 ".",
+ currentTime);
+ bool firstIteration = (_logPeriod.second == 0);
+ // For a slow system to still be doing metrics tasks each n'th
+ // second, rather than each n'th + time to do something seconds,
+ // we constantly add next time to do something from the last timer.
+ // For that to work, we need to initialize timers on first iteration
+ // to set them to current time.
+ time_t nextWorkTime;
+ if (firstIteration) {
+ // Setting next log period to now, such that we log metrics
+ // straight away
+ _logPeriod.second = currentTime;
+ for (uint32_t i=0; i<_snapshots.size(); ++i) {
+ _snapshots[i]->setFromTime(currentTime);
+ }
+ for (auto hook : _periodicUpdateHooks) {
+ hook->_nextCall = currentTime;
+ }
+ // Ensure correct time for first snapshot
+ _snapshots[0]->getSnapshot().setToTime(currentTime);
+ }
+
+ // Check for new config and reconfigure
+ if (_configSubscriber.get() && _configSubscriber->nextConfig(0)) {
+ configure(guard, _configHandle->getConfig());
+ }
+
+ // If metrics have changed since last time we did a snapshot,
+ // work that out before taking the snapshot, such that new
+ // metric can be included
+ checkMetricsAltered(guard);
+
+ // Set next work time to the time we want to take next snapshot.
+ nextWorkTime = _snapshots[0]->getToTime() + _snapshots[0]->getPeriod();
+ time_t nextUpdateHookTime;
+ if (nextWorkTime <= currentTime || _forceEventLogging) {
+ // If taking a new snapshot or logging, force calls to all
+ // update hooks.
+ LOG(debug, "%s. Calling update hooks.", nextWorkTime <= currentTime
+ ? "Time to do snapshot" : "Out of sequence event logging");
+ nextUpdateHookTime = updatePeriodicMetrics(guard, currentTime, true);
+ updateSnapshotMetrics(guard);
+ } else {
+ // If not taking a new snapshot. Only give update hooks to
+ // periodic hooks wanting it.
+ nextUpdateHookTime = updatePeriodicMetrics(guard, currentTime, false);
+ }
+ // Do snapshotting if it is time
+ if (nextWorkTime <= currentTime) takeSnapshots(guard, nextWorkTime);
+ // Log if it is time
+ if (_logPeriod.second <= currentTime || _forceEventLogging) {
+ LogMetricVisitor totalVisitor(true);
+ LogMetricVisitor fiveMinVisitor(false);
+ if (_logPeriod.second <= currentTime) {
+ LOG(debug, "Logging total metrics.");
+ visit(guard, *_totalMetrics, totalVisitor, "log");
+ visit(guard, _snapshots[0]->getSnapshot(), fiveMinVisitor , "log");
+ if (_logPeriod.second + _logPeriod.first < currentTime) {
+ uint64_t next = _snapshots[0]->getFromTime()
+ + _logPeriod.first;
+ LOG(warning, "Logged events at time %" PRIu64 " for time %"
+ PRIu64 ". Since this is more than a period %u "
+ "in the past, next run has been set to next "
+ "snapshot time at %" PRIu64 ".",
+ currentTime, _logPeriod.second, _logPeriod.first, next);
+ _logPeriod.second = next;
+ } else {
+ _logPeriod.second += _logPeriod.first;
+ }
+ } else {
+ LOG(debug, "Logging total metrics out of sequence.");
+ metrics::MetricSnapshot snapshot(
+ "Total out of sequence metrics from start until "
+ "current time", 0, _totalMetrics->getMetrics(),
+ _snapshotUnsetMetrics);
+ _activeMetrics.addToSnapshot(snapshot, false, currentTime);
+ snapshot.setFromTime(_totalMetrics->getFromTime());
+ visit(guard, snapshot, totalVisitor, "log");
+ visit(guard, snapshot, fiveMinVisitor, "log");
+ }
+ _forceEventLogging = false;
+ }
+ _lastProcessedTime = (nextWorkTime <= currentTime ? nextWorkTime
+ : currentTime);
+ LOG(spam, "Worker thread done with processing for time %" PRIu64 ".",
+ _lastProcessedTime);
+ time_t next = std::min(
+ _logPeriod.second,
+ _snapshots[0]->getPeriod() + _snapshots[0]->getToTime());
+ next = std::min(next, nextUpdateHookTime);
+ return next;
+}
+
+void
+MetricManager::takeSnapshots(const MetricLockGuard &, time_t timeToProcess)
+{
+ // If not time to do dump data from active snapshot yet, nothing to do
+ if (!_snapshots[0]->timeForAnotherSnapshot(timeToProcess)) {
+ LOG(spam, "Not time to process snapshot %s at time %" PRIu64 ". Current "
+ "first period (%u) snapshot goes from %" PRIu64 " to %" PRIu64,
+ _snapshots[0]->getName().c_str(), timeToProcess,
+ _snapshots[0]->getPeriod(), _snapshots[0]->getFromTime(),
+ _snapshots[0]->getToTime());
+ return;
+ }
+ time_t preTime = _timer->getTimeInMilliSecs();
+ LOG(debug, "Updating %s snapshot and total metrics at time %" PRIu64 ".",
+ _snapshots[0]->getName().c_str(), timeToProcess);
+ MetricSnapshot& firstTarget(_snapshots[0]->getNextTarget());
+ firstTarget.reset(_activeMetrics.getFromTime());
+ _activeMetrics.addToSnapshot(firstTarget, false, timeToProcess);
+ _activeMetrics.addToSnapshot(*_totalMetrics, false, timeToProcess);
+ _activeMetrics.reset(timeToProcess);
+ LOG(debug, "After snapshotting, "
+ "active metrics goes from %" PRIu64 " to %" PRIu64", "
+ "and 5 minute metrics goes from %" PRIu64 " to %" PRIu64".",
+ _activeMetrics.getFromTime(), _activeMetrics.getToTime(),
+ firstTarget.getFromTime(), firstTarget.getToTime());
+
+ // Update later snapshots if it is time for it
+ for (uint32_t i=1; i<_snapshots.size(); ++i) {
+ LOG(debug, "Adding data from last snapshot to building snapshot of "
+ "next period snapshot %s.",
+ _snapshots[i]->getName().c_str());
+ MetricSnapshot& target(_snapshots[i]->getNextTarget());
+ _snapshots[i-1]->getSnapshot().addToSnapshot(
+ target, false, timeToProcess);
+ target.setToTime(timeToProcess);
+ if (!_snapshots[i]->haveCompletedNewPeriod(timeToProcess)) {
+ LOG(debug, "Not time to roll snapshot %s yet. %u of %u snapshot "
+ "taken at time %" PRIu64 ", and period of %u is not up "
+ "yet as we're currently processing for time %" PRIu64 ".",
+ _snapshots[i]->getName().c_str(),
+ _snapshots[i]->getBuilderCount(),
+ _snapshots[i]->getCount(),
+ _snapshots[i]->getBuilderCount() * _snapshots[i]->getPeriod()
+ + _snapshots[i]->getFromTime(),
+ _snapshots[i]->getPeriod(),
+ timeToProcess);
+ break;
+ } else {
+ LOG(debug, "Rolled snapshot %s at time %" PRIu64 ".",
+ _snapshots[i]->getName().c_str(),
+ timeToProcess);
+ }
+ }
+ time_t postTime = _timer->getTimeInMilliSecs();
+ _snapshotLatency.addValue(postTime - preTime);
+}
+
+MemoryConsumption::UP
+MetricManager::getMemoryConsumption(const MetricLockGuard & guard) const
+{
+ (void) guard;
+ MemoryConsumption::UP mc(new MemoryConsumption);
+ mc->_consumerCount += _consumerConfig.size();
+ mc->_consumerMeta += (sizeof(ConsumerSpec::SP) + sizeof(ConsumerSpec))
+ * _consumerConfig.size();
+ for (auto it = _consumerConfig.begin(); it != _consumerConfig.end(); ++it) {
+ mc->_consumerId += mc->getStringMemoryUsage(
+ it->first, mc->_consumerIdUnique)
+ + sizeof(Metric::String);
+ it->second->addMemoryUsage(*mc);
+ }
+ uint32_t preTotal = mc->getTotalMemoryUsage();
+ _activeMetrics.addMemoryUsage(*mc);
+ uint32_t postTotal = mc->getTotalMemoryUsage();
+ mc->addSnapShotUsage("active", postTotal - preTotal);
+ preTotal = postTotal;
+ for (uint32_t i=0; i<_snapshots.size(); ++i) {
+ _snapshots[i]->addMemoryUsage(*mc);
+ postTotal = mc->getTotalMemoryUsage();
+ mc->addSnapShotUsage(_snapshots[i]->getName(), postTotal - preTotal);
+ preTotal = postTotal;
+ }
+ _totalMetrics->addMemoryUsage(*mc);
+ postTotal = mc->getTotalMemoryUsage();
+ mc->addSnapShotUsage("total", postTotal - preTotal);
+ _nameHash.addMemoryUsage(*mc);
+ return mc;
+}
+
+} // metrics
diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h
new file mode 100644
index 00000000000..467e97469c1
--- /dev/null
+++ b/metrics/src/vespa/metrics/metricmanager.h
@@ -0,0 +1,322 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class metrics::MetricManager
+ *
+ * A metrics-enabled application should have a single MetricManager.
+ * You can register a number of MetricSets in the MetricManager. Each
+ * metric in the metrics sets can be used by zero or more consumers,
+ * configurable using readConfig().
+ *
+ * The consumers get their data by calling the getSnapshot() method,
+ * which gives them a snapshot of all the current metrics which are
+ * configured for the given name.
+ *
+ * Locking strategy:
+ *
+ * There are three locks in this class:
+ *
+ * Config lock:
+ * - This protects the class on config changes. It protects the _config and
+ * _consumerConfig members.
+ *
+ * Thread monitor (waiter):
+ * - This lock is kept by the worker thread while it is doing a work cycle,
+ * and it uses this monitor to sleep. It is used to make shutdown quick by
+ * interrupting thread, and to let functions called by clients be able to
+ * do a change while the worker thread is idle.
+ * - The log period is protected by the thread monitor.
+ * - The update hooks is protected by the thread monitor.
+ *
+ * Metric lock:
+ * - The metric log protects the active metric set when adding or removing
+ * metrics. Clients need to grab this lock before altering active metrics.
+ * The metric manager needs to grab this lock everytime it visits active
+ * metrics.
+ * - The metric log protects the snapshots. The snapshot writer is the metric
+ * worker thread and will grab the lock while editing them. Readers that
+ * aren't the worker thread itself must grab lock to be sure.
+ *
+ * If multiple locks is taken, the allowed locking order is:
+ * 1. Thread monitor.
+ * 2. Metric lock.
+ * 3. Config lock.
+ */
+#pragma once
+
+#include <string>
+#include <boost/weak_ptr.hpp>
+#include <vespa/vespalib/util/document_runnable.h>
+#include <vespa/metrics/config-metricsmanager.h>
+#include <vespa/metrics/metricset.h>
+#include <vespa/metrics/metricsnapshot.h>
+#include <vespa/metrics/namehash.h>
+#include <vespa/metrics/valuemetric.h>
+#include <vespa/vespalib/stllike/hash_set.h>
+#include <vespa/vespalib/util/sync.h>
+#include <map>
+#include <list>
+#include <vespa/vespalib/util/jsonwriter.h>
+#include <vespa/config/config.h>
+
+namespace metrics {
+
+typedef vespalib::MonitorGuard MetricLockGuard;
+
+class MetricManager : private document::Runnable
+{
+public:
+ class UpdateHook {
+ const char* _name;
+ time_t _nextCall;
+ uint32_t _period;
+ friend class MetricManager;
+
+ public:
+ typedef vespalib::LinkedPtr<UpdateHook> LP;
+ using MetricLockGuard = metrics::MetricLockGuard;
+
+ UpdateHook(const char* name) : _name(name), _nextCall(0), _period(0) {}
+ virtual ~UpdateHook() {}
+ virtual void updateMetrics(const MetricLockGuard & guard) = 0;
+ const char* getName() const { return _name; }
+ };
+
+ struct Timer {
+ virtual ~Timer() {}
+ virtual time_t getTime() const { return time(0); }
+ virtual time_t getTimeInMilliSecs() const { return getTime() * 1000; }
+ };
+
+ /**
+ * Spec saved from config. If metricSetChildren has content, metric pointed
+ * to is a metric set.
+ */
+ struct ConsumerSpec : public vespalib::Printable {
+ typedef std::shared_ptr<ConsumerSpec> SP;
+
+ vespalib::hash_set<Metric::String> includedMetrics;
+ ConsumerSpec() : includedMetrics() {}
+
+ bool contains(const Metric& m) const {
+ return (includedMetrics.find(m.getPath()) != includedMetrics.end());
+ }
+
+ void print(std::ostream& out, bool verbose,
+ const std::string& indent) const;
+
+ void addMemoryUsage(MemoryConsumption&) const;
+ };
+
+private:
+ MetricSnapshot _activeMetrics;
+ std::unique_ptr<config::ConfigSubscriber> _configSubscriber;
+ config::ConfigHandle<MetricsmanagerConfig>::UP _configHandle;
+ std::unique_ptr<MetricsmanagerConfig> _config;
+ std::map<Metric::String, ConsumerSpec::SP> _consumerConfig;
+ std::list<UpdateHook*> _periodicUpdateHooks;
+ std::list<UpdateHook*> _snapshotUpdateHooks;
+ vespalib::Monitor _waiter;
+ typedef std::pair<uint32_t, time_t> PeriodTimePair;
+ PeriodTimePair _logPeriod;
+ std::vector<MetricSnapshotSet::SP> _snapshots;
+ MetricSnapshot::SP _totalMetrics;
+ std::unique_ptr<Timer> _timer;
+ time_t _lastProcessedTime;
+ bool _forceEventLogging;
+ // Should be added to config, but wont now due to problems with
+ // upgrading
+ bool _snapshotUnsetMetrics;
+ bool _consumerConfigChanged;
+
+ MetricSet _metricManagerMetrics;
+ LongAverageMetric _periodicHookLatency;
+ LongAverageMetric _snapshotHookLatency;
+ LongAverageMetric _resetLatency;
+ LongAverageMetric _snapshotLatency;
+ LongAverageMetric _sleepTimes;
+
+ // Name hash trying to ensure string ref counting works as well as can be
+ // expected
+ NameHash _nameHash;
+
+public:
+ MetricManager(std::unique_ptr<Timer> timer = std::unique_ptr<Timer>(new Timer));
+ ~MetricManager();
+
+ void stop();
+
+ void snapshotUnsetMetrics(bool doIt) { _snapshotUnsetMetrics = doIt; }
+
+ /**
+ * Add a metric update hook. This will always be called prior to
+ * snapshotting and metric logging, to make the metrics the best as they can
+ * be at those occasions.
+ *
+ * @param period Period in seconds for how often callback should be called.
+ * The default value of 0, means only before snapshotting or
+ * logging, while another value will give callbacks each
+ * period seconds. Expensive metrics to calculate will
+ * typically only want to do it before snapshotting, while
+ * inexpensive metrics might want to log their value every 5
+ * seconds or so. Any value of period >= the smallest snapshot
+ * time will behave identically as if period is set to 0.
+ */
+ void addMetricUpdateHook(UpdateHook&, uint32_t period = 0);
+
+ /** Remove a metric update hook so it won't get any more updates. */
+ void removeMetricUpdateHook(UpdateHook&);
+
+ /**
+ * Force a metric update for all update hooks. Useful if you want to ensure
+ * nice values before reporting something.
+ * This function can not be called from an update hook callback.
+ */
+ void updateMetrics(bool includeSnapshotOnlyHooks = false);
+
+ /**
+ * Force event logging to happen now.
+ * This function can not be called from an update hook callback.
+ */
+ void forceEventLogging();
+
+ /**
+ * Register a new metric to be included in the active metric set. You need
+ * to have grabbed the metric lock in order to do this. (You also need to
+ * grab that lock if you alter registration of already registered metric
+ * set.) This function can not be called from an update hook callback.
+ */
+ void registerMetric(const MetricLockGuard& l, Metric& m) {
+ assertMetricLockLocked(l);
+ _activeMetrics.getMetrics().registerMetric(m);
+ }
+
+ /**
+ * Unregister a metric from the active metric set. You need to have grabbed
+ * the metric lock in order to do this. (You also need to grab that lock
+ * if you alter registration of already registered metric set.)
+ * This function can not be called from an update hook callback.
+ */
+ void unregisterMetric(const MetricLockGuard& l, Metric& m) {
+ assertMetricLockLocked(l);
+ _activeMetrics.getMetrics().unregisterMetric(m);
+ }
+
+ /**
+ * Reset all metrics including all snapshots.
+ * This function can not be called from an update hook callback.
+ */
+ void reset(time_t currentTime);
+
+ /**
+ * Read configuration. Before reading config, all metrics should be set
+ * up first. By doing this, the metrics manager can optimize reporting
+ * of consumers. readConfig() will start a config subscription. It should
+ * not be called multiple times.
+ */
+ void init(const config::ConfigUri & uri, FastOS_ThreadPool&,
+ bool startThread = true);
+
+ /**
+ * Visit a given snapshot for a given consumer. (Empty consumer name means
+ * all metrics). This function can be used for various printing by using
+ * various writer visitors in the metrics module, or your own.
+ */
+ void visit(const MetricLockGuard & guard, const MetricSnapshot&,
+ MetricVisitor&, const std::string& consumer) const;
+
+ /**
+ * The metric lock protects against changes in metric structure. After
+ * metric manager init, you need to take this lock if you want to add or
+ * remove metrics from registered metric sets, to avoid that happening at
+ * the same time as metrics are being visited. Also, when accessing
+ * snapshots, you need to have this lock to prevent metric manager to alter
+ * snapshots while you are accessing them.
+ */
+ MetricLockGuard getMetricLock() const {
+ MetricLockGuard m(_waiter);
+ return m;
+ }
+
+ /** While accessing the active metrics you should have the metric lock. */
+ MetricSnapshot& getActiveMetrics(const MetricLockGuard& l) {
+ assertMetricLockLocked(l);
+ return _activeMetrics;
+ }
+ const MetricSnapshot& getActiveMetrics(const MetricLockGuard& l) const {
+ assertMetricLockLocked(l);
+ return _activeMetrics;
+ }
+
+ /** While accessing the total metrics you should have the metric lock. */
+ const MetricSnapshot& getTotalMetricSnapshot(const MetricLockGuard& l) const
+ {
+ assertMetricLockLocked(l);
+ return *_totalMetrics;
+ }
+ /** While accessing snapshots you should have the metric lock. */
+ const MetricSnapshot& getMetricSnapshot(
+ const MetricLockGuard&,
+ uint32_t period, bool getInProgressSet = false) const;
+ const MetricSnapshotSet& getMetricSnapshotSet(
+ const MetricLockGuard&, uint32_t period) const;
+ bool hasTemporarySnapshot(const MetricLockGuard& l, uint32_t period) const
+ { return getMetricSnapshotSet(l, period).hasTemporarySnapshot(); }
+
+ std::vector<uint32_t> getSnapshotPeriods(const MetricLockGuard& l) const;
+
+ ConsumerSpec::SP getConsumerSpec(const MetricLockGuard & guard, const Metric::String& consumer) const;
+
+ /**
+ * If you add or remove metrics from the active metric sets, normally,
+ * snapshots will be recreated next snapshot period. However, if you want
+ * to see the effects of such changes in status pages ahead of that, you
+ * can call this function in order to check whether snapshots needs to be
+ * regenerated and regenerate them if needed.
+ */
+ void checkMetricsAltered(const MetricLockGuard &);
+
+ /** Used by unit tests to verify that we have processed for a given time. */
+ time_t getLastProcessedTime() const { return _lastProcessedTime; }
+
+ /** Used by unit tests to wake waiters after altering time. */
+ void timeChangedNotification() const;
+
+ MemoryConsumption::UP getMemoryConsumption(const MetricLockGuard & guard) const;
+
+private:
+ void takeSnapshots(const MetricLockGuard &, time_t timeToProcess);
+
+ friend class MetricManagerTest;
+ friend class SnapshotTest;
+
+ void configure(const MetricLockGuard & guard, std::unique_ptr<MetricsmanagerConfig> conf);
+ void run() override;
+ time_t tick(const MetricLockGuard & guard, time_t currentTime);
+ /**
+ * Utility function for updating periodic metrics.
+ *
+ * @param updateTime Update metrics timed to update at this time.
+ * @param outOfSchedule Force calls to all hooks. Don't screw up normal
+ * schedule though. If not time to update yet, update
+ * without adjusting schedule for next update.
+ * @return Time of next hook to be called in the future.
+ */
+ time_t updatePeriodicMetrics(const MetricLockGuard & guard, time_t updateTime, bool outOfSchedule);
+ void updateSnapshotMetrics(const MetricLockGuard & guard);
+
+ void handleMetricsAltered(const MetricLockGuard &);
+
+ typedef std::pair<uint32_t, std::string> SnapSpec;
+ static std::vector<SnapSpec> createSnapshotPeriods(
+ const MetricsmanagerConfig& config);
+ void assertMetricLockLocked(const MetricLockGuard& g) const {
+ if (!g.monitors(_waiter)) {
+ throw vespalib::IllegalArgumentException(
+ "Given lock does not lock the metric lock.", VESPA_STRLOC);
+ }
+ }
+
+};
+
+} // metrics
+
diff --git a/metrics/src/vespa/metrics/metrics.h b/metrics/src/vespa/metrics/metrics.h
new file mode 100644
index 00000000000..4b21dfb8579
--- /dev/null
+++ b/metrics/src/vespa/metrics/metrics.h
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * File to include if you want to get all the most used metrics classes
+ * included.
+ */
+#pragma once
+
+#include <vespa/metrics/metric.h>
+#include <vespa/metrics/countmetric.h>
+#include <vespa/metrics/valuemetric.h>
+#include <vespa/metrics/loadmetric.h>
+#include <vespa/metrics/summetric.h>
+#include <vespa/metrics/metricset.h>
+#include <vespa/metrics/metricmanager.h>
+#include <vespa/metrics/metricsnapshot.h>
+#include <vespa/metrics/metrictimer.h>
+
diff --git a/metrics/src/vespa/metrics/metricset.cpp b/metrics/src/vespa/metrics/metricset.cpp
new file mode 100644
index 00000000000..cbddccfa787
--- /dev/null
+++ b/metrics/src/vespa/metrics/metricset.cpp
@@ -0,0 +1,366 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/metrics/metricset.h>
+#include <vespa/vespalib/stllike/hash_map.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include <list>
+
+#include <vespa/log/log.h>
+
+LOG_SETUP(".metrics.metricsset");
+
+namespace metrics {
+
+MetricSet::MetricSet(const String& name, const String& tags,
+ const String& description, MetricSet* owner,
+ const std::string& dimensionKey)
+ : Metric(name, tags, description, owner),
+ _metricOrder(),
+ _registrationAltered(false),
+ _dimensionKey(dimensionKey)
+{
+}
+
+MetricSet::MetricSet(const String& name, Tags dimensions,
+ const String& description, MetricSet* owner)
+ : Metric(name, std::move(dimensions), description, owner),
+ _metricOrder(),
+ _registrationAltered(false),
+ _dimensionKey()
+{
+}
+
+MetricSet::MetricSet(const MetricSet& other,
+ std::vector<Metric::LP>& ownerList,
+ CopyType copyType,
+ MetricSet* owner,
+ bool includeUnused)
+ : Metric(other, owner),
+ _metricOrder(),
+ _registrationAltered(false),
+ _dimensionKey(other._dimensionKey)
+{
+ if (copyType == INACTIVE && owner == NULL && includeUnused) {
+ _dimensionKey = "";
+ }
+
+ for (const Metric* metric : other._metricOrder) {
+ if (copyType != INACTIVE || includeUnused || metric->used()) {
+ Metric* m = metric->clone(ownerList, copyType, this, includeUnused);
+ ownerList.push_back(Metric::LP(m));
+ }
+ }
+}
+
+const Metric*
+MetricSet::getMetricInternal(const String& name) const
+{
+ for (const Metric* metric : _metricOrder) {
+ if (metric->getMangledName() == name) {
+ return metric;
+ }
+ }
+ return 0;
+}
+
+const Metric*
+MetricSet::getMetric(const String& name) const
+{
+ std::string::size_type pos = name.find('.');
+ if (pos == std::string::npos) {
+ return getMetricInternal(name);
+ } else {
+ std::string child(name.substr(0, pos));
+ std::string rest(name.substr(pos+1));
+ const Metric* m(getMetricInternal(child));
+ if (m == 0) return 0;
+ if (!m->isMetricSet()) {
+ throw vespalib::IllegalStateException(
+ "Metric " + child + " is not a metric set. Cannot retrieve "
+ "metric at path " + name + " within metric " + getPath(),
+ VESPA_STRLOC);
+ }
+ return static_cast<const MetricSet*>(m)->getMetric(rest);
+ }
+}
+
+namespace {
+ struct MetricSetFinder : public MetricVisitor {
+ std::list<MetricSet*> _metricSets;
+
+ virtual bool visitMetricSet(const MetricSet& set, bool autoGenerated)
+ {
+ if (autoGenerated) return false;
+ _metricSets.push_back(const_cast<MetricSet*>(&set));
+ return true;
+ }
+ virtual bool visitMetric(const Metric&, bool autoGenerated)
+ {
+ (void) autoGenerated;
+ return true;
+ }
+ };
+}
+
+void
+MetricSet::clearRegistrationAltered()
+{
+ MetricSetFinder finder;
+ visit(finder);
+ for (MetricSet* metricSet : finder._metricSets) {
+ metricSet->_registrationAltered = false;
+ }
+}
+
+void
+MetricSet::tagRegistrationAltered()
+{
+ _registrationAltered = true;
+ if (_owner != 0) {
+ _owner->tagRegistrationAltered();
+ }
+}
+
+void
+MetricSet::registerMetric(Metric& metric)
+{
+ if (metric.isRegistered()) {
+ throw vespalib::IllegalStateException(
+ "Metric " + metric.getMangledName() +
+ " is already registered in a metric set. "
+ "Cannot register it twice.", VESPA_STRLOC);
+ }
+ const Metric* existing(getMetricInternal(metric.getMangledName()));
+ if (existing != 0) {
+ throw vespalib::IllegalStateException(
+ "A metric named " + metric.getMangledName() +
+ " is already registered in metric set " + getPath(),
+ VESPA_STRLOC);
+ }
+ _metricOrder.push_back(&metric);
+ metric.setRegistered(this);
+ tagRegistrationAltered();
+ if (metric.isMetricSet()) {
+ static_cast<MetricSet &>(metric)._owner = this;
+ }
+ LOG(spam, "Registered metric%s %s in metric set %s.",
+ metric.isMetricSet() ? "set" : "",
+ metric.getMangledName().c_str(),
+ getPath().c_str());
+}
+
+void
+MetricSet::unregisterMetric(Metric& metric)
+{
+ // In case of abrubt shutdowns, don't die hard on attempts to unregister
+ // non-registered metrics. Just warn and ignore.
+ const Metric* existing(getMetricInternal(metric.getMangledName()));
+ if (existing == 0) {
+ LOG(warning, "Attempt to unregister metric %s in metric set %s, where "
+ "it wasn't registered to begin with.",
+ metric.getMangledName().c_str(),
+ getPath().c_str());
+ return;
+ }
+ bool found = false;
+ for (std::vector<Metric*>::iterator it = _metricOrder.begin();
+ it != _metricOrder.end(); ++it)
+ {
+ if (*it == &metric) {
+ _metricOrder.erase(it);
+ found = true;
+ break;
+ }
+ }
+ assert(found); // We check above for existence.
+ (void) found;
+ metric.setRegistered(NULL);
+ tagRegistrationAltered();
+ if (metric.isMetricSet()) {
+ static_cast<MetricSet &>(metric)._owner = this;
+ }
+ LOG(spam, "Unregistered metric%s %s from metric set %s.",
+ metric.isMetricSet() ? "set" : "",
+ metric.getMangledName().c_str(),
+ getPath().c_str());
+}
+
+namespace {
+ typedef vespalib::stringref TmpString;
+ class StringMetric {
+ public:
+ StringMetric(const TmpString & s, Metric * m) : first(s), second(m) { }
+ bool operator == (const StringMetric & b) const { return first == b.first; }
+ bool operator == (const TmpString & b) const { return first == b; }
+ bool operator <(const StringMetric & b) const { return first < b.first; }
+ bool operator <(const TmpString & b) const { return first < b; }
+ TmpString first;
+ Metric * second;
+ };
+ bool operator < (const TmpString & a, const StringMetric & b) { return a < b.first; }
+
+ typedef std::vector<StringMetric> SortedVector;
+
+ void createMetricMap(SortedVector& metricMap,
+ const std::vector<Metric*>& orderedList)
+ {
+ metricMap.reserve(orderedList.size());
+ for (Metric* metric : orderedList) {
+ metricMap.push_back(StringMetric(metric->getMangledName(), metric));
+ }
+ std::sort(metricMap.begin(), metricMap.end());
+ }
+}
+
+void
+MetricSet::addTo(Metric& other, std::vector<Metric::LP>* ownerList) const
+{
+ bool mustAdd = (ownerList == 0);
+ MetricSet& o(static_cast<MetricSet&>(other));
+ SortedVector map1, map2;
+ createMetricMap(map1, _metricOrder);
+ createMetricMap(map2, o._metricOrder);
+ SortedVector::iterator source(map1.begin());
+ SortedVector::iterator target(map2.begin());
+ typedef vespalib::hash_map<TmpString, Metric*> HashMap;
+ HashMap newMetrics;
+ while (source != map1.end()) {
+ if (target == map2.end() || source->first < target->first) {
+ // Source missing in snapshot to add to. Lets create and add.
+ if (!mustAdd && source->second->used()) {
+ Metric::LP copy(source->second->clone(
+ *ownerList, INACTIVE, &o));
+ ownerList->push_back(copy);
+ newMetrics[source->first] = copy.get();
+ }
+ ++source;
+ } else if (source->first == target->first) {
+ if (mustAdd) {
+ source->second->addToPart(*target->second);
+ } else {
+ source->second->addToSnapshot(*target->second, *ownerList);
+ }
+ ++source;
+ ++target;
+ } else {
+ ++target;
+ }
+ }
+ // If we added metrics, reorder target order list to equal source
+ if (!newMetrics.empty()) {
+ std::vector<Metric*> newOrder;
+ newOrder.reserve(o._metricOrder.size() + newMetrics.size());
+ for (const Metric* metric : _metricOrder) {
+ TmpString v(metric->getMangledName());
+ target = std::lower_bound(map2.begin(), map2.end(), v);
+ if ((target != map2.end()) && (target->first == v)) {
+ newOrder.push_back(target->second);
+ } else {
+ HashMap::iterator starget = newMetrics.find(v);
+ if (starget != newMetrics.end()) {
+ newOrder.push_back(starget->second);
+ }
+ }
+ }
+ // If target had unique metrics, add them at the end
+ for (Metric* metric : o._metricOrder) {
+ TmpString v(metric->getMangledName());
+ if ( ! std::binary_search(map1.begin(), map1.end(), v) ) {
+ LOG(debug, "Metric %s exist in one snapshot but not other."
+ "Order will be messed up. Adding target unique "
+ "metrics to end.",
+ metric->getPath().c_str());
+ newOrder.push_back(metric);
+ }
+ }
+ o._metricOrder.swap(newOrder);
+ }
+}
+
+bool
+MetricSet::logEvent(const String& fullName) const
+{
+ (void) fullName;
+ throw vespalib::IllegalStateException(
+ "logEvent() cannot be called on metrics set.", VESPA_STRLOC);
+}
+
+void
+MetricSet::reset()
+{
+ for (Metric* metric : _metricOrder) {
+ metric->reset();
+ }
+}
+
+bool
+MetricSet::visit(MetricVisitor& visitor, bool tagAsAutoGenerated) const
+{
+ if (!visitor.visitMetricSet(*this, tagAsAutoGenerated)) return true;
+ for (const Metric* metric : _metricOrder) {
+ if (!metric->visit(visitor, tagAsAutoGenerated)) {
+ break;
+ }
+ }
+ visitor.doneVisitingMetricSet(*this);
+ return true;
+}
+
+void
+MetricSet::print(std::ostream& out, bool verbose,
+ const std::string& indent, uint64_t secondsPassed) const
+{
+ out << _name << ":";
+ for (const Metric* metric : _metricOrder) {
+ out << "\n" << indent << " ";
+ metric->print(out, verbose, indent + " ", secondsPassed);
+ }
+}
+
+bool
+MetricSet::used() const
+{
+ for (const Metric* metric : _metricOrder) {
+ if (metric->used()) return true;
+ }
+ return false;
+}
+
+void
+MetricSet::addMemoryUsage(MemoryConsumption& mc) const
+{
+ Metric::addMemoryUsage(mc);
+ ++mc._metricSetCount;
+ mc._metricSetMeta += sizeof(MetricSet) - sizeof(Metric);
+ mc._metricSetOrder += _metricOrder.size() * 3 * sizeof(void*);
+ for (const Metric* metric : _metricOrder) {
+ metric->addMemoryUsage(mc);
+ }
+}
+
+void
+MetricSet::updateNames(NameHash& hash) const
+{
+ Metric::updateNames(hash);
+ for (const Metric* metric : _metricOrder) {
+ metric->updateNames(hash);
+ }
+}
+
+void
+MetricSet::printDebug(std::ostream& out, const std::string& indent) const
+{
+ out << "set ";
+ Metric::printDebug(out, indent);
+ if (_registrationAltered) out << ", regAltered";
+ out << " {";
+ for (const Metric* metric : _metricOrder) {
+ out << "\n" << indent << " ";
+ metric->printDebug(out, indent + " ");
+ }
+ out << "}";
+}
+
+} // metrics
+
diff --git a/metrics/src/vespa/metrics/metricset.h b/metrics/src/vespa/metrics/metricset.h
new file mode 100644
index 00000000000..257bb699690
--- /dev/null
+++ b/metrics/src/vespa/metrics/metricset.h
@@ -0,0 +1,123 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class metrics::MetricSet
+ * \ingroup metrics
+ *
+ * \brief Class containing a set of metrics.
+ *
+ * This is a class to bundle related metrics. Note that the metricsset is
+ * itself a metric so this generates a tree where metricssets are leaf nodes.
+ */
+#pragma once
+
+#include <boost/weak_ptr.hpp>
+#include <map>
+#include <vespa/metrics/metric.h>
+
+namespace metrics {
+
+class MetricSet : public Metric
+{
+ std::vector<Metric*> _metricOrder; // Keep added order for reporting
+ bool _registrationAltered; // Set to true if metrics have been
+ // registered/unregistered since last time
+ // it was reset
+ std::string _dimensionKey; // If this metric is part of a dimension (yamas-speak),
+ // the key of the dimension should be set here.
+ // If so, the name of the metric is used as dimension value.
+
+public:
+ typedef std::unique_ptr<MetricSet> UP;
+ typedef std::shared_ptr<MetricSet> SP;
+
+ MetricSet(const String& name, const String& tags,
+ const String& description, MetricSet* owner = 0,
+ const std::string& dimensionKey = "");
+
+ MetricSet(const String& name, Tags dimensions,
+ const String& description, MetricSet* owner = 0);
+
+ MetricSet(const MetricSet&, std::vector<Metric::LP>& ownerList,
+ CopyType, MetricSet* owner = 0, bool includeUnused = false);
+ virtual ~MetricSet() {}
+
+ // If no path, this metric is not registered within another
+ bool isTopSet() const { return _owner == 0; }
+
+ /**
+ * Returns true if registration has been altered since it was last
+ * cleared. Used by the metric manager to know when it needs to recalculate
+ * which consumers will see what.
+ */
+ bool isRegistrationAltered() const { return _registrationAltered; }
+ /** Clear all registration altered flags. */
+ void clearRegistrationAltered();
+
+ void registerMetric(Metric& m);
+ void unregisterMetric(Metric& m);
+
+ virtual Metric* clone(std::vector<Metric::LP>& ownerList,
+ CopyType type, MetricSet* owner,
+ bool includeUnused = false) const
+ { return new MetricSet(*this, ownerList, type, owner, includeUnused); }
+
+ void reset();
+
+ virtual bool visit(MetricVisitor&, bool tagAsAutoGenerated = false) const;
+
+ virtual bool logEvent(const String& fullName) const;
+
+ virtual void print(std::ostream&, bool verbose,
+ const std::string& indent, uint64_t secondsPassed) const;
+
+ // These should never be called on metrics set.
+ virtual int64_t getLongValue(const stringref & id) const
+ { (void) id; assert(false); return 0; }
+ virtual double getDoubleValue(const stringref & id) const
+ { (void) id; assert(false); return 0; }
+
+ const Metric* getMetric(const String& name) const;
+ Metric* getMetric(const String& name) {
+ return const_cast<Metric*>(
+ const_cast<const MetricSet*>(this)->getMetric(name));
+ }
+
+ virtual void addToSnapshot(Metric& m, std::vector<Metric::LP>& o) const
+ { addTo(m, &o); }
+
+ const std::vector<Metric*>& getRegisteredMetrics() const
+ { return _metricOrder; }
+
+ virtual bool used() const;
+
+ virtual void addMemoryUsage(MemoryConsumption&) const;
+
+ /** Update names using the given name hash, to utilize ref counting. */
+ virtual void updateNames(NameHash&) const;
+
+ virtual void printDebug(std::ostream&, const std::string& indent="") const;
+
+ virtual bool isMetricSet() const { return true; }
+
+ virtual void addToPart(Metric& m) const
+ { addTo(m, 0); }
+
+ /**
+ * Returns the key of the dimension this metric is part of (if any).
+ */
+ const std::string& getDimensionKey() const { return _dimensionKey; }
+
+private:
+ // Do not generate default copy constructor or assignment operator
+ // These would screw up metric registering
+ MetricSet(const MetricSet&);
+ MetricSet& operator=(const MetricSet&);
+
+ void tagRegistrationAltered();
+ const Metric* getMetricInternal(const String& name) const;
+
+ virtual void addTo(Metric&, std::vector<Metric::LP>* ownerList) const;
+};
+
+} // metrics
+
diff --git a/metrics/src/vespa/metrics/metricsmanager.def b/metrics/src/vespa/metrics/metricsmanager.def
new file mode 100644
index 00000000000..fdf6d1b78e5
--- /dev/null
+++ b/metrics/src/vespa/metrics/metricsmanager.def
@@ -0,0 +1,64 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+version=4
+namespace=metrics
+
+# If any snapshot periods is set, these override all the default ones.
+# The array sets the snapshot periods in number of seconds. Note that when
+# sorted in rising time length order, it is required that the snapshot length of
+# the next snapshot is a multiplum of the previous snapshot.
+## restart flag was added automatically and needs to be verified.
+snapshot.periods[] int restart
+
+# The name of the consumer that should include the given metrics specified.
+## restart flag was added automatically and needs to be verified.
+consumer[].name string restart
+
+# Include metrics that match the given tags.
+#
+# A tag specification either adds a single tag by given the full tag name, or
+# any tag, by specifying an asterix as wildcard ('*'). An asterix will not match
+# metrics without any tags set, but you can match these by specifying an empty
+# string.
+#
+# A metric set matched by tags, will also include every metric contained in it
+# regardless of tags.
+## restart flag was added automatically and needs to be verified.
+consumer[].tags[] string restart
+
+# Do not include metrics that match the specific tags. This has higher
+# presedence than the adding of tags, but lower presedence than name patterns.
+#
+# Removed tags are specified as added tags, with a small exception. Removing
+# wildcard or untagged metrics does not make sense, as metrics are not added by
+# default. If you don not want to include these metrics, just not specify them
+# as tags to be added. Thus, these are not allowed in remove tags spec.
+#
+# A metric set matched by remove tag will remove all metrics below that tag.
+## restart flag was added automatically and needs to be verified.
+consumer[].removedtags[] string restart
+
+# Include all metrics that matches the names given here.
+# Metrics added here have presedence above tag specifications.
+#
+# Names are specified as a complete dot separated path to metric. Asterix can
+# be used to match any part. An asterix will only match a complete name without
+# a dot. So if you have an average metric called 'queuesize' in your top level
+# metric set called 'myapp', you can add this metric with the spec
+# 'myapp.queuesize'. You can also add non-default average metric entries by
+# specifying which, such as 'myapp.queuesize.last' or 'myapp.queuesize.max'.
+# Patterns can be used, to for instance specify '*.queuesize', 'myapp.*', '*.*',
+# 'myapp.*.max' or similar.
+#
+# A metric set added with specific name will add all metrics within it.
+## restart flag was added automatically and needs to be verified.
+consumer[].addedmetrics[] string restart
+
+# Do not include metrics with the below names.
+# This has highest presedence. Metrics removed like this will override any
+# specification otherwise that would include them.
+#
+# The remove metrics are specified exactly the same as the added metrics.
+#
+# A metric set removed with specific name will remove all metrics within it.
+## restart flag was added automatically and needs to be verified.
+consumer[].removedmetrics[] string restart
diff --git a/metrics/src/vespa/metrics/metricsnapshot.cpp b/metrics/src/vespa/metrics/metricsnapshot.cpp
new file mode 100644
index 00000000000..afb4aa305d4
--- /dev/null
+++ b/metrics/src/vespa/metrics/metricsnapshot.cpp
@@ -0,0 +1,164 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/metrics/metricsnapshot.h>
+
+#include <vespa/log/log.h>
+#include <vespa/metrics/metricmanager.h>
+
+LOG_SETUP(".metrics.snapshot");
+
+namespace metrics {
+
+MetricSnapshot::MetricSnapshot(const Metric::String& name)
+ : _name(name),
+ _period(0),
+ _fromTime(0),
+ _toTime(0),
+ _snapshot(new MetricSet("top", "", "")),
+ _metrics()
+{
+}
+
+MetricSnapshot::MetricSnapshot(
+ const Metric::String& name, uint32_t period, const MetricSet& source,
+ bool copyUnset)
+ : _name(name),
+ _period(period),
+ _fromTime(0),
+ _toTime(0),
+ _snapshot(),
+ _metrics()
+{
+ Metric* m = source.clone(_metrics, Metric::INACTIVE, 0, copyUnset);
+ assert(m->isMetricSet());
+ _snapshot.reset(static_cast<MetricSet*>(m));
+ trim(_metrics);
+}
+
+void
+MetricSnapshot::reset(time_t currentTime)
+{
+ _fromTime = currentTime;
+ _toTime = 0;
+ _snapshot->reset();
+}
+
+void
+MetricSnapshot::recreateSnapshot(const MetricSet& metrics, bool copyUnset)
+{
+ std::vector<Metric::LP> newMetrics;
+ Metric* m = metrics.clone(newMetrics, Metric::INACTIVE, 0, copyUnset);
+ assert(m->isMetricSet());
+ MetricSet::UP newSnapshot(static_cast<MetricSet*>(m));
+ newSnapshot->reset();
+ _snapshot->addToSnapshot(*newSnapshot, newMetrics);
+ _snapshot = std::move(newSnapshot);
+ _metrics.swap(newMetrics);
+ trim(_metrics);
+}
+
+void
+MetricSnapshot::addMemoryUsage(MemoryConsumption& mc) const
+{
+ ++mc._snapshotCount;
+ mc._snapshotName += mc.getStringMemoryUsage(_name, mc._snapshotNameUnique);
+ mc._snapshotMeta += sizeof(MetricSnapshot)
+ + _metrics.capacity() * sizeof(Metric::SP);
+ _snapshot->addMemoryUsage(mc);
+}
+
+MetricSnapshotSet::MetricSnapshotSet(
+ const Metric::String& name, uint32_t period,
+ uint32_t count, const MetricSet& source, bool snapshotUnsetMetrics)
+ : _count(count),
+ _builderCount(0),
+ _current(new MetricSnapshot(name, period, source, snapshotUnsetMetrics)),
+ _building(count == 1 ? 0 : new MetricSnapshot(
+ name, period, source, snapshotUnsetMetrics))
+{
+ _current->reset(0);
+ if (_building.get()) _building->reset(0);
+}
+
+MetricSnapshot&
+MetricSnapshotSet::getNextTarget()
+{
+ if (_count == 1) return *_current;
+ return *_building;
+}
+
+bool
+MetricSnapshotSet::haveCompletedNewPeriod(time_t newFromTime)
+{
+ if (_count == 1) {
+ _current->setToTime(newFromTime);
+ return true;
+ }
+ _building->setToTime(newFromTime);
+ // If not time to roll yet, just return
+ if (++_builderCount < _count) return false;
+ // Building buffer done. Use that as current and reset current.
+ MetricSnapshot::UP tmp(std::move(_current));
+ _current = std::move(_building);
+ _building = std::move(tmp);
+ _building->reset(newFromTime);
+ _builderCount = 0;
+ return true;
+}
+
+bool
+MetricSnapshotSet::timeForAnotherSnapshot(time_t currentTime) {
+ time_t lastTime = getToTime();
+ if (currentTime >= lastTime + getPeriod()) {
+ if (currentTime >= lastTime + 2 * getPeriod()) {
+ LOG(warning, "Metric snapshot set %s was asked if it was time for "
+ "another snapshot, a whole period beyond when it "
+ "should have been done (Last update was at time %"
+ PRIu64 ", current time is %" PRIu64" and period is %u). "
+ "Clearing data and updating time to current time.",
+ getName().c_str(), lastTime, currentTime, getPeriod());
+ reset(currentTime);
+ }
+ return true;
+ }
+ return false;
+}
+
+void
+MetricSnapshotSet::reset(time_t currentTime) {
+ if (_count != 1) _building->reset(currentTime);
+ _current->reset(currentTime);
+ _builderCount = 0;
+}
+
+void
+MetricSnapshotSet::recreateSnapshot(const MetricSet& metrics, bool copyUnset)
+{
+ if (_count != 1) _building->recreateSnapshot(metrics, copyUnset);
+ _current->recreateSnapshot(metrics, copyUnset);
+}
+
+void
+MetricSnapshotSet::addMemoryUsage(MemoryConsumption& mc) const
+{
+ ++mc._snapshotSetCount;
+ mc._snapshotSetMeta += sizeof(MetricSnapshotSet);
+ if (_count != 1) _building->addMemoryUsage(mc);
+ _current->addMemoryUsage(mc);
+}
+
+void
+MetricSnapshotSet::updateNames(NameHash& hash) const
+{
+ if (_count != 1) _building->updateNames(hash);
+ _current->updateNames(hash);
+}
+
+void
+MetricSnapshotSet::setFromTime(time_t fromTime)
+{
+ if (_count != 1) _building->setFromTime(fromTime);
+ _current->setFromTime(fromTime);
+}
+
+} // metrics
diff --git a/metrics/src/vespa/metrics/metricsnapshot.h b/metrics/src/vespa/metrics/metricsnapshot.h
new file mode 100644
index 00000000000..910d97906de
--- /dev/null
+++ b/metrics/src/vespa/metrics/metricsnapshot.h
@@ -0,0 +1,116 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+/**
+ * \class metrics::MetricSnapshot
+ * \ingroup metrics
+ *
+ * \brief Represents a snapshot of a metric set.
+ */
+#pragma once
+
+#include <boost/weak_ptr.hpp>
+#include <map>
+#include <vespa/metrics/metric.h>
+#include <vespa/metrics/metricset.h>
+
+namespace metrics {
+
+class MetricManager;
+
+class MetricSnapshot
+{
+ Metric::String _name;
+ // Period length of this snapshot
+ uint32_t _period;
+ // Time this snapshot was last updated.
+ time_t _fromTime;
+ // If set to 0, use _fromTime + _period.
+ time_t _toTime;
+ // Keeps the metrics set view of the snapshot
+ MetricSet::UP _snapshot;
+ // Snapshots must own their own metrics
+ mutable std::vector<Metric::LP> _metrics;
+
+public:
+ typedef std::unique_ptr<MetricSnapshot> UP;
+ typedef std::shared_ptr<MetricSnapshot> SP;
+
+ /** Create a fresh empty top level snapshot. */
+ MetricSnapshot(const Metric::String& name);
+ /** Create a snapshot of another metric source. */
+ MetricSnapshot(const Metric::String& name, uint32_t period,
+ const MetricSet& source, bool copyUnset);
+ virtual ~MetricSnapshot() {}
+
+ void addToSnapshot(MetricSnapshot& other, bool reset_, time_t currentTime) {
+ _snapshot->addToSnapshot(other.getMetrics(), other._metrics);
+ if (reset_) reset(currentTime);
+ other._toTime = currentTime;
+ }
+ void addToSnapshot(MetricSnapshot& other, time_t currentTime) const {
+ _snapshot->addToSnapshot(other.getMetrics(), other._metrics);
+ other._toTime = currentTime;
+ }
+ void setFromTime(time_t fromTime) { _fromTime = fromTime; }
+ void setToTime(time_t toTime) { _toTime = toTime; }
+
+ const Metric::String& getName() const { return _name; }
+ uint32_t getPeriod() const { return _period; }
+ time_t getFromTime() const { return _fromTime; }
+ time_t getToTime() const { return _toTime; }
+ time_t getLength() const
+ { return (_toTime != 0 ? _toTime - _fromTime : _fromTime + _period); }
+ const MetricSet& getMetrics() const { return *_snapshot; }
+ MetricSet& getMetrics() { return *_snapshot; }
+ void reset(time_t currentTime);
+ /**
+ * Recreate snapshot by cloning given metric set and then add the data
+ * from the old one. New metrics have been added.
+ */
+ void recreateSnapshot(const MetricSet& metrics, bool copyUnset);
+
+ void addMemoryUsage(MemoryConsumption&) const;
+ void updateNames(NameHash& hash) const { _snapshot->updateNames(hash); }
+};
+
+class MetricSnapshotSet {
+ uint32_t _count; // Number of times we need to add to building period
+ // before we have a full time window.
+ uint32_t _builderCount; // Number of times we've currently added to the
+ // building instance.
+ MetricSnapshot::UP _current; // The last full period
+ MetricSnapshot::UP _building; // The building period
+public:
+ typedef std::shared_ptr<MetricSnapshotSet> SP;
+
+ MetricSnapshotSet(const Metric::String& name, uint32_t period,
+ uint32_t count, const MetricSet& source,
+ bool snapshotUnsetMetrics);
+
+ const Metric::String& getName() const { return _current->getName(); }
+ uint32_t getPeriod() const { return _current->getPeriod(); }
+ time_t getFromTime() const { return _current->getFromTime(); }
+ time_t getToTime() const { return _current->getToTime(); }
+ uint32_t getCount() const { return _count; }
+ uint32_t getBuilderCount() const { return _builderCount; }
+ bool hasTemporarySnapshot() const { return (_building.get() != 0); }
+ MetricSnapshot& getSnapshot(bool temporary = false)
+ { return *((temporary && _count > 1) ? _building : _current); }
+ const MetricSnapshot& getSnapshot(bool temporary = false) const
+ { return *((temporary && _count > 1) ? _building : _current); }
+ MetricSnapshot& getNextTarget();
+ bool timeForAnotherSnapshot(time_t currentTime);
+ bool haveCompletedNewPeriod(time_t newFromTime);
+ void reset(time_t currentTime);
+ /**
+ * Recreate snapshot by cloning given metric set and then add the data
+ * from the old one. New metrics have been added.
+ */
+ void recreateSnapshot(const MetricSet& metrics, bool copyUnset);
+ void addMemoryUsage(MemoryConsumption&) const;
+ void updateNames(NameHash& hash) const;
+ void setFromTime(time_t fromTime);
+};
+
+} // metrics
+
diff --git a/metrics/src/vespa/metrics/metrictimer.cpp b/metrics/src/vespa/metrics/metrictimer.cpp
new file mode 100644
index 00000000000..dfaa982d809
--- /dev/null
+++ b/metrics/src/vespa/metrics/metrictimer.cpp
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/metrics/metrictimer.h>
+
+namespace metrics {
+
+MetricTimer::MetricTimer()
+{
+ // Amusingly enough, steady_clock was not actually steady by default on
+ // GCC < 4.8.1, so add a bit of compile-time paranoia just to make sure.
+ static_assert(std::chrono::steady_clock::is_steady,
+ "Old/broken STL implementation; steady_clock not steady");
+ _startTime = std::chrono::steady_clock::now();
+}
+
+} // metrics
diff --git a/metrics/src/vespa/metrics/metrictimer.h b/metrics/src/vespa/metrics/metrictimer.h
new file mode 100644
index 00000000000..7228daced34
--- /dev/null
+++ b/metrics/src/vespa/metrics/metrictimer.h
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class MetricTimer
+ * @ingruop metrics
+ *
+ * @brief Used to add time values to metrics.
+ */
+
+#pragma once
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/metrics/valuemetric.h>
+#include <chrono>
+
+namespace metrics {
+
+class MetricTimer
+{
+public:
+ MetricTimer();
+
+ /**
+ * Adds ms passed since this timer was constructed to given value metric.
+ * Returns that value as well.
+ *
+ * Uses a steady (monotonic) clock internally so value should never
+ * underflow or be affected by system clock changes.
+ */
+ template<typename AvgVal, typename TotVal, bool SumOnAdd>
+ uint64_t stop(ValueMetric<AvgVal, TotVal, SumOnAdd>& metric) {
+ const auto delta = std::chrono::steady_clock::now() - _startTime;
+ const uint64_t deltaMs(
+ std::chrono::duration_cast<std::chrono::milliseconds>(delta)
+ .count());
+ metric.addValue(deltaMs);
+ return deltaMs;
+ }
+
+private:
+ std::chrono::time_point<std::chrono::steady_clock> _startTime;
+};
+
+} // metrics
+
diff --git a/metrics/src/vespa/metrics/metricvalueset.h b/metrics/src/vespa/metrics/metricvalueset.h
new file mode 100644
index 00000000000..98bc6160833
--- /dev/null
+++ b/metrics/src/vespa/metrics/metricvalueset.h
@@ -0,0 +1,182 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class MetricValueSet
+ * \ingroup metrics
+ *
+ * \brief Utility for doing lockless metric updates and reads.
+ *
+ * We don't want to use regular locking while updating metrics due to overhead.
+ * We use this class to make metric updates as safe as possible without
+ * requiring locks.
+ *
+ * It keeps the set of values a metric wants to set. Thus it is templated on
+ * the class keeping the values. All that is required of this class is that it
+ * has an empty constructor and a copy constructor.
+ *
+ * The locking works, by keeping a set of values, with an active pointer into
+ * the value vector. Assuming only one thread calls setValues(), it can update
+ * the active pointer safely. We assume updating the active pointer is a
+ * non-interruptable operations, such that other threads will see either the new
+ * or the old value correctly. This should be the case on our platforms.
+ *
+ * Due to the reset functionality, it is possible to miss out on a metrics
+ * added during the reset, but this is very unlikely. For that to happen, when
+ * someone sets the reset flag, the writer thread must be in setValues(),
+ * having already passed the check for the reset flag, but not finished setting
+ * the values yet.
+ */
+#pragma once
+
+#include <sstream>
+#include <vector>
+#include <atomic>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/jsonstream.h>
+
+namespace metrics {
+
+struct MetricValueClass {
+ typedef std::unique_ptr<MetricValueClass> UP;
+ virtual ~MetricValueClass() {}
+
+ virtual double getDoubleValue(const vespalib::stringref & id) const = 0;
+ virtual uint64_t getLongValue(const vespalib::stringref & id) const = 0;
+ virtual void output(const std::string& id, std::ostream&) const = 0;
+ virtual void output(const std::string& id, vespalib::JsonStream&) const = 0;
+ std::string toString(const std::string& id) {
+ std::ostringstream ost;
+ output(id, ost);
+ return ost.str();
+ }
+};
+
+template<typename ValueClass>
+class MetricValueSet {
+ using AtomicValues = typename ValueClass::AtomicImpl;
+ std::vector<AtomicValues> _values;
+ std::atomic<uint32_t> _activeValueIndex;
+ std::atomic<uint32_t> _flags;
+
+ enum Flag { RESET = 1 };
+ bool isReset() const { return hasFlag(RESET); }
+
+ void validateCorrectValueSuperClass(const MetricValueClass&) {}
+
+public:
+ MetricValueSet(uint32_t copyCount = 3)
+ : _values(copyCount),
+ _activeValueIndex(0),
+ _flags(0)
+ {
+ }
+
+ MetricValueSet(const MetricValueSet& other, uint32_t copyCount = 3)
+ : _values(copyCount),
+ _activeValueIndex(0),
+ _flags(other._flags.load(std::memory_order_relaxed))
+ {
+ setValues(other.getValues());
+ }
+
+ MetricValueSet& operator=(const MetricValueSet& other)
+ {
+ setValues(other.getValues());
+ return *this;
+ }
+
+
+ /** Get the current values. */
+ ValueClass getValues() const {
+ ValueClass v{};
+ if (!isReset()) {
+ // Must load with acquire to match release store in setValues.
+ // Note that despite being atomic on _individual fields_, this
+ // does not guarantee reading a consistent snapshot _across_
+ // fields for any given metric.
+ const size_t readIndex(
+ _activeValueIndex.load(std::memory_order_acquire));
+ v.relaxedLoadFrom(_values[readIndex]);
+ }
+ return v;
+ }
+
+ /**
+ * Get the current values from the metric. This function should not be
+ * called in parallel. Only call it from a single thread or use external
+ * locking. If it returns false, it means the metric have just been reset.
+ * In which case, redo getValues(), apply the update again, and call
+ * setValues() again.
+ */
+ bool setValues(const ValueClass& values) {
+ validateCorrectValueSuperClass(values);
+ // Only setter-thread can write _activeValueIndex, so relaxed
+ // load suffices.
+ uint32_t nextIndex = (_activeValueIndex.load(std::memory_order_relaxed)
+ + 1) % _values.size();
+ // Reset flag is loaded/stored with relaxed semantics since it does not
+ // carry data dependencies. _activeValueIndex has a dependency on
+ // _values, however, so we must ensure that stores are published
+ // and loads acquired.
+ if (isReset()) {
+ removeFlag(RESET);
+ ValueClass resetValues{};
+ resetValues.relaxedStoreInto(_values[nextIndex]);
+ _activeValueIndex.store(nextIndex, std::memory_order_release);
+ return false;
+ } else {
+ values.relaxedStoreInto(_values[nextIndex]);
+ _activeValueIndex.store(nextIndex, std::memory_order_release);
+ return true;
+ }
+ }
+
+ /**
+ * Retrieve and reset in a single operation, to minimize chance of
+ * alteration in the process.
+ */
+ ValueClass getValuesAndReset() {
+ ValueClass result(getValues());
+ setFlag(RESET);
+ return result;
+ }
+
+ void reset() {
+ setFlag(RESET);
+ }
+
+ std::string toString() {
+ std::ostringstream ost;
+ ost << "MetricValueSet(reset=" << (isReset() ? "true" : "false")
+ << ", active " << _activeValueIndex;
+ ost << "\n empty: " << ValueClass().toString();
+ for (uint32_t i=0; i<_values.size(); ++i) {
+ ost << "\n " << _values[i].toString();
+ }
+ ost << "\n)";
+ return ost.str();
+ }
+
+ uint32_t getMemoryUsageAllocatedInternally() const {
+ return _values.capacity() * sizeof(ValueClass);
+ }
+
+ uint32_t size() const { return _values.size(); }
+
+ bool hasFlag(uint32_t flags) const {
+ return ((_flags.load(std::memory_order_relaxed) & flags) != 0);
+ }
+ void setFlag(uint32_t flags) {
+ _flags.store(_flags.load(std::memory_order_relaxed) | flags,
+ std::memory_order_relaxed);
+ }
+ void removeFlag(uint32_t flags) {
+ _flags.store(_flags.load(std::memory_order_relaxed) & ~flags,
+ std::memory_order_relaxed);
+ }
+ uint32_t getFlags() const {
+ return _flags.load(std::memory_order_relaxed);
+ }
+};
+
+} // metrics
+
diff --git a/metrics/src/vespa/metrics/namehash.h b/metrics/src/vespa/metrics/namehash.h
new file mode 100644
index 00000000000..9a281e39111
--- /dev/null
+++ b/metrics/src/vespa/metrics/namehash.h
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * \class metrics::NameHash
+ * \ingroup metrics
+ *
+ * \brief Simple class to enable string reference counting to work better.
+ *
+ * When creating metrics, it is easy to use const char references from code,
+ * for instance having a for loop setting up metrics for each thread, this will
+ * not actually generate ref counted strings, but rather unique strings.
+ *
+ * Also, with ref counted strings, it is easy to screw it up if you access the
+ * string in a way requiring copy.
+ *
+ * This class is used to just keep a set of strings, and having a class for
+ * users to input their strings and get the "master" string with that content.
+ *
+ * Metrics use this after having registered metrics, to ensure we dont keep more
+ * copies of non-unique strings than needed.
+ */
+#pragma once
+
+#include <boost/utility.hpp>
+#include <vespa/metrics/memoryconsumption.h>
+#include <vespa/vespalib/stllike/hash_set.h>
+
+namespace metrics {
+
+class NameHash : private boost::noncopyable {
+ vespalib::hash_set<std::string> _hash;
+ uint32_t _unifiedCounter;
+ uint32_t _checkedCounter;
+
+public:
+ NameHash() : _unifiedCounter(0), _checkedCounter(0) {}
+
+ void updateName(std::string& name) {
+ ++_checkedCounter;
+ vespalib::hash_set<std::string>::const_iterator it(_hash.find(name));
+ if (it != _hash.end()) {
+ if (name.c_str() != it->c_str()) {
+ name = *it;
+ ++_unifiedCounter;
+ }
+ } else {
+ _hash.insert(name);
+ }
+ }
+
+ uint32_t getUnifiedStringCount() const { return _unifiedCounter; }
+ uint32_t getCheckedStringCount() const { return _checkedCounter; }
+ void resetCounts() { _unifiedCounter = 0; _checkedCounter = 0; }
+ void addMemoryUsage(MemoryConsumption& mc) const {
+ mc._nameHash += sizeof(NameHash)
+ + _hash.getMemoryConsumption()
+ - sizeof(vespalib::hash_set<std::string>);
+ for (vespalib::hash_set<std::string>::const_iterator it(_hash.begin());
+ it != _hash.end(); ++it)
+ {
+ mc._nameHashStrings
+ += mc.getStringMemoryUsage(*it, mc._nameHashUnique);
+ }
+ }
+};
+
+
+} // metrics
+
diff --git a/metrics/src/vespa/metrics/printutils.h b/metrics/src/vespa/metrics/printutils.h
new file mode 100644
index 00000000000..42f640e83d2
--- /dev/null
+++ b/metrics/src/vespa/metrics/printutils.h
@@ -0,0 +1,418 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+/**
+ * This file contains utility functions to help print out metric snapshots in
+ * a user friendly way. It defines value types, functions for retrieving and
+ * doing algorithmics with the values, and printing them in an HTML table.
+ *
+ * This is used by storage to print HTML metrics report for its status page.
+ */
+
+#pragma once
+
+#include <vespa/vespalib/text/stringtokenizer.h>
+#include <vespa/metrics/metricmanager.h>
+
+namespace metrics {
+namespace printutils {
+
+typedef std::pair<int64_t, bool> LongValue;
+typedef std::pair<double, bool> DoubleValue;
+
+struct MetricSource {
+ typedef Metric::String String;
+
+ const MetricSnapshot& _snapshot;
+ String _metricsPrefix;
+ // If no map is supplied, this map will own the data in the metrics
+ // accessed map.
+ std::map<String, Metric::SP> _metricsAccessedOwner;
+ std::map<String, Metric::SP>& _metricsAccessed;
+
+ MetricSource(const MetricSnapshot& s,
+ const String& metricsPrefix,
+ std::map<String, Metric::SP>* metricsAccessed = 0)
+ : _snapshot(s),
+ _metricsPrefix(metricsPrefix),
+ _metricsAccessedOwner(),
+ _metricsAccessed(metricsAccessed == 0 ? _metricsAccessedOwner
+ : *metricsAccessed)
+ {
+ }
+
+ String createAbsoluteMetricName(const String& name) const {
+ String prefix = _metricsPrefix;
+ String addition = name;
+ while (addition.find("../") == 0) {
+ String::size_type pos1 = prefix.rfind('.');
+ if (pos1 == String::npos)
+ throw vespalib::IllegalArgumentException(
+ "Cannot go back anymore in path " + prefix,
+ VESPA_STRLOC);
+ prefix = prefix.substr(0, pos1);
+ addition = addition.substr(3);
+ }
+ return (prefix.empty() ? addition : prefix + "." + addition);
+ }
+
+ struct SourceMetricVisitor : public metrics::MetricVisitor {
+ String _stringPath;
+ vespalib::StringTokenizer::TokenList _path;
+ int32_t _pathIndex;
+ Metric::UP _resultMetric;
+ bool _prefixMatch;
+ std::vector<String> _prefixMatches;
+
+ SourceMetricVisitor(const String& path, bool prefixMatch)
+ : _stringPath(path),
+ _path(vespalib::StringTokenizer(path, ".").getTokens()),
+ _pathIndex(-1),
+ _resultMetric(),
+ _prefixMatch(prefixMatch),
+ _prefixMatches()
+ {
+ }
+
+ void checkForPrefixMatch(const Metric& metric) {
+ if (metric.getName().size() >= _path[_pathIndex].size()) {
+ if (metric.getName().find(_path[_pathIndex]) == 0) {
+ _prefixMatches.push_back(metric.getName());
+ }
+ }
+ }
+
+ bool visitMetricSet(const MetricSet& set, bool) {
+ if (_pathIndex == -1) {
+ _pathIndex = 0;
+ return true;
+ }
+ if (_prefixMatch
+ && static_cast<size_t>(_pathIndex + 1) == _path.size())
+ {
+ for (std::vector<Metric*>::const_iterator it
+ = set.getRegisteredMetrics().begin();
+ it != set.getRegisteredMetrics().end(); ++it)
+ {
+ checkForPrefixMatch(**it);
+ }
+ return false;
+ }
+ if (set.getName() != _path[_pathIndex]) return false;
+ if (static_cast<size_t>(++_pathIndex) >= _path.size()) {
+ throw vespalib::IllegalArgumentException(
+ "Path " + _stringPath + " points to a metric set. "
+ "Only primitive metrics can be retrieved.",
+ VESPA_STRLOC);
+ }
+ return true;
+ }
+ void doneVisitingMetricSet(const MetricSet&) { --_pathIndex; }
+ bool visitMetric(const Metric& metric, bool) {
+ if (_prefixMatch) {
+ checkForPrefixMatch(metric);
+ }
+ if (_path[_pathIndex] != metric.getName()) {
+ return true;
+ }
+ if (_prefixMatch) {
+ throw vespalib::IllegalArgumentException(
+ "Cannot find existing entries with prefix "
+ + _stringPath + " since element " + metric.getName()
+ + " is not a metric set", VESPA_STRLOC);
+ }
+ if (static_cast<size_t>(_pathIndex + 1) < _path.size()) {
+ throw vespalib::IllegalArgumentException(
+ "Path " + _stringPath + " cannot exist since element "
+ + _path[_pathIndex] + " is not a metric set: "
+ + metric.toString(),
+ VESPA_STRLOC);
+ }
+ std::vector<Metric::LP> ownerList;
+ _resultMetric.reset(metric.clone(ownerList, Metric::INACTIVE, 0));
+ if (!ownerList.empty()) {
+ throw vespalib::IllegalArgumentException(
+ "Metric " + metric.getName() + " added entries to "
+ "owners list when cloning. This should not happen "
+ "for primitive metrics.", VESPA_STRLOC);
+ }
+ return false;
+ }
+
+ };
+
+ const Metric* getMetric(const String& name) {
+ String path = createAbsoluteMetricName(name);
+ std::map<String, Metric::SP>::const_iterator it(
+ _metricsAccessed.find(path));
+ if (it != _metricsAccessed.end()) {
+ return it->second.get();
+ }
+ SourceMetricVisitor visitor(path, false);
+ _snapshot.getMetrics().visit(visitor);
+ if (visitor._resultMetric.get() == 0) {
+ throw vespalib::IllegalArgumentException(
+ "Metric " + path + " was not found.", VESPA_STRLOC);
+ }
+ Metric::SP metric(visitor._resultMetric.release());
+ _metricsAccessed[path] = metric;
+ return metric.get();
+ }
+
+ std::vector<String>
+ getPathsMatchingPrefix(const String& prefix) const
+ {
+ String path = createAbsoluteMetricName(prefix);
+ SourceMetricVisitor visitor(path, true);
+ _snapshot.getMetrics().visit(visitor);
+ return visitor._prefixMatches;
+ }
+};
+
+// Addition functions. Ensure that if floating point value is used,
+// result ends up as floating point too.
+LongValue operator+(LongValue addend1, LongValue addend2)
+{
+ return LongValue(addend1.first + addend2.first,
+ addend1.second && addend2.second);
+}
+
+template<typename ValueType1, typename ValueType2>
+DoubleValue operator+(std::pair<ValueType1, bool> addend1,
+ std::pair<ValueType2, bool> addend2)
+{
+ return DoubleValue(addend1.first + addend2.first,
+ addend1.second && addend2.second);
+}
+
+// Subtraction functions. Ensure that if floating point value is used,
+// result ends up as floating point too.
+LongValue operator-(LongValue minuend, LongValue subtrahend)
+{
+ return LongValue(minuend.first - subtrahend.first,
+ minuend.second && subtrahend.second);
+}
+
+template<typename ValueType1, typename ValueType2>
+DoubleValue operator-(std::pair<ValueType1, bool> minuend,
+ std::pair<ValueType2, bool> subtrahend)
+{
+ return DoubleValue(minuend.first - subtrahend.first,
+ minuend.second && subtrahend.second);
+}
+
+// Multiplication functions. Ensure that if floating point value is used,
+// result ends up as floating point too.
+
+LongValue operator*(LongValue factor1, LongValue factor2)
+{
+ return std::pair<int64_t, bool>(factor1.first * factor2.first,
+ factor1.second && factor2.second);
+}
+
+template<typename ValueType1, typename ValueType2>
+DoubleValue operator*(std::pair<ValueType1, bool> factor1,
+ std::pair<ValueType2, bool> factor2)
+{
+ return std::pair<double, bool>(factor1.first * factor2.first,
+ factor1.second && factor2.second);
+}
+
+// Division functions. Ensure that if floating point value is used,
+// result ends up as floating point too.
+
+LongValue operator/(LongValue dividend, LongValue divisor)
+{
+ if (dividend.first == 0) return LongValue(
+ 0, dividend.second && divisor.second);
+ if (divisor.first == 0) return LongValue(
+ std::numeric_limits<int64_t>().max(),
+ dividend.second && divisor.second);
+ return LongValue(dividend.first / divisor.first,
+ dividend.second && divisor.second);
+}
+
+template<typename ValueType1, typename ValueType2>
+DoubleValue operator/(std::pair<ValueType1, bool> dividend,
+ std::pair<ValueType2, bool> divisor)
+{
+ // In case divisor is integer, we will core if we attempt to divide
+ // with it.
+ if (dividend.first == 0) return DoubleValue(
+ 0, dividend.second && divisor.second);
+ if (divisor.first == 0) return DoubleValue(
+ std::numeric_limits<double>().infinity(),
+ dividend.second && divisor.second);
+ return DoubleValue(
+ dividend.first / static_cast<double>(divisor.first),
+ dividend.second && divisor.second);
+}
+
+// Min/Max functions
+template<typename ValueType>
+std::pair<ValueType, bool> getMin(std::pair<ValueType, bool> val1,
+ std::pair<ValueType, bool> val2)
+{
+ if (!val1.second) return val2;
+ if (!val2.second) return val1;
+ return std::pair<ValueType, bool>(std::min(val1.first, val2.first), true);
+}
+
+template<typename ValueType>
+std::pair<ValueType, bool> getMax(std::pair<ValueType, bool> val1,
+ std::pair<ValueType, bool> val2)
+{
+ if (!val1.second) return val2;
+ if (!val2.second) return val1;
+ return std::pair<ValueType, bool>(std::max(val1.first, val2.first), true);
+}
+
+// Wrapper types for primitives. Would be nice to allow primitives directly, but
+// using a wrapper we can get by with less operator overloads above.
+
+template<typename ValueType>
+struct VW : public std::pair<ValueType, bool> {
+ VW(ValueType val) : std::pair<ValueType, bool>(val, true) {}
+};
+
+typedef VW<int64_t> LVW;
+typedef VW<double> DVW;
+
+
+/** Get metric with given name from source. Set bool true if existing. */
+LongValue getLongMetric(const std::string& name, MetricSource& source)
+{
+ std::string::size_type pos = name.rfind('.');
+ const Metric* metric = (pos == std::string::npos
+ ? 0 : source.getMetric(name.substr(0, pos)));
+ try{
+ return LongValue(metric == 0
+ ? 0 : metric->getLongValue(name.substr(pos+1)), metric != 0);
+ } catch (vespalib::IllegalArgumentException& e) {
+ return LongValue(0, false);
+ }
+}
+
+/** Get metric with given name from source. Set bool true if existing. */
+DoubleValue getDoubleMetric(const std::string& name, MetricSource& source)
+{
+ std::string::size_type pos = name.rfind('.');
+ const Metric* metric = (pos == std::string::npos
+ ? 0 : source.getMetric(name.substr(0, pos)));
+ try{
+ return DoubleValue(metric == 0
+ ? 0.0 : metric->getDoubleValue(name.substr(pos+1)), metric != 0);
+ } catch (vespalib::IllegalArgumentException& e) {
+ return DoubleValue(0, false);
+ }
+}
+
+std::string getValueString(LongValue value, const char* format = "%'lld")
+{
+ if (!value.second) return "na";
+ std::vector<char> buffer(30);
+ snprintf(&buffer[0], 30, format, value.first);
+ return std::string(&buffer[0]);
+}
+
+std::string getValueString(DoubleValue value, const char* format = "%'f")
+{
+ if (!value.second) return "na";
+ std::vector<char> buffer(30);
+ snprintf(&buffer[0], 30, format, value.first);
+ return std::string(&buffer[0]);
+}
+
+template<typename ValueType>
+std::string getByteValueString(std::pair<ValueType, bool> val)
+{
+ static const int64_t k = (1ul << 10);
+ static const int64_t m = (1ul << 20);
+ static const int64_t g = (1ul << 30);
+
+ if (!val.second) return "na";
+ std::pair<int64_t, bool> value(
+ static_cast<int64_t>(val.first), val.second);
+ if (value.first < 64 * k) {
+ return getValueString(value, "%'llu B");
+ }
+ if (value.first < 64 * m) {
+ value.first /= k;
+ return getValueString(value, "%'llu kB");
+ }
+ if (value.first < 64 * g) {
+ value.first /= m;
+ return getValueString(value, "%'llu MB");
+ }
+ value.first /= g;
+ return getValueString(value, "%'llu GB");
+}
+
+struct HttpTable {
+ std::string title;
+ std::string topLeftText;
+ std::vector<std::string> colNames;
+ std::vector<std::string> rowNames;
+ struct Cell {
+ bool set;
+ std::string value;
+
+ Cell() : set(false), value() {}
+
+ void operator=(const std::string& val) { value = val; set = true; }
+ };
+ struct Row {
+ std::vector<Cell> cells;
+ Cell& operator[](uint32_t i) {
+ if (i >= cells.size()) cells.resize(i + 1);
+ return cells[i];
+ }
+ };
+ std::vector<Row> cells;
+
+ HttpTable(const std::string& title_, const std::string& topLeftText_)
+ : title(title_), topLeftText(topLeftText_) {}
+
+ Row& operator[](uint32_t i) {
+ if (i >= cells.size()) cells.resize(i + 1);
+ return cells[i];
+ }
+
+ void fillInEmptyHoles() {
+ if (rowNames.size() < cells.size()) rowNames.resize(cells.size());
+ if (rowNames.size() > cells.size()) cells.resize(rowNames.size());
+ for (uint32_t i=0; i<cells.size(); ++i) {
+ if (colNames.size() < cells[i].cells.size())
+ colNames.resize(cells[i].cells.size());
+ if (colNames.size() > cells[i].cells.size())
+ cells[i].cells.resize(colNames.size());
+ }
+ }
+
+ void print(std::ostream& out) {
+ out << "<h3>" << title << "</h3>\n";
+ out << "<table border=\"1\">\n";
+ fillInEmptyHoles();
+ for (uint32_t i=0; i<=rowNames.size(); ++i) {
+ if (i == 0) {
+ out << "<tr><th>" << topLeftText << "</th>";
+ for (uint32_t j=0; j<colNames.size(); ++j) {
+ out << "<th>" << colNames[j] << "</th>";
+ }
+ out << "</tr>\n";
+ } else {
+ out << "<tr><td>" << rowNames[i - 1] << "</td>";
+ for (uint32_t j=0; j<colNames.size(); ++j) {
+ out << "<td align=\"right\">"
+ << (cells[i - 1][j].set ? cells[i - 1][j].value : "-")
+ << "</td>";
+ }
+ out << "</tr>\n";
+ }
+ }
+ out << "</table>\n";
+ }
+};
+
+} // printutils
+} // metrics
+
diff --git a/metrics/src/vespa/metrics/state_api_adapter.cpp b/metrics/src/vespa/metrics/state_api_adapter.cpp
new file mode 100644
index 00000000000..606f57a87fe
--- /dev/null
+++ b/metrics/src/vespa/metrics/state_api_adapter.cpp
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "state_api_adapter.h"
+#include "jsonwriter.h"
+
+namespace metrics {
+
+vespalib::string
+StateApiAdapter::getMetrics(const vespalib::string &consumer)
+{
+ metrics::MetricLockGuard guard(_manager.getMetricLock());
+ std::vector<uint32_t> periods = _manager.getSnapshotPeriods(guard);
+ if (periods.empty()) {
+ return ""; // no configuration yet
+ }
+ const metrics::MetricSnapshot &snapshot(
+ _manager.getMetricSnapshot(guard, periods[0]));
+ vespalib::asciistream json;
+ vespalib::JsonStream stream(json);
+ metrics::JsonWriter metricJsonWriter(stream);
+ _manager.visit(guard, snapshot, metricJsonWriter, consumer);
+ stream.finalize();
+ return json.str();
+}
+
+vespalib::string
+StateApiAdapter::getTotalMetrics(const vespalib::string &consumer)
+{
+ _manager.updateMetrics(true);
+ metrics::MetricLockGuard guard(_manager.getMetricLock());
+ _manager.checkMetricsAltered(guard);
+ time_t currentTime = time(0);
+ std::unique_ptr<metrics::MetricSnapshot> generated(
+ new metrics::MetricSnapshot(
+ "Total metrics from start until current time", 0,
+ _manager.getTotalMetricSnapshot(guard).getMetrics(),
+ true));
+ _manager.getActiveMetrics(guard)
+ .addToSnapshot(*generated, false, currentTime);
+ generated->setFromTime(
+ _manager.getTotalMetricSnapshot(guard).getFromTime());
+ const metrics::MetricSnapshot &snapshot = *generated;
+ vespalib::asciistream json;
+ vespalib::JsonStream stream(json);
+ metrics::JsonWriter metricJsonWriter(stream);
+ _manager.visit(guard, snapshot, metricJsonWriter, consumer);
+ stream.finalize();
+ return json.str();
+}
+
+} // namespace metrics
diff --git a/metrics/src/vespa/metrics/state_api_adapter.h b/metrics/src/vespa/metrics/state_api_adapter.h
new file mode 100644
index 00000000000..dd041a3c37c
--- /dev/null
+++ b/metrics/src/vespa/metrics/state_api_adapter.h
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/net/metrics_producer.h>
+#include "metricmanager.h"
+
+namespace metrics {
+
+/**
+ * This is an adapter class that implements the metrics producer
+ * interface defined by the state api implementation in vespalib by
+ * extracting metrics in json format from a metric manager.
+ **/
+class StateApiAdapter : public vespalib::MetricsProducer
+{
+private:
+ MetricManager &_manager;
+
+public:
+ StateApiAdapter(MetricManager &manager) : _manager(manager) {}
+
+ vespalib::string getMetrics(const vespalib::string &consumer) override;
+ vespalib::string getTotalMetrics(const vespalib::string &consumer) override;
+};
+
+
+} // namespace metrics
+
diff --git a/metrics/src/vespa/metrics/summetric.h b/metrics/src/vespa/metrics/summetric.h
new file mode 100644
index 00000000000..6f1c92e5b4f
--- /dev/null
+++ b/metrics/src/vespa/metrics/summetric.h
@@ -0,0 +1,422 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class metrics::CounterMetric
+ * @ingroup metrics
+ *
+ * @brief Counts a value that only moves upwards.
+ *
+ * NB! If you have a MetricSet subclass you want to create a sum for, use
+ * MetricSet itself as the template argument. Otherwise you'll need to override
+ * clone(...) in order to make it return the correct type for your
+ * implementation.
+ */
+
+#pragma once
+
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/metrics/metric.h>
+#include <vespa/metrics/metricset.h>
+
+namespace metrics {
+
+template<typename AddendMetric>
+class SumMetric : public Metric
+{
+public:
+ class StartValue
+ {
+ private:
+ std::vector<Metric::LP> _startValueChildren;
+ std::unique_ptr<AddendMetric> _startValue;
+
+ public:
+ typedef std::shared_ptr<StartValue> SP;
+ StartValue(const AddendMetric &metric)
+ : _startValueChildren(),
+ _startValue(metric.clone(_startValueChildren, CLONE, 0, false)) {}
+ const AddendMetric &getStartValue() const { return *_startValue; }
+ };
+
+private:
+ typename StartValue::SP _startValue;
+ std::vector<const AddendMetric*> _metricsToSum;
+
+public:
+ SumMetric(const String& name, const String& tags,
+ const String& description, MetricSet* owner = 0);
+ SumMetric(const SumMetric<AddendMetric>& other,
+ std::vector<Metric::LP>& ownerList, MetricSet* owner = 0);
+
+ virtual Metric* clone(
+ std::vector<Metric::LP>&, CopyType, MetricSet* owner,
+ bool includeUnused = false) const;
+
+ /**
+ * If you want to support sums of collections of metrics that may
+ * be empty, you must supply a start value for the sum operation
+ * by calling this function.
+ **/
+ void setStartValue(const AddendMetric &metric) { _startValue.reset(new StartValue(metric)); }
+ typename StartValue::SP getStartValue() const { return _startValue; }
+
+ void addMetricToSum(const AddendMetric&);
+ void removeMetricFromSum(const AddendMetric&);
+
+ virtual bool logEvent(const String& fullName) const;
+
+ virtual void print(std::ostream&, bool verbose,
+ const std::string& indent, uint64_t secondsPassed) const;
+
+ virtual int64_t getLongValue(const stringref & id) const;
+ virtual double getDoubleValue(const stringref & id) const;
+
+ virtual void reset() {}
+
+ virtual bool visit(MetricVisitor& visitor,
+ bool tagAsAutoGenerated = false) const;
+
+ virtual bool used() const;
+
+ virtual void addMemoryUsage(MemoryConsumption&) const;
+
+ virtual void printDebug(std::ostream&, const std::string& indent="") const;
+
+ virtual void addToPart(metrics::Metric&) const;
+ virtual void addToSnapshot(metrics::Metric&,
+ std::vector<Metric::LP>&) const;
+
+private:
+ friend class MetricManagerTest;
+ std::pair<std::vector<Metric::LP>, Metric::LP> generateSum() const;
+
+ virtual void addTo(metrics::Metric&,
+ std::vector<Metric::LP>* ownerList) const;
+ bool isAddendType(const Metric* m) const;
+};
+
+template<typename AddendMetric>
+bool
+SumMetric<AddendMetric>::visit(MetricVisitor& visitor,
+ bool tagAsAutoGenerated) const
+{
+ (void) tagAsAutoGenerated;
+ if (_metricsToSum.empty()) return true;
+ std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum());
+ return sum.second->visit(visitor, true);
+}
+
+template<typename AddendMetric>
+SumMetric<AddendMetric>::SumMetric(const String& name, const String& tags,
+ const String& description, MetricSet* owner)
+ : Metric(name, tags, description, owner),
+ _startValue(),
+ _metricsToSum()
+{
+}
+
+template<typename AddendMetric>
+SumMetric<AddendMetric>::SumMetric(const SumMetric<AddendMetric>& other,
+ std::vector<Metric::LP>& ownerList,
+ MetricSet* owner)
+ : Metric(other, owner),
+ _startValue(other._startValue),
+ _metricsToSum()
+{
+ (void) ownerList;
+ if (other._owner == 0) {
+ throw vespalib::IllegalStateException(
+ "Cannot copy a sum metric not registered in a metric set, as "
+ "we need to use parent to detect new metrics to point to.",
+ VESPA_STRLOC);
+ }
+ if (owner == 0) {
+ throw vespalib::IllegalStateException(
+ "Cannot copy a sum metric directly. One needs to at least "
+ "include metric set above it in order to include metrics "
+ "summed.", VESPA_STRLOC);
+ }
+ std::vector<String> parentPath(other._owner->getPathVector());
+ _metricsToSum.reserve(other._metricsToSum.size());
+ for (typename std::vector<const AddendMetric*>::const_iterator it
+ = other._metricsToSum.begin();
+ it != other._metricsToSum.end(); ++it)
+ {
+ std::vector<String> addendPath((**it).getPathVector());
+ MetricSet* newAddendParent = owner;
+ for (uint32_t i=parentPath.size(), n=addendPath.size() - 1; i<n; ++i) {
+ Metric* child = newAddendParent->getMetric(addendPath[i]);
+ if (child == 0) {
+ throw vespalib::IllegalStateException(
+ "Metric " + addendPath[i] + " in metric set "
+ + newAddendParent->getPath() + " was expected to "
+ "exist. This sounds like a bug.", VESPA_STRLOC);
+ }
+ if (!child->isMetricSet()) {
+ throw vespalib::IllegalStateException(
+ "Metric " + addendPath[i] + " in metric set "
+ + newAddendParent->getPath() + " was expected to be a "
+ "metric set. This sounds like a bug.", VESPA_STRLOC);
+ }
+ newAddendParent = static_cast<MetricSet*>(child);
+ }
+ Metric* child = newAddendParent->getMetric(
+ addendPath[addendPath.size() - 1]);
+ if (child == 0) {
+ throw vespalib::IllegalStateException(
+ "Metric " + addendPath[addendPath.size() - 1] + " in "
+ "metric set " + newAddendParent->getPath() + " was "
+ "expected to exist. This sounds like a bug.", VESPA_STRLOC);
+ }
+ AddendMetric* am(dynamic_cast<AddendMetric*>(child));
+ if (am == 0) {
+ throw vespalib::IllegalStateException(
+ "Metric " + child->getPath() + " is of wrong type for sum "
+ + other.getPath() + ". This sounds like a bug.",
+ VESPA_STRLOC);
+ }
+ _metricsToSum.push_back(am);
+ }
+}
+
+template<typename AddendMetric>
+Metric*
+SumMetric<AddendMetric>::clone(std::vector<Metric::LP>& ownerList,
+ CopyType copyType, MetricSet* owner,
+ bool includeUnused) const
+{
+ (void) includeUnused;
+ if (_metricsToSum.empty() && _startValue.get() == 0) {
+ throw vespalib::IllegalStateException(
+ "Attempted to clone sum metric without any children or start value. "
+ "This is currently illegal, to avoid needing to be able to "
+ "construct a metric of appropriate type without having a "
+ "template. (Hard to know how to construct any metric.",
+ VESPA_STRLOC);
+ }
+ if (copyType == CLONE) {
+ return new SumMetric<AddendMetric>(*this, ownerList, owner);
+ }
+ // Else we're generating an inactive copy by evaluating sum
+ typename std::vector<const AddendMetric*>::const_iterator it(
+ _metricsToSum.begin());
+ // Clone start value or first child and use as accumulator
+ // As the metric cloned will have wrong info, we have to wait to
+ // register it in parent until we have fixed that.
+ Metric *m = 0;
+ if (_startValue.get() != 0) {
+ m = _startValue->getStartValue().clone(ownerList, INACTIVE, 0, true);
+ } else {
+ m = (**it).clone(ownerList, INACTIVE, 0, true);
+ ++it;
+ }
+ m->setName(getName());
+ m->setDescription(getDescription());
+ m->setTags(getTags());
+ if (owner != 0) owner->registerMetric(*m);
+ // Add the others to the metric cloned.
+ for (; it != _metricsToSum.end(); ++it) {
+ (**it).addToPart(*m);
+ }
+ return m;
+}
+
+template<typename AddendMetric>
+void
+SumMetric<AddendMetric>::addToPart(metrics::Metric& m) const
+{
+ std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum());
+ sum.second->addToPart(m);
+}
+
+template<typename AddendMetric>
+void
+SumMetric<AddendMetric>::addToSnapshot(
+ metrics::Metric& m, std::vector<Metric::LP>& ownerList) const
+{
+ if (isAddendType(&m)) {
+ // If the type to add to is an addend metric, it is part of an inactive
+ // copy we need to add data to.
+ std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum());
+ sum.second->addToSnapshot(m, ownerList);
+ }
+}
+
+template<typename AddendMetric>
+void
+SumMetric<AddendMetric>::addTo(metrics::Metric& m,
+ std::vector<Metric::LP>* ownerList) const
+{
+ if (ownerList == 0) {
+ std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum());
+ sum.second->addToPart(m);
+ } else {
+ if (isAddendType(&m)) {
+ // If the type to add to is an addend metric, it is part of an
+ // inactive copy we need to add data to.
+ std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum());
+ sum.second->addToSnapshot(m, *ownerList);
+ }
+ }
+}
+
+template<typename AddendMetric>
+void
+SumMetric<AddendMetric>::addMetricToSum(const AddendMetric& metric)
+{
+ if (_owner == 0) {
+ throw vespalib::IllegalStateException(
+ "Sum metric needs to be registered in a parent metric set "
+ "prior to adding metrics to sum.", VESPA_STRLOC);
+ }
+ std::vector<String> sumParentPath(_owner->getPathVector());
+ std::vector<String> addedPath(metric.getPathVector());
+ bool error = false;
+ if (addedPath.size() <= sumParentPath.size()) {
+ error = true;
+ } else for (uint32_t i=0; i<sumParentPath.size(); ++i) {
+ if (sumParentPath[i] != addedPath[i]) {
+ error = true;
+ break;
+ }
+ }
+ if (error) {
+ throw vespalib::IllegalStateException(
+ "Metric added to sum is required to be a child of the sum's "
+ "direct parent metric set. (Need not be a direct child) "
+ "Metric set " + metric.getPath() + " is not a child of "
+ + _owner->getPath(), VESPA_STRLOC);
+ }
+ std::vector<const AddendMetric*> metrics(_metricsToSum.size() + 1);
+ for (uint32_t i=0; i<_metricsToSum.size(); ++i) {
+ metrics[i] = _metricsToSum[i];
+ }
+ metrics[metrics.size() - 1] = &metric;
+ _metricsToSum.swap(metrics);
+ // Ensure we don't use extra memory
+ assert(_metricsToSum.capacity() == _metricsToSum.size());
+}
+
+template<typename AddendMetric>
+void
+SumMetric<AddendMetric>::removeMetricFromSum(const AddendMetric &metric)
+{
+ _metricsToSum.erase(remove(_metricsToSum.begin(), _metricsToSum.end(),
+ &metric));
+}
+
+template<typename AddendMetric>
+std::pair<std::vector<Metric::LP>, Metric::LP>
+SumMetric<AddendMetric>::generateSum() const
+{
+ std::pair<std::vector<Metric::LP>, Metric::LP> retVal;
+ Metric* m = clone(retVal.first, INACTIVE, 0, true);
+ m->setRegistered(_owner);
+ retVal.second.reset(m);
+ return retVal;
+}
+
+template<typename AddendMetric>
+int64_t
+SumMetric<AddendMetric>::getLongValue(const stringref & id) const
+{
+ std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum());
+ if (sum.second.get() == 0) return 0;
+ return sum.second->getLongValue(id);
+}
+
+template<typename AddendMetric>
+double
+SumMetric<AddendMetric>::getDoubleValue(const stringref & id) const
+{
+ std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum());
+ if (sum.second.get() == 0) return 0.0;
+ return sum.second->getDoubleValue(id);
+}
+
+template<typename AddendMetric>
+bool
+SumMetric<AddendMetric>::logEvent(const String & fullName) const
+{
+ std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum());
+ if (sum.second.get() == 0) return false;
+ return sum.second->logEvent(fullName);
+}
+
+template<typename AddendMetric>
+void
+SumMetric<AddendMetric>::print(std::ostream& out, bool verbose,
+ const std::string& indent,
+ uint64_t secondsPassed) const
+{
+ std::pair<std::vector<Metric::LP>, Metric::LP> sum(generateSum());
+ if (sum.second.get() == 0) return;
+ sum.second->print(out, verbose, indent, secondsPassed);
+}
+
+template<typename AddendMetric>
+bool
+SumMetric<AddendMetric>::used() const
+{
+ for(typename std::vector<const AddendMetric*>::const_iterator it(
+ _metricsToSum.begin()); it != _metricsToSum.end(); ++it)
+ {
+ if ((**it).used()) return true;
+ }
+ return false;
+}
+
+template<typename AddendMetric>
+void
+SumMetric<AddendMetric>::addMemoryUsage(MemoryConsumption& mc) const
+{
+ ++mc._sumMetricCount;
+ mc._sumMetricMeta += sizeof(SumMetric<AddendMetric>) - sizeof(Metric)
+ + _metricsToSum.capacity() * sizeof(Metric*);
+ Metric::addMemoryUsage(mc);
+}
+
+template<typename AddendMetric>
+void
+SumMetric<AddendMetric>::printDebug(std::ostream& out,
+ const std::string& indent) const
+{
+ out << "sum ";
+ Metric::printDebug(out, indent);
+ out << " {";
+ for(typename std::vector<const AddendMetric*>::const_iterator it(
+ _metricsToSum.begin()); it != _metricsToSum.end(); ++it)
+ {
+ out << "\n" << indent << " ";
+ (**it).printDebug(out, indent + " ");
+ }
+ out << "}";
+}
+
+template<typename AddendMetric>
+bool
+SumMetric<AddendMetric>::isAddendType(const Metric* m) const
+{
+ // If metric to addend it a metric set, we can only check if target also is
+ // a metric set, as other type will be lost when going inactive. Is there a
+ // way to do this without using an actual instance?
+ if (_metricsToSum.empty() && _startValue.get() == 0) {
+ throw vespalib::IllegalStateException(
+ "Attempted to verify addend type for sum metric without any "
+ "children or start value.", VESPA_STRLOC);
+ }
+ const Metric* wantedType;
+ if (!_metricsToSum.empty()) {
+ wantedType = _metricsToSum[0];
+ } else {
+ wantedType = &_startValue->getStartValue();
+ }
+ if (wantedType->isMetricSet()) {
+ return (m->isMetricSet());
+ } else {
+ return (dynamic_cast<const AddendMetric*>(m) != 0);
+ }
+}
+
+} // metrics
+
diff --git a/metrics/src/vespa/metrics/textwriter.cpp b/metrics/src/vespa/metrics/textwriter.cpp
new file mode 100644
index 00000000000..0b5cb9a287b
--- /dev/null
+++ b/metrics/src/vespa/metrics/textwriter.cpp
@@ -0,0 +1,84 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/metrics/textwriter.h>
+
+#include <vespa/metrics/countmetric.h>
+#include <vespa/metrics/metricset.h>
+#include <vespa/metrics/metricsnapshot.h>
+#include <vespa/metrics/valuemetric.h>
+
+namespace metrics {
+
+TextWriter::TextWriter(std::ostream& out, uint32_t period,
+ const std::string& regex, bool verbose)
+ : _period(period), _out(out), _regex(regex), _verbose(verbose)
+{
+}
+
+bool
+TextWriter::visitSnapshot(const MetricSnapshot& snapshot)
+{
+ _out << "snapshot \"" << snapshot.getName() << "\" from "
+ << snapshot.getFromTime() << " to " << snapshot.getToTime()
+ << " period " << snapshot.getPeriod();
+ return true;
+}
+
+void
+TextWriter::doneVisitingSnapshot(const MetricSnapshot&)
+{
+}
+
+bool
+TextWriter::visitMetricSet(const MetricSet& set, bool)
+{
+ _path.push_back(set.getMangledName());
+ return true;
+}
+void
+TextWriter::doneVisitingMetricSet(const MetricSet&) {
+ _path.pop_back();
+}
+
+bool
+TextWriter::writeCommon(const Metric& metric)
+{
+ std::ostringstream path;
+ for (uint32_t i=0; i<_path.size(); ++i) {
+ path << _path[i] << ".";
+ }
+ std::string mypath(path.str());
+ path << metric.getMangledName();
+ if (_regex.match(path.str())) {
+ if (metric.used() || _verbose) {
+ _out << "\n" << mypath;
+ return true;
+ }
+ }
+ return false;
+}
+
+bool
+TextWriter::visitCountMetric(const AbstractCountMetric& m, bool)
+{
+ if (writeCommon(m)) {
+ if (_verbose || m.used()) {
+ MetricValueClass::UP values(m.getValues());
+ _out << m.getMangledName()
+ << (m.sumOnAdd() ? " count=" : " value=");
+ values->output("count", _out);
+ }
+ }
+ return true;
+}
+
+bool
+TextWriter::visitValueMetric(const AbstractValueMetric& m, bool)
+{
+ if (writeCommon(m)) {
+ m.print(_out, _verbose, " ", _period);
+ }
+ return true;
+}
+
+} // metrics
diff --git a/metrics/src/vespa/metrics/textwriter.h b/metrics/src/vespa/metrics/textwriter.h
new file mode 100644
index 00000000000..dadeb70a7fc
--- /dev/null
+++ b/metrics/src/vespa/metrics/textwriter.h
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/util/regexp.h>
+#include <vespa/metrics/metric.h>
+
+namespace metrics {
+
+class TextWriter : public MetricVisitor {
+ uint32_t _period;
+ std::ostream& _out;
+ std::vector<std::string> _path;
+ vespalib::Regexp _regex;
+ bool _verbose;
+
+public:
+ TextWriter(std::ostream& out, uint32_t period,
+ const std::string& regex, bool verbose);
+
+ virtual bool visitSnapshot(const MetricSnapshot&);
+ virtual void doneVisitingSnapshot(const MetricSnapshot&);
+ virtual bool visitMetricSet(const MetricSet& set, bool);
+ virtual void doneVisitingMetricSet(const MetricSet&);
+ virtual bool visitCountMetric(const AbstractCountMetric&,
+ bool autoGenerated);
+ virtual bool visitValueMetric(const AbstractValueMetric&,
+ bool autoGenerated);
+
+private:
+ bool writeCommon(const Metric& m);
+};
+
+}
+
diff --git a/metrics/src/vespa/metrics/valuemetric.cpp b/metrics/src/vespa/metrics/valuemetric.cpp
new file mode 100644
index 00000000000..3da48885054
--- /dev/null
+++ b/metrics/src/vespa/metrics/valuemetric.cpp
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/metrics/valuemetric.h>
+#include <atomic>
+
+#include <vespa/log/log.h>
+
+LOG_SETUP(".metrics.metric.value");
+
+namespace metrics {
+
+namespace {
+
+std::atomic<bool> hasWarned {false};
+
+}
+
+void
+AbstractValueMetric::logWarning(const char* msg) const
+{
+ LOG(warning, "%s", msg);
+}
+
+void
+AbstractValueMetric::sendLogEvent(Metric::String name, double value) const
+{
+ EV_VALUE(name.c_str(), value);
+}
+
+void
+AbstractValueMetric::logNonFiniteValueWarning() const
+{
+ if (hasWarned.exchange(true, std::memory_order_relaxed) == false) {
+ LOG(warning,
+ "Metric '%s' attempted updated with a value that is NaN or "
+ "Infinity; update ignored! No further warnings will be printed for "
+ "such updates on any metrics, but they can be observed with debug "
+ "logging enabled on component 'metrics.metric.value'.",
+ getPath().c_str());
+ } else {
+ LOG(debug,
+ "Metric '%s' attempted updated with a value that is NaN/Infinity; "
+ "update ignored!",
+ getPath().c_str());
+ }
+}
+
+} // metrics
diff --git a/metrics/src/vespa/metrics/valuemetric.h b/metrics/src/vespa/metrics/valuemetric.h
new file mode 100644
index 00000000000..9c9d750a1a8
--- /dev/null
+++ b/metrics/src/vespa/metrics/valuemetric.h
@@ -0,0 +1,532 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @class metrics::ValueMetric
+ * @ingroup metrics
+ *
+ * @brief Creates a metric measuring any value.
+ *
+ * A value metric have the following properties:
+ * - Logs the average as a value event. (It is not strictly increasing)
+ * - When summing average metrics together, the sum becomes the average of
+ * all values added to both.
+ */
+
+#pragma once
+
+#include <vespa/vespalib/util/sync.h>
+#include <vespa/vespalib/util/exceptions.h>
+#include <sstream>
+#include <type_traits>
+#include <cmath>
+#include <vespa/metrics/metric.h>
+#include <vespa/metrics/metricvalueset.h>
+#include <vespa/vespalib/util/jsonstream.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <atomic>
+
+namespace metrics {
+
+struct AbstractValueMetric : public Metric {
+ virtual bool visit(MetricVisitor& visitor,
+ bool tagAsAutoGenerated = false) const
+ {
+ return visitor.visitValueMetric(*this, tagAsAutoGenerated);
+ }
+ virtual MetricValueClass::UP getValues() const = 0;
+ virtual bool inUse(const MetricValueClass& v) const = 0;
+ virtual bool summedAverage() const = 0;
+
+protected:
+ AbstractValueMetric(const String& name, const String& tags,
+ const String& description, MetricSet* owner)
+ : Metric(name, tags, description, owner) {}
+
+ AbstractValueMetric(const String& name, Tags dimensions,
+ const String& description, MetricSet* owner)
+ : Metric(name, std::move(dimensions), description, owner) {}
+
+ AbstractValueMetric(const AbstractValueMetric& other, MetricSet* owner)
+ : Metric(other, owner) {}
+
+ void logWarning(const char* msg) const;
+ void logNonFiniteValueWarning() const;
+ void sendLogEvent(Metric::String name, double value) const;
+};
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+class ValueMetric
+ : public AbstractValueMetric
+{
+ typedef Metric::String String; // Redefine so template finds it easy
+
+ struct Values : MetricValueClass {
+ uint32_t _count;
+ AvgVal _min, _max, _last;
+ TotVal _total;
+
+ struct AtomicImpl {
+ std::atomic<uint32_t> _count {0};
+ std::atomic<AvgVal> _min {std::numeric_limits<AvgVal>::max()};
+ std::atomic<AvgVal> _max {std::numeric_limits<AvgVal>::min()};
+ std::atomic<AvgVal> _last {0};
+ std::atomic<TotVal> _total {0};
+ };
+
+ Values()
+ : _count(0),
+ _min(std::numeric_limits<AvgVal>::max()),
+ _max(std::numeric_limits<AvgVal>::min()),
+ _last(0),
+ _total(0)
+ {
+ // numeric_limits min() returns smallest positive number
+ // for signed floating point types. Haven't found a way to
+ // get minimum negative value, so using -1 * positive for those.
+ if (std::numeric_limits<AvgVal>::is_signed
+ && !std::numeric_limits<AvgVal>::is_integer)
+ {
+ _max = -1 * std::numeric_limits<AvgVal>::max();
+ }
+ }
+
+ void relaxedStoreInto(AtomicImpl& target) const noexcept {
+ target._count.store(_count, std::memory_order_relaxed);
+ target._min.store(_min, std::memory_order_relaxed);
+ target._max.store(_max, std::memory_order_relaxed);
+ target._last.store(_last, std::memory_order_relaxed);
+ target._total.store(_total, std::memory_order_relaxed);
+ }
+
+ void relaxedLoadFrom(const AtomicImpl& source) noexcept {
+ _count = source._count.load(std::memory_order_relaxed);
+ _min = source._min.load(std::memory_order_relaxed);
+ _max = source._max.load(std::memory_order_relaxed);
+ _last = source._last.load(std::memory_order_relaxed);
+ _total = source._total.load(std::memory_order_relaxed);
+ }
+
+ template<typename T>
+ T getValue(const stringref & id) const {
+ if (id == "last") return static_cast<T>(_last);
+ if (id == "count") return static_cast<T>(_count);
+ if (id == "total") return static_cast<T>(_total);
+ if (id == "min") return static_cast<T>(_count > 0 ? _min : 0);
+ if (id == "max") return static_cast<T>( _count > 0 ? _max : 0);
+ throw vespalib::IllegalArgumentException(
+ "No value " + vespalib::string(id) + " in value metric.", VESPA_STRLOC);
+ }
+
+ double getDoubleValue(const stringref & id) const {
+ return getValue<double>(id);
+ }
+ uint64_t getLongValue(const stringref & id) const {
+ return getValue<uint64_t>(id);
+ }
+ void output(const std::string& id, std::ostream& out) const {
+ if (id == "last") { out << _last; return; }
+ if (id == "count") { out << _count; return; }
+ if (id == "total") { out << _total; return; }
+ if (id == "min") { out << (_count > 0 ? _min : 0); return; }
+ if (id == "max") { out << (_count > 0 ? _max : 0); return; }
+ throw vespalib::IllegalArgumentException(
+ "No value " + id + " in value metric.", VESPA_STRLOC);
+ }
+ void output(const std::string& id, vespalib::JsonStream& stream) const {
+ if (id == "last") { stream << _last; return; }
+ if (id == "count") { stream << _count; return; }
+ if (id == "total") { stream << _total; return; }
+ if (id == "min") { stream << (_count > 0 ? _min : 0); return; }
+ if (id == "max") { stream << (_count > 0 ? _max : 0); return; }
+ throw vespalib::IllegalArgumentException(
+ "No value " + id + " in value metric.", VESPA_STRLOC);
+ }
+ friend std::ostream & operator << (std::ostream & os, const Values & v) {
+ os << "count=" << v._count;
+ os << ", total=" << v._total;
+ return os;
+ }
+ };
+
+ MetricValueSet<Values> _values;
+
+ enum Flag { SUMMED_AVERAGE = 2, UNSET_ON_ZERO_VALUE = 4, LOG_IF_UNSET = 8 };
+
+ virtual bool summedAverage() const
+ { return _values.hasFlag(SUMMED_AVERAGE); }
+ bool unsetOnZeroValue() const
+ { return _values.hasFlag(UNSET_ON_ZERO_VALUE); }
+ bool logIfUnset() const { return _values.hasFlag(LOG_IF_UNSET); }
+
+ void add(const Values& values, bool sumOnAdd);
+ void dec(const Values& values);
+
+ void addValueWithCount(AvgVal avg, TotVal tot, uint32_t count);
+
+ // Finite number (not infinity/NaN) check using type trait tag dispatch.
+ // 2nd param is instance of std::true_type iff AvgVal is floating point.
+ bool checkFinite(AvgVal v, std::true_type) {
+ if (!std::isfinite(v)) {
+ logNonFiniteValueWarning();
+ return false;
+ }
+ return true;
+ }
+ bool checkFinite(AvgVal, std::false_type) { return true; }
+
+public:
+ ValueMetric(const String& name, const String& tags,
+ const String& description, MetricSet* owner = 0);
+
+ ValueMetric(const ValueMetric<AvgVal, TotVal, SumOnAdd>&,
+ CopyType, MetricSet* owner);
+
+ ValueMetric(const String& name, Tags dimensions,
+ const String& description, MetricSet* owner = 0);
+
+ virtual MetricValueClass::UP getValues() const
+ { return MetricValueClass::UP(new Values(_values.getValues())); }
+
+ void unsetOnZeroValue() { _values.setFlag(UNSET_ON_ZERO_VALUE); }
+ void logOnlyIfSet() { _values.removeFlag(LOG_IF_UNSET); }
+
+ virtual ValueMetric<AvgVal, TotVal, SumOnAdd>* clone(
+ std::vector<Metric::LP>&, CopyType type, MetricSet* owner,
+ bool /*includeUnused*/) const
+ { return new ValueMetric<AvgVal,TotVal,SumOnAdd>(*this, type, owner); }
+
+ ValueMetric & operator+=(const ValueMetric &);
+
+ friend ValueMetric operator+(const ValueMetric & a, const ValueMetric & b) {
+ ValueMetric t(a); t += b; return t;
+ }
+
+ void addAvgValueWithCount(AvgVal avg, uint32_t count)
+ { if (count) { addValueWithCount(avg, avg * count, count); } }
+ void addTotalValueWithCount(TotVal tot, uint32_t count)
+ { if (count) { addValueWithCount(tot / count, tot, count); } }
+ void addValueBatch(AvgVal avg, uint32_t count) {
+ addAvgValueWithCount(avg, count);
+ }
+ virtual void addValue(AvgVal avg) { addAvgValueWithCount(avg, 1); }
+ virtual void set(AvgVal avg) { addValue(avg); }
+ virtual void inc(AvgVal val = 1);
+ virtual void dec(AvgVal val = 1);
+
+ double getAverage() const;
+ AvgVal getMinimum() const { return _values.getValues()._min; }
+ AvgVal getMaximum() const { return _values.getValues()._max; }
+ AvgVal getCount() const { return _values.getValues()._count; }
+ AvgVal getLast() const { return _values.getValues()._last; }
+
+ void reset() { _values.reset(); }
+
+ virtual bool logEvent(const String& fullName) const;
+
+ virtual void print(std::ostream&, bool verbose,
+ const std::string& indent, uint64_t secondsPassed) const;
+
+ virtual int64_t getLongValue(const stringref & id) const;
+ virtual double getDoubleValue(const stringref & id) const;
+
+
+ virtual bool inUse(const MetricValueClass& v) const {
+ const Values& values(static_cast<const Values&>(v));
+ return (values._total != 0
+ || (values._count != 0 && !unsetOnZeroValue()));
+ }
+ virtual bool used() const {
+ return inUse(_values.getValues());
+ }
+
+ virtual void addMemoryUsage(MemoryConsumption&) const;
+
+ virtual void printDebug(std::ostream&, const std::string& indent) const;
+
+ virtual void addToPart(Metric&) const;
+ virtual void addToSnapshot(Metric&, std::vector<Metric::LP>&) const;
+};
+
+typedef ValueMetric<double, double, true> DoubleValueMetric;
+typedef ValueMetric<double, double, false> DoubleAverageMetric;
+typedef ValueMetric<int64_t, int64_t, true> LongValueMetric;
+typedef ValueMetric<int64_t, int64_t, false> LongAverageMetric;
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+ValueMetric<AvgVal, TotVal, SumOnAdd>::ValueMetric(
+ const String& name, const String& tags,
+ const String& description, MetricSet* owner)
+ : AbstractValueMetric(name, tags, description, owner),
+ _values()
+{
+ _values.setFlag(LOG_IF_UNSET);
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+ValueMetric<AvgVal, TotVal, SumOnAdd>::ValueMetric(
+ const String& name, const Tags dimensions,
+ const String& description, MetricSet* owner)
+ : AbstractValueMetric(name, std::move(dimensions), description, owner),
+ _values()
+{
+ _values.setFlag(LOG_IF_UNSET);
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+ValueMetric<AvgVal, TotVal, SumOnAdd>::ValueMetric(
+ const ValueMetric<AvgVal, TotVal, SumOnAdd>& other,
+ CopyType copyType, MetricSet* owner)
+ : AbstractValueMetric(other, owner),
+ _values(other._values, copyType == CLONE ? other._values.size() : 1)
+{
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+void ValueMetric<AvgVal, TotVal, SumOnAdd>::inc(AvgVal incVal)
+{
+ if (!checkFinite(incVal, std::is_floating_point<AvgVal>())) {
+ return;
+ }
+ Values values;
+ do {
+ values = _values.getValues();
+ AvgVal val = values._last + incVal;
+ ++values._count;
+ values._total += val;
+ if (val < values._min) values._min = val;
+ if (val > values._max) values._max = val;
+ values._last = val;
+ } while (!_values.setValues(values));
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+void ValueMetric<AvgVal, TotVal, SumOnAdd>::dec(AvgVal decVal)
+{
+ if (!checkFinite(decVal, std::is_floating_point<AvgVal>())) {
+ return;
+ }
+ Values values;
+ do {
+ values = _values.getValues();
+ AvgVal val = values._last - decVal;
+ ++values._count;
+ values._total += val;
+ if (val < values._min) values._min = val;
+ if (val > values._max) values._max = val;
+ values._last = val;
+ } while (!_values.setValues(values));
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+void
+ValueMetric<AvgVal, TotVal, SumOnAdd>::addToSnapshot(
+ Metric& other, std::vector<Metric::LP>&) const
+{
+ ValueMetric<AvgVal, TotVal, SumOnAdd>& o(
+ reinterpret_cast<ValueMetric<AvgVal, TotVal, SumOnAdd>&>(other));
+ if (_values.getValues()._count == 0) return; // Don't add if not set
+ o.add(_values.getValues(), false);
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+void
+ValueMetric<AvgVal, TotVal, SumOnAdd>::addToPart(Metric& other) const
+{
+ ValueMetric<AvgVal, TotVal, SumOnAdd>& o(
+ reinterpret_cast<ValueMetric<AvgVal, TotVal, SumOnAdd>&>(
+ other));
+ o.add(_values.getValues(), SumOnAdd);
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+void
+ValueMetric<AvgVal, TotVal, SumOnAdd>::add(const Values& values2, bool sumOnAdd)
+{
+ bool overflow;
+ Values values;
+ do {
+ values = _values.getValues();
+ overflow = values._count > values._count + values2._count
+ || (values2._total >= 0
+ ? values._total > values._total + values2._total
+ : values._total < values._total + values2._total);
+ if (values._count == 0) {
+ values = values2;
+ } else if (values2._count == 0) {
+ // Do nothing
+ } else if (sumOnAdd) {
+ double totalAverage
+ = static_cast<double>(values._total) / values._count
+ + static_cast<double>(values2._total) / values2._count;
+ values._count += values2._count;
+ values._total = static_cast<TotVal>(totalAverage * values._count);
+ values._last += values2._last;
+ _values.setFlag(SUMMED_AVERAGE);
+ } else {
+ values._count += values2._count;
+ values._total += values2._total;
+ values._last = values2._last;
+ }
+ if (values._min > values2._min) values._min = values2._min;
+ if (values._max < values2._max) values._max = values2._max;
+ } while (!_values.setValues(values));
+ if (overflow) {
+ std::ostringstream ost;
+ ost << "ValueMetric " << getPath() << " overflowed. Resetting it. Old value = (";
+ ost << values << "), Incoming = (" << values2 << ")";
+ logWarning(ost.str().c_str());
+ _values.reset();
+ }
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+void
+ValueMetric<AvgVal, TotVal, SumOnAdd>::dec(const Values& values2)
+{
+ bool underflow;
+ Values values;
+ do {
+ values = _values.getValues();
+ underflow = values._count < values._count - values2._count
+ || values._total < values._total - values2._total;
+ values._count -= values2._count;
+ values._total -= values2._total;
+ } while (!_values.setValues(values));
+ if (underflow) {
+ _values.reset();
+ std::ostringstream ost;
+ ost << "ValueMetric " << getPath() << " underflowed. Resetting it.";
+ logWarning(ost.str().c_str());
+ }
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+void ValueMetric<AvgVal, TotVal, SumOnAdd>::addValueWithCount(
+ AvgVal avg, TotVal tot, uint32_t count)
+{
+ if (!checkFinite(avg, std::is_floating_point<AvgVal>())) {
+ return;
+ }
+ Values values;
+ do {
+ values = _values.getValues();
+ values._count += count;
+ values._total += tot;
+ if (avg < values._min) values._min = avg;
+ if (avg > values._max) values._max = avg;
+ values._last = avg;
+ } while (!_values.setValues(values));
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+ValueMetric<AvgVal, TotVal, SumOnAdd>&
+ValueMetric<AvgVal, TotVal, SumOnAdd>::operator+=(
+ const ValueMetric<AvgVal, TotVal, SumOnAdd>& other)
+{
+ add(other._values.getValues(), SumOnAdd);
+ return *this;
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+double
+ValueMetric<AvgVal, TotVal, SumOnAdd>::getAverage() const
+{
+ Values values(_values.getValues());
+ if (values._count == 0) return 0;
+ return static_cast<double>(values._total) / values._count;
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+bool
+ValueMetric<AvgVal, TotVal, SumOnAdd>::logEvent(const String& fullName) const
+{
+ Values values(_values.getValues());
+ if (!logIfUnset() && !inUse(values)) return false;
+ sendLogEvent(fullName, SumOnAdd
+ ? static_cast<double>(values._last)
+ : static_cast<double>(values._total) / values._count);
+ return true;
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+void
+ValueMetric<AvgVal, TotVal, SumOnAdd>::print(
+ std::ostream& out, bool verbose, const std::string& indent,
+ uint64_t secondsPassed) const
+{
+ (void) indent;
+ (void) secondsPassed;
+ Values values(_values.getValues());
+ if (!inUse(values) && !verbose) return;
+ out << this->_name << " average=" << (values._count == 0
+ ? 0 : static_cast<double>(values._total) / values._count)
+ << " last=" << values._last;
+ if (!summedAverage()) {
+ if (values._count > 0) {
+ out << " min=" << values._min << " max=" << values._max;
+ }
+ out << " count=" << values._count << " total=" << values._total;
+ }
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+int64_t
+ValueMetric<AvgVal, TotVal, SumOnAdd>::getLongValue(const stringref & id) const
+{
+ Values values(_values.getValues());
+ if (id == "last" || (SumOnAdd && id == "value"))
+ return static_cast<int64_t>(values._last);
+ if (id == "average" || (!SumOnAdd && id == "value"))
+ return static_cast<int64_t>(getAverage());
+ if (id == "count") return static_cast<int64_t>(values._count);
+ if (id == "total") return static_cast<int64_t>(values._total);
+ if (id == "min") return static_cast<int64_t>(
+ values._count > 0 ? values._min : 0);
+ if (id == "max") return static_cast<int64_t>(
+ values._count > 0 ? values._max : 0);
+ throw vespalib::IllegalArgumentException(
+ "No value " + id + " in average metric.", VESPA_STRLOC);
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+double
+ValueMetric<AvgVal, TotVal, SumOnAdd>::getDoubleValue(const stringref & id) const
+{
+ Values values(_values.getValues());
+ if (id == "last" || (SumOnAdd && id == "value"))
+ return static_cast<double>(values._last);
+ if (id == "average" || (!SumOnAdd && id == "value"))
+ return getAverage();
+ if (id == "count") return static_cast<double>(values._count);
+ if (id == "total") return static_cast<double>(values._total);
+ if (id == "min") return static_cast<double>(
+ values._count > 0 ? values._min : 0);
+ if (id == "max") return static_cast<double>(
+ values._count > 0 ? values._max : 0);
+ throw vespalib::IllegalArgumentException(
+ "No value " + vespalib::string(id) + " in average metric.", VESPA_STRLOC);
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+void
+ValueMetric<AvgVal, TotVal, SumOnAdd>::addMemoryUsage(
+ MemoryConsumption& mc) const
+{
+ ++mc._valueMetricCount;
+ mc._valueMetricValues += _values.getMemoryUsageAllocatedInternally();
+ mc._valueMetricMeta += sizeof(ValueMetric<AvgVal, TotVal, SumOnAdd>)
+ - sizeof(Metric);
+ Metric::addMemoryUsage(mc);
+}
+
+template<typename AvgVal, typename TotVal, bool SumOnAdd>
+void
+ValueMetric<AvgVal, TotVal, SumOnAdd>::printDebug(
+ std::ostream& out, const std::string& indent) const
+{
+ Values values(_values.getValues());
+ out << "value=" << values._last << " ";
+ Metric::printDebug(out, indent);
+}
+
+} // metrics
+
diff --git a/metrics/src/vespa/metrics/xmlwriter.cpp b/metrics/src/vespa/metrics/xmlwriter.cpp
new file mode 100644
index 00000000000..01c509030ff
--- /dev/null
+++ b/metrics/src/vespa/metrics/xmlwriter.cpp
@@ -0,0 +1,111 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/metrics/xmlwriter.h>
+
+#include <vespa/metrics/countmetric.h>
+#include <vespa/metrics/metricset.h>
+#include <vespa/metrics/metricsnapshot.h>
+#include <vespa/metrics/valuemetric.h>
+
+namespace metrics {
+
+XmlWriter::XmlWriter(vespalib::xml::XmlOutputStream& xos,
+ uint32_t period, int verbosity)
+ : _period(period), _xos(xos), _verbosity(verbosity) {}
+
+bool
+XmlWriter::visitSnapshot(const MetricSnapshot& snapshot)
+{
+ using namespace vespalib::xml;
+ _xos << XmlTag("snapshot") << XmlAttribute("name", snapshot.getName())
+ << XmlAttribute("from", snapshot.getFromTime())
+ << XmlAttribute("to", snapshot.getToTime())
+ << XmlAttribute("period", snapshot.getPeriod());
+ return true;
+}
+
+void
+XmlWriter::doneVisitingSnapshot(const MetricSnapshot&)
+{
+ using namespace vespalib::xml;
+ _xos << XmlEndTag();
+}
+
+bool
+XmlWriter::visitMetricSet(const MetricSet& set, bool)
+{
+ using namespace vespalib::xml;
+ if (set.used() || _verbosity >= 2) {
+ _xos << XmlTag(set.getName(), CONVERT_ILLEGAL_CHARACTERS);
+ printCommonXmlParts(set);
+ return true;
+ }
+ return false;
+}
+void
+XmlWriter::doneVisitingMetricSet(const MetricSet&) {
+ using namespace vespalib::xml;
+ _xos << XmlEndTag();
+}
+
+bool
+XmlWriter::visitCountMetric(const AbstractCountMetric& metric, bool)
+{
+ MetricValueClass::UP values(metric.getValues());
+ if (!metric.inUse(*values) && _verbosity < 2) return true;
+ using namespace vespalib::xml;
+ std::ostringstream ost;
+ _xos << XmlTag(metric.getName(), CONVERT_ILLEGAL_CHARACTERS)
+ << XmlAttribute(metric.sumOnAdd()
+ ? "count" : "value", values->toString("count"));
+ printCommonXmlParts(metric);
+ _xos << XmlEndTag();
+ return true;
+}
+
+bool
+XmlWriter::visitValueMetric(const AbstractValueMetric& metric, bool)
+{
+ MetricValueClass::UP values(metric.getValues());
+ if (!metric.inUse(*values) && _verbosity < 2) return true;
+ using namespace vespalib::xml;
+ _xos << XmlTag(metric.getName(), CONVERT_ILLEGAL_CHARACTERS)
+ << XmlAttribute("average", values->getLongValue("count") == 0
+ ? 0 : values->getDoubleValue("total")
+ / values->getDoubleValue("count"))
+ << XmlAttribute("last", values->toString("last"));
+ if (!metric.summedAverage()) {
+ if (values->getLongValue("count") > 0) {
+ _xos << XmlAttribute("min", values->toString("min"))
+ << XmlAttribute("max", values->toString("max"));
+ }
+ _xos << XmlAttribute("count", values->getLongValue("count"));
+ if (_verbosity >= 2) {
+ _xos << XmlAttribute("total", values->toString("total"));
+ }
+ }
+ printCommonXmlParts(metric);
+ _xos << XmlEndTag();
+ return true;
+}
+
+void
+XmlWriter::printCommonXmlParts(const Metric& metric) const
+{
+ using namespace vespalib::xml;
+ const Metric::Tags& tags(metric.getTags());
+ if (_verbosity >= 3 && tags.size() > 0) {
+ std::ostringstream ost;
+ // XXX print tag values as well
+ ost << tags[0].key;
+ for (uint32_t i=1; i<tags.size(); ++i) {
+ ost << "," << tags[i].key;
+ }
+ _xos << XmlAttribute("tags", ost.str());
+ }
+ if (_verbosity >= 1 && !metric.getDescription().empty()) {
+ _xos << XmlAttribute("description", metric.getDescription());
+ }
+}
+
+} // metrics
diff --git a/metrics/src/vespa/metrics/xmlwriter.h b/metrics/src/vespa/metrics/xmlwriter.h
new file mode 100644
index 00000000000..c55ed82f093
--- /dev/null
+++ b/metrics/src/vespa/metrics/xmlwriter.h
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/metrics/metric.h>
+#include <vespa/vespalib/util/xmlserializable.h>
+
+namespace metrics {
+
+class XmlWriter : public MetricVisitor {
+ uint32_t _period;
+ vespalib::xml::XmlOutputStream& _xos;
+ int _verbosity;
+
+public:
+ XmlWriter(vespalib::xml::XmlOutputStream& xos,
+ uint32_t period, int verbosity);
+
+ virtual bool visitSnapshot(const MetricSnapshot&);
+ virtual void doneVisitingSnapshot(const MetricSnapshot&);
+ virtual bool visitMetricSet(const MetricSet& set, bool);
+ virtual void doneVisitingMetricSet(const MetricSet&);
+ virtual bool visitCountMetric(const AbstractCountMetric&,
+ bool autoGenerated);
+ virtual bool visitValueMetric(const AbstractValueMetric&,
+ bool autoGenerated);
+
+private:
+ void printCommonXmlParts(const Metric& metric) const;
+};
+
+}
+