summaryrefslogtreecommitdiffstats
path: root/statistics
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
committerJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
commit72231250ed81e10d66bfe70701e64fa5fe50f712 (patch)
tree2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /statistics
Publish
Diffstat (limited to 'statistics')
-rw-r--r--statistics/.gitignore10
-rw-r--r--statistics/OWNERS1
-rw-r--r--statistics/pom.xml87
-rw-r--r--statistics/src/main/java/com/yahoo/.gitignore1
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/.gitignore0
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/Axis.java30
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/Bucket.java21
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/Callback.java25
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/Counter.java210
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/CounterGroup.java167
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/CounterProxy.java31
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/Group.java14
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/Handle.java106
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/Histogram.java240
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/HistogramType.java42
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/Limits.java69
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/Proxy.java31
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java59
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/SampleSet.java102
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/Statistics.java59
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java154
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/Sum.java74
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/Value.java846
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/ValueGroup.java120
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/ValueProxy.java83
-rw-r--r--statistics/src/main/java/com/yahoo/statistics/package-info.java7
-rw-r--r--statistics/src/main/resources/configdefinitions/statistics.def23
-rw-r--r--statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java132
-rw-r--r--statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java103
-rw-r--r--statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java75
-rw-r--r--statistics/src/test/java/com/yahoo/statistics/ProxyTestCase.java62
-rw-r--r--statistics/src/test/java/com/yahoo/statistics/StatisticsImplTestCase.java108
-rw-r--r--statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java116
-rw-r--r--statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java284
34 files changed, 3492 insertions, 0 deletions
diff --git a/statistics/.gitignore b/statistics/.gitignore
new file mode 100644
index 00000000000..85d32008813
--- /dev/null
+++ b/statistics/.gitignore
@@ -0,0 +1,10 @@
+statistics.iml
+archive
+build
+log
+target
+.settings
+.classpath
+.project
+
+/pom.xml.build
diff --git a/statistics/OWNERS b/statistics/OWNERS
new file mode 100644
index 00000000000..67cd2820bb8
--- /dev/null
+++ b/statistics/OWNERS
@@ -0,0 +1 @@
+arnej27959
diff --git a/statistics/pom.xml b/statistics/pom.xml
new file mode 100644
index 00000000000..09c81dccb71
--- /dev/null
+++ b/statistics/pom.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>parent</artifactId>
+ <version>6-SNAPSHOT</version>
+ <relativePath>../parent/pom.xml</relativePath>
+ </parent>
+ <artifactId>statistics</artifactId>
+ <version>6-SNAPSHOT</version>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-library</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>vespalog</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-bundle</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>component</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.osgi</groupId>
+ <artifactId>org.osgi.core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.yahoo.vespa</groupId>
+ <artifactId>config-class-plugin</artifactId>
+ <version>${project.version}</version>
+ <executions>
+ <execution>
+ <id>config-gen</id>
+ <goals>
+ <goal>config-gen</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/statistics/src/main/java/com/yahoo/.gitignore b/statistics/src/main/java/com/yahoo/.gitignore
new file mode 100644
index 00000000000..04204c7c9d0
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/.gitignore
@@ -0,0 +1 @@
+config
diff --git a/statistics/src/main/java/com/yahoo/statistics/.gitignore b/statistics/src/main/java/com/yahoo/statistics/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/.gitignore
diff --git a/statistics/src/main/java/com/yahoo/statistics/Axis.java b/statistics/src/main/java/com/yahoo/statistics/Axis.java
new file mode 100644
index 00000000000..baec19c8f24
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/Axis.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+import java.util.Arrays;
+
+
+/**
+ * A wrapper class for representing a single axis of an n-dimensional
+ * histogram.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+class Axis {
+ private final double[] limits;
+ private final String name;
+
+ Axis(String name, double[] limits) {
+ this.limits = Arrays.copyOf(limits, limits.length);
+ this.name = name;
+ }
+
+ double[] getLimits() {
+ return limits;
+ }
+
+ String getName() {
+ return name;
+ }
+
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/Bucket.java b/statistics/src/main/java/com/yahoo/statistics/Bucket.java
new file mode 100644
index 00000000000..3bd9dbe3301
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/Bucket.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+import java.util.List;
+
+
+/**
+ * A bucket in a multidimensional histogram.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+interface Bucket {
+ void put(double[] value, int dim);
+ void reset();
+ double lowerLimit();
+ double upperLimit();
+ boolean isLeaf();
+ List<Bucket> getBuckets();
+ long getSum();
+ void add(long n);
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/Callback.java b/statistics/src/main/java/com/yahoo/statistics/Callback.java
new file mode 100644
index 00000000000..8f826cb4097
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/Callback.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+/**
+ * Definition of the callback interface for the statistics API. It is a common
+ * use case to need a reference to the Handle (e.g. Value or Counter) which a
+ * callback is related to. Since everything in a Handle since 5.1.4 is fully
+ * initialized from the constructor, it became cumbersome to use Runnable for
+ * the callback and this interface came into use.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @since 5.1.4
+ */
+public interface Callback {
+ /**
+ * Invoked each logging cycle right before the events for a Handle are
+ * emitted to the log.
+ *
+ * @param h
+ * the handle which invoked this callback
+ * @param firstTime
+ * true the first time the method is invoked from h, false later
+ */
+ public void run(Handle h, boolean firstTime);
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/Counter.java b/statistics/src/main/java/com/yahoo/statistics/Counter.java
new file mode 100644
index 00000000000..a17ef5824dc
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/Counter.java
@@ -0,0 +1,210 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.yahoo.log.event.Event;
+import com.yahoo.container.StatisticsConfig;
+
+
+/**
+ * A single integer value which can be incremented.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class Counter extends Handle {
+ // The current value of this counter
+ private AtomicLong current = new AtomicLong(0L);
+
+ // Whether or not this counter shall be reset between each logging
+ // interval
+ private final boolean resetCounter;
+
+ /**
+ * A monotonically increasing 64 bit integer value.
+ *
+ * @param name
+ * The name of this counter, for use in logging.
+ * @param manager
+ * the statistics manager acquired by injection
+ * @param fetchParametersFromConfig
+ * Whether or not this counter should be initialized from config.
+ */
+ public Counter(String name, Statistics manager, boolean fetchParametersFromConfig) {
+ this(name, manager, fetchParametersFromConfig, null, false, true);
+ }
+
+ /**
+ * A monotonically increasing 64 bit integer value.
+ *
+ * @param name
+ * The name of this counter, for use in logging.
+ * @param manager
+ * the statistics manager acquired by injection
+ * @param fetchParametersFromConfig
+ * Whether or not this counter should be initialized from config.
+ * @param callback
+ * will be invoked each time this counter is written to the log
+ * @param resetCounter
+ * Control for if this Counter should be reset between each
+ * logging interval.
+ */
+ public Counter(String name, Statistics manager,
+ boolean fetchParametersFromConfig, Callback callback, boolean resetCounter) {
+ this(name, manager, fetchParametersFromConfig, callback,
+ resetCounter, true);
+ }
+
+ /**
+ * A monotonically increasing 64 bit integer value. Do not make this
+ * constructor public, it is used for creating unregistered counters.
+ *
+ * @param name
+ * The name of this counter, for use in logging.
+ * @param manager
+ * the statistics manager acquired by injection
+ * @param fetchParametersFromConfig
+ * Whether or not this counter should be initialized from config.
+ * @param callback
+ * will be invoked each time this counter is written to the log
+ * @param resetCounter
+ * Control for if this Counter should be reset between each
+ * logging interval.
+ * @param register
+ * Whether to register the counter in the statistics manager
+ */
+ private Counter(String name, Statistics manager,
+ boolean fetchParametersFromConfig, Callback callback,
+ boolean resetCounter, boolean register) {
+ super(name, manager, callback);
+ if (fetchParametersFromConfig) {
+ StatisticsConfig config = manager.getConfig();
+ this.resetCounter = getResetCounter(name, config);
+ } else {
+ this.resetCounter = resetCounter;
+ }
+ if (register) {
+ manager.register(this);
+ }
+ }
+
+
+ /**
+ * Get a Counter instance not registered in the statistics manager. This is
+ * used by CounterGroup and should not be made public.
+ *
+ * @param name
+ * The name of this counter, for use in logging.
+ * @param resetCounter
+ * Control for if this Counter should be reset between each
+ * logging interval.
+ */
+ static Counter intializeUnregisteredCounter(String name,
+ boolean resetCounter) {
+ return new Counter(name, null, false, null, resetCounter, false);
+ }
+
+ /**
+ * If this Counter is set up to read config, configure it
+ * according to the config given.
+ */
+ private static boolean getResetCounter(String name, StatisticsConfig config) {
+ for (int i = 0; i < config.counterresets().size(); i++) {
+ String configName = config.counterresets(i).name();
+ if (configName.equals(name)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Increment by 1.
+ */
+ public void increment() {
+ current.incrementAndGet();
+ }
+
+ /**
+ * Increment by n.
+ */
+ public void increment(long n) {
+ current.addAndGet(n);
+ }
+
+ /**
+ * @return current value of this counter
+ */
+ public long get() {
+ return current.get();
+ }
+
+ /**
+ * The reset counter is true if this is counter is reset to 0 between each
+ * logging interval.
+ *
+ * @return whether this counter is reset between each logging interval.
+ */
+ public boolean getResetCounter() {
+ return resetCounter;
+ }
+
+ /**
+ * If this counter should be set to 0 between each logging interval,
+ * do that.
+ */
+ public void reset() {
+ if (resetCounter) {
+ current.set(0L);
+ }
+ }
+
+ /**
+ * Log current state and reset.
+ */
+ @Override
+ public void runHandle() {
+ String name = getName();
+ long lastCurrent;
+ boolean resetState = getResetCounter();
+
+ if (resetState) {
+ lastCurrent = current.getAndSet(0L);
+ Event.value(name, lastCurrent);
+ } else {
+ lastCurrent = current.get();
+ Event.count(name, lastCurrent);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + " " + getName() + " " + current;
+ }
+
+ CounterProxy getProxyAndReset() {
+ CounterProxy c = new CounterProxy(getName());
+ if (getResetCounter()) {
+ c.setRaw(current.getAndSet(0L));
+ } else {
+ c.setRaw(current.get());
+ }
+ return c;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o.getClass() != this.getClass()) {
+ return false;
+ }
+ Counter other = (Counter) o;
+ return getName().equals(other.getName());
+ }
+
+ @Override
+ public int hashCode() {
+ return getName().hashCode() + 31 * "Counter".hashCode();
+ }
+
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java b/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java
new file mode 100644
index 00000000000..8dfdf21cc1a
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java
@@ -0,0 +1,167 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+import com.yahoo.container.StatisticsConfig;
+import com.yahoo.log.event.Event;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Iterator;
+
+
+/**
+ * A set of associated counters.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class CounterGroup extends Group {
+ private final boolean resetCounter;
+
+ // A map for names of subevents and Value instances
+ private Map<String, Counter> subEvents = new HashMap<>();
+
+ /**
+ * @param name The symbolic name of this group of counters.
+ */
+ public CounterGroup(String name, Statistics manager) {
+ this(name, manager, true);
+ }
+
+ /**
+ * Create a basic group of counter which may or may not depend on config.
+ *
+ * @param name
+ * The symbolic name of this group of counters.
+ * @param manager
+ * the statistics manager acquired by injection
+ * @param fetchParametersFromConfig
+ * Whether this Group should be configured from config.
+ */
+ public CounterGroup(String name, Statistics manager,
+ boolean fetchParametersFromConfig) {
+ this(name, manager, fetchParametersFromConfig, null, false);
+ }
+
+ /**
+ * Create a group of counters with a callback included.
+ *
+ * @param name
+ * The symbolic name of this group of counters.
+ * @param manager
+ * the statistics manager acquired by injection
+ * @param fetchParametersFromConfig
+ * Whether this Group should be configured from config.
+ * @param callback
+ * will be invoked each time data is written to the log
+ * @param resetCounter
+ * Control for if this group should be reset between each
+ * logging interval.
+ */
+ public CounterGroup(String name, Statistics manager,
+ boolean fetchParametersFromConfig, Callback callback, boolean resetCounter) {
+
+ super(name, manager, callback);
+ if (fetchParametersFromConfig) {
+ StatisticsConfig config = manager.getConfig();
+ this.resetCounter = getResetCounter(name, config);
+ } else {
+ this.resetCounter = resetCounter;
+ }
+ manager.register(this);
+ }
+
+ private static boolean getResetCounter(String name, StatisticsConfig config) {
+ for (int i = 0; i < config.counterresets().size(); i++) {
+ String configName = config.counterresets(i).name();
+ if (configName.equals(name)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+
+ /**
+ * Increment named contained counter by 1.
+ */
+ public void increment(String name) {
+ Counter c = getCounter(name);
+ c.increment();
+ }
+
+ /**
+ * Increment named contained counter by n.
+ */
+ public void increment(String name, long n) {
+ Counter c = getCounter(name);
+ c.increment(n);
+ }
+
+ /**
+ * Get a counter with a given name, creates a new counter if no
+ * counter with the name given exists.
+ */
+ synchronized Counter getCounter(String name) {
+ Counter c = subEvents.get(name);
+ if (c == null) {
+ c = getNewCounter(name);
+ }
+ return c;
+ }
+
+ private Counter getNewCounter(String subName) {
+ Counter c = Counter.intializeUnregisteredCounter(subName, resetCounter);
+ subEvents.put(subName, c);
+ return c;
+ }
+
+ /**
+ * Dump contained counters to log and reset.
+ */
+ @Override
+ public void runHandle() {
+ StringBuilder multi = new StringBuilder();
+ CounterProxy[] proxies;
+ int i = 0;
+
+ // this is to make sure the number of events does not change while logging
+ synchronized (this) {
+ proxies = new CounterProxy[subEvents.size()];
+ i = 0;
+ for (Iterator<Counter> j = subEvents.values().iterator(); j
+ .hasNext();) {
+ Counter c = j.next();
+ proxies[i] = c.getProxyAndReset();
+ i++;
+ }
+ }
+
+ while (i > 0) {
+ i--;
+ if (multi.length() > 0) {
+ multi.append(", ");
+ }
+ multi.append(proxies[i].getName());
+ multi.append("=");
+ multi.append(proxies[i].getRaw());
+ }
+
+ Event.countGroup(getName(), multi.toString());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o.getClass() != this.getClass()) {
+ return false;
+ }
+ CounterGroup other = (CounterGroup) o;
+ return getName().equals(other.getName());
+ }
+
+ @Override
+ public int hashCode() {
+ return getName().hashCode() + 31 * "CounterGroup".hashCode();
+ }
+}
+
diff --git a/statistics/src/main/java/com/yahoo/statistics/CounterProxy.java b/statistics/src/main/java/com/yahoo/statistics/CounterProxy.java
new file mode 100644
index 00000000000..5c3475bc640
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/CounterProxy.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+/**
+ * To be able to cache events concerning Counters internally, group them
+ * together and similar.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+class CounterProxy extends Proxy {
+ private long raw;
+ private boolean hasRaw = false;
+
+ CounterProxy(String name) {
+ super(name);
+ }
+
+ boolean hasRaw() {
+ return hasRaw;
+ }
+ long getRaw() {
+ return raw;
+ }
+ void setRaw(long raw) {
+ hasRaw = true;
+ this.raw = raw;
+ }
+
+}
+
diff --git a/statistics/src/main/java/com/yahoo/statistics/Group.java b/statistics/src/main/java/com/yahoo/statistics/Group.java
new file mode 100644
index 00000000000..6400b38d687
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/Group.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+/**
+ * The base class for groups of counters and values.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+abstract class Group extends Handle {
+ Group(String name, Statistics manager, Callback parametrizedCallback) {
+ super(name, manager, parametrizedCallback);
+ }
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/Handle.java b/statistics/src/main/java/com/yahoo/statistics/Handle.java
new file mode 100644
index 00000000000..6c3d94dad41
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/Handle.java
@@ -0,0 +1,106 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+import java.util.TimerTask;
+
+
+/**
+ * Base class for the interface to the statistics framework.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public abstract class Handle {
+
+ private TimerTask task;
+
+ private final String name;
+ private boolean cancelled;
+ private final Statistics manager;
+ private final Callback parametrizedCallback;
+ private boolean firstTime;
+
+ Handle(String name, Statistics manager, Callback parametrizedCallback) {
+ this.name = name;
+ this.manager = manager;
+ this.parametrizedCallback = parametrizedCallback;
+ firstTime = true;
+ }
+
+ String getName() {
+ return name;
+ }
+
+ TimerTask makeTask() {
+ final Handle self = this;
+ synchronized (self) {
+ if (task != null) {
+ task.cancel();
+ }
+ task = new TimerTask() {
+ public void run() {
+ self.run();
+ }
+ };
+ return task;
+ }
+ }
+
+ /**
+ * Run the callback object.
+ *
+ * This will happen at the start of each invocation of a Handle's
+ * run() method. The callback is presumed to be exception safe.
+ * If no callback is set, this is a no-op. The callback will need
+ * to handle any necessary synchronization itself.
+ */
+ public final void runCallback() {
+ if (parametrizedCallback == null) {
+ return;
+ }
+ parametrizedCallback.run(this, firstTime);
+ firstTime = false;
+ }
+
+ /**
+ * Run the callback object first, then invoke runHandle().
+ */
+ public final void run() {
+ runCallback();
+ runHandle();
+ }
+
+ /**
+ * Invoke an action to be performed periodically for a statistics Handle.
+ *
+ * <p>Synchronization has to be handled by the method itself.
+ */
+ public abstract void runHandle();
+
+ /**
+ * Cancel this Handle and remove it from internal state in Statistics.
+ *
+ * @return value of java.util.TimerTask.cancel()
+ */
+ public final boolean cancel() {
+ boolean ok = (task == null ? false : task.cancel());
+ cancelled = true;
+ manager.purge();
+ return ok;
+ }
+
+ /**
+ * Returns whether this object has been cancelled or not.
+ *
+ * @return true if cancelled
+ */
+ public final boolean isCancelled() {
+ return cancelled;
+ }
+
+ @Override
+ public abstract boolean equals(Object o);
+
+ @Override
+ public abstract int hashCode();
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/Histogram.java b/statistics/src/main/java/com/yahoo/statistics/Histogram.java
new file mode 100644
index 00000000000..83960058dd6
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/Histogram.java
@@ -0,0 +1,240 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ArrayList;
+
+
+/**
+ * A set of sums or other histograms.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class Histogram implements Bucket {
+ // The upper and lower limit for the bucket in another histogram
+ // this histogram represents. The "outermost" histogram in a
+ // multidimensional histogram will effectively ignore this, but set
+ // them to -Inf and Inf for consistency.
+ private final double lower;
+ private final double upper;
+
+ // The names of all the axes, only used in "outermost" histogram in
+ // multi dimensional histogram.
+ private String axes = null;
+
+ private List<Bucket> buckets = new ArrayList<>();
+
+ /**
+ * Build a new histogram using bucket limits from the given Limits
+ * object.
+ */
+ public Histogram(Limits limits) {
+ // lower and upper will never be used here,
+ // but it's nicer with defined values
+ this(limits, 0, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY);
+
+ // generate axis names, but only if there is more than one
+ int guard = limits.getDimensions();
+ if (guard == 1) {
+ return;
+ }
+ StringBuffer names = new StringBuffer();
+ int i = 0;
+ while (i < guard) {
+ names.append(limits.getAxis(i).getName());
+ i++;
+ if (i < guard) {
+ names.append(",");
+ }
+ }
+ axes = names.toString();
+ }
+
+ private Histogram(Limits limits, int dim, double lower, double upper) {
+ this.lower = lower;
+ this.upper = upper;
+ // last axis, create sum objects instead of histograms
+ boolean lastAxis = dim == (limits.getDimensions() - 1);
+ Axis next = limits.getAxis(dim);
+ double[] delimiters = next.getLimits();
+ int i = 0;
+ double previousBucket = Double.NEGATIVE_INFINITY;
+ dim++;
+ while (i < delimiters.length) {
+ if (lastAxis) {
+ buckets.add(new Sum(previousBucket, delimiters[i]));
+ } else {
+ buckets.add(new Histogram(limits, dim,
+ previousBucket, delimiters[i]));
+ }
+ previousBucket = delimiters[i];
+ i++;
+ }
+ if (lastAxis) {
+ buckets.add(new Sum(previousBucket, Double.POSITIVE_INFINITY));
+ } else {
+ buckets.add(new Histogram(limits, dim,
+ previousBucket,
+ Double.POSITIVE_INFINITY));
+ }
+
+ }
+
+ /**
+ * Increment the corresponding bucket for this data point by 1.
+ */
+ public synchronized void put(double[] value) {
+ put(value, 0);
+ }
+
+ /**
+ * Increment the corresponding bucket for this data point by 1.
+ *
+ * @param dim the index of the first value to consider in value array
+ */
+ @Override
+ public void put(double[] value, int dim) {
+ Bucket bucket = findBucket(0, buckets.size(), value[dim]);
+ bucket.put(value, ++dim);
+ }
+
+ private Bucket findBucket(int offset, int limit, double value) {
+ int index = offset + (limit - offset) / 2;
+ Bucket bucket = buckets.get(index);
+ if (bucket.lowerLimit() <= value && value < bucket.upperLimit()) {
+ return bucket;
+ } else if (bucket.upperLimit() <= value) {
+ return findBucket(index + 1, limit, value);
+ } else { // value < bucket.lowerLimit()
+ return findBucket(offset, index, value);
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer s;
+ int i, t;
+
+ s = new StringBuffer();
+ if (axes != null) {
+ s.append(axes).append(" ");
+ }
+ s.append("(");
+ s.append(buckets.get(0).toString());
+ s.append(")");
+
+ t = buckets.size();
+ i = 1;
+ while (i < t) {
+ Bucket b = buckets.get(i);
+ s.append(" < ");
+ s.append(b.lowerLimit());
+ s.append(" (");
+ s.append(b.toString());
+ s.append(")");
+ i += 1;
+ }
+
+ return s.toString();
+ }
+
+ /**
+ * Reset all contained buckets.
+ */
+ @Override
+ public void reset() {
+ for (Iterator<Bucket> i = buckets.iterator(); i.hasNext(); ) {
+ i.next().reset();
+ }
+ }
+
+ /**
+ * The lower limit for the bucket this histogram represents.
+ *
+ * @return the lower limit for the bucket this histogram represents
+ */
+ @Override
+ public double lowerLimit() {
+ return lower;
+ }
+
+ /**
+ * The upper limit for the bucket this histogram represents.
+ *
+ * @return the upper limit for the bucket this histogram represents
+ */
+ @Override
+ public double upperLimit() {
+ return upper;
+ }
+
+ @Override
+ public List<Bucket> getBuckets() {
+ return buckets;
+ }
+
+ private List<Bucket> getLeaves() {
+ final class Bookmark {
+ final int i;
+ final List<Bucket> buckets;
+ Bookmark(int i, List<Bucket> buckets) {
+ this.i = i;
+ this.buckets = buckets;
+ }
+ }
+ List<Bucket> sums = new ArrayList<>();
+ Deque<Bookmark> stack = new ArrayDeque<>();
+ List<Bucket> current;
+ int i = 0;
+ stack.addFirst(new Bookmark(i, buckets));
+ while (stack.size() > 0) {
+ Bookmark currentMark = stack.removeFirst();
+ i = currentMark.i;
+ current = currentMark.buckets;
+ while (i < current.size()) {
+ Bucket b = current.get(i++);
+ if (b.isLeaf()) {
+ sums.add(b);
+ } else {
+ Bookmark marker = new Bookmark(i, current);
+ stack.addFirst(marker);
+ i = 0;
+ current = b.getBuckets();
+ }
+ }
+ }
+ return sums;
+ }
+
+ void merge(Histogram source) {
+ List<Bucket> src = source.getLeaves();
+ List<Bucket> dst = getLeaves();
+ if (dst.size() != src.size()) {
+ throw new IllegalStateException(
+ "Number of buckets in destination and source not equal. (Source "
+ + src.size() + ", destination " + dst.size() + ".");
+ }
+ for (int i = 0; i < dst.size(); ++i) {
+ dst.get(i).add(src.get(i).getSum());
+ }
+ }
+
+ @Override
+ public long getSum() {
+ throw new RuntimeException("Not implemented.");
+ }
+
+ @Override
+ public boolean isLeaf() {
+ return false;
+ }
+
+ @Override
+ public void add(long n) {
+ throw new IllegalStateException("Can not add directly to a Histogram instance.");
+ }
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/HistogramType.java b/statistics/src/main/java/com/yahoo/statistics/HistogramType.java
new file mode 100644
index 00000000000..911069902c7
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/HistogramType.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+/**
+ * Enumeration of how a histogram should be represented from
+ * admin server.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public final class HistogramType {
+ private final String representation;
+
+ private HistogramType(String representation) {
+ this.representation = representation;
+ }
+
+ public String toString() { return representation; }
+
+ /**
+ * Basic histograms, each bucket is count representing the bucket's
+ * defined interval.
+ */
+ public static final HistogramType REGULAR = new HistogramType("REGULAR");
+
+ /**
+ * Cumulative histograms, that is, a given bucket contains the count
+ * for values corresponding to "itself" and all preceding buckets.
+ */
+ public static final HistogramType CUMULATIVE =
+ new HistogramType("CUMULATIVE");
+
+
+ /**
+ * Reverse cumulative histograms, that is, a given bucket contains
+ * the count for values corresponding to "itself" and all following
+ * buckets.
+ */
+ public static final HistogramType REVERSE_CUMULATIVE =
+ new HistogramType("REVERSE_CUMULATIVE");
+
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/Limits.java b/statistics/src/main/java/com/yahoo/statistics/Limits.java
new file mode 100644
index 00000000000..fb792eede14
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/Limits.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+import java.util.List;
+import java.util.ArrayList;
+
+
+/**
+ * Limits for a histogram, this is only a wrapper for initializing
+ * a histogram.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class Limits {
+ private final List<Axis> axes = new ArrayList<>(1);
+ private boolean frozen = false;
+
+ /**
+ * Create an empty Limits instance.
+ */
+ public Limits() {
+ }
+
+ /**
+ * Create a Limits instance suitable for use in a Value.Parameters instance.
+ * The instance will be frozen.
+ */
+ public Limits(double[] limits) {
+ addAxis(null, limits);
+ freeze();
+ }
+
+ /**
+ * @param name the name of the variable for this axis
+ * @param limits the bucket limits for this axis
+ */
+ public void addAxis(String name, double[] limits) {
+ if (frozen) {
+ throw new IllegalStateException("Can not add more axes to a frozen Limits instance.");
+ }
+ axes.add(new Axis(name, limits));
+ }
+
+ int getDimensions() {
+ return axes.size();
+ }
+
+ Axis getAxis(int i) {
+ return axes.get(i);
+ }
+
+ /**
+ * Prevent adding any mores axes.
+ */
+ public void freeze() {
+ this.frozen = true;
+ }
+
+ /**
+ * True if further change is not permitted.
+ *
+ * @return whether this object now should be considered immutable
+ */
+ public boolean isFrozen() {
+ return frozen;
+ }
+
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/Proxy.java b/statistics/src/main/java/com/yahoo/statistics/Proxy.java
new file mode 100644
index 00000000000..46fa4a3190c
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/Proxy.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+/**
+ * Base class for event proxies, which are used to cache and group
+ * events internally.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+abstract class Proxy {
+ private long timestamp;
+ private String name;
+
+ Proxy(String name) {
+ this(name, System.currentTimeMillis());
+ }
+
+ Proxy(String name, long timestamp) {
+ this.timestamp = timestamp;
+ this.name = name;
+ }
+
+ long getTimestamp() {
+ return timestamp;
+ }
+
+ String getName() {
+ return name;
+ }
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java b/statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java
new file mode 100644
index 00000000000..c920c35ca13
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.yahoo.statistics.SampleSet.Sampling;
+
+/**
+ * Book-keeping class to know which SampleSet instances have unlogged data.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+final class SampleDirectory {
+ private final Object directoryLock = new Object();
+ private List<SampleSet> directory = new ArrayList<>(200);
+
+ void put(SampleSet s) {
+ synchronized (directoryLock) {
+ directory.add(s);
+ s.setRegisteredForLogging(true);
+ }
+ }
+
+ /**
+ * Get a view of the current generation of data and instantiate a new
+ * generation. This does the memory barrier two-step for the
+ * client.
+ */
+ Sampling[] fetchValues() {
+ Sampling[] copyToReturn;
+ synchronized (directoryLock) {
+ List<SampleSet> tmpDir = directory;
+ copyToReturn = new Sampling[tmpDir.size()];
+ List<SampleSet> newDir = new ArrayList<>(200);
+ for (int i = 0; i < copyToReturn.length; ++i) {
+ copyToReturn[i] = tmpDir.get(i).getAndReset();
+ }
+ directory = newDir;
+ }
+ return copyToReturn;
+ }
+
+ /**
+ * Return a view of the current generation of data. This does the memory
+ * barrier two-step for the client.
+ */
+ Sampling[] viewValues() {
+ Sampling[] copy;
+ synchronized (directoryLock) {
+ copy = new Sampling[directory.size()];
+ for (int i = 0; i < copy.length; ++i) {
+ copy[i] = directory.get(i).values;
+ }
+ }
+ return copy;
+ }
+
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/SampleSet.java b/statistics/src/main/java/com/yahoo/statistics/SampleSet.java
new file mode 100644
index 00000000000..572d87aea5e
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/SampleSet.java
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+/**
+ * A running set of samples for a Value instance. It is
+ * used only in a very specific context between the sampling threads (each instance
+ * is only used by on sampling thread) and the single logging thread.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+final class SampleSet {
+ Sampling values;
+ final Limits histogramLimits;
+ private boolean isRegisteredForLogging = false;
+
+ SampleSet(Limits histogramLimits) {
+ this.histogramLimits = histogramLimits;
+ values = new Sampling(0.0d, 0L, 0.0d, 0.0d, histogramLimits);
+ }
+
+ static final class Sampling {
+ final double sum;
+ final long insertions;
+ final double max;
+ final double min;
+ final Histogram histogram;
+
+ Sampling(double sum, long insertions, double max, double min, Limits histogramLimits) {
+ if (histogramLimits != null) {
+ this.histogram = new Histogram(histogramLimits);
+ } else {
+ this.histogram = null;
+ }
+ this.sum = sum;
+ this.insertions = insertions;
+ this.max = max;
+ this.min = min;
+ }
+
+ Sampling(double sum, long insertions, double max, double min, Histogram histogram) {
+ this.histogram = histogram;
+ this.sum = sum;
+ this.insertions = insertions;
+ this.max = max;
+ this.min = min;
+ }
+ }
+
+
+ private Sampling createSampling(double x, Sampling previous) {
+ double sum = previous.sum;
+ long insertions = previous.insertions;
+ double max = previous.max;
+ double min = previous.min;
+
+ sum += x;
+ if (insertions == 0) {
+ max = x;
+ min = x;
+ } else {
+ max = Math.max(x, max);
+ min = Math.min(x, min);
+ }
+ insertions++;
+ return new Sampling(sum, insertions, max, min, previous.histogram);
+ }
+
+ /**
+ * Insert x, do all pertinent operations. (Update histogram, update
+ * insertion count for calculating mean, etc.)
+ *
+ * @return whether this is registered for logging
+ */
+ synchronized boolean put(double x) {
+ Sampling previous = values;
+ Histogram histogram = previous.histogram;
+ if (histogram != null) {
+ histogram.put(new double[] {x});
+ }
+ values = createSampling(x, previous);
+ return isRegisteredForLogging;
+ }
+
+ /**
+ * Get state and reset it.
+ */
+ synchronized Sampling getAndReset() {
+ Sampling previous = values;
+ values = new Sampling(0.0d, 0L, 0.0d, 0.0d, histogramLimits);
+ setRegisteredForLogging(false);
+ return previous;
+ }
+
+ // Setting this state is protected by SampleDirectory.directoryLock. It is
+ // either set from the logging thread (protected by directoryLock and using
+ // "this" as a memory barrier through getAndReset()), or from the sampling
+ // thread protected by directoryLock and without a mem barrier.
+ void setRegisteredForLogging(boolean isRegisteredForLogging) {
+ this.isRegisteredForLogging = isRegisteredForLogging;
+ }
+
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/Statistics.java b/statistics/src/main/java/com/yahoo/statistics/Statistics.java
new file mode 100644
index 00000000000..6ad3f484d79
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/Statistics.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+import com.yahoo.container.StatisticsConfig;
+
+/**
+ * Interface used for registering statistics values and counters for logging.
+ *
+ * @author steinar
+ * @author tonytv
+ */
+public interface Statistics {
+ /**
+ * Add a new handle to be scheduled for periodic logging. If a handle
+ * already exists with the same name, it will be cancelled and removed from
+ * the internal state of this object.
+ */
+ void register(Handle h);
+
+ /**
+ * Remove a named handler from the set of working handlers.
+ */
+ void remove(String name);
+
+ /**
+ * Get current config used. This may be a null reference, depending on how
+ * the instance was constructed.
+ */
+ StatisticsConfig getConfig();
+
+ /**
+ * Purges all cancelled Handles from internal Map and Timer.
+ *
+ * @return return value from java.util.Timer.purge()
+ */
+ int purge();
+
+ /** A null implementation which ignores all calls and returns the default config */
+ public static Statistics nullImplementation=new NullImplementation();
+
+ static class NullImplementation implements Statistics {
+
+ private StatisticsConfig nullConfig=new StatisticsConfig(new StatisticsConfig.Builder());
+
+ @Override
+ public void register(Handle h) { }
+
+ @Override
+ public void remove(String name) { }
+
+ @Override
+ public StatisticsConfig getConfig() { return nullConfig; }
+
+ @Override
+ public int purge() { return 0; }
+
+ }
+
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java b/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java
new file mode 100644
index 00000000000..c2a8444da27
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java
@@ -0,0 +1,154 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Timer;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.container.StatisticsConfig;
+
+/**
+ * Centralized book keeping for statistics module. Times logging and reads
+ * config. It is normally obtained through injection, but may be initialized
+ * manually for testing or when an injection mechanism is unavailable. Logging
+ * will be disabled by initializing the Statistics class with a null config
+ * object.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public final class StatisticsImpl extends AbstractComponent implements Statistics {
+ private final Timer worker;
+ private final StatisticsConfig config;
+ private static final Logger log = Logger.getLogger(StatisticsImpl.class
+ .getName());
+ private final int collectioninterval;
+ private final int logginginterval;
+ // default access for testing only
+ final Map<String, Handle> handles = new HashMap<>();
+
+ /**
+ * Build a statistics manager based on the given config values. Use a config
+ * builder for testing if logging is necessary, set the config argument to
+ * the constructor to null is logging is not necessary.
+ *
+ * @since 5.1.4
+ * @param config
+ * settings for logging interval and configured events. Setting
+ * it to null disables logging.
+ * @throws IllegalArgumentException
+ * if logging interval is smaller than collection interval, or
+ * collection interval is not a multiplum of logging interval
+ */
+ public StatisticsImpl(final StatisticsConfig config) {
+ final int l = (int) config.loggingintervalsec();
+ final int c = (int) config.collectionintervalsec();
+
+ if (l != 0 && l < c) {
+ throw new IllegalArgumentException(
+ "Logging interval (" + l + ") smaller than collection interval (" + c + ")."
+ + " New config ignored.");
+ }
+ if ((l % c) != 0) {
+ throw new IllegalArgumentException(
+ "Collection interval (" + c + ") not multiplum of logging interval (" + l + ")."
+ + " New config ignored.");
+ }
+ this.logginginterval = l;
+ this.collectioninterval = c;
+ this.config = config;
+ this.worker = new Timer(true);
+ }
+
+ /**
+ * Cancel internal worker thread and do any other necessary cleanup. The
+ * internal worker thread is a daemon thread, so not calling this will not
+ * hamper a clean exit from the VM.
+ *
+ * @since 5.1.4
+ */
+ @Override
+ public void deconstruct() {
+ worker.cancel();
+ }
+
+ private void schedule(final Handle h) {
+ if (logginginterval != 0) {
+ h.run();
+ // We use the rather creative assumption that there is
+ // exactly 24h pr day+night.
+ final Date d = new Date();
+ final long ms = collectioninterval * 1000L;
+ final long delay = ms - (d.getTime() % (ms));
+ worker.scheduleAtFixedRate(h.makeTask(), delay, ms);
+ }
+ }
+
+ /**
+ * Add a new handle to be scheduled for periodic logging. If a handle
+ * already exists with the same name, it will be cancelled and removed from
+ * the internal state of this object.
+ */
+ @Override
+ public void register(final Handle h) {
+ synchronized (handles) {
+ final Handle oldHandle = handles.get(h.getName());
+ if (oldHandle == h) {
+ log.log(Level.WARNING, "Handle [" + h + "] already registered");
+ return;
+ }
+ if (oldHandle != null) {
+ oldHandle.cancel();
+ }
+ handles.put(h.getName(), h);
+ if (worker != null) {
+ schedule(h);
+ }
+ }
+ }
+
+ /**
+ * Remove a named handler from the set of working handlers.
+ */
+ @Override
+ public void remove(final String name) {
+ synchronized (handles) {
+ final Handle oldHandle = handles.remove(name);
+ if (oldHandle != null) {
+ oldHandle.cancel();
+ }
+ }
+ }
+
+ /**
+ * Get current config used. This may be a null reference, depending on how
+ * the instance was constructed.
+ */
+ @Override
+ public StatisticsConfig getConfig() {
+ return config;
+ }
+
+ /**
+ * Purges all cancelled Handles from internal Map and Timer.
+ *
+ * @return return value from java.util.Timer.purge()
+ */
+ @Override
+ public int purge() {
+ synchronized (handles) {
+ final Iterator<Handle> it = handles.values().iterator();
+ while (it.hasNext()) {
+ final Handle h = it.next();
+ if (h.isCancelled()) {
+ it.remove();
+ }
+ }
+ return worker == null ? 0 : worker.purge();
+ }
+ }
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/Sum.java b/statistics/src/main/java/com/yahoo/statistics/Sum.java
new file mode 100644
index 00000000000..87662e38560
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/Sum.java
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+import java.util.List;
+
+
+/**
+ * The innermost part of a histogram, a bucket which only contains a
+ * counter.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+class Sum implements Bucket {
+ private long sum = 0L;
+ private double lower;
+ private double upper;
+
+ Sum(double lower, double upper) {
+ this.lower = lower;
+ this.upper = upper;
+ }
+
+ /**
+ * Increment this bucket.
+ */
+ public void put(double[] value, int dim) {
+ sum += 1;
+ }
+
+ /**
+ * Set this bucket's count to 0.
+ */
+ public void reset() {
+ sum = 0L;
+ }
+
+ /**
+ * The lower limit for values counted by this bucket.
+ */
+ public double lowerLimit() {
+ return lower;
+ }
+
+ /**
+ * The upper limit for values counted by this bucket.
+ */
+ public double upperLimit() {
+ return upper;
+ }
+
+ public String toString() {
+ return Long.toString(sum);
+ }
+
+ @Override
+ public List<Bucket> getBuckets() {
+ return null;
+ }
+
+ @Override
+ public boolean isLeaf() {
+ return true;
+ }
+
+ @Override
+ public long getSum() {
+ return sum;
+ }
+
+ @Override
+ public void add(long n) {
+ sum += n;
+ }
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/Value.java b/statistics/src/main/java/com/yahoo/statistics/Value.java
new file mode 100644
index 00000000000..88ce66990c3
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/Value.java
@@ -0,0 +1,846 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.logging.Logger;
+
+import com.yahoo.container.StatisticsConfig;
+import com.yahoo.container.StatisticsConfig.Values.Operations;
+import com.yahoo.log.LogLevel;
+import com.yahoo.log.event.Event;
+import com.yahoo.statistics.SampleSet.Sampling;
+
+
+/**
+ * A statistical variable, typically representing a sampling of an
+ * arbitrarily changing parameter.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class Value extends Handle {
+ // For accumulated values, SampleSet instances are mem barriers between {n
+ // sampling threads} and {logging thread}.
+
+ // lastValue is a memory barrier between {n sampling threads} and {n
+ // sampling threads, logging thread}.
+
+ // Therefore, the logging thread first locks SampleDirectory.directoryLock,
+ // then locks each SampleSet, one by one. The sampling threads _either_ lock
+ // a single SampleSet _or_ lock SampleDirectory.directoryLock.
+
+ // It is necessary to create a memory relationship between the logging
+ // threads to ensure the newest sample ends up in the log for logRaw = true.
+
+ private final ThreadLocal<SampleSet> sample = new ThreadLocal<>();
+ private final SampleDirectory directory = new SampleDirectory();
+
+ // This must _only_ be touched if logRaw is true
+ private volatile double lastValue = 0.0d;
+
+ private final boolean logRaw;
+ private final boolean logMean;
+ private final boolean logSum;
+ private final boolean logInsertions;
+ private final boolean logMax;
+ private final boolean logMin;
+ private final boolean logHistogram;
+
+ private final Limits histogram;
+ private final boolean nameExtension;
+ final HistogramType histogramId;
+ private final char appendChar;
+
+ private static final Logger log = Logger.getLogger(Value.class.getName());
+ static final String HISTOGRAM_TYPE_WARNING = "Histogram types other than REGULAR currently not supported."
+ +" Reverting to regular histogram for statistics event";
+
+ /**
+ * Parameters for building Value instances. All settings are classes instead
+ * of primitive types to allow tri-state logic (true, false, unset).
+ */
+ public static class Parameters {
+ /**
+ * Log raw values. Raw values are basically the last value logged.
+ */
+ Boolean logRaw;
+ /**
+ * Log the sum of all data points for each interval.
+ */
+ Boolean logSum;
+ /**
+ * Log the mean value for each interval.
+ */
+ Boolean logMean;
+ /**
+ * Log the maximal value observed for each interval.
+ */
+ Boolean logMax;
+ /**
+ * Log the minimal value observed for each interval.
+ */
+ Boolean logMin;
+ /**
+ * Log the number of observations for each interval.
+ */
+ Boolean logInsertions;
+ /**
+ * Whether or not to add an identifying extension (like mean) to event
+ * names when logging derived values.
+ *
+ * It is useful to disable extensions if a only a single dervied value,
+ * e.g. the mean, is the only thing to be logged. The default is to use
+ * extensions.
+ *
+ * If extensions are disabled, the ability to log more than one of raw
+ * value, min, max, mean (i.e. the raw value and derived values of the
+ * same type) is disabled to avoid confusion. Since histograms are not
+ * Value events, these never have a name extension and are always
+ * available.
+ */
+ Boolean nameExtension;
+ /**
+ * Log a data histogram.
+ */
+ Boolean logHistogram;
+ /**
+ * What kind of histogram to log.
+ *
+ * @see HistogramType
+ */
+ HistogramType histogramId;
+ /**
+ * The limits to use if logging as a histogram. The Limits instance must
+ * be frozen before using the Parameters instance in a Value constructor
+ * call.
+ *
+ * @see Limits
+ */
+ Limits limits;
+ /**
+ * Separator character to use between event name and type of
+ * nameExtension is set to true.
+ */
+ Character appendChar;
+
+ /**
+ * This is invoked each time a value is dumped to the log.
+ *
+ * @see Handle#runCallback()
+ */
+ Callback callback;
+
+ /**
+ * Whether to register in the Statistics manager. This is not touched by
+ * merge and also has no undefined state. In general, a Value should
+ * always register and not doing so explicitly should not be part of the
+ * public API.
+ */
+ private boolean register = true;
+
+ /**
+ * Get a fresh Parameters instance with all features turned/unset.
+ * Parameters instances may be recycled for construction multiple Value
+ * instances, but do note any Limits instance added must be frozen.
+ */
+ public Parameters() {
+ }
+
+ /**
+ * (De-)Activate logging of raw values. Raw values are basically the
+ * last value logged.
+ *
+ * @return "this" for call chaining
+ */
+ public Parameters setLogRaw(Boolean logRaw) {
+ this.logRaw = logRaw;
+ return this;
+ }
+
+ /**
+ * (De-)Activate logging the sum of all data points for each interval.
+ *
+ * @return "this" for call chaining
+ */
+ public Parameters setLogSum(Boolean logSum) {
+ this.logSum = logSum;
+ return this;
+ }
+
+ /**
+ * (De)-activate loging the mean value for each interval.
+ *
+ * @return "this" for call chaining
+ */
+ public Parameters setLogMean(Boolean logMean) {
+ this.logMean = logMean;
+ return this;
+ }
+
+ /**
+ * (De-)Activate logging the maximal value observed for each interval.
+ *
+ * @return "this" for call chaining
+ */
+ public Parameters setLogMax(Boolean logMax) {
+ this.logMax = logMax;
+ return this;
+ }
+
+ /**
+ * (De-)Activate logging the minimal value observed for each interval.
+ *
+ * @return "this" for call chaining
+ */
+ public Parameters setLogMin(Boolean logMin) {
+ this.logMin = logMin;
+ return this;
+ }
+
+ /**
+ * (De-)Activate loging the number of observations for each interval.
+ *
+ * @return "this" for call chaining
+ */
+ public Parameters setLogInsertions(Boolean logInsertions) {
+ this.logInsertions = logInsertions;
+ return this;
+ }
+
+ /**
+ * Whether or not to add an identifying extension (like mean) to event
+ * names when logging derived values.
+ *
+ * It is useful to disable extensions if a only a single dervied value,
+ * e.g. the mean, is the only thing to be logged. The default is to use
+ * extensions.
+ *
+ * If extensions are disabled, the ability to log more than one of raw
+ * value, min, max, mean (i.e. the raw value and derived values of the
+ * same type) is disabled to avoid confusion. Since histograms are not
+ * Value events, these never have a name extension and are always
+ * available.
+ *
+ * @return "this" for call chaining
+ */
+ public Parameters setNameExtension(Boolean nameExtension) {
+ this.nameExtension = nameExtension;
+ return this;
+ }
+
+ /**
+ * (De-)Activate logging a data histogram.
+ *
+ * @return "this" for call chaining
+ */
+ public Parameters setLogHistogram(Boolean logHistogram) {
+ this.logHistogram = logHistogram;
+ return this;
+ }
+
+ /**
+ * What kind of histogram to log.
+ *
+ * @see HistogramType
+ *
+ * @return "this" for call chaining
+ */
+ public Parameters setHistogramId(HistogramType histogramId) {
+ this.histogramId = histogramId;
+ return this;
+ }
+
+ /**
+ * The limits to use if logging as a histogram. The Limits instance must
+ * be frozen before using the Parameters instance in a Value constructor
+ * call.
+ *
+ * @return "this" for call chaining*
+ * @see Limits
+ */
+ public Parameters setLimits(Limits limits) {
+ this.limits = limits;
+ return this;
+ }
+
+ /**
+ * Separator character to use between event name and type of
+ * nameExtension is set to true. The default is '.'.
+ *
+ * @return "this" for call chaining
+ */
+ public Parameters setAppendChar(Character appendChar) {
+ this.appendChar = appendChar;
+ return this;
+ }
+
+ /**
+ * Set a callback to be invoked each time this Value is written to the
+ * log.
+ *
+ * @param callback
+ * to be invoked each time the Value is written to the log
+ * @return "this" for call chaining
+ */
+ public Parameters setCallback(Callback callback) {
+ this.callback = callback;
+ return this;
+ }
+
+ /**
+ * Set whether to register in the statistics manager. Do note the
+ * package private access for this method opposed to all the other
+ * setters.
+ *
+ * @param register
+ * set to false to avoid registering
+ * @return "this" for call chaining
+ */
+ private Parameters setRegister(boolean register) {
+ this.register = register;
+ return this;
+ }
+
+ /**
+ * If a member is not set in this, add it from defaults. Do note, this
+ * applies for both true and false settings, in other words, the default
+ * may also be used to turn off something if it is undefined in this.
+ */
+ void merge(Parameters defaults) {
+ if (defaults == null) {
+ return;
+ }
+ this.logRaw = this.logRaw == null ? defaults.logRaw : this.logRaw;
+ this.logSum = this.logSum == null ? defaults.logSum : this.logSum;
+ this.logMean = this.logMean == null ? defaults.logMean
+ : this.logMean;
+ this.logMax = this.logMax == null ? defaults.logMax : this.logMax;
+ this.logMin = this.logMin == null ? defaults.logMin : this.logMin;
+ this.logInsertions = this.logInsertions == null ? defaults.logInsertions
+ : this.logInsertions;
+ this.nameExtension = this.nameExtension == null ? defaults.nameExtension
+ : this.nameExtension;
+ this.logHistogram = this.logHistogram == null ? defaults.logHistogram
+ : this.logHistogram;
+ this.histogramId = this.histogramId == null ? defaults.histogramId
+ : this.histogramId;
+ this.limits = this.limits == null ? defaults.limits : this.limits;
+ this.appendChar = this.appendChar == null ? defaults.appendChar
+ : this.appendChar;
+ this.callback = this.callback == null ? defaults.callback
+ : this.callback;
+ }
+
+ }
+
+ /**
+ * Configure a Value instance fully, no raw config access.
+ *
+ * @param name
+ * tag for logging
+ * @param manager
+ * the statistics manager acquired by injection
+ * @param parameters
+ * all the parameters necessary for initializing a Value
+ * @throws IllegalStateException
+ * if Parameters.limits exists and is not frozen
+ */
+ public Value(String name, Statistics manager, Parameters parameters) {
+ super(name, manager, parameters.callback);
+ this.logHistogram = isTrue(parameters.logHistogram);
+ this.logMax = isTrue(parameters.logMax);
+ this.logMean = isTrue(parameters.logMean);
+ this.logMin = isTrue(parameters.logMin);
+ this.logRaw = isTrue(parameters.logRaw);
+ this.logSum = isTrue(parameters.logSum);
+ this.logInsertions = isTrue(parameters.logInsertions);
+ this.nameExtension = isTrue(parameters.nameExtension);
+ if (logHistogram) {
+ if (!parameters.limits.isFrozen()) {
+ throw new IllegalStateException("The Limits instance must be frozen.");
+ }
+ if (parameters.histogramId != HistogramType.REGULAR) {
+ log.log(LogLevel.WARNING, HISTOGRAM_TYPE_WARNING
+ + " '" + name + "'");
+ }
+ this.histogramId = HistogramType.REGULAR;
+ this.histogram = parameters.limits;
+ } else {
+ this.histogram = null;
+ this.histogramId = HistogramType.REGULAR;
+ }
+ Character appendChar = parameters.appendChar;
+ if (appendChar == null) {
+ this.appendChar = '.';
+ } else {
+ this.appendChar = appendChar.charValue();
+ }
+ if (parameters.register) {
+ manager.register(this);
+ }
+ }
+
+ private static boolean isTrue(Boolean b) {
+ return b != null && b.booleanValue();
+ }
+
+ /**
+ * Build a Value which should be initialized from config.
+ *
+ * @param name
+ * the name of the event in the log
+ * @param manager
+ * the current Statistics manager, acquired by injection
+ * @param defaults
+ * defaults for values not defined by config, this may be null
+ */
+ public static Value buildValue(String name, Statistics manager, Parameters defaults) {
+ return new Value(name, manager, buildParameters(name, manager, defaults));
+ }
+
+ /**
+ * Get a Value instance not registered in the statistics manager. This is
+ * used by ValueGroup and should not be made public.
+ *
+ * @param name
+ * The name of this counter, for use in logging.
+ * @param parameters
+ * setting for the new Value
+ */
+ static Value initializeUnregisteredValue(String name, Parameters parameters) {
+ return new Value(name, null, parameters.setRegister(false));
+ }
+
+ private static Parameters buildParameters(String name, Statistics manager, Parameters defaults) {
+ Parameters p = null;
+ StatisticsConfig config = manager.getConfig();
+ if (config != null) {
+ for (int i = 0; i < config.values().size(); i++) {
+ String configName = config.values(i).name();
+ if (configName.equals(name)) {
+ p = parametersFromConfig(config.values(i).operations());
+ break;
+ }
+ }
+ }
+ if (p == null) {
+ if (defaults == null) {
+ p = defaultParameters();
+ } else {
+ p = defaults;
+ }
+ } else {
+ p.merge(defaults);
+ }
+ return p;
+ }
+
+ static Parameters defaultParameters() {
+ return new Parameters().setLogRaw(true).setNameExtension(true);
+ }
+
+ private static Parameters parametersFromConfig(List<Operations> o) {
+ Parameters p = new Parameters().setNameExtension(true);
+
+ for (Operations operation : o) {
+ Operations.Name.Enum opName = operation.name();
+
+ HashMap<String, String> args = new HashMap<>();
+ for (int j = 0; j < operation.arguments().size(); j++) {
+ args.put(operation.arguments(j).key(), operation.arguments(j).value());
+ }
+
+ if (opName == Operations.Name.MEAN) {
+ p.setLogMean(true);
+ } else if (opName == Operations.Name.MAX) {
+ p.setLogMax(true);
+ } else if (opName == Operations.Name.MIN) {
+ p.setLogMin(true);
+ } else if (opName == Operations.Name.RAW) {
+ p.setLogRaw(true);
+ } else if (opName == Operations.Name.SUM) {
+ p.setLogSum(true);
+ } else if (opName == Operations.Name.INSERTIONS) {
+ p.setLogInsertions(true);
+ } else if (opName == Operations.Name.REGULAR) {
+ p.setLogHistogram(true);
+ p.setHistogramId(HistogramType.REGULAR);
+ p.setLimits(initHistogram(args.get("axes"), args.get("limits")));
+ } else if (opName == Operations.Name.CUMULATIVE) {
+ p.setLogHistogram(true);
+ p.setHistogramId(HistogramType.CUMULATIVE);
+ p.setLimits(initHistogram(args.get("axes"), args.get("limits")));
+ } else if (opName == Operations.Name.REVERSE_CUMULATIVE) {
+ p.setLogHistogram(true);
+ p.setHistogramId(HistogramType.REVERSE_CUMULATIVE);
+ p.setLimits(initHistogram(args.get("axes"),args.get("limits")));
+ }
+ }
+ return p;
+ }
+
+ private static Limits initHistogram(String axes, String limits) {
+ String[] borders;
+ double[] vanillaLimits;
+ Limits l = new Limits();
+ int i = 0;
+
+ if (axes != null) {
+ throw new RuntimeException("Config of multidimensional histograms not yet implemented.");
+ }
+ if (limits == null) {
+ throw new RuntimeException("Config of histograms needs a list of limits.");
+ }
+ borders = limits.split(",");
+ vanillaLimits = new double[borders.length];
+ while (i < vanillaLimits.length) {
+ vanillaLimits[i] = Double.parseDouble(borders[i].trim());
+ ++i;
+ }
+ l.addAxis(null, vanillaLimits);
+ l.freeze();
+ return l;
+ }
+
+ private SampleSet getSample() {
+ SampleSet s = sample.get();
+ if (s == null) {
+ s = new SampleSet(histogram);
+ sample.set(s);
+ }
+ return s;
+ }
+
+ private void putComposite(double x) {
+ SampleSet s = getSample();
+ boolean isInDir = s.put(x);
+ if (!isInDir) {
+ directory.put(s);
+ }
+ }
+
+ /**
+ * Insert x, do all pertinent operations. (Update histogram, update
+ * insertion count for calculating mean, etc.)
+ */
+ public void put(double x) {
+ if (logComposite()) {
+ putComposite(x);
+ }
+ if (logRaw) {
+ lastValue = x;
+ }
+ }
+
+ private boolean logComposite() {
+ return logMean || logMin || logMax || logSum || logInsertions || logHistogram;
+ }
+
+ /**
+ * Get mean value since last reset.
+ */
+ public double getMean() {
+ Sampling[] values = directory.viewValues();
+ long insertions = 0L;
+ double sum = 0.0d;
+ for (Sampling x : values) {
+ insertions += x.insertions;
+ sum += x.sum;
+ }
+ if (insertions == 0) {
+ return 0.0d;
+ }
+ return sum/insertions;
+ }
+
+ /**
+ * Get minimal value logged since last reset.
+ */
+ public double getMin() {
+ Sampling[] values = directory.viewValues();
+ long insertions = 0L;
+ double min = 0.0d;
+ for (Sampling x : values) {
+ if (x.insertions == 0) {
+ continue;
+ }
+ if (insertions == 0) {
+ min = x.min;
+ } else {
+ min = Math.min(x.min, min);
+ }
+ insertions += x.insertions;
+ }
+ return min;
+ }
+
+ /**
+ * Get maximum value logged since last reset.
+ */
+ public double getMax() {
+ Sampling[] values = directory.viewValues();
+ long insertions = 0L;
+ double max = 0.0d;
+ for (Sampling x : values) {
+ if (x.insertions == 0) {
+ continue;
+ }
+ if (insertions == 0) {
+ max = x.max;
+ } else {
+ max = Math.max(x.max, max);
+ }
+ insertions += x.insertions;
+ }
+ return max;
+ }
+
+ private Histogram getHistogram() {
+ if (histogram == null) {
+ return null;
+ } else {
+ Sampling[] values = directory.viewValues();
+ Histogram merged = new Histogram(histogram);
+ for (Sampling s : values) {
+ merged.merge(s.histogram);
+ }
+ return merged;
+ }
+ }
+
+ /**
+ * Get last value logged, 0 if nothing logged since reset.
+ */
+ public double get() {
+ return lastValue;
+ }
+
+ /**
+ * Set last value logged container to 0, reset histogram and set all
+ * counters and derived statistics to 0.
+ */
+ public void reset() {
+ if (logComposite()) {
+ directory.fetchValues();
+ }
+ if (logRaw) {
+ lastValue = 0.0d;
+ }
+ }
+
+ /**
+ * Dump state to log and reset.
+ */
+ @Override
+ public void runHandle() {
+ String rawImage = null;
+ String meanImage = null;
+ String minImage = null;
+ String maxImage = null;
+ String sumImage = null;
+ String insertionsImage = null;
+ String histImage = null;
+ String lastHist = null;
+ String histType = null;
+ Snapshot now = getCurrentState();
+
+ if (nameExtension) {
+ if (logRaw) {
+ rawImage = getName();
+ }
+ if (logMean) {
+ meanImage = getName() + appendChar + "mean";
+ }
+ if (logMin) {
+ minImage = getName() + appendChar + "min";
+ }
+ if (logMax) {
+ maxImage = getName() + appendChar + "max";
+ }
+ if (logSum) {
+ sumImage = getName() + appendChar + "sum";
+ }
+ if (logInsertions) {
+ insertionsImage = getName() + appendChar + "insertions";
+ }
+ } else {
+ if (logRaw) {
+ rawImage = getName();
+ } else if (logMean) {
+ meanImage = getName();
+ } else if (logMin) {
+ minImage = getName();
+ } else if (logMax) {
+ maxImage = getName();
+ } else if (logSum) {
+ sumImage = getName();
+ } else if (logInsertions) {
+ insertionsImage = getName();
+ }
+ }
+ if (logHistogram) {
+ histImage = getName();
+ lastHist = now.histogram.toString();
+ histType = histogramId.toString();
+ }
+ if (rawImage != null) {
+ Event.value(rawImage, now.raw);
+ }
+ if (meanImage != null) {
+ Event.value(meanImage, now.mean);
+ }
+ if (minImage != null) {
+ Event.value(minImage, now.min);
+ }
+ if (maxImage != null) {
+ Event.value(maxImage, now.max);
+ }
+ if (histImage != null) {
+ Event.histogram(histImage, lastHist, histType);
+ }
+ if (sumImage != null) {
+ Event.value(sumImage, now.sum);
+ }
+ if (insertionsImage != null) {
+ Event.value(insertionsImage, now.insertions);
+ }
+ }
+
+ public String toString() {
+ if (histogram == null) {
+ return super.toString() + " " + getName();
+ } else {
+ return super.toString() + " " + getName() + " " + getHistogram().toString();
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o.getClass() != this.getClass()) {
+ return false;
+ }
+ Value other = (Value) o;
+ return getName().equals(other.getName());
+ }
+
+ @Override
+ public int hashCode() {
+ return getName().hashCode() + 31 * "Value".hashCode();
+ }
+
+ static class Snapshot {
+ double insertions;
+ double max;
+ double min;
+ double mean;
+ double sum;
+ double raw;
+ Histogram histogram = null;
+
+ Snapshot insertions(double lastInsertions) {
+ this.insertions = lastInsertions;
+ return this;
+ }
+
+ Snapshot max(double lastMax) {
+ this.max = lastMax;
+ return this;
+ }
+
+ Snapshot min(double lastMin) {
+ this.min = lastMin;
+ return this;
+ }
+
+ Snapshot mean(double lastMean) {
+ this.mean = lastMean;
+ return this;
+ }
+
+ Snapshot sum(double lastSum) {
+ this.sum = lastSum;
+ return this;
+ }
+
+ Snapshot raw(double lastRaw) {
+ this.raw = lastRaw;
+ return this;
+ }
+
+ Snapshot histogram(Histogram mergedHistogram) {
+ this.histogram = mergedHistogram;
+ return this;
+ }
+ }
+
+ private Snapshot getCurrentState() {
+ double lastInsertions = 0L;
+ double lastMax = 0.0d;
+ double lastMin = 0.0d;
+ double lastMean = 0.0d;
+ double lastSum = 0.0d;
+ double lastRaw = 0.0d;
+ Histogram mergedHistogram = null;
+
+ if (logRaw) {
+ lastRaw = lastValue;
+ }
+ if (logComposite()) {
+ Sampling[] lastInterval = directory.fetchValues();
+ if (histogram != null) {
+ mergedHistogram = new Histogram(histogram);
+ }
+ for (Sampling threadData : lastInterval) {
+ if (threadData.insertions == 0) {
+ continue;
+ }
+ if (lastInsertions == 0L) {
+ lastMax = threadData.max;
+ lastMin = threadData.min;
+ } else {
+ lastMax = Math.max(threadData.max, lastMax);
+ lastMin = Math.min(threadData.min, lastMin);
+ }
+ lastSum += threadData.sum;
+ if (mergedHistogram != null) {
+ mergedHistogram.merge(threadData.histogram);
+ }
+ lastInsertions += threadData.insertions;
+ }
+ if (lastInsertions == 0L) {
+ lastMean = 0.0d;
+ } else {
+ lastMean = lastSum / lastInsertions;
+ }
+ }
+ return new Snapshot().insertions(lastInsertions)
+ .max(lastMax).mean(lastMean).min(lastMin)
+ .raw(lastRaw).sum(lastSum)
+ .histogram(mergedHistogram);
+ }
+
+ ValueProxy getProxyAndReset() {
+ ValueProxy p = new ValueProxy(getName());
+ Snapshot now = getCurrentState();
+
+ if (logRaw) {
+ p.setRaw(now.raw);
+ }
+ if (logMean) {
+ p.setMean(now.mean);
+ }
+ if (logMin) {
+ p.setMin(now.min);
+ }
+ if (logMax) {
+ p.setMax(now.max);
+ }
+ if (logHistogram) {
+ p.setHistogram(now.histogram);
+ }
+
+ return p;
+ }
+}
+
diff --git a/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java b/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java
new file mode 100644
index 00000000000..e9886c7757c
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java
@@ -0,0 +1,120 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+import com.yahoo.log.event.Event;
+import com.yahoo.statistics.Value.Parameters;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.HashMap;
+
+/**
+ * A set of related values which should be logged together.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class ValueGroup extends Group {
+ // A map for names of subevents and Value instances
+ private Map<String, Value> subEvents = new HashMap<>();
+
+ /**
+ * Create a ValueGroup.
+ *
+ * @param name
+ * The symbolic name of this group of values
+ * @param manager
+ * the statistics manager acquired by injection
+ */
+ public ValueGroup(String name, Statistics manager) {
+ this(name, manager, null);
+ }
+
+ /**
+ * Create a ValueGroup.
+ *
+ * @param name
+ * The symbolic name of this group of values
+ * @param manager
+ * the statistics manager acquired by injection
+ * @param callback
+ * will be invoked each time data is written to the log
+ */
+ public ValueGroup(String name, Statistics manager, Callback callback) {
+ super(name, manager, callback);
+ manager.register(this);
+ }
+
+ /**
+ * Put a value into the named value in the group.
+ */
+ public void put(String name, double x) {
+ Value v = getValue(name);
+ v.put(x);
+ }
+
+ /**
+ * Get a value with a given name, creates a new value if no
+ * value with the name given exists.
+ *
+ */
+ synchronized Value getValue(String name) {
+ Value v = subEvents.get(name);
+ if (v == null) {
+ v = getNewValue(name);
+ }
+ return v;
+ }
+
+ private Value getNewValue(String subName) {
+ Value v = Value.initializeUnregisteredValue(subName, new Parameters().setLogRaw(true));
+ subEvents.put(subName, v);
+ return v;
+ }
+
+ /**
+ * Dump state to log and reset.
+ */
+ @Override
+ public void runHandle() {
+ StringBuilder multi = new StringBuilder();
+ ValueProxy[] proxies;
+ int i = 0;
+
+ synchronized (this) {
+ proxies = new ValueProxy[subEvents.size()];
+ i = 0;
+ for (Iterator<Value> j = subEvents.values().iterator(); j.hasNext();) {
+ Value v = j.next();
+ proxies[i] = v.getProxyAndReset();
+ i++;
+ }
+ }
+
+ while (i > 0) {
+ i--;
+ if (multi.length() > 0) {
+ multi.append(", ");
+ }
+ multi.append(proxies[i].getName());
+ multi.append("=");
+ multi.append(proxies[i].getRaw());
+ }
+
+ Event.valueGroup(getName(), multi.toString());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o.getClass() != this.getClass()) {
+ return false;
+ }
+ ValueGroup other = (ValueGroup) o;
+ return getName().equals(other.getName());
+ }
+
+ @Override
+ public int hashCode() {
+ return getName().hashCode() + 31 * "ValueGroup".hashCode();
+ }
+}
diff --git a/statistics/src/main/java/com/yahoo/statistics/ValueProxy.java b/statistics/src/main/java/com/yahoo/statistics/ValueProxy.java
new file mode 100644
index 00000000000..1d36d6fe796
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/ValueProxy.java
@@ -0,0 +1,83 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+/**
+ * To be able to cache events concerning Values internally, group them
+ * together and similar.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+class ValueProxy extends Proxy {
+ private double raw;
+ private boolean hasRaw = false;
+ private double min;
+ private boolean hasMin = false;
+ private double mean;
+ private boolean hasMean = false;
+ private double max;
+ private boolean hasMax = false;
+ private Histogram histogram;
+ private boolean hasHistogram;
+
+ ValueProxy(String name) {
+ super(name);
+ }
+
+ boolean hasRaw() {
+ return hasRaw;
+ }
+ double getRaw() {
+ return raw;
+ }
+ void setRaw(double raw) {
+ hasRaw = true;
+ this.raw = raw;
+ }
+
+ boolean hasMin() {
+ return hasMin;
+ }
+ double getMin() {
+ return min;
+ }
+ void setMin(double min) {
+ hasMin = true;
+ this.min = min;
+ }
+
+ boolean hasMean() {
+ return hasMean;
+ }
+ double getMean() {
+ return mean;
+ }
+ void setMean(double mean) {
+ hasMean = true;
+ this.mean = mean;
+ }
+
+ boolean hasMax() {
+ return hasMax;
+ }
+ double getMax() {
+ return max;
+ }
+ void setMax(double max) {
+ hasMax = true;
+ this.max = max;
+ }
+
+ boolean hasHistogram() {
+ return hasHistogram;
+ }
+ Histogram getHistogram() {
+ return histogram;
+ }
+ void setHistogram(Histogram histogram) {
+ hasHistogram = true;
+ this.histogram = histogram;
+ }
+
+}
+
diff --git a/statistics/src/main/java/com/yahoo/statistics/package-info.java b/statistics/src/main/java/com/yahoo/statistics/package-info.java
new file mode 100644
index 00000000000..07c1644aea4
--- /dev/null
+++ b/statistics/src/main/java/com/yahoo/statistics/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.statistics;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/statistics/src/main/resources/configdefinitions/statistics.def b/statistics/src/main/resources/configdefinitions/statistics.def
new file mode 100644
index 00000000000..586a32eedf7
--- /dev/null
+++ b/statistics/src/main/resources/configdefinitions/statistics.def
@@ -0,0 +1,23 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+version=6
+namespace=container
+
+## Interval between internal sample points measured in seconds
+collectionintervalsec double default=300.0
+
+## Interval between each dump to log measured in seconds
+loggingintervalsec double default=300.0
+
+## Configuration for measured values
+values[].name string
+
+## An array of what operations should be performed on gathered data
+values[].operations[].name enum { RAW, MEAN, MIN, MAX, REGULAR, CUMULATIVE, REVERSE_CUMULATIVE, SUM, INSERTIONS } default=RAW
+
+## Arguments for the operations, typically limits for arrays.
+values[].operations[].arguments[].key string
+values[].operations[].arguments[].value string
+
+## The counters which should be reset between each collection interval,
+## and therefore be logged as values.
+counterresets[].name string
diff --git a/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java b/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java
new file mode 100644
index 00000000000..5e51c3b4754
--- /dev/null
+++ b/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java
@@ -0,0 +1,132 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+import java.util.Arrays;
+import java.util.logging.Handler;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+
+import com.yahoo.container.StatisticsConfig;
+
+/**
+ * Test set for groups of counters.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class CounterGroupTestCase extends junit.framework.TestCase {
+ private volatile boolean gotRecord = false;
+
+ private class CounterGroupHandler extends Handler {
+ // This is for testing CounterProxy
+ @Override
+ public void publish(LogRecord record) {
+ com.yahoo.log.event.CountGroup msg = (com.yahoo.log.event.CountGroup) ((Object[]) record.getParameters())[0];
+ assertEquals("test", msg.getValue("name"));
+ String values = msg.getValue("values");
+ assertFalse("Unexpected value for a.", values.indexOf("a=500") == -1);
+ assertFalse("Unexpected value for b.", values.indexOf("b=1") == -1);
+ assertFalse("Unexpected value for c.", values.indexOf("c=0") == -1);
+ gotRecord = true;
+
+ }
+
+ @Override
+ public void flush() {
+ }
+
+ @Override
+ public void close() throws SecurityException {
+ }
+ }
+
+ public void testBasic() {
+ Logger logger = Logger.getLogger(CounterGroup.class.getName());
+ boolean initUseParentHandlers = logger.getUseParentHandlers();
+ Handler logChecker = new CounterGroupHandler();
+ logger.setUseParentHandlers(false);
+ CounterGroup c = new CounterGroup("test", Statistics.nullImplementation, false);
+ Counter n;
+ c.increment("a");
+ c.increment("b");
+ c.increment("a", 499);
+ n = c.getCounter("a");
+ assertEquals(500, n.get());
+ n = c.getCounter("b");
+ assertEquals(1, n.get());
+ n = c.getCounter("c");
+ assertEquals(0, n.get());
+ logger.addHandler(logChecker);
+ c.run();
+ assertFalse("The logging handler did not really run.", gotRecord == false);
+ // cleanup:
+ logger.removeHandler(logChecker);
+ logger.setUseParentHandlers(initUseParentHandlers);
+ }
+
+ public void testObjectContracts() {
+ CounterGroup c = new CounterGroup("test", Statistics.nullImplementation, false);
+ CounterGroup c2 = new CounterGroup("test", Statistics.nullImplementation, false);
+ c2.increment("nalle");
+ assertEquals(c, c2);
+ assertEquals(c.hashCode(), c2.hashCode());
+ c2 = new CounterGroup("nalle", Statistics.nullImplementation, false);
+ assertFalse("Different names should lead to different hashcodes",
+ c.hashCode() == c2.hashCode());
+ assertFalse("Different names should lead to equals() return false",
+ c.equals(c2));
+ }
+
+ public void testConfigStuff() {
+ Logger logger = Logger.getLogger(CounterGroup.class.getName());
+ boolean initUseParentHandlers = logger.getUseParentHandlers();
+ logger.setUseParentHandlers(false);
+ MockStatistics m = new MockStatistics();
+ final String joppe = "joppe";
+ StatisticsConfig config = new StatisticsConfig(
+ new StatisticsConfig.Builder().counterresets(Arrays
+ .asList(new StatisticsConfig.Counterresets.Builder[] {
+ new StatisticsConfig.Counterresets.Builder().name(joppe) })));
+ m.config = config;
+ CounterGroup c = new CounterGroup("nalle", m);
+ CounterGroup c2 = new CounterGroup(joppe, m);
+ final String bamse = "bamse";
+ c.increment(bamse);
+ c2.increment(bamse);
+ assertEquals(1L, c.getCounter(bamse).get());
+ assertEquals(1L, c2.getCounter(bamse).get());
+ c2.increment(bamse);
+ assertEquals(1L, c.getCounter(bamse).get());
+ assertEquals(2L, c2.getCounter(bamse).get());
+ c.run();
+ c2.run();
+ assertEquals(1L, c.getCounter(bamse).get());
+ assertEquals(0L, c2.getCounter(bamse).get());
+ logger.setUseParentHandlers(initUseParentHandlers);
+ }
+
+ public class MockStatistics implements Statistics {
+ public StatisticsConfig config = null;
+ public int registerCount = 0;
+
+ @Override
+ public void register(Handle h) {
+ registerCount += 1;
+ }
+
+ @Override
+ public void remove(String name) {
+ }
+
+ @Override
+ public StatisticsConfig getConfig() {
+ return config;
+ }
+
+ @Override
+ public int purge() {
+ return 0;
+ }
+ }
+
+}
diff --git a/statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java b/statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java
new file mode 100644
index 00000000000..800e993bb0f
--- /dev/null
+++ b/statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+import java.util.Arrays;
+import java.util.logging.Logger;
+
+import com.yahoo.container.StatisticsConfig;
+
+/**
+ * Check counters work.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+
+public class CounterTestCase extends junit.framework.TestCase {
+
+ public void testBasic() {
+ Counter c = new Counter("test", Statistics.nullImplementation, false);
+ c.increment();
+ assertEquals(1, c.get());
+ c.increment(499);
+ assertEquals(500, c.get());
+ c.reset();
+ assertEquals(500, c.get());
+ c = new Counter("test", Statistics.nullImplementation, false, null, true);
+ c.increment();
+ assertEquals(1, c.get());
+ c.increment(499);
+ assertEquals(500, c.get());
+ c.reset();
+ assertEquals(0, c.get());
+ }
+
+ public void testObjectContracts() {
+ final String counterName = "test";
+ Counter c = new Counter(counterName, Statistics.nullImplementation, false);
+ Counter c2 = new Counter(counterName, Statistics.nullImplementation, false);
+ c2.increment();
+ assertEquals(c, c2);
+ assertEquals(c.hashCode(), c2.hashCode());
+ c2 = new Counter("nalle", Statistics.nullImplementation, false);
+ assertFalse("Different names should lead to different hashcodes",
+ c.hashCode() == c2.hashCode());
+ assertFalse("Different names should lead to equals() return false",
+ c.equals(c2));
+ String prefix = "com.yahoo.statistics.Counter";
+ String suffix = counterName + " 0";
+ String image = c.toString();
+ assertEquals(suffix, image.substring(image.length() - suffix.length()));
+ assertEquals(prefix, image.substring(0, prefix.length()));
+ }
+
+ public void testConfigStuff() {
+ Logger logger = Logger.getLogger(Counter.class.getName());
+ boolean initUseParentHandlers = logger.getUseParentHandlers();
+ logger.setUseParentHandlers(false);
+ MockStatistics m = new MockStatistics();
+ final String joppe = "joppe";
+ StatisticsConfig config = new StatisticsConfig(
+ new StatisticsConfig.Builder().counterresets(Arrays
+ .asList(new StatisticsConfig.Counterresets.Builder[] { new StatisticsConfig.Counterresets.Builder()
+ .name(joppe) })));
+ m.config = config;
+ Counter c = new Counter("nalle", m, true);
+ Counter c2 = new Counter(joppe, m, true);
+ c.increment();
+ c2.increment();
+ assertEquals(1L, c.get());
+ assertEquals(1L, c2.get());
+ c.run();
+ c2.run();
+ assertEquals(1L, c.get());
+ assertEquals(0L, c2.get());
+ logger.setUseParentHandlers(initUseParentHandlers);
+
+ }
+
+ public class MockStatistics implements Statistics {
+ public StatisticsConfig config = null;
+ public int registerCount = 0;
+
+ @Override
+ public void register(Handle h) {
+ registerCount += 1;
+ }
+
+ @Override
+ public void remove(String name) {
+ }
+
+ @Override
+ public StatisticsConfig getConfig() {
+ return config;
+ }
+
+ @Override
+ public int purge() {
+ return 0;
+ }
+ }
+
+}
diff --git a/statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java b/statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java
new file mode 100644
index 00000000000..575feaa93f8
--- /dev/null
+++ b/statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java
@@ -0,0 +1,75 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+
+
+/**
+ * Some low level checking of the histograms.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+
+public class HistogramTestCase extends junit.framework.TestCase {
+
+ public HistogramTestCase(String name) {
+ super(name);
+ }
+
+ public void testFindBucket() {
+ Limits l = new Limits();
+ double[] thresholds = {.5, 1.0, 5.0};
+ double[] value = {.5, .5};
+ l.addAxis("latency", thresholds);
+ thresholds = new double[] {500.0, 1000.0, 5000.0};
+ l.addAxis("size", thresholds);
+ Histogram h = new Histogram(l);
+ assertEquals("latency,size ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 0.5 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 1.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 5.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0))",
+ h.toString());
+ h.put(value);
+ assertEquals("latency,size ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 0.5 ((1) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 1.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 5.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0))",
+ h.toString());
+
+ }
+
+ public void testMerge() {
+ Limits l = new Limits();
+ double[] thresholds = {.5, 1.0, 5.0};
+ double[] value = {.75};
+ l.addAxis("latency", thresholds);
+ Histogram h = new Histogram(l);
+ Histogram h2 = new Histogram(l);
+ h.put(value);
+ h.put(value);
+ h2.put(value);
+ h2.merge(h);
+ assertEquals("(0) < 0.5 (3) < 1.0 (0) < 5.0 (0)", h2.toString());
+ }
+
+ public void testMultiDimensionalMerge() {
+ Limits l = new Limits();
+ double[] thresholds = {.5, 1.0, 5.0};
+ double[] value = {.5, .5};
+ l.addAxis("latency", thresholds);
+ thresholds = new double[] {500.0, 1000.0, 5000.0};
+ l.addAxis("size", thresholds);
+ Histogram h = new Histogram(l);
+ Histogram h2 = new Histogram(l);
+ h.put(value);
+ assertEquals("latency,size ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 0.5 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 1.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 5.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0))",
+ h2.toString());
+ h2.merge(h);
+ assertEquals("latency,size ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 0.5 ((1) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 1.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 5.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0))",
+ h.toString());
+ }
+
+ public void testEmptyHistogram() {
+ try {
+ new Histogram(new Limits());
+ } catch (IndexOutOfBoundsException e) {
+ return;
+ }
+ fail("Got no exception when trying to create an empty histogram.");
+ }
+
+}
diff --git a/statistics/src/test/java/com/yahoo/statistics/ProxyTestCase.java b/statistics/src/test/java/com/yahoo/statistics/ProxyTestCase.java
new file mode 100644
index 00000000000..a7fc02a1fbf
--- /dev/null
+++ b/statistics/src/test/java/com/yahoo/statistics/ProxyTestCase.java
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+import static org.junit.Assert.*;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tabular checking of statistics proxies.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class ProxyTestCase {
+ private static final double MAX = 2.0d;
+ private static final double MEAN = 1.0d;
+ private static final double MIN = -1.0d;
+ private static final double RAW = 0.5d;
+ private static final long C_RAW = 3;
+ ValueProxy vp;
+ CounterProxy cp;
+ Histogram h = new Histogram(new Limits(new double[] { 1.0d }));
+
+ @Before
+ public void setUp() throws Exception {
+ vp = new ValueProxy("nalle");
+ vp.setRaw(RAW);
+ vp.setMin(MIN);
+ vp.setMean(MEAN);
+ vp.setMax(MAX);
+ vp.setHistogram(h);
+ cp = new CounterProxy("nalle");
+ cp.setRaw(C_RAW);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ @Test
+ public final void test() {
+ assertFalse(vp.hasHistogram() == false);
+ assertFalse(vp.hasRaw() == false);
+ assertFalse(vp.hasMin() == false);
+ assertFalse(vp.hasMean() == false);
+ assertFalse(vp.hasMax() == false);
+ assertFalse(cp.hasRaw() == false);
+ assertEquals(C_RAW, cp.getRaw());
+ assertEquals(MAX, vp.getMax(), 1e-9);
+ assertEquals(MEAN, vp.getMean(), 1e-9);
+ assertEquals(MIN, vp.getMin(), 1e-9);
+ assertEquals(RAW, vp.getRaw(), 1e-9);
+ assertSame(h, vp.getHistogram());
+
+ final long t = 11L;
+ Proxy p = new Proxy("nalle", t) {
+ };
+ assertEquals(t, p.getTimestamp());
+ }
+
+}
diff --git a/statistics/src/test/java/com/yahoo/statistics/StatisticsImplTestCase.java b/statistics/src/test/java/com/yahoo/statistics/StatisticsImplTestCase.java
new file mode 100644
index 00000000000..e6694a0b654
--- /dev/null
+++ b/statistics/src/test/java/com/yahoo/statistics/StatisticsImplTestCase.java
@@ -0,0 +1,108 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+
+import java.util.logging.Logger;
+
+import org.junit.Test;
+
+import com.yahoo.container.StatisticsConfig;
+
+/**
+ * Check register/remove semantics.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class StatisticsImplTestCase {
+ private static class TestHandle extends Handle {
+
+ TestHandle(final String name, final Statistics manager,
+ final Callback parametrizedCallback) {
+ super(name, manager, parametrizedCallback);
+ }
+
+ @Override
+ public void runHandle() {
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (o == this) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getName().hashCode();
+ }
+
+ }
+
+ @Test
+ public final void testRegister() {
+ final StatisticsConfig config = new StatisticsConfig(
+ new StatisticsConfig.Builder().collectionintervalsec(31e9)
+ .loggingintervalsec(31e9));
+ final StatisticsImpl s = new StatisticsImpl(config);
+ final Logger logger = Logger.getLogger(TestHandle.class.getName());
+ final boolean initUseParentHandlers = logger.getUseParentHandlers();
+ logger.setUseParentHandlers(false);
+ final String firstHandle = "a";
+ final Handle a = new TestHandle(firstHandle, s, null);
+ final Handle a2 = new TestHandle(firstHandle, s, null);
+ final String secondHandle = "b";
+ final Handle b = new TestHandle(secondHandle, s, null);
+ s.register(a);
+ s.register(a2);
+ assertFalse("Old handle should be cancelled.", a.isCancelled() == false);
+ assertFalse("New handle should not be cancelled.", a2.isCancelled());
+ assertEquals("Internal handles map semantics have been changed?", 1,
+ s.handles.size());
+ s.register(b);
+ s.remove(secondHandle);
+ assertFalse("Removed handle should be cancelled.",
+ b.isCancelled() == false);
+ a2.cancel();
+ s.purge();
+ assertEquals("Cancelled tasks should be removed.", 0, s.handles.size());
+ s.deconstruct();
+ assertSame(config, s.getConfig());
+ logger.setUseParentHandlers(initUseParentHandlers);
+ }
+
+ @Test
+ public void freezeNullImplementationBehavior() {
+ Statistics s = Statistics.nullImplementation;
+ assertEquals(0, s.purge());
+ // invoke s.register
+ Handle h = new Handle("nalle", s, null) {
+ @Override
+ public void runHandle() {
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ };
+ assertEquals(0, s.purge());
+ s.register(h);
+ s.remove("nalle");
+ s.register(h);
+ h.cancel();
+ assertEquals(0, s.purge());
+ }
+
+}
diff --git a/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java b/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java
new file mode 100644
index 00000000000..1491575626f
--- /dev/null
+++ b/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java
@@ -0,0 +1,116 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+
+import java.util.logging.Handler;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+
+import com.yahoo.container.StatisticsConfig;
+
+/**
+ * Test set for groups of values.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class ValueGroupTestCase extends junit.framework.TestCase {
+ private volatile boolean gotRecord = false;
+
+ private class ValueGroupHandler extends Handler {
+ // this is for testing ValueProxy
+
+ @Override
+ public void publish(LogRecord record) {
+ com.yahoo.log.event.ValueGroup msg = (com.yahoo.log.event.ValueGroup) ((Object[]) record.getParameters())[0];
+ assertEquals("test", msg.getValue("name"));
+ String values = msg.getValue("values");
+ assertFalse("Unexpected value for a.", values.indexOf("a=-50.0") == -1);
+ assertFalse("Unexpected value for b.", values.indexOf("b=40.0") == -1);
+ assertFalse("Unexpected value for c.", values.indexOf("c=0.0") == -1);
+ gotRecord = true;
+ }
+
+ @Override
+ public void flush() {
+ }
+
+ @Override
+ public void close() throws SecurityException {
+ }
+ }
+
+ public void testBasic() {
+ Logger logger = Logger.getLogger(ValueGroup.class.getName());
+ boolean initUseParentHandlers = logger.getUseParentHandlers();
+ Handler logChecker = new ValueGroupHandler();
+ logger.setUseParentHandlers(false);
+ final MockStatistics manager = new MockStatistics();
+ ValueGroup v = new ValueGroup("test", manager);
+ v.put("a", 50.0);
+ v.put("b", 40.0);
+ v.put("a", -50.0);
+ assertTrue("Last value inserted to a was -50",
+ -50.0 == v.getValue("a").get());
+ assertTrue("Last value inserted to b was 40.",
+ 40.0 == v.getValue("b").get());
+ assertTrue("c has not been used yet",
+ 0.0 == v.getValue("c").get());
+ logger.addHandler(logChecker);
+ v.run();
+ assertFalse("The logging handler did not really run.", gotRecord == false);
+ assertEquals(1, manager.registerCount);
+ // cleanup:
+ logger.removeHandler(logChecker);
+ logger.setUseParentHandlers(initUseParentHandlers);
+ }
+
+ public void testOverlappingSubnames() {
+ final MockStatistics manager = new MockStatistics();
+ ValueGroup v = new ValueGroup("jappe", manager);
+ ValueGroup v2 = new ValueGroup("nalle", manager);
+ final String name = "mobil";
+ v.put(name, 50.0);
+ v2.put(name, 40.0);
+ assertEquals(50.0, v.getValue(name).get(), 1e-9);
+ assertEquals(40.0, v2.getValue(name).get(), 1e-9);
+ assertEquals(2, manager.registerCount);
+ }
+
+ public void testObjectContracts() {
+ ValueGroup v = new ValueGroup("test", new MockStatistics());
+ ValueGroup v2 = new ValueGroup("test", new MockStatistics());
+ v2.put("nalle", 2.0);
+ assertEquals(v, v2);
+ assertEquals(v.hashCode(), v2.hashCode());
+ v2 = new ValueGroup("nalle", new MockStatistics());
+ assertFalse("Different names should lead to different hashcodes",
+ v.hashCode() == v2.hashCode());
+ assertFalse("Different names should lead to equals() return false",
+ v.equals(v2));
+ }
+
+ public class MockStatistics implements Statistics {
+ public StatisticsConfig config = null;
+ public int registerCount = 0;
+
+ @Override
+ public void register(Handle h) {
+ registerCount += 1;
+ }
+
+ @Override
+ public void remove(String name) {
+ }
+
+ @Override
+ public StatisticsConfig getConfig() {
+ return config;
+ }
+
+ @Override
+ public int purge() {
+ return 0;
+ }
+ }
+
+}
diff --git a/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java b/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java
new file mode 100644
index 00000000000..b837b775040
--- /dev/null
+++ b/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java
@@ -0,0 +1,284 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.statistics;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Handler;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+
+import com.yahoo.container.StatisticsConfig;
+import static com.yahoo.container.StatisticsConfig.Values.Operations;
+import com.yahoo.statistics.Value.Parameters;
+
+/**
+ * Check correct statistics are generated for basic values.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class ValueTestCase extends junit.framework.TestCase {
+
+ private static final String NALLE = "nalle";
+ private static final double SECOND = 43.0d;
+ private static final double FIRST = 42.0d;
+
+ private static class TrivialCallback implements Callback {
+
+ @Override
+ public void run(Handle h, boolean firstRun) {
+ Value v = (Value) h;
+ if (firstRun) {
+ v.put(FIRST);
+ } else {
+ v.put(SECOND);
+ }
+ }
+
+ }
+
+ public void testMean() {
+ Value v = new Value("thingie", Statistics.nullImplementation, new Parameters().setLogMean(true));
+ v.put(1.0);
+ v.put(2.0);
+ v.put(4.0);
+ v.put(-1.0);
+ assertTrue("Mean should be 1.5", 1.5 == v.getMean());
+ ValueProxy vp = v.getProxyAndReset();
+ assertTrue("Proxy mean should be 1.5", 1.5 == vp.getMean());
+ assertTrue("Value should have been reset.", 0.0d == v.getMean());
+ }
+
+ public void testMin() {
+ Value v = new Value("thingie", Statistics.nullImplementation, new Parameters().setLogMin(true));
+ v.put(2.0);
+ assertTrue("Min should be 2.0", 2.0 == v.getMin());
+ v.put(1.0);
+ assertTrue("Min should be 1.0", 1.0 == v.getMin());
+ v.put(-1.0);
+ v.put(4.0);
+ assertTrue("Min should be -1.0", -1.0 == v.getMin());
+ }
+
+ public void testMax() {
+ Value v = new Value("thingie", Statistics.nullImplementation, new Parameters().setLogMax(true));
+ v.put(-1.0);
+ assertTrue("Max should be -1.0", -1.0 == v.getMax());
+ v.put(1.0);
+ v.put(2.0);
+ assertTrue("Max should be 2.0", 2.0 == v.getMax());
+ v.put(4.0);
+ v.put(-1.0);
+ assertTrue("Max should be 4.0", 4.0 == v.getMax());
+ }
+
+ public void testHistogram() {
+ Value v = new Value("thingie", Statistics.nullImplementation, new Parameters()
+ .setLogHistogram(true).setHistogramId(HistogramType.REGULAR)
+ .setLimits(new Limits(new double[] { 0.0, 1.0, 2.0 })));
+ v.put(-1.0);
+ v.put(0.0);
+ v.put(1.0);
+ v.put(2.0);
+ v.put(3.0);
+ assertTrue(v.toString().endsWith(
+ " thingie (1) < 0.0 (1) < 1.0 (1) < 2.0 (2)"));
+ }
+
+ public void testCallback() {
+ Logger logger = Logger.getLogger(Value.class.getName());
+ boolean initUseParentHandlers = logger.getUseParentHandlers();
+ logger.setUseParentHandlers(false);
+ Value v = new Value("thingie", Statistics.nullImplementation, new Parameters()
+ .setLogRaw(true).setCallback(new TrivialCallback()));
+ v.run();
+ assertEquals(FIRST, v.get());
+ v.run();
+ assertEquals(SECOND, v.get());
+ v.run();
+ assertEquals(SECOND, v.get());
+ logger.setUseParentHandlers(initUseParentHandlers);
+ }
+
+ public void testParameter() {
+ Value.Parameters p = new Value.Parameters().setLogInsertions(true)
+ .setNameExtension(true).setAppendChar('_');
+ Value.Parameters p2 = new Value.Parameters().setLogSum(true);
+ assertNull(p2.appendChar);
+ assertNull(p.logSum);
+ p2.merge(p);
+ assertEquals(new Character('_'), p2.appendChar);
+ assertNull(p2.logMax);
+ assertEquals(Boolean.TRUE, p2.logSum);
+ }
+
+ private class CheckHistogram extends Handler {
+ volatile boolean gotRecord = false;
+ volatile boolean gotWarning = false;
+ final String histogram;
+ final String representation;
+
+ public CheckHistogram(String histogram, String representation) {
+ this.histogram = histogram;
+ this.representation = representation;
+ }
+
+ @Override
+ public void publish(LogRecord record) {
+ if (record.getParameters() == null) {
+ assertEquals(Value.HISTOGRAM_TYPE_WARNING + " '" + NALLE + "'", record.getMessage());
+ gotWarning = true;
+ return;
+ }
+ if (!(record.getParameters()[0] instanceof com.yahoo.log.event.Histogram)) {
+ return;
+ }
+ com.yahoo.log.event.Histogram msg = (com.yahoo.log.event.Histogram) ((Object[]) record.getParameters())[0];
+ assertEquals(NALLE, msg.getValue("name"));
+ assertEquals(histogram, msg.getValue("counts"));
+ assertEquals(representation, msg.getValue("representation"));
+ gotRecord = true;
+ }
+
+ @Override
+ public void flush() {
+ }
+
+ @Override
+ public void close() throws SecurityException {
+ }
+ }
+
+ public void testParameterFromConfig() {
+ Logger logger = Logger.getLogger(Value.class.getName());
+ boolean initUseParentHandlers = logger.getUseParentHandlers();
+ logger.setUseParentHandlers(false);
+ CheckHistogram h = new CheckHistogram("(0) < 0.0 (0) < 1.0 (0) < 2.0 (1)", "REGULAR");
+ logger.addHandler(h);
+ List<Operations.Arguments.Builder> histogram = Arrays.asList(new Operations.Arguments.Builder[] {
+ new Operations.Arguments.Builder().key("limits").value("0, 1, 2")});
+ List<Operations.Builder> ops = Arrays.asList(new Operations.Builder[] {
+ new Operations.Builder().name(Operations.Name.Enum.MEAN),
+ new Operations.Builder().name(Operations.Name.Enum.MIN),
+ new Operations.Builder().name(Operations.Name.Enum.MAX),
+ new Operations.Builder().name(Operations.Name.Enum.RAW),
+ new Operations.Builder().name(Operations.Name.Enum.INSERTIONS),
+ new Operations.Builder().name(Operations.Name.Enum.REGULAR).arguments(histogram),
+ new Operations.Builder().name(Operations.Name.Enum.SUM) });
+ StatisticsConfig c = new StatisticsConfig(
+ new StatisticsConfig.Builder()
+ .values(new StatisticsConfig.Values.Builder().name(
+ NALLE).operations(ops)));
+ MockStatistics m = new MockStatistics();
+ m.config = c;
+ Value v = Value.buildValue(NALLE, m, null);
+ final double x = 79.0d;
+ v.put(x);
+ assertEquals(x, v.getMean());
+ v.run();
+ assertEquals(true, h.gotRecord);
+ logger.removeHandler(h);
+ logger.setUseParentHandlers(initUseParentHandlers);
+ }
+
+ public void testReverseHistogram() {
+ Logger logger = Logger.getLogger(Value.class.getName());
+ boolean initUseParentHandlers = logger.getUseParentHandlers();
+ logger.setUseParentHandlers(false);
+ CheckHistogram h = new CheckHistogram("(0) < 0.0 (2) < 1.0 (2) < 2.0 (0)", "REGULAR");
+ logger.addHandler(h);
+ List<Operations.Arguments.Builder> histogram = Arrays.asList(new Operations.Arguments.Builder[] {
+ new Operations.Arguments.Builder().key("limits").value("0, 1, 2")});
+ List<Operations.Builder> ops = Arrays.asList(new Operations.Builder[] {
+ new Operations.Builder().name(Operations.Name.Enum.REVERSE_CUMULATIVE).arguments(histogram) });
+ StatisticsConfig c = new StatisticsConfig(
+ new StatisticsConfig.Builder()
+ .values(new StatisticsConfig.Values.Builder().name(
+ NALLE).operations(ops)));
+ MockStatistics m = new MockStatistics();
+ m.config = c;
+ Value v = Value.buildValue(NALLE, m, null);
+ assertEquals(HistogramType.REGULAR.toString(), v.histogramId.toString());
+ v.put(.5d);
+ v.put(.5d);
+ v.put(1.5d);
+ v.put(1.5d);
+ v.run();
+ assertEquals(true, h.gotRecord);
+ assertEquals(true, h.gotWarning);
+ logger.removeHandler(h);
+ logger.setUseParentHandlers(initUseParentHandlers);
+ }
+
+ public void testCumulativeHistogram() {
+ Logger logger = Logger.getLogger(Value.class.getName());
+ boolean initUseParentHandlers = logger.getUseParentHandlers();
+ logger.setUseParentHandlers(false);
+ CheckHistogram h = new CheckHistogram("(0) < 0.0 (2) < 1.0 (2) < 2.0 (0)", "REGULAR");
+ logger.addHandler(h);
+ List<Operations.Arguments.Builder> histogram = Arrays.asList(new Operations.Arguments.Builder[] {
+ new Operations.Arguments.Builder().key("limits").value("0, 1, 2")});
+ List<Operations.Builder> ops = Arrays.asList(new Operations.Builder[] {
+ new Operations.Builder().name(Operations.Name.Enum.CUMULATIVE).arguments(histogram) });
+ StatisticsConfig c = new StatisticsConfig(
+ new StatisticsConfig.Builder()
+ .values(new StatisticsConfig.Values.Builder().name(
+ NALLE).operations(ops)));
+ MockStatistics m = new MockStatistics();
+ m.config = c;
+ Value v = Value.buildValue(NALLE, m, null);
+ assertEquals(HistogramType.REGULAR.toString(), v.histogramId.toString());
+ v.put(.5d);
+ v.put(.5d);
+ v.put(1.5d);
+ v.put(1.5d);
+ v.run();
+ assertEquals(true, h.gotRecord);
+ assertEquals(true, h.gotWarning);
+ logger.removeHandler(h);
+ logger.setUseParentHandlers(initUseParentHandlers);
+ }
+
+ public void testObjectContracts() {
+ final String valueName = "test";
+ Value v = new Value(valueName, Statistics.nullImplementation, Value.defaultParameters());
+ Value v2 = new Value(valueName, Statistics.nullImplementation, Value.defaultParameters());
+ v2.put(1.0);
+ assertEquals(v, v2);
+ assertEquals(v.hashCode(), v2.hashCode());
+ v2 = new Value("nalle", Statistics.nullImplementation, Value.defaultParameters());
+ assertFalse("Different names should lead to different hashcodes",
+ v.hashCode() == v2.hashCode());
+ assertFalse("Different names should lead to equals() return false",
+ v.equals(v2));
+ String image = v.toString();
+ String prefix = "com.yahoo.statistics.Value";
+ assertEquals(prefix, image.substring(0, prefix.length()));
+ assertEquals(valueName, image.substring(image.length() - valueName.length()));
+ }
+
+
+ public class MockStatistics implements Statistics {
+ public StatisticsConfig config = null;
+ public int registerCount = 0;
+
+ @Override
+ public void register(Handle h) {
+ registerCount += 1;
+ }
+
+ @Override
+ public void remove(String name) {
+ }
+
+ @Override
+ public StatisticsConfig getConfig() {
+ return config;
+ }
+
+ @Override
+ public int purge() {
+ return 0;
+ }
+ }
+
+}