From 72231250ed81e10d66bfe70701e64fa5fe50f712 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 15 Jun 2016 23:09:44 +0200 Subject: Publish --- statistics/src/main/java/com/yahoo/.gitignore | 1 + .../src/main/java/com/yahoo/statistics/.gitignore | 0 .../src/main/java/com/yahoo/statistics/Axis.java | 30 + .../src/main/java/com/yahoo/statistics/Bucket.java | 21 + .../main/java/com/yahoo/statistics/Callback.java | 25 + .../main/java/com/yahoo/statistics/Counter.java | 210 +++++ .../java/com/yahoo/statistics/CounterGroup.java | 167 ++++ .../java/com/yahoo/statistics/CounterProxy.java | 31 + .../src/main/java/com/yahoo/statistics/Group.java | 14 + .../src/main/java/com/yahoo/statistics/Handle.java | 106 +++ .../main/java/com/yahoo/statistics/Histogram.java | 240 ++++++ .../java/com/yahoo/statistics/HistogramType.java | 42 + .../src/main/java/com/yahoo/statistics/Limits.java | 69 ++ .../src/main/java/com/yahoo/statistics/Proxy.java | 31 + .../java/com/yahoo/statistics/SampleDirectory.java | 59 ++ .../main/java/com/yahoo/statistics/SampleSet.java | 102 +++ .../main/java/com/yahoo/statistics/Statistics.java | 59 ++ .../java/com/yahoo/statistics/StatisticsImpl.java | 154 ++++ .../src/main/java/com/yahoo/statistics/Sum.java | 74 ++ .../src/main/java/com/yahoo/statistics/Value.java | 846 +++++++++++++++++++++ .../main/java/com/yahoo/statistics/ValueGroup.java | 120 +++ .../main/java/com/yahoo/statistics/ValueProxy.java | 83 ++ .../java/com/yahoo/statistics/package-info.java | 7 + 23 files changed, 2491 insertions(+) create mode 100644 statistics/src/main/java/com/yahoo/.gitignore create mode 100644 statistics/src/main/java/com/yahoo/statistics/.gitignore create mode 100644 statistics/src/main/java/com/yahoo/statistics/Axis.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/Bucket.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/Callback.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/Counter.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/CounterGroup.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/CounterProxy.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/Group.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/Handle.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/Histogram.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/HistogramType.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/Limits.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/Proxy.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/SampleSet.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/Statistics.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/Sum.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/Value.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/ValueGroup.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/ValueProxy.java create mode 100644 statistics/src/main/java/com/yahoo/statistics/package-info.java (limited to 'statistics/src/main/java/com/yahoo') diff --git a/statistics/src/main/java/com/yahoo/.gitignore b/statistics/src/main/java/com/yahoo/.gitignore new file mode 100644 index 00000000000..04204c7c9d0 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/.gitignore @@ -0,0 +1 @@ +config diff --git a/statistics/src/main/java/com/yahoo/statistics/.gitignore b/statistics/src/main/java/com/yahoo/statistics/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/statistics/src/main/java/com/yahoo/statistics/Axis.java b/statistics/src/main/java/com/yahoo/statistics/Axis.java new file mode 100644 index 00000000000..baec19c8f24 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/Axis.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + +import java.util.Arrays; + + +/** + * A wrapper class for representing a single axis of an n-dimensional + * histogram. + * + * @author Steinar Knutsen + */ +class Axis { + private final double[] limits; + private final String name; + + Axis(String name, double[] limits) { + this.limits = Arrays.copyOf(limits, limits.length); + this.name = name; + } + + double[] getLimits() { + return limits; + } + + String getName() { + return name; + } + +} diff --git a/statistics/src/main/java/com/yahoo/statistics/Bucket.java b/statistics/src/main/java/com/yahoo/statistics/Bucket.java new file mode 100644 index 00000000000..3bd9dbe3301 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/Bucket.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + +import java.util.List; + + +/** + * A bucket in a multidimensional histogram. + * + * @author Steinar Knutsen + */ +interface Bucket { + void put(double[] value, int dim); + void reset(); + double lowerLimit(); + double upperLimit(); + boolean isLeaf(); + List getBuckets(); + long getSum(); + void add(long n); +} diff --git a/statistics/src/main/java/com/yahoo/statistics/Callback.java b/statistics/src/main/java/com/yahoo/statistics/Callback.java new file mode 100644 index 00000000000..8f826cb4097 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/Callback.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + +/** + * Definition of the callback interface for the statistics API. It is a common + * use case to need a reference to the Handle (e.g. Value or Counter) which a + * callback is related to. Since everything in a Handle since 5.1.4 is fully + * initialized from the constructor, it became cumbersome to use Runnable for + * the callback and this interface came into use. + * + * @author Steinar Knutsen + * @since 5.1.4 + */ +public interface Callback { + /** + * Invoked each logging cycle right before the events for a Handle are + * emitted to the log. + * + * @param h + * the handle which invoked this callback + * @param firstTime + * true the first time the method is invoked from h, false later + */ + public void run(Handle h, boolean firstTime); +} diff --git a/statistics/src/main/java/com/yahoo/statistics/Counter.java b/statistics/src/main/java/com/yahoo/statistics/Counter.java new file mode 100644 index 00000000000..a17ef5824dc --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/Counter.java @@ -0,0 +1,210 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + + +import java.util.concurrent.atomic.AtomicLong; + +import com.yahoo.log.event.Event; +import com.yahoo.container.StatisticsConfig; + + +/** + * A single integer value which can be incremented. + * + * @author Steinar Knutsen + */ +public class Counter extends Handle { + // The current value of this counter + private AtomicLong current = new AtomicLong(0L); + + // Whether or not this counter shall be reset between each logging + // interval + private final boolean resetCounter; + + /** + * A monotonically increasing 64 bit integer value. + * + * @param name + * The name of this counter, for use in logging. + * @param manager + * the statistics manager acquired by injection + * @param fetchParametersFromConfig + * Whether or not this counter should be initialized from config. + */ + public Counter(String name, Statistics manager, boolean fetchParametersFromConfig) { + this(name, manager, fetchParametersFromConfig, null, false, true); + } + + /** + * A monotonically increasing 64 bit integer value. + * + * @param name + * The name of this counter, for use in logging. + * @param manager + * the statistics manager acquired by injection + * @param fetchParametersFromConfig + * Whether or not this counter should be initialized from config. + * @param callback + * will be invoked each time this counter is written to the log + * @param resetCounter + * Control for if this Counter should be reset between each + * logging interval. + */ + public Counter(String name, Statistics manager, + boolean fetchParametersFromConfig, Callback callback, boolean resetCounter) { + this(name, manager, fetchParametersFromConfig, callback, + resetCounter, true); + } + + /** + * A monotonically increasing 64 bit integer value. Do not make this + * constructor public, it is used for creating unregistered counters. + * + * @param name + * The name of this counter, for use in logging. + * @param manager + * the statistics manager acquired by injection + * @param fetchParametersFromConfig + * Whether or not this counter should be initialized from config. + * @param callback + * will be invoked each time this counter is written to the log + * @param resetCounter + * Control for if this Counter should be reset between each + * logging interval. + * @param register + * Whether to register the counter in the statistics manager + */ + private Counter(String name, Statistics manager, + boolean fetchParametersFromConfig, Callback callback, + boolean resetCounter, boolean register) { + super(name, manager, callback); + if (fetchParametersFromConfig) { + StatisticsConfig config = manager.getConfig(); + this.resetCounter = getResetCounter(name, config); + } else { + this.resetCounter = resetCounter; + } + if (register) { + manager.register(this); + } + } + + + /** + * Get a Counter instance not registered in the statistics manager. This is + * used by CounterGroup and should not be made public. + * + * @param name + * The name of this counter, for use in logging. + * @param resetCounter + * Control for if this Counter should be reset between each + * logging interval. + */ + static Counter intializeUnregisteredCounter(String name, + boolean resetCounter) { + return new Counter(name, null, false, null, resetCounter, false); + } + + /** + * If this Counter is set up to read config, configure it + * according to the config given. + */ + private static boolean getResetCounter(String name, StatisticsConfig config) { + for (int i = 0; i < config.counterresets().size(); i++) { + String configName = config.counterresets(i).name(); + if (configName.equals(name)) { + return true; + } + } + return false; + } + + /** + * Increment by 1. + */ + public void increment() { + current.incrementAndGet(); + } + + /** + * Increment by n. + */ + public void increment(long n) { + current.addAndGet(n); + } + + /** + * @return current value of this counter + */ + public long get() { + return current.get(); + } + + /** + * The reset counter is true if this is counter is reset to 0 between each + * logging interval. + * + * @return whether this counter is reset between each logging interval. + */ + public boolean getResetCounter() { + return resetCounter; + } + + /** + * If this counter should be set to 0 between each logging interval, + * do that. + */ + public void reset() { + if (resetCounter) { + current.set(0L); + } + } + + /** + * Log current state and reset. + */ + @Override + public void runHandle() { + String name = getName(); + long lastCurrent; + boolean resetState = getResetCounter(); + + if (resetState) { + lastCurrent = current.getAndSet(0L); + Event.value(name, lastCurrent); + } else { + lastCurrent = current.get(); + Event.count(name, lastCurrent); + } + } + + @Override + public String toString() { + return super.toString() + " " + getName() + " " + current; + } + + CounterProxy getProxyAndReset() { + CounterProxy c = new CounterProxy(getName()); + if (getResetCounter()) { + c.setRaw(current.getAndSet(0L)); + } else { + c.setRaw(current.get()); + } + return c; + } + + @Override + public boolean equals(Object o) { + if (o.getClass() != this.getClass()) { + return false; + } + Counter other = (Counter) o; + return getName().equals(other.getName()); + } + + @Override + public int hashCode() { + return getName().hashCode() + 31 * "Counter".hashCode(); + } + +} diff --git a/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java b/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java new file mode 100644 index 00000000000..8dfdf21cc1a --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java @@ -0,0 +1,167 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + + +import com.yahoo.container.StatisticsConfig; +import com.yahoo.log.event.Event; + +import java.util.Map; +import java.util.HashMap; +import java.util.Iterator; + + +/** + * A set of associated counters. + * + * @author Steinar Knutsen + */ +public class CounterGroup extends Group { + private final boolean resetCounter; + + // A map for names of subevents and Value instances + private Map subEvents = new HashMap<>(); + + /** + * @param name The symbolic name of this group of counters. + */ + public CounterGroup(String name, Statistics manager) { + this(name, manager, true); + } + + /** + * Create a basic group of counter which may or may not depend on config. + * + * @param name + * The symbolic name of this group of counters. + * @param manager + * the statistics manager acquired by injection + * @param fetchParametersFromConfig + * Whether this Group should be configured from config. + */ + public CounterGroup(String name, Statistics manager, + boolean fetchParametersFromConfig) { + this(name, manager, fetchParametersFromConfig, null, false); + } + + /** + * Create a group of counters with a callback included. + * + * @param name + * The symbolic name of this group of counters. + * @param manager + * the statistics manager acquired by injection + * @param fetchParametersFromConfig + * Whether this Group should be configured from config. + * @param callback + * will be invoked each time data is written to the log + * @param resetCounter + * Control for if this group should be reset between each + * logging interval. + */ + public CounterGroup(String name, Statistics manager, + boolean fetchParametersFromConfig, Callback callback, boolean resetCounter) { + + super(name, manager, callback); + if (fetchParametersFromConfig) { + StatisticsConfig config = manager.getConfig(); + this.resetCounter = getResetCounter(name, config); + } else { + this.resetCounter = resetCounter; + } + manager.register(this); + } + + private static boolean getResetCounter(String name, StatisticsConfig config) { + for (int i = 0; i < config.counterresets().size(); i++) { + String configName = config.counterresets(i).name(); + if (configName.equals(name)) { + return true; + } + } + return false; + } + + + /** + * Increment named contained counter by 1. + */ + public void increment(String name) { + Counter c = getCounter(name); + c.increment(); + } + + /** + * Increment named contained counter by n. + */ + public void increment(String name, long n) { + Counter c = getCounter(name); + c.increment(n); + } + + /** + * Get a counter with a given name, creates a new counter if no + * counter with the name given exists. + */ + synchronized Counter getCounter(String name) { + Counter c = subEvents.get(name); + if (c == null) { + c = getNewCounter(name); + } + return c; + } + + private Counter getNewCounter(String subName) { + Counter c = Counter.intializeUnregisteredCounter(subName, resetCounter); + subEvents.put(subName, c); + return c; + } + + /** + * Dump contained counters to log and reset. + */ + @Override + public void runHandle() { + StringBuilder multi = new StringBuilder(); + CounterProxy[] proxies; + int i = 0; + + // this is to make sure the number of events does not change while logging + synchronized (this) { + proxies = new CounterProxy[subEvents.size()]; + i = 0; + for (Iterator j = subEvents.values().iterator(); j + .hasNext();) { + Counter c = j.next(); + proxies[i] = c.getProxyAndReset(); + i++; + } + } + + while (i > 0) { + i--; + if (multi.length() > 0) { + multi.append(", "); + } + multi.append(proxies[i].getName()); + multi.append("="); + multi.append(proxies[i].getRaw()); + } + + Event.countGroup(getName(), multi.toString()); + } + + @Override + public boolean equals(Object o) { + if (o.getClass() != this.getClass()) { + return false; + } + CounterGroup other = (CounterGroup) o; + return getName().equals(other.getName()); + } + + @Override + public int hashCode() { + return getName().hashCode() + 31 * "CounterGroup".hashCode(); + } +} + diff --git a/statistics/src/main/java/com/yahoo/statistics/CounterProxy.java b/statistics/src/main/java/com/yahoo/statistics/CounterProxy.java new file mode 100644 index 00000000000..5c3475bc640 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/CounterProxy.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + + +/** + * To be able to cache events concerning Counters internally, group them + * together and similar. + * + * @author Steinar Knutsen + */ +class CounterProxy extends Proxy { + private long raw; + private boolean hasRaw = false; + + CounterProxy(String name) { + super(name); + } + + boolean hasRaw() { + return hasRaw; + } + long getRaw() { + return raw; + } + void setRaw(long raw) { + hasRaw = true; + this.raw = raw; + } + +} + diff --git a/statistics/src/main/java/com/yahoo/statistics/Group.java b/statistics/src/main/java/com/yahoo/statistics/Group.java new file mode 100644 index 00000000000..6400b38d687 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/Group.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + + +/** + * The base class for groups of counters and values. + * + * @author Steinar Knutsen + */ +abstract class Group extends Handle { + Group(String name, Statistics manager, Callback parametrizedCallback) { + super(name, manager, parametrizedCallback); + } +} diff --git a/statistics/src/main/java/com/yahoo/statistics/Handle.java b/statistics/src/main/java/com/yahoo/statistics/Handle.java new file mode 100644 index 00000000000..6c3d94dad41 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/Handle.java @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + + +import java.util.TimerTask; + + +/** + * Base class for the interface to the statistics framework. + * + * @author Steinar Knutsen + */ +public abstract class Handle { + + private TimerTask task; + + private final String name; + private boolean cancelled; + private final Statistics manager; + private final Callback parametrizedCallback; + private boolean firstTime; + + Handle(String name, Statistics manager, Callback parametrizedCallback) { + this.name = name; + this.manager = manager; + this.parametrizedCallback = parametrizedCallback; + firstTime = true; + } + + String getName() { + return name; + } + + TimerTask makeTask() { + final Handle self = this; + synchronized (self) { + if (task != null) { + task.cancel(); + } + task = new TimerTask() { + public void run() { + self.run(); + } + }; + return task; + } + } + + /** + * Run the callback object. + * + * This will happen at the start of each invocation of a Handle's + * run() method. The callback is presumed to be exception safe. + * If no callback is set, this is a no-op. The callback will need + * to handle any necessary synchronization itself. + */ + public final void runCallback() { + if (parametrizedCallback == null) { + return; + } + parametrizedCallback.run(this, firstTime); + firstTime = false; + } + + /** + * Run the callback object first, then invoke runHandle(). + */ + public final void run() { + runCallback(); + runHandle(); + } + + /** + * Invoke an action to be performed periodically for a statistics Handle. + * + *

Synchronization has to be handled by the method itself. + */ + public abstract void runHandle(); + + /** + * Cancel this Handle and remove it from internal state in Statistics. + * + * @return value of java.util.TimerTask.cancel() + */ + public final boolean cancel() { + boolean ok = (task == null ? false : task.cancel()); + cancelled = true; + manager.purge(); + return ok; + } + + /** + * Returns whether this object has been cancelled or not. + * + * @return true if cancelled + */ + public final boolean isCancelled() { + return cancelled; + } + + @Override + public abstract boolean equals(Object o); + + @Override + public abstract int hashCode(); +} diff --git a/statistics/src/main/java/com/yahoo/statistics/Histogram.java b/statistics/src/main/java/com/yahoo/statistics/Histogram.java new file mode 100644 index 00000000000..83960058dd6 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/Histogram.java @@ -0,0 +1,240 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.ArrayList; + + +/** + * A set of sums or other histograms. + * + * @author Steinar Knutsen + */ +public class Histogram implements Bucket { + // The upper and lower limit for the bucket in another histogram + // this histogram represents. The "outermost" histogram in a + // multidimensional histogram will effectively ignore this, but set + // them to -Inf and Inf for consistency. + private final double lower; + private final double upper; + + // The names of all the axes, only used in "outermost" histogram in + // multi dimensional histogram. + private String axes = null; + + private List buckets = new ArrayList<>(); + + /** + * Build a new histogram using bucket limits from the given Limits + * object. + */ + public Histogram(Limits limits) { + // lower and upper will never be used here, + // but it's nicer with defined values + this(limits, 0, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); + + // generate axis names, but only if there is more than one + int guard = limits.getDimensions(); + if (guard == 1) { + return; + } + StringBuffer names = new StringBuffer(); + int i = 0; + while (i < guard) { + names.append(limits.getAxis(i).getName()); + i++; + if (i < guard) { + names.append(","); + } + } + axes = names.toString(); + } + + private Histogram(Limits limits, int dim, double lower, double upper) { + this.lower = lower; + this.upper = upper; + // last axis, create sum objects instead of histograms + boolean lastAxis = dim == (limits.getDimensions() - 1); + Axis next = limits.getAxis(dim); + double[] delimiters = next.getLimits(); + int i = 0; + double previousBucket = Double.NEGATIVE_INFINITY; + dim++; + while (i < delimiters.length) { + if (lastAxis) { + buckets.add(new Sum(previousBucket, delimiters[i])); + } else { + buckets.add(new Histogram(limits, dim, + previousBucket, delimiters[i])); + } + previousBucket = delimiters[i]; + i++; + } + if (lastAxis) { + buckets.add(new Sum(previousBucket, Double.POSITIVE_INFINITY)); + } else { + buckets.add(new Histogram(limits, dim, + previousBucket, + Double.POSITIVE_INFINITY)); + } + + } + + /** + * Increment the corresponding bucket for this data point by 1. + */ + public synchronized void put(double[] value) { + put(value, 0); + } + + /** + * Increment the corresponding bucket for this data point by 1. + * + * @param dim the index of the first value to consider in value array + */ + @Override + public void put(double[] value, int dim) { + Bucket bucket = findBucket(0, buckets.size(), value[dim]); + bucket.put(value, ++dim); + } + + private Bucket findBucket(int offset, int limit, double value) { + int index = offset + (limit - offset) / 2; + Bucket bucket = buckets.get(index); + if (bucket.lowerLimit() <= value && value < bucket.upperLimit()) { + return bucket; + } else if (bucket.upperLimit() <= value) { + return findBucket(index + 1, limit, value); + } else { // value < bucket.lowerLimit() + return findBucket(offset, index, value); + } + } + + @Override + public String toString() { + StringBuffer s; + int i, t; + + s = new StringBuffer(); + if (axes != null) { + s.append(axes).append(" "); + } + s.append("("); + s.append(buckets.get(0).toString()); + s.append(")"); + + t = buckets.size(); + i = 1; + while (i < t) { + Bucket b = buckets.get(i); + s.append(" < "); + s.append(b.lowerLimit()); + s.append(" ("); + s.append(b.toString()); + s.append(")"); + i += 1; + } + + return s.toString(); + } + + /** + * Reset all contained buckets. + */ + @Override + public void reset() { + for (Iterator i = buckets.iterator(); i.hasNext(); ) { + i.next().reset(); + } + } + + /** + * The lower limit for the bucket this histogram represents. + * + * @return the lower limit for the bucket this histogram represents + */ + @Override + public double lowerLimit() { + return lower; + } + + /** + * The upper limit for the bucket this histogram represents. + * + * @return the upper limit for the bucket this histogram represents + */ + @Override + public double upperLimit() { + return upper; + } + + @Override + public List getBuckets() { + return buckets; + } + + private List getLeaves() { + final class Bookmark { + final int i; + final List buckets; + Bookmark(int i, List buckets) { + this.i = i; + this.buckets = buckets; + } + } + List sums = new ArrayList<>(); + Deque stack = new ArrayDeque<>(); + List current; + int i = 0; + stack.addFirst(new Bookmark(i, buckets)); + while (stack.size() > 0) { + Bookmark currentMark = stack.removeFirst(); + i = currentMark.i; + current = currentMark.buckets; + while (i < current.size()) { + Bucket b = current.get(i++); + if (b.isLeaf()) { + sums.add(b); + } else { + Bookmark marker = new Bookmark(i, current); + stack.addFirst(marker); + i = 0; + current = b.getBuckets(); + } + } + } + return sums; + } + + void merge(Histogram source) { + List src = source.getLeaves(); + List dst = getLeaves(); + if (dst.size() != src.size()) { + throw new IllegalStateException( + "Number of buckets in destination and source not equal. (Source " + + src.size() + ", destination " + dst.size() + "."); + } + for (int i = 0; i < dst.size(); ++i) { + dst.get(i).add(src.get(i).getSum()); + } + } + + @Override + public long getSum() { + throw new RuntimeException("Not implemented."); + } + + @Override + public boolean isLeaf() { + return false; + } + + @Override + public void add(long n) { + throw new IllegalStateException("Can not add directly to a Histogram instance."); + } +} diff --git a/statistics/src/main/java/com/yahoo/statistics/HistogramType.java b/statistics/src/main/java/com/yahoo/statistics/HistogramType.java new file mode 100644 index 00000000000..911069902c7 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/HistogramType.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + + +/** + * Enumeration of how a histogram should be represented from + * admin server. + * + * @author Steinar Knutsen + */ +public final class HistogramType { + private final String representation; + + private HistogramType(String representation) { + this.representation = representation; + } + + public String toString() { return representation; } + + /** + * Basic histograms, each bucket is count representing the bucket's + * defined interval. + */ + public static final HistogramType REGULAR = new HistogramType("REGULAR"); + + /** + * Cumulative histograms, that is, a given bucket contains the count + * for values corresponding to "itself" and all preceding buckets. + */ + public static final HistogramType CUMULATIVE = + new HistogramType("CUMULATIVE"); + + + /** + * Reverse cumulative histograms, that is, a given bucket contains + * the count for values corresponding to "itself" and all following + * buckets. + */ + public static final HistogramType REVERSE_CUMULATIVE = + new HistogramType("REVERSE_CUMULATIVE"); + +} diff --git a/statistics/src/main/java/com/yahoo/statistics/Limits.java b/statistics/src/main/java/com/yahoo/statistics/Limits.java new file mode 100644 index 00000000000..fb792eede14 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/Limits.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + + +import java.util.List; +import java.util.ArrayList; + + +/** + * Limits for a histogram, this is only a wrapper for initializing + * a histogram. + * + * @author Steinar Knutsen + */ +public class Limits { + private final List axes = new ArrayList<>(1); + private boolean frozen = false; + + /** + * Create an empty Limits instance. + */ + public Limits() { + } + + /** + * Create a Limits instance suitable for use in a Value.Parameters instance. + * The instance will be frozen. + */ + public Limits(double[] limits) { + addAxis(null, limits); + freeze(); + } + + /** + * @param name the name of the variable for this axis + * @param limits the bucket limits for this axis + */ + public void addAxis(String name, double[] limits) { + if (frozen) { + throw new IllegalStateException("Can not add more axes to a frozen Limits instance."); + } + axes.add(new Axis(name, limits)); + } + + int getDimensions() { + return axes.size(); + } + + Axis getAxis(int i) { + return axes.get(i); + } + + /** + * Prevent adding any mores axes. + */ + public void freeze() { + this.frozen = true; + } + + /** + * True if further change is not permitted. + * + * @return whether this object now should be considered immutable + */ + public boolean isFrozen() { + return frozen; + } + +} diff --git a/statistics/src/main/java/com/yahoo/statistics/Proxy.java b/statistics/src/main/java/com/yahoo/statistics/Proxy.java new file mode 100644 index 00000000000..46fa4a3190c --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/Proxy.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + + +/** + * Base class for event proxies, which are used to cache and group + * events internally. + * + * @author Steinar Knutsen + */ +abstract class Proxy { + private long timestamp; + private String name; + + Proxy(String name) { + this(name, System.currentTimeMillis()); + } + + Proxy(String name, long timestamp) { + this.timestamp = timestamp; + this.name = name; + } + + long getTimestamp() { + return timestamp; + } + + String getName() { + return name; + } +} diff --git a/statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java b/statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java new file mode 100644 index 00000000000..c920c35ca13 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + +import java.util.ArrayList; +import java.util.List; + +import com.yahoo.statistics.SampleSet.Sampling; + +/** + * Book-keeping class to know which SampleSet instances have unlogged data. + * + * @author Steinar Knutsen + */ +final class SampleDirectory { + private final Object directoryLock = new Object(); + private List directory = new ArrayList<>(200); + + void put(SampleSet s) { + synchronized (directoryLock) { + directory.add(s); + s.setRegisteredForLogging(true); + } + } + + /** + * Get a view of the current generation of data and instantiate a new + * generation. This does the memory barrier two-step for the + * client. + */ + Sampling[] fetchValues() { + Sampling[] copyToReturn; + synchronized (directoryLock) { + List tmpDir = directory; + copyToReturn = new Sampling[tmpDir.size()]; + List newDir = new ArrayList<>(200); + for (int i = 0; i < copyToReturn.length; ++i) { + copyToReturn[i] = tmpDir.get(i).getAndReset(); + } + directory = newDir; + } + return copyToReturn; + } + + /** + * Return a view of the current generation of data. This does the memory + * barrier two-step for the client. + */ + Sampling[] viewValues() { + Sampling[] copy; + synchronized (directoryLock) { + copy = new Sampling[directory.size()]; + for (int i = 0; i < copy.length; ++i) { + copy[i] = directory.get(i).values; + } + } + return copy; + } + +} diff --git a/statistics/src/main/java/com/yahoo/statistics/SampleSet.java b/statistics/src/main/java/com/yahoo/statistics/SampleSet.java new file mode 100644 index 00000000000..572d87aea5e --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/SampleSet.java @@ -0,0 +1,102 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + +/** + * A running set of samples for a Value instance. It is + * used only in a very specific context between the sampling threads (each instance + * is only used by on sampling thread) and the single logging thread. + * + * @author Steinar Knutsen + */ +final class SampleSet { + Sampling values; + final Limits histogramLimits; + private boolean isRegisteredForLogging = false; + + SampleSet(Limits histogramLimits) { + this.histogramLimits = histogramLimits; + values = new Sampling(0.0d, 0L, 0.0d, 0.0d, histogramLimits); + } + + static final class Sampling { + final double sum; + final long insertions; + final double max; + final double min; + final Histogram histogram; + + Sampling(double sum, long insertions, double max, double min, Limits histogramLimits) { + if (histogramLimits != null) { + this.histogram = new Histogram(histogramLimits); + } else { + this.histogram = null; + } + this.sum = sum; + this.insertions = insertions; + this.max = max; + this.min = min; + } + + Sampling(double sum, long insertions, double max, double min, Histogram histogram) { + this.histogram = histogram; + this.sum = sum; + this.insertions = insertions; + this.max = max; + this.min = min; + } + } + + + private Sampling createSampling(double x, Sampling previous) { + double sum = previous.sum; + long insertions = previous.insertions; + double max = previous.max; + double min = previous.min; + + sum += x; + if (insertions == 0) { + max = x; + min = x; + } else { + max = Math.max(x, max); + min = Math.min(x, min); + } + insertions++; + return new Sampling(sum, insertions, max, min, previous.histogram); + } + + /** + * Insert x, do all pertinent operations. (Update histogram, update + * insertion count for calculating mean, etc.) + * + * @return whether this is registered for logging + */ + synchronized boolean put(double x) { + Sampling previous = values; + Histogram histogram = previous.histogram; + if (histogram != null) { + histogram.put(new double[] {x}); + } + values = createSampling(x, previous); + return isRegisteredForLogging; + } + + /** + * Get state and reset it. + */ + synchronized Sampling getAndReset() { + Sampling previous = values; + values = new Sampling(0.0d, 0L, 0.0d, 0.0d, histogramLimits); + setRegisteredForLogging(false); + return previous; + } + + // Setting this state is protected by SampleDirectory.directoryLock. It is + // either set from the logging thread (protected by directoryLock and using + // "this" as a memory barrier through getAndReset()), or from the sampling + // thread protected by directoryLock and without a mem barrier. + void setRegisteredForLogging(boolean isRegisteredForLogging) { + this.isRegisteredForLogging = isRegisteredForLogging; + } + +} diff --git a/statistics/src/main/java/com/yahoo/statistics/Statistics.java b/statistics/src/main/java/com/yahoo/statistics/Statistics.java new file mode 100644 index 00000000000..6ad3f484d79 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/Statistics.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + +import com.yahoo.container.StatisticsConfig; + +/** + * Interface used for registering statistics values and counters for logging. + * + * @author steinar + * @author tonytv + */ +public interface Statistics { + /** + * Add a new handle to be scheduled for periodic logging. If a handle + * already exists with the same name, it will be cancelled and removed from + * the internal state of this object. + */ + void register(Handle h); + + /** + * Remove a named handler from the set of working handlers. + */ + void remove(String name); + + /** + * Get current config used. This may be a null reference, depending on how + * the instance was constructed. + */ + StatisticsConfig getConfig(); + + /** + * Purges all cancelled Handles from internal Map and Timer. + * + * @return return value from java.util.Timer.purge() + */ + int purge(); + + /** A null implementation which ignores all calls and returns the default config */ + public static Statistics nullImplementation=new NullImplementation(); + + static class NullImplementation implements Statistics { + + private StatisticsConfig nullConfig=new StatisticsConfig(new StatisticsConfig.Builder()); + + @Override + public void register(Handle h) { } + + @Override + public void remove(String name) { } + + @Override + public StatisticsConfig getConfig() { return nullConfig; } + + @Override + public int purge() { return 0; } + + } + +} diff --git a/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java b/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java new file mode 100644 index 00000000000..c2a8444da27 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java @@ -0,0 +1,154 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Timer; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.container.StatisticsConfig; + +/** + * Centralized book keeping for statistics module. Times logging and reads + * config. It is normally obtained through injection, but may be initialized + * manually for testing or when an injection mechanism is unavailable. Logging + * will be disabled by initializing the Statistics class with a null config + * object. + * + * @author Steinar Knutsen + */ +public final class StatisticsImpl extends AbstractComponent implements Statistics { + private final Timer worker; + private final StatisticsConfig config; + private static final Logger log = Logger.getLogger(StatisticsImpl.class + .getName()); + private final int collectioninterval; + private final int logginginterval; + // default access for testing only + final Map handles = new HashMap<>(); + + /** + * Build a statistics manager based on the given config values. Use a config + * builder for testing if logging is necessary, set the config argument to + * the constructor to null is logging is not necessary. + * + * @since 5.1.4 + * @param config + * settings for logging interval and configured events. Setting + * it to null disables logging. + * @throws IllegalArgumentException + * if logging interval is smaller than collection interval, or + * collection interval is not a multiplum of logging interval + */ + public StatisticsImpl(final StatisticsConfig config) { + final int l = (int) config.loggingintervalsec(); + final int c = (int) config.collectionintervalsec(); + + if (l != 0 && l < c) { + throw new IllegalArgumentException( + "Logging interval (" + l + ") smaller than collection interval (" + c + ")." + + " New config ignored."); + } + if ((l % c) != 0) { + throw new IllegalArgumentException( + "Collection interval (" + c + ") not multiplum of logging interval (" + l + ")." + + " New config ignored."); + } + this.logginginterval = l; + this.collectioninterval = c; + this.config = config; + this.worker = new Timer(true); + } + + /** + * Cancel internal worker thread and do any other necessary cleanup. The + * internal worker thread is a daemon thread, so not calling this will not + * hamper a clean exit from the VM. + * + * @since 5.1.4 + */ + @Override + public void deconstruct() { + worker.cancel(); + } + + private void schedule(final Handle h) { + if (logginginterval != 0) { + h.run(); + // We use the rather creative assumption that there is + // exactly 24h pr day+night. + final Date d = new Date(); + final long ms = collectioninterval * 1000L; + final long delay = ms - (d.getTime() % (ms)); + worker.scheduleAtFixedRate(h.makeTask(), delay, ms); + } + } + + /** + * Add a new handle to be scheduled for periodic logging. If a handle + * already exists with the same name, it will be cancelled and removed from + * the internal state of this object. + */ + @Override + public void register(final Handle h) { + synchronized (handles) { + final Handle oldHandle = handles.get(h.getName()); + if (oldHandle == h) { + log.log(Level.WARNING, "Handle [" + h + "] already registered"); + return; + } + if (oldHandle != null) { + oldHandle.cancel(); + } + handles.put(h.getName(), h); + if (worker != null) { + schedule(h); + } + } + } + + /** + * Remove a named handler from the set of working handlers. + */ + @Override + public void remove(final String name) { + synchronized (handles) { + final Handle oldHandle = handles.remove(name); + if (oldHandle != null) { + oldHandle.cancel(); + } + } + } + + /** + * Get current config used. This may be a null reference, depending on how + * the instance was constructed. + */ + @Override + public StatisticsConfig getConfig() { + return config; + } + + /** + * Purges all cancelled Handles from internal Map and Timer. + * + * @return return value from java.util.Timer.purge() + */ + @Override + public int purge() { + synchronized (handles) { + final Iterator it = handles.values().iterator(); + while (it.hasNext()) { + final Handle h = it.next(); + if (h.isCancelled()) { + it.remove(); + } + } + return worker == null ? 0 : worker.purge(); + } + } +} diff --git a/statistics/src/main/java/com/yahoo/statistics/Sum.java b/statistics/src/main/java/com/yahoo/statistics/Sum.java new file mode 100644 index 00000000000..87662e38560 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/Sum.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + +import java.util.List; + + +/** + * The innermost part of a histogram, a bucket which only contains a + * counter. + * + * @author Steinar Knutsen + */ +class Sum implements Bucket { + private long sum = 0L; + private double lower; + private double upper; + + Sum(double lower, double upper) { + this.lower = lower; + this.upper = upper; + } + + /** + * Increment this bucket. + */ + public void put(double[] value, int dim) { + sum += 1; + } + + /** + * Set this bucket's count to 0. + */ + public void reset() { + sum = 0L; + } + + /** + * The lower limit for values counted by this bucket. + */ + public double lowerLimit() { + return lower; + } + + /** + * The upper limit for values counted by this bucket. + */ + public double upperLimit() { + return upper; + } + + public String toString() { + return Long.toString(sum); + } + + @Override + public List getBuckets() { + return null; + } + + @Override + public boolean isLeaf() { + return true; + } + + @Override + public long getSum() { + return sum; + } + + @Override + public void add(long n) { + sum += n; + } +} diff --git a/statistics/src/main/java/com/yahoo/statistics/Value.java b/statistics/src/main/java/com/yahoo/statistics/Value.java new file mode 100644 index 00000000000..88ce66990c3 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/Value.java @@ -0,0 +1,846 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + + +import java.util.HashMap; +import java.util.List; +import java.util.logging.Logger; + +import com.yahoo.container.StatisticsConfig; +import com.yahoo.container.StatisticsConfig.Values.Operations; +import com.yahoo.log.LogLevel; +import com.yahoo.log.event.Event; +import com.yahoo.statistics.SampleSet.Sampling; + + +/** + * A statistical variable, typically representing a sampling of an + * arbitrarily changing parameter. + * + * @author Steinar Knutsen + */ +public class Value extends Handle { + // For accumulated values, SampleSet instances are mem barriers between {n + // sampling threads} and {logging thread}. + + // lastValue is a memory barrier between {n sampling threads} and {n + // sampling threads, logging thread}. + + // Therefore, the logging thread first locks SampleDirectory.directoryLock, + // then locks each SampleSet, one by one. The sampling threads _either_ lock + // a single SampleSet _or_ lock SampleDirectory.directoryLock. + + // It is necessary to create a memory relationship between the logging + // threads to ensure the newest sample ends up in the log for logRaw = true. + + private final ThreadLocal sample = new ThreadLocal<>(); + private final SampleDirectory directory = new SampleDirectory(); + + // This must _only_ be touched if logRaw is true + private volatile double lastValue = 0.0d; + + private final boolean logRaw; + private final boolean logMean; + private final boolean logSum; + private final boolean logInsertions; + private final boolean logMax; + private final boolean logMin; + private final boolean logHistogram; + + private final Limits histogram; + private final boolean nameExtension; + final HistogramType histogramId; + private final char appendChar; + + private static final Logger log = Logger.getLogger(Value.class.getName()); + static final String HISTOGRAM_TYPE_WARNING = "Histogram types other than REGULAR currently not supported." + +" Reverting to regular histogram for statistics event"; + + /** + * Parameters for building Value instances. All settings are classes instead + * of primitive types to allow tri-state logic (true, false, unset). + */ + public static class Parameters { + /** + * Log raw values. Raw values are basically the last value logged. + */ + Boolean logRaw; + /** + * Log the sum of all data points for each interval. + */ + Boolean logSum; + /** + * Log the mean value for each interval. + */ + Boolean logMean; + /** + * Log the maximal value observed for each interval. + */ + Boolean logMax; + /** + * Log the minimal value observed for each interval. + */ + Boolean logMin; + /** + * Log the number of observations for each interval. + */ + Boolean logInsertions; + /** + * Whether or not to add an identifying extension (like mean) to event + * names when logging derived values. + * + * It is useful to disable extensions if a only a single dervied value, + * e.g. the mean, is the only thing to be logged. The default is to use + * extensions. + * + * If extensions are disabled, the ability to log more than one of raw + * value, min, max, mean (i.e. the raw value and derived values of the + * same type) is disabled to avoid confusion. Since histograms are not + * Value events, these never have a name extension and are always + * available. + */ + Boolean nameExtension; + /** + * Log a data histogram. + */ + Boolean logHistogram; + /** + * What kind of histogram to log. + * + * @see HistogramType + */ + HistogramType histogramId; + /** + * The limits to use if logging as a histogram. The Limits instance must + * be frozen before using the Parameters instance in a Value constructor + * call. + * + * @see Limits + */ + Limits limits; + /** + * Separator character to use between event name and type of + * nameExtension is set to true. + */ + Character appendChar; + + /** + * This is invoked each time a value is dumped to the log. + * + * @see Handle#runCallback() + */ + Callback callback; + + /** + * Whether to register in the Statistics manager. This is not touched by + * merge and also has no undefined state. In general, a Value should + * always register and not doing so explicitly should not be part of the + * public API. + */ + private boolean register = true; + + /** + * Get a fresh Parameters instance with all features turned/unset. + * Parameters instances may be recycled for construction multiple Value + * instances, but do note any Limits instance added must be frozen. + */ + public Parameters() { + } + + /** + * (De-)Activate logging of raw values. Raw values are basically the + * last value logged. + * + * @return "this" for call chaining + */ + public Parameters setLogRaw(Boolean logRaw) { + this.logRaw = logRaw; + return this; + } + + /** + * (De-)Activate logging the sum of all data points for each interval. + * + * @return "this" for call chaining + */ + public Parameters setLogSum(Boolean logSum) { + this.logSum = logSum; + return this; + } + + /** + * (De)-activate loging the mean value for each interval. + * + * @return "this" for call chaining + */ + public Parameters setLogMean(Boolean logMean) { + this.logMean = logMean; + return this; + } + + /** + * (De-)Activate logging the maximal value observed for each interval. + * + * @return "this" for call chaining + */ + public Parameters setLogMax(Boolean logMax) { + this.logMax = logMax; + return this; + } + + /** + * (De-)Activate logging the minimal value observed for each interval. + * + * @return "this" for call chaining + */ + public Parameters setLogMin(Boolean logMin) { + this.logMin = logMin; + return this; + } + + /** + * (De-)Activate loging the number of observations for each interval. + * + * @return "this" for call chaining + */ + public Parameters setLogInsertions(Boolean logInsertions) { + this.logInsertions = logInsertions; + return this; + } + + /** + * Whether or not to add an identifying extension (like mean) to event + * names when logging derived values. + * + * It is useful to disable extensions if a only a single dervied value, + * e.g. the mean, is the only thing to be logged. The default is to use + * extensions. + * + * If extensions are disabled, the ability to log more than one of raw + * value, min, max, mean (i.e. the raw value and derived values of the + * same type) is disabled to avoid confusion. Since histograms are not + * Value events, these never have a name extension and are always + * available. + * + * @return "this" for call chaining + */ + public Parameters setNameExtension(Boolean nameExtension) { + this.nameExtension = nameExtension; + return this; + } + + /** + * (De-)Activate logging a data histogram. + * + * @return "this" for call chaining + */ + public Parameters setLogHistogram(Boolean logHistogram) { + this.logHistogram = logHistogram; + return this; + } + + /** + * What kind of histogram to log. + * + * @see HistogramType + * + * @return "this" for call chaining + */ + public Parameters setHistogramId(HistogramType histogramId) { + this.histogramId = histogramId; + return this; + } + + /** + * The limits to use if logging as a histogram. The Limits instance must + * be frozen before using the Parameters instance in a Value constructor + * call. + * + * @return "this" for call chaining* + * @see Limits + */ + public Parameters setLimits(Limits limits) { + this.limits = limits; + return this; + } + + /** + * Separator character to use between event name and type of + * nameExtension is set to true. The default is '.'. + * + * @return "this" for call chaining + */ + public Parameters setAppendChar(Character appendChar) { + this.appendChar = appendChar; + return this; + } + + /** + * Set a callback to be invoked each time this Value is written to the + * log. + * + * @param callback + * to be invoked each time the Value is written to the log + * @return "this" for call chaining + */ + public Parameters setCallback(Callback callback) { + this.callback = callback; + return this; + } + + /** + * Set whether to register in the statistics manager. Do note the + * package private access for this method opposed to all the other + * setters. + * + * @param register + * set to false to avoid registering + * @return "this" for call chaining + */ + private Parameters setRegister(boolean register) { + this.register = register; + return this; + } + + /** + * If a member is not set in this, add it from defaults. Do note, this + * applies for both true and false settings, in other words, the default + * may also be used to turn off something if it is undefined in this. + */ + void merge(Parameters defaults) { + if (defaults == null) { + return; + } + this.logRaw = this.logRaw == null ? defaults.logRaw : this.logRaw; + this.logSum = this.logSum == null ? defaults.logSum : this.logSum; + this.logMean = this.logMean == null ? defaults.logMean + : this.logMean; + this.logMax = this.logMax == null ? defaults.logMax : this.logMax; + this.logMin = this.logMin == null ? defaults.logMin : this.logMin; + this.logInsertions = this.logInsertions == null ? defaults.logInsertions + : this.logInsertions; + this.nameExtension = this.nameExtension == null ? defaults.nameExtension + : this.nameExtension; + this.logHistogram = this.logHistogram == null ? defaults.logHistogram + : this.logHistogram; + this.histogramId = this.histogramId == null ? defaults.histogramId + : this.histogramId; + this.limits = this.limits == null ? defaults.limits : this.limits; + this.appendChar = this.appendChar == null ? defaults.appendChar + : this.appendChar; + this.callback = this.callback == null ? defaults.callback + : this.callback; + } + + } + + /** + * Configure a Value instance fully, no raw config access. + * + * @param name + * tag for logging + * @param manager + * the statistics manager acquired by injection + * @param parameters + * all the parameters necessary for initializing a Value + * @throws IllegalStateException + * if Parameters.limits exists and is not frozen + */ + public Value(String name, Statistics manager, Parameters parameters) { + super(name, manager, parameters.callback); + this.logHistogram = isTrue(parameters.logHistogram); + this.logMax = isTrue(parameters.logMax); + this.logMean = isTrue(parameters.logMean); + this.logMin = isTrue(parameters.logMin); + this.logRaw = isTrue(parameters.logRaw); + this.logSum = isTrue(parameters.logSum); + this.logInsertions = isTrue(parameters.logInsertions); + this.nameExtension = isTrue(parameters.nameExtension); + if (logHistogram) { + if (!parameters.limits.isFrozen()) { + throw new IllegalStateException("The Limits instance must be frozen."); + } + if (parameters.histogramId != HistogramType.REGULAR) { + log.log(LogLevel.WARNING, HISTOGRAM_TYPE_WARNING + + " '" + name + "'"); + } + this.histogramId = HistogramType.REGULAR; + this.histogram = parameters.limits; + } else { + this.histogram = null; + this.histogramId = HistogramType.REGULAR; + } + Character appendChar = parameters.appendChar; + if (appendChar == null) { + this.appendChar = '.'; + } else { + this.appendChar = appendChar.charValue(); + } + if (parameters.register) { + manager.register(this); + } + } + + private static boolean isTrue(Boolean b) { + return b != null && b.booleanValue(); + } + + /** + * Build a Value which should be initialized from config. + * + * @param name + * the name of the event in the log + * @param manager + * the current Statistics manager, acquired by injection + * @param defaults + * defaults for values not defined by config, this may be null + */ + public static Value buildValue(String name, Statistics manager, Parameters defaults) { + return new Value(name, manager, buildParameters(name, manager, defaults)); + } + + /** + * Get a Value instance not registered in the statistics manager. This is + * used by ValueGroup and should not be made public. + * + * @param name + * The name of this counter, for use in logging. + * @param parameters + * setting for the new Value + */ + static Value initializeUnregisteredValue(String name, Parameters parameters) { + return new Value(name, null, parameters.setRegister(false)); + } + + private static Parameters buildParameters(String name, Statistics manager, Parameters defaults) { + Parameters p = null; + StatisticsConfig config = manager.getConfig(); + if (config != null) { + for (int i = 0; i < config.values().size(); i++) { + String configName = config.values(i).name(); + if (configName.equals(name)) { + p = parametersFromConfig(config.values(i).operations()); + break; + } + } + } + if (p == null) { + if (defaults == null) { + p = defaultParameters(); + } else { + p = defaults; + } + } else { + p.merge(defaults); + } + return p; + } + + static Parameters defaultParameters() { + return new Parameters().setLogRaw(true).setNameExtension(true); + } + + private static Parameters parametersFromConfig(List o) { + Parameters p = new Parameters().setNameExtension(true); + + for (Operations operation : o) { + Operations.Name.Enum opName = operation.name(); + + HashMap args = new HashMap<>(); + for (int j = 0; j < operation.arguments().size(); j++) { + args.put(operation.arguments(j).key(), operation.arguments(j).value()); + } + + if (opName == Operations.Name.MEAN) { + p.setLogMean(true); + } else if (opName == Operations.Name.MAX) { + p.setLogMax(true); + } else if (opName == Operations.Name.MIN) { + p.setLogMin(true); + } else if (opName == Operations.Name.RAW) { + p.setLogRaw(true); + } else if (opName == Operations.Name.SUM) { + p.setLogSum(true); + } else if (opName == Operations.Name.INSERTIONS) { + p.setLogInsertions(true); + } else if (opName == Operations.Name.REGULAR) { + p.setLogHistogram(true); + p.setHistogramId(HistogramType.REGULAR); + p.setLimits(initHistogram(args.get("axes"), args.get("limits"))); + } else if (opName == Operations.Name.CUMULATIVE) { + p.setLogHistogram(true); + p.setHistogramId(HistogramType.CUMULATIVE); + p.setLimits(initHistogram(args.get("axes"), args.get("limits"))); + } else if (opName == Operations.Name.REVERSE_CUMULATIVE) { + p.setLogHistogram(true); + p.setHistogramId(HistogramType.REVERSE_CUMULATIVE); + p.setLimits(initHistogram(args.get("axes"),args.get("limits"))); + } + } + return p; + } + + private static Limits initHistogram(String axes, String limits) { + String[] borders; + double[] vanillaLimits; + Limits l = new Limits(); + int i = 0; + + if (axes != null) { + throw new RuntimeException("Config of multidimensional histograms not yet implemented."); + } + if (limits == null) { + throw new RuntimeException("Config of histograms needs a list of limits."); + } + borders = limits.split(","); + vanillaLimits = new double[borders.length]; + while (i < vanillaLimits.length) { + vanillaLimits[i] = Double.parseDouble(borders[i].trim()); + ++i; + } + l.addAxis(null, vanillaLimits); + l.freeze(); + return l; + } + + private SampleSet getSample() { + SampleSet s = sample.get(); + if (s == null) { + s = new SampleSet(histogram); + sample.set(s); + } + return s; + } + + private void putComposite(double x) { + SampleSet s = getSample(); + boolean isInDir = s.put(x); + if (!isInDir) { + directory.put(s); + } + } + + /** + * Insert x, do all pertinent operations. (Update histogram, update + * insertion count for calculating mean, etc.) + */ + public void put(double x) { + if (logComposite()) { + putComposite(x); + } + if (logRaw) { + lastValue = x; + } + } + + private boolean logComposite() { + return logMean || logMin || logMax || logSum || logInsertions || logHistogram; + } + + /** + * Get mean value since last reset. + */ + public double getMean() { + Sampling[] values = directory.viewValues(); + long insertions = 0L; + double sum = 0.0d; + for (Sampling x : values) { + insertions += x.insertions; + sum += x.sum; + } + if (insertions == 0) { + return 0.0d; + } + return sum/insertions; + } + + /** + * Get minimal value logged since last reset. + */ + public double getMin() { + Sampling[] values = directory.viewValues(); + long insertions = 0L; + double min = 0.0d; + for (Sampling x : values) { + if (x.insertions == 0) { + continue; + } + if (insertions == 0) { + min = x.min; + } else { + min = Math.min(x.min, min); + } + insertions += x.insertions; + } + return min; + } + + /** + * Get maximum value logged since last reset. + */ + public double getMax() { + Sampling[] values = directory.viewValues(); + long insertions = 0L; + double max = 0.0d; + for (Sampling x : values) { + if (x.insertions == 0) { + continue; + } + if (insertions == 0) { + max = x.max; + } else { + max = Math.max(x.max, max); + } + insertions += x.insertions; + } + return max; + } + + private Histogram getHistogram() { + if (histogram == null) { + return null; + } else { + Sampling[] values = directory.viewValues(); + Histogram merged = new Histogram(histogram); + for (Sampling s : values) { + merged.merge(s.histogram); + } + return merged; + } + } + + /** + * Get last value logged, 0 if nothing logged since reset. + */ + public double get() { + return lastValue; + } + + /** + * Set last value logged container to 0, reset histogram and set all + * counters and derived statistics to 0. + */ + public void reset() { + if (logComposite()) { + directory.fetchValues(); + } + if (logRaw) { + lastValue = 0.0d; + } + } + + /** + * Dump state to log and reset. + */ + @Override + public void runHandle() { + String rawImage = null; + String meanImage = null; + String minImage = null; + String maxImage = null; + String sumImage = null; + String insertionsImage = null; + String histImage = null; + String lastHist = null; + String histType = null; + Snapshot now = getCurrentState(); + + if (nameExtension) { + if (logRaw) { + rawImage = getName(); + } + if (logMean) { + meanImage = getName() + appendChar + "mean"; + } + if (logMin) { + minImage = getName() + appendChar + "min"; + } + if (logMax) { + maxImage = getName() + appendChar + "max"; + } + if (logSum) { + sumImage = getName() + appendChar + "sum"; + } + if (logInsertions) { + insertionsImage = getName() + appendChar + "insertions"; + } + } else { + if (logRaw) { + rawImage = getName(); + } else if (logMean) { + meanImage = getName(); + } else if (logMin) { + minImage = getName(); + } else if (logMax) { + maxImage = getName(); + } else if (logSum) { + sumImage = getName(); + } else if (logInsertions) { + insertionsImage = getName(); + } + } + if (logHistogram) { + histImage = getName(); + lastHist = now.histogram.toString(); + histType = histogramId.toString(); + } + if (rawImage != null) { + Event.value(rawImage, now.raw); + } + if (meanImage != null) { + Event.value(meanImage, now.mean); + } + if (minImage != null) { + Event.value(minImage, now.min); + } + if (maxImage != null) { + Event.value(maxImage, now.max); + } + if (histImage != null) { + Event.histogram(histImage, lastHist, histType); + } + if (sumImage != null) { + Event.value(sumImage, now.sum); + } + if (insertionsImage != null) { + Event.value(insertionsImage, now.insertions); + } + } + + public String toString() { + if (histogram == null) { + return super.toString() + " " + getName(); + } else { + return super.toString() + " " + getName() + " " + getHistogram().toString(); + } + } + + @Override + public boolean equals(Object o) { + if (o.getClass() != this.getClass()) { + return false; + } + Value other = (Value) o; + return getName().equals(other.getName()); + } + + @Override + public int hashCode() { + return getName().hashCode() + 31 * "Value".hashCode(); + } + + static class Snapshot { + double insertions; + double max; + double min; + double mean; + double sum; + double raw; + Histogram histogram = null; + + Snapshot insertions(double lastInsertions) { + this.insertions = lastInsertions; + return this; + } + + Snapshot max(double lastMax) { + this.max = lastMax; + return this; + } + + Snapshot min(double lastMin) { + this.min = lastMin; + return this; + } + + Snapshot mean(double lastMean) { + this.mean = lastMean; + return this; + } + + Snapshot sum(double lastSum) { + this.sum = lastSum; + return this; + } + + Snapshot raw(double lastRaw) { + this.raw = lastRaw; + return this; + } + + Snapshot histogram(Histogram mergedHistogram) { + this.histogram = mergedHistogram; + return this; + } + } + + private Snapshot getCurrentState() { + double lastInsertions = 0L; + double lastMax = 0.0d; + double lastMin = 0.0d; + double lastMean = 0.0d; + double lastSum = 0.0d; + double lastRaw = 0.0d; + Histogram mergedHistogram = null; + + if (logRaw) { + lastRaw = lastValue; + } + if (logComposite()) { + Sampling[] lastInterval = directory.fetchValues(); + if (histogram != null) { + mergedHistogram = new Histogram(histogram); + } + for (Sampling threadData : lastInterval) { + if (threadData.insertions == 0) { + continue; + } + if (lastInsertions == 0L) { + lastMax = threadData.max; + lastMin = threadData.min; + } else { + lastMax = Math.max(threadData.max, lastMax); + lastMin = Math.min(threadData.min, lastMin); + } + lastSum += threadData.sum; + if (mergedHistogram != null) { + mergedHistogram.merge(threadData.histogram); + } + lastInsertions += threadData.insertions; + } + if (lastInsertions == 0L) { + lastMean = 0.0d; + } else { + lastMean = lastSum / lastInsertions; + } + } + return new Snapshot().insertions(lastInsertions) + .max(lastMax).mean(lastMean).min(lastMin) + .raw(lastRaw).sum(lastSum) + .histogram(mergedHistogram); + } + + ValueProxy getProxyAndReset() { + ValueProxy p = new ValueProxy(getName()); + Snapshot now = getCurrentState(); + + if (logRaw) { + p.setRaw(now.raw); + } + if (logMean) { + p.setMean(now.mean); + } + if (logMin) { + p.setMin(now.min); + } + if (logMax) { + p.setMax(now.max); + } + if (logHistogram) { + p.setHistogram(now.histogram); + } + + return p; + } +} + diff --git a/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java b/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java new file mode 100644 index 00000000000..e9886c7757c --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java @@ -0,0 +1,120 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + + +import com.yahoo.log.event.Event; +import com.yahoo.statistics.Value.Parameters; + +import java.util.Iterator; +import java.util.Map; +import java.util.HashMap; + +/** + * A set of related values which should be logged together. + * + * @author Steinar Knutsen + */ +public class ValueGroup extends Group { + // A map for names of subevents and Value instances + private Map subEvents = new HashMap<>(); + + /** + * Create a ValueGroup. + * + * @param name + * The symbolic name of this group of values + * @param manager + * the statistics manager acquired by injection + */ + public ValueGroup(String name, Statistics manager) { + this(name, manager, null); + } + + /** + * Create a ValueGroup. + * + * @param name + * The symbolic name of this group of values + * @param manager + * the statistics manager acquired by injection + * @param callback + * will be invoked each time data is written to the log + */ + public ValueGroup(String name, Statistics manager, Callback callback) { + super(name, manager, callback); + manager.register(this); + } + + /** + * Put a value into the named value in the group. + */ + public void put(String name, double x) { + Value v = getValue(name); + v.put(x); + } + + /** + * Get a value with a given name, creates a new value if no + * value with the name given exists. + * + */ + synchronized Value getValue(String name) { + Value v = subEvents.get(name); + if (v == null) { + v = getNewValue(name); + } + return v; + } + + private Value getNewValue(String subName) { + Value v = Value.initializeUnregisteredValue(subName, new Parameters().setLogRaw(true)); + subEvents.put(subName, v); + return v; + } + + /** + * Dump state to log and reset. + */ + @Override + public void runHandle() { + StringBuilder multi = new StringBuilder(); + ValueProxy[] proxies; + int i = 0; + + synchronized (this) { + proxies = new ValueProxy[subEvents.size()]; + i = 0; + for (Iterator j = subEvents.values().iterator(); j.hasNext();) { + Value v = j.next(); + proxies[i] = v.getProxyAndReset(); + i++; + } + } + + while (i > 0) { + i--; + if (multi.length() > 0) { + multi.append(", "); + } + multi.append(proxies[i].getName()); + multi.append("="); + multi.append(proxies[i].getRaw()); + } + + Event.valueGroup(getName(), multi.toString()); + } + + @Override + public boolean equals(Object o) { + if (o.getClass() != this.getClass()) { + return false; + } + ValueGroup other = (ValueGroup) o; + return getName().equals(other.getName()); + } + + @Override + public int hashCode() { + return getName().hashCode() + 31 * "ValueGroup".hashCode(); + } +} diff --git a/statistics/src/main/java/com/yahoo/statistics/ValueProxy.java b/statistics/src/main/java/com/yahoo/statistics/ValueProxy.java new file mode 100644 index 00000000000..1d36d6fe796 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/ValueProxy.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.statistics; + + +/** + * To be able to cache events concerning Values internally, group them + * together and similar. + * + * @author Steinar Knutsen + */ +class ValueProxy extends Proxy { + private double raw; + private boolean hasRaw = false; + private double min; + private boolean hasMin = false; + private double mean; + private boolean hasMean = false; + private double max; + private boolean hasMax = false; + private Histogram histogram; + private boolean hasHistogram; + + ValueProxy(String name) { + super(name); + } + + boolean hasRaw() { + return hasRaw; + } + double getRaw() { + return raw; + } + void setRaw(double raw) { + hasRaw = true; + this.raw = raw; + } + + boolean hasMin() { + return hasMin; + } + double getMin() { + return min; + } + void setMin(double min) { + hasMin = true; + this.min = min; + } + + boolean hasMean() { + return hasMean; + } + double getMean() { + return mean; + } + void setMean(double mean) { + hasMean = true; + this.mean = mean; + } + + boolean hasMax() { + return hasMax; + } + double getMax() { + return max; + } + void setMax(double max) { + hasMax = true; + this.max = max; + } + + boolean hasHistogram() { + return hasHistogram; + } + Histogram getHistogram() { + return histogram; + } + void setHistogram(Histogram histogram) { + hasHistogram = true; + this.histogram = histogram; + } + +} + diff --git a/statistics/src/main/java/com/yahoo/statistics/package-info.java b/statistics/src/main/java/com/yahoo/statistics/package-info.java new file mode 100644 index 00000000000..07c1644aea4 --- /dev/null +++ b/statistics/src/main/java/com/yahoo/statistics/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.statistics; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; -- cgit v1.2.3