diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /vespajlib/src/main/java/com |
Publish
Diffstat (limited to 'vespajlib/src/main/java/com')
213 files changed, 21285 insertions, 0 deletions
diff --git a/vespajlib/src/main/java/com/yahoo/api/annotations/.gitignore b/vespajlib/src/main/java/com/yahoo/api/annotations/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/api/annotations/.gitignore diff --git a/vespajlib/src/main/java/com/yahoo/api/annotations/PackageMarker.java b/vespajlib/src/main/java/com/yahoo/api/annotations/PackageMarker.java new file mode 100644 index 00000000000..cbd09fbf69f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/api/annotations/PackageMarker.java @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.api.annotations; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.SOURCE) +public @interface PackageMarker { } diff --git a/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryPrefix.java b/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryPrefix.java new file mode 100644 index 00000000000..e207e584115 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryPrefix.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.binaryprefix; + +/** + * Represents binary prefixes. + * @author tonytv + */ +public enum BinaryPrefix { + //represents the binary prefix 2^(k*10) + unit(0), + kilo(1, 'K'), + mega(2, 'M'), + giga(3, 'G'), + tera(4, 'T'), + peta(5, 'P'), + exa(6, 'E'), + zetta(7, 'Z'), + yotta(8, 'Y'); + + private final int k; + public final char symbol; + + private BinaryPrefix(int k, char symbol) { + this.k = k; + this.symbol = symbol; + } + + private BinaryPrefix(int k) { + this(k, (char)0); + } + + /* In most cases, BinaryScaledAmount should be prefered instead of this */ + public double convertFrom(double value, BinaryPrefix binaryPrefix) { + return value * Math.pow(2, + 10 * (binaryPrefix.k - k)); + } + + public static BinaryPrefix fromSymbol(char c) { + for (BinaryPrefix binaryPrefix : values()) { + if (binaryPrefix.symbol == c) + return binaryPrefix; + } + throw new RuntimeException("No such binary prefix: " + c); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryScaledAmount.java b/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryScaledAmount.java new file mode 100644 index 00000000000..303674bb504 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryScaledAmount.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.binaryprefix; + +/** + * An amount scaled by a binary prefix. + * + * <p> + * Examples: 2 kilo, 2 mega, ... + * </p> + * + * @author tonytv + */ +public final class BinaryScaledAmount { + public final double amount; + public final BinaryPrefix binaryPrefix; + + public BinaryScaledAmount(double amount, BinaryPrefix binaryPrefix) { + this.amount = amount; + this.binaryPrefix = binaryPrefix; + } + + public BinaryScaledAmount() { + this(0, BinaryPrefix.unit); + } + + public long as(BinaryPrefix newBinaryPrefix) { + return Math.round(newBinaryPrefix.convertFrom(amount, binaryPrefix)); + } + + public boolean equals(BinaryScaledAmount candidate) { + return BinaryPrefix.unit.convertFrom(amount, binaryPrefix) == + BinaryPrefix.unit.convertFrom(candidate.amount, candidate.binaryPrefix); + } + + public BinaryScaledAmount multiply(double d) { + return new BinaryScaledAmount(d*amount, binaryPrefix); + } + + public BinaryScaledAmount divide(double d) { + return multiply(1/d); + } + + @Override + public boolean equals(Object candidate) { + if (!(candidate instanceof BinaryScaledAmount)) { + return false; + } else { + return equals((BinaryScaledAmount)candidate); + } + } + + @Override + public int hashCode() { + return (int)BinaryPrefix.unit.convertFrom(amount, binaryPrefix); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/binaryprefix/package-info.java b/vespajlib/src/main/java/com/yahoo/binaryprefix/package-info.java new file mode 100644 index 00000000000..e1dada71ade --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/binaryprefix/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.binaryprefix; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/cache/Cache.java b/vespajlib/src/main/java/com/yahoo/cache/Cache.java new file mode 100644 index 00000000000..bfc3f3010aa --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/cache/Cache.java @@ -0,0 +1,276 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.cache; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * <p>A generic cache which keeps the total memory consumed by its content + * below a configured maximum.</p> + * + * <p>Thread safe.</p> + * + * @author vegardh + */ +public class Cache<K, V> { + private Map<CacheKey<K>,CacheValue<K, V>> content=new LinkedHashMap<>(12500, 1.0f, true); + private SizeCalculator calc = new SizeCalculator(); + private long maxSizeBytes; + + private long currentSizeBytes=0; + + /** The time an element is allowed to live, negative for indefinite lifespan */ + private long timeToLiveMillis=-1; + + /** The max allowed size of an entry, negative for no limit */ + private long maxEntrySizeBytes=10000; + + /** + * Creates a new cache + * + * @param maxSizeBytes the max size in bytes this cache is permitted to consume, + * including Result objects and Query keys + * @param timeToLiveMillis a negative value means unlimited time + * @param maxEntrySizeBytes never cache objects bigger than this, negative for no such limit + */ + public Cache(long maxSizeBytes,long timeToLiveMillis, long maxEntrySizeBytes) { + this.maxSizeBytes=maxSizeBytes; + this.timeToLiveMillis=timeToLiveMillis; + this.maxEntrySizeBytes=maxEntrySizeBytes; + } + + private synchronized CacheValue<K, V> synchGet(CacheKey<K> k) { + return content.get(k); + } + + private synchronized boolean synchPut(K key,V value, long keySizeBytes, long valueSizeBytes) { + // log.info("Put "+key.toString()+ " key size:"+keySizeBytes+" val size:"+valueSizeBytes); + if ((valueSizeBytes+keySizeBytes)>maxSizeBytes) { + return false; + } + makeRoomForBytes(valueSizeBytes+keySizeBytes); + CacheKey<K> cacheKey = new CacheKey<>(keySizeBytes, key); + CacheValue<K, V> cacheValue; + if (timeToLiveMillis<0) { + cacheValue=new CacheValue<>(valueSizeBytes,value, cacheKey); + } else { + cacheValue=new AgingCacheValue<>(valueSizeBytes,value, cacheKey); + } + currentSizeBytes+=(valueSizeBytes+keySizeBytes); + content.put(cacheKey, cacheValue); + return true; + } + + /** + * Attempts to add a value to the cache + * + * @param key the key of the value + * @param value the value to add + * @return true if the value was added, false if it could not be added + */ + public boolean put(K key,V value) { + long keySizeBytes=calc.sizeOf(key); + long valueSizeBytes=calc.sizeOf(value); + if (tooBigToCache(keySizeBytes+valueSizeBytes)) { + return false; + } + return synchPut(key, value, keySizeBytes, valueSizeBytes); + } + + /** + * Don't cache elems that are too big, even if there's space + * @return true if the argument is too big to cache. + */ + private boolean tooBigToCache(long totalSize) { + if (maxEntrySizeBytes<0) { + return false; + } + if (totalSize > maxEntrySizeBytes) { + return true; + } + return false; + } + + private void makeRoomForBytes(long bytes) { + if ((maxSizeBytes-currentSizeBytes) > bytes) { + return; + } + if (content.isEmpty()) { + return; + } + for (Iterator<Map.Entry<CacheKey<K>, CacheValue<K, V>>> i = content.entrySet().iterator() ; i.hasNext() ; ) { + Map.Entry<CacheKey<K>, CacheValue<K, V>> entry = i.next(); + CacheKey<K> key = entry.getKey(); + CacheValue<K, V> value = entry.getValue(); + // Can't call this.remove(), breaks iterator. + i.remove(); // Access order: first ones are LRU. + currentSizeBytes-=key.sizeBytes(); + currentSizeBytes-=value.sizeBytes(); + if ((maxSizeBytes-currentSizeBytes) > bytes) { + break; + } + } + } + + public boolean containsKey(K k) { + return content.containsKey(new CacheKey<>(-1, k)); + } + + /** Returns a value, if it is present in the cache */ + public V get(K key) { + // Currently it works to make a new CacheKey object without size + // because we have changed hashCode() there. + CacheKey<K> cacheKey = new CacheKey<>(-1, key); + CacheValue<K, V> value=synchGet(cacheKey); + if (value==null) { + return null; + } + if (timeToLiveMillis<0) { + return value.value(); + } + + if (value.expired(timeToLiveMillis)) { + // There was a value, which has now expired + remove(key); + return null; + } else { + return value.value(); + } + } + + /** + * Removes a cache value if present + * + * @return true if the value was removed, false if it was not present + */ + public synchronized boolean remove(K key) { + CacheValue<K, V> value=content.remove(key); + if (value==null) { + return false; + } + currentSizeBytes-=value.sizeBytes(); + currentSizeBytes-=value.getKey().sizeBytes(); + return true; + } + + public long getTimeToLiveMillis() { + return timeToLiveMillis; + } + + public int size() { + return content.size(); + } + + private static class CacheKey<K> { + private long sizeBytes; + private K key; + public CacheKey(long sizeBytes,K key) { + this.sizeBytes=sizeBytes; + this.key=key; + } + + public long sizeBytes() { + return sizeBytes; + } + + public K getKey() { + return key; + } + + public int hashCode() { + return key.hashCode(); + } + + @SuppressWarnings("rawtypes") + public boolean equals(Object k) { + if (key==null) { + return false; + } + if (k==null) { + return false; + } + if (k instanceof CacheKey) { + return key.equals(((CacheKey)k).getKey()); + } + return false; + } + } + + private static class CacheValue<K, V> { + private long sizeBytes; + private V value; + private CacheKey<K> key; + public CacheValue(long sizeBytes, V value, CacheKey<K> key) { + this.sizeBytes=sizeBytes; + this.value=value; + this.key = key; + } + + public boolean expired(long ttl) { + return false; + } + + public V value() { + return value; + } + + public long sizeBytes() { + return sizeBytes; + } + + public CacheKey<K> getKey() { + return key; + } + } + + private static class AgingCacheValue<K, V> extends CacheValue<K, V> { + private long birthTimeMillis; + + public AgingCacheValue(long sizeBytes,V value, CacheKey<K> key) { + super(sizeBytes,value, key); + this.birthTimeMillis=System.currentTimeMillis(); + } + + public long ageMillis() { + return System.currentTimeMillis()-birthTimeMillis; + } + + public boolean expired(long ttl) { + return (ageMillis() >= ttl); + } + } + + /** + * Empties the cache + */ + public synchronized void clear() { + content.clear(); + currentSizeBytes=0; + } + + /** + * Collection of keys. + */ + public Collection<K> getKeys() { + Collection<K> ret = new ArrayList<>(); + for (Iterator<CacheKey<K>> i = content.keySet().iterator(); i.hasNext();) { + ret.add(i.next().getKey()); + } + return ret; + } + + /** + * Collection of values. + */ + public Collection<V> getValues() { + Collection<V> ret = new ArrayList<>(); + for (Iterator<CacheValue<K, V>> i = content.values().iterator(); i.hasNext();) { + ret.add(i.next().value()); + } + return ret; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/cache/SizeCalculator.java b/vespajlib/src/main/java/com/yahoo/cache/SizeCalculator.java new file mode 100644 index 00000000000..677a3fb07e6 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/cache/SizeCalculator.java @@ -0,0 +1,175 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.cache; + +import java.lang.reflect.Array; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; + +/** + * Size calculator for objects. + * Thread safe. + * @author vegardh + * @see <a href="http://www.javaspecialists.co.za/archive/Issue078.html">MemoryCounter by Dr H M Kabutz</a> + */ +public class SizeCalculator { + + private static class ObjectSet { + private final Map<Object, Object> map = new IdentityHashMap<>(); + + public boolean had(Object obj) { + if (map.containsKey(obj)) { + return true; + } + map.put(obj, null); + return false; + } + } + + private int getPointerSize() { + return 4; + } + + private int getClassSize() { + return 8; + } + + private int getArraySize() { + return 16; + } + + @SuppressWarnings("serial") + private final IdentityHashMap<Class<?>, Integer> primitiveSizes = new IdentityHashMap<Class<?>, Integer>() { + { + put(boolean.class, 1); + put(byte.class, 1); + put(char.class, 2); + put(short.class, 2); + put(int.class, 4); + put(float.class, 4); + put(double.class, 8); + put(long.class, 8); + } + }; + + // Only called on un-visited objects and only with array. + private long sizeOfArray(Object a, ObjectSet visitedObjects) { + long sum = getArraySize(); + int length = Array.getLength(a); + if (length == 0) { + return sum; + } + Class<?> elementClass = a.getClass().getComponentType(); + if (elementClass.isPrimitive()) { + sum += length * (primitiveSizes.get(elementClass)); + return sum; + } else { + for (int i = 0; i < length; i++) { + Object val = Array.get(a, i); + sum += getPointerSize(); + sum += sizeOfObject(val, visitedObjects); + } + return sum; + } + } + + private long getSumOfFields(Class<?> clas, Object obj, + ObjectSet visitedObjects) { + long sum = 0; + Field[] fields = clas.getDeclaredFields(); + for (Field field : fields) { + if (!Modifier.isStatic(field.getModifiers())) { + if (field.getType().isPrimitive()) { + sum += primitiveSizes.get(field.getType()); + } else { + sum += getPointerSize(); + field.setAccessible(true); + try { + sum += sizeOfObject(field.get(obj), visitedObjects); + } catch (IllegalArgumentException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + } + } + return sum; + } + + // Skip literal strings + private boolean isIntern(Object obj) { + if (obj instanceof String) { + if (obj == ((String) obj).intern()) { + return true; + } + } + return false; + } + + // Only called on non-visited non-arrays. + private long sizeOfNonArray(Class<?> clas, Object obj, + ObjectSet visitedObjects) { + if (isIntern(obj)) { + return 0; + } + long sum = getClassSize(); + while (clas != null) { + sum += getSumOfFields(clas, obj, visitedObjects); + clas = clas.getSuperclass(); + } + return sum; + } + + private long sizeOfObject(Object obj, ObjectSet visitedObjects) { + if (obj == null) { + return 0; + } + if (visitedObjects.had(obj)) { + return 0; + } + Class<?> clas = obj.getClass(); + if (clas.isArray()) { + return sizeOfArray(obj, visitedObjects); + } + return sizeOfNonArray(clas, obj, visitedObjects); + } + + /** + * Returns the heap size of an object/array + * + * @return Number of bytes for object, approximately + */ + public long sizeOf(Object value) { + ObjectSet visitedObjects = new ObjectSet(); + return sizeOfObject(value, visitedObjects); + } + + /** + * Returns the heap size of two objects/arrays, common objects counted only + * once + * + * @return Number of bytes for objects, approximately + */ + public long sizeOf(Object value1, Object value2) { + ObjectSet visitedObjects = new ObjectSet(); + return sizeOfObject(value1, visitedObjects) + + sizeOfObject(value2, visitedObjects); + } + + /** + * The approximate size in bytes for a list of objects, viewed as a closure, + * ie. common objects are counted only once. + * + * @return total number of bytes + */ + public long sizeOf(List<?> objects) { + ObjectSet visitedObjects = new ObjectSet(); + long sum = 0; + for (Object o : objects) { + sum += sizeOfObject(o, visitedObjects); + } + return sum; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/ArraySet.java b/vespajlib/src/main/java/com/yahoo/collections/ArraySet.java new file mode 100644 index 00000000000..8df46e113a2 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/ArraySet.java @@ -0,0 +1,251 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Set; + +/** + * A Set implementation with low allocation cost. It should only be used for + * small number of objects, as it is implemented as scanning an ArrayList for + * equality matches. In other words: Performance will only be acceptable for + * <i>small</i> sets. + * + * <p> + * The rationale for this class is the high cost of the object identifier used + * in IdentityHashMap, where the key set is often used as an identity set. + * </p> + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author balder + * @since 5.1.4 + * + * @param <E> + * the type contained in the Set + */ +public final class ArraySet<E> implements Set<E> { + private class ArrayIterator<T> implements Iterator<E> { + private int i = -1; + private boolean removed = false; + + @Override + public boolean hasNext() { + return i + 1 < size; + } + + @SuppressWarnings("unchecked") + @Override + public E next() { + if (!hasNext()) { + throw new NoSuchElementException("No more elements available"); + } + removed = false; + return (E) entries[++i]; + } + + @Override + public void remove() { + if (removed) { + throw new IllegalStateException( + "Trying to remove same element twice."); + } + if (i == -1) { + throw new IllegalStateException( + "Trying to remove before entering iterator."); + } + delete(i--); + removed = true; + } + + } + + private Object[] entries; + private int size = 0; + + /** + * Create a set with an initial capacity of initSize. The internal array + * will grow automatically with a linear growth rate if more elements than + * initSize are added. + * + * @param initSize + * initial size of internal element array + */ + public ArraySet(final int initSize) { + entries = new Object[initSize]; + } + + /** + * Expose the index in the internal array of a given object. -1 is returned + * if the object is not present in the internal array. + * + * @param e + * an object to check whether exists in this set + * @return the index of the argument e in the internal array, or -1 if the + * object is not present + */ + public int indexOf(final Object e) { + for (int i = 0; i < size; ++i) { + if (e.equals(entries[i])) { + return i; + } + } + return -1; + } + + private void clean() { + int offset = 0; + for (int i = 0; i < size; ++i) { + if (entries[i] == null) { + ++offset; + } else { + entries[i - offset] = entries[i]; + } + } + size -= offset; + } + + private void grow() { + entries = Arrays.copyOf(entries, entries.length * 2 + 1); + } + + private void append(final Object arg) { + if (size == entries.length) { + grow(); + } + entries[size++] = arg; + } + + @Override + public boolean add(final E arg) { + final int i = indexOf(arg); + if (i >= 0) { + return false; + } + append(arg); + return true; + } + + @Override + public boolean addAll(final Collection<? extends E> arg) { + boolean changed = false; + for (final E entry : arg) { + changed |= add(entry); + } + return changed; + } + + @Override + public void clear() { + size = 0; + } + + @Override + public boolean contains(final Object arg) { + return indexOf(arg) >= 0; + } + + /** + * This is an extremely expensive implementation of + * {@link Set#containsAll(Collection)}. It is implemented as O(n**2). + */ + @Override + public boolean containsAll(final Collection<?> arg) { + for (final Object entry : arg) { + if (indexOf(entry) < 0) { + return false; + } + } + return true; + } + + @Override + public boolean isEmpty() { + return size == 0; + } + + @Override + public Iterator<E> iterator() { + return new ArrayIterator<E>(); + } + + private void delete(int i) { + if (i < 0 || i >= size) { + return; + } + --size; + while (i < size) { + entries[i] = entries[i + 1]; + ++i; + } + entries[i] = null; + } + + @Override + public boolean remove(final Object arg) { + final int i = indexOf(arg); + if (i < 0) { + return false; + } + delete(i); + return true; + } + + /** + * This is an extremely expensive implementation of + * {@link Set#removeAll(Collection)}. It is implemented as O(n**2). + */ + @Override + public boolean removeAll(final Collection<?> arg) { + boolean changed = false; + for (final Object entry : arg) { + final int i = indexOf(entry); + if (i >= 0) { + entries[i] = null; + changed = true; + } + } + if (changed) { + clean(); + } + return changed; + } + + /** + * This is an extremely expensive implementation of + * {@link Set#retainAll(Collection)}. It is implemented as O(n**2). + */ + @Override + public boolean retainAll(final Collection<?> arg) { + boolean changed = false; + for (int i = 0; i < size; ++i) { + final Object entry = entries[i]; + if ( !arg.contains(entry)) { + entries[i] = null; + changed = true; + } + } + if (changed) { + clean(); + } + return changed; + } + + @Override + public int size() { + return size; + } + + @Override + public Object[] toArray() { + return Arrays.copyOf(entries, size); + } + + @SuppressWarnings("unchecked") + @Override + public <T> T[] toArray(final T[] arg) { + return Arrays.copyOf(entries, size, (Class<T[]>) arg.getClass()); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/BobHash.java b/vespajlib/src/main/java/com/yahoo/collections/BobHash.java new file mode 100644 index 00000000000..b942c4e78f0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/BobHash.java @@ -0,0 +1,200 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import com.yahoo.text.Utf8; + +/** + * <p>A Java port of Michael Susag's BobHash in FastLib. This version is + * specifically done to be bit compatible with the one in FastLib, as it + * is used in decoding packets from FastServer.</p> + * + * <p>Hash function based on + * <a href="http://burtleburtle.net/bob/hash/index.html"> + * http://burtleburtle.net/bob/hash/index.html</a> + * by Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. You may use this + * code any way you wish, private, educational, or commercial. It's free.</p> + * + * @author Michael Susag + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * + * + */ + +public class BobHash { + + /** + * mix -- mix 3 32-bit values reversibly. + * For every delta with one or two bits set, and the deltas of all three + * high bits or all three low bits, whether the original value of a,b,c + * is almost all zero or is uniformly distributed, + * If mix() is run forward or backward, at least 32 bits in a,b,c + * have at least 1/4 probability of changing. + * If mix() is run forward, every bit of c will change between 1/3 and + * 2/3 of the time. (Well, 22/100 and 78/100 for some 2-bit deltas.) + * mix() was built out of 36 single-cycle latency instructions in a + * structure that could supported 2x parallelism, like so: + * + * <pre> + * a -= b; + * a -= c; x = (c>>13); + * b -= c; a ^= x; + * b -= a; x = (a<<8); + * c -= a; b ^= x; + * c -= b; x = (b>>13); + * ... + * </pre> + * + * <p> + * Unfortunately, superscalar Pentiums and Sparcs can't take advantage + * of that parallelism. They've also turned some of those single-cycle + * latency instructions into multi-cycle latency instructions. Still, + * this is the fastest good hash I could find. There were about 2^^68 + * to choose from. I only looked at a billion or so. + */ + private static int[] mix(int a, int b, int c) { + a -= b; a -= c; a ^= (c >>> 13); + b -= c; b -= a; b ^= (a << 8); + c -= a; c -= b; c ^= (b >>> 13); + a -= b; a -= c; a ^= (c >>> 12); + b -= c; b -= a; b ^= (a << 16); + c -= a; c -= b; c ^= (b >>> 5); + a -= b; a -= c; a ^= (c >>> 3); + b -= c; b -= a; b ^= (a << 10); + c -= a; c -= b; c ^= (b >>> 15); + + return new int[]{ a, b, c }; + } + + /** + * Transform a byte to an int viewed as an unsigned byte. + */ + private static int unsign(byte x) { + int y; + + y = 0xFF & x; + return y; + } + + /** + * Hashes a string, by calling hash(byte[] key,int initval) with + * the utf-8 bytes of the string as key and 0 as initval. + * Note: This is copying the string content, change implementation to + * use efficiently on large strings. + * + * <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ + public static int hash(String key) { + return hash(Utf8.toBytes(key), 0); + } + + /** + * The hash function + * + * <p> + * hash() -- hash a variable-length key into a 32-bit value<br> + * k : the key (the unaligned variable-length array of bytes)<br> + * len : the length of the key, counting by bytes<br> + * initval : can be any 4-byte value + * + * <p> + * Returns a 32-bit value. Every bit of the key affects every bit of + * the return value. Every 1-bit and 2-bit delta achieves avalanche. + * About 6*len+35 instructions. + * + * <p> + * The best hash table sizes are powers of 2. There is no need to do + * mod a prime (mod is sooo slow!). If you need less than 32 bits, + * use a bitmask. For example, if you need only 10 bits, do + * h = (h & hashmask(10)); + * In which case, the hash table should have hashsize(10) elements. + * + * If you are hashing n strings (ub1 **)k, do it like this: + * for (i=0, h=0; i<n; ++i) h = hash( k[i], len[i], h); + * + * <p> + * By Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. You may use this + * code any way you wish, private, educational, or commercial. It's free. + * + * <p> + * See http://burtleburtle.net/bob/hash/evahash.html + * Use for hash table lookup, or anything where one collision in 2^^32 is + * acceptable. Do NOT use for cryptographic purposes. + * + * @param k the key + * @param initval the previous hash, or an arbitrary value + * @return A 32 bit hash value + */ + @SuppressWarnings("fallthrough") + public static int hash(byte[] k, int initval) { + int a, b, c, len; + int offset = 0; + int[] abcBuffer; + + /* Set up the internal state */ + len = k.length; + a = b = 0x9e3779b9; /* the golden ratio; an arbitrary value */ + c = initval; /* the previous hash value */ + + // handle most of the key + while (len >= 12) { + a += (unsign(k[offset + 0]) + (unsign(k[offset + 1]) << 8) + + (unsign(k[offset + 2]) << 16) + + (unsign(k[offset + 3]) << 24)); + b += (unsign(k[offset + 4]) + (unsign(k[offset + 5]) << 8) + + (unsign(k[offset + 6]) << 16) + + (unsign(k[offset + 7]) << 24)); + c += (unsign(k[offset + 8]) + (unsign(k[offset + 9]) << 8) + + (unsign(k[offset + 10]) << 16) + + (unsign(k[offset + 11]) << 24)); + abcBuffer = mix(a, b, c); + a = abcBuffer[0]; + b = abcBuffer[1]; + c = abcBuffer[2]; + offset += 12; + len -= 12; + } + + // handle the last 11 bytes + c += k.length; + switch (len) { + // all the case statements fall through + case 11: + c += (unsign(k[offset + 10]) << 24); + + case 10: + c += (unsign(k[offset + 9]) << 16); + + case 9: + c += (unsign(k[offset + 8]) << 8); + + /* the first byte of c is reserved for the length */ + case 8: + b += (unsign(k[offset + 7]) << 24); + + case 7: + b += (unsign(k[offset + 6]) << 16); + + case 6: + b += (unsign(k[offset + 5]) << 8); + + case 5: + b += unsign(k[offset + 4]); + + case 4: + a += (unsign(k[offset + 3]) << 24); + + case 3: + a += (unsign(k[offset + 2]) << 16); + + case 2: + a += (unsign(k[offset + 1]) << 8); + + case 1: + a += unsign(k[offset + 0]); + + /* case 0: nothing left to add */ + } + abcBuffer = mix(a, b, c); + return abcBuffer[2]; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/ByteArrayComparator.java b/vespajlib/src/main/java/com/yahoo/collections/ByteArrayComparator.java new file mode 100644 index 00000000000..c0f630a92e5 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/ByteArrayComparator.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +/** + * Utility class which is useful when implementing <code>Comparable</code> and one needs to + * compare byte arrays as instance variables. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class ByteArrayComparator { + /** + * Compare the arguments. Shorter arrays are always considered + * smaller than longer arrays. For arrays of equal lengths, the elements + * are compared one-by-one. Whenever two corresponding elements in the + * two arrays are non-equal, the method returns. If all elements at + * corresponding positions in the two arrays are equal, the arrays + * are considered equal. + * + * @param first a byte array to be compared + * @param second a byte array to be compared + * @return 0 if the arguments are equal, -1 if the first argument is smaller, 1 if the second argument is smaller + * @throws NullPointerException if any of the arguments are null + */ + public static int compare(byte[] first, byte[] second) { + if (first.length < second.length) { + return -1; + } + if (first.length > second.length) { + return 1; + } + + //lengths are equal, compare contents + for (int i = 0; i < first.length; i++) { + if (first[i] < second[i]) { + return -1; + } else if (first[i] > second[i]) { + return 1; + } + //values at index i are equal, continue... + } + + //we haven't returned yet; contents must be equal: + return 0; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/CollectionComparator.java b/vespajlib/src/main/java/com/yahoo/collections/CollectionComparator.java new file mode 100644 index 00000000000..8999f0cac9c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/CollectionComparator.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.Collection; +import java.util.Iterator; + +/** + * Utility class which is useful when implementing <code>Comparable</code> and one needs to + * compare Collections of Comparables as instance variables. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class CollectionComparator { + /** + * Compare the arguments. Shorter Collections are always considered + * smaller than longer Collections. For Collections of equal lengths, the elements + * are compared one-by-one. Whenever two corresponding elements in the + * two Collections are non-equal, the method returns. If all elements at + * corresponding positions in the two Collections are equal, the Collections + * are considered equal. + * + * @param first a Collection of Comparables to be compared + * @param second a Collection of Comparables to be compared + * @return 0 if the arguments are equal, -1 if the first argument is smaller, 1 if the second argument is smaller + * @throws NullPointerException if any of the arguments are null + */ + @SuppressWarnings({"rawtypes", "unchecked"}) + public static int compare(Collection<? extends Comparable> first, Collection<? extends Comparable> second) { + if (first.size() < second.size()) { + return -1; + } + if (first.size() > second.size()) { + return 1; + } + + //sizes are equal, compare contents + Iterator<? extends Comparable> firstIt = first.iterator(); + Iterator<? extends Comparable> secondIt = second.iterator(); + + while (firstIt.hasNext()) { + // FIXME: unchecked casting + Comparable itemFirst = firstIt.next(); + Comparable itemSecond = secondIt.next(); + int comp = itemFirst.compareTo(itemSecond); + if (comp != 0) { + return comp; + } + //values are equal, continue... + } + + //we haven't returned yet; contents must be equal: + return 0; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/CollectionUtil.java b/vespajlib/src/main/java/com/yahoo/collections/CollectionUtil.java new file mode 100644 index 00000000000..ddcc6e97dff --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/CollectionUtil.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + * Utilities for java collections + * + * @author tonytv + * @author gjoranv + * @since 5.1.8 + */ +public class CollectionUtil { + + /** + * Returns a String containing the string representation of all elements from + * the given collection, separated by the separator string. + * + * @param collection The collection + * @param sep The separator string + * @return A string: elem(0) + sep + ... + elem(N) + */ + public static String mkString(Collection<?> collection, String sep) { + return mkString(collection, "", sep, ""); + } + + /** + * Returns a String containing the string representation of all elements from + * the given collection, using a start string, separator strings, and an end string. + * + * @param collection The collection + * @param start The start string + * @param sep The separator string + * @param end The end string + * @param <T> The element type + * @return A string: start + elem(0) + sep + ... + elem(N) + end + */ + public static <T> String mkString(Collection<T> collection, String start, String sep, String end) { + return collection.stream() + .map(T::toString) + .collect(Collectors.joining(sep, start, end)); + } + + /** + * Returns true if the contents of the two given collections are equal, ignoring order. + */ + public static boolean equalContentsIgnoreOrder(Collection<?> c1, Collection<?> c2) { + return c1.size() == c2.size() && + c1.containsAll(c2); + } + + /** + * Returns the symmetric difference between two collections, i.e. the set of elements + * that occur in exactly one of the collections. + */ + public static <T> Set<T> symmetricDifference(Collection<? extends T> c1, Collection<? extends T> c2) { + Set<T> diff1 = new HashSet<>(c1); + diff1.removeAll(c2); + + Set<T> diff2 = new HashSet<>(c2); + diff2.removeAll(c1); + + diff1.addAll(diff2); + return diff1; + } + + /** + * Returns the subset of elements from the given collection that can be cast to the reference + * type, defined by the given Class object. + */ + public static <T> Collection<T> filter(Collection<?> collection, Class<T> lowerBound) { + List<T> result = new ArrayList<>(); + for (Object element : collection) { + if (lowerBound.isInstance(element)) { + result.add(lowerBound.cast(element)); + } + } + return result; + } + + /** + * Returns the first element in a collection according to iteration order. + * Returns null if the collection is empty. + */ + public static <T> T first(Collection<T> collection) { + return collection.isEmpty()? null: collection.iterator().next(); + } + + public static <T> Optional<T> firstMatching(T[] array, Predicate<? super T> predicate) { + for (T t: array) { + if (predicate.test(t)) + return Optional.of(t); + } + return Optional.empty(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/ConcurrentResourcePool.java b/vespajlib/src/main/java/com/yahoo/collections/ConcurrentResourcePool.java new file mode 100644 index 00000000000..98cc443fd71 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/ConcurrentResourcePool.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 13.11.12 + * Time: 20:57 + * To change this template use File | Settings | File Templates. + */ +public class ConcurrentResourcePool<T> implements Iterable<T> { + + private final Queue<T> pool = new ConcurrentLinkedQueue<>(); + private final ResourceFactory<T> factory; + + public ConcurrentResourcePool(ResourceFactory<T> factory) { + this.factory = factory; + } + + public final T alloc() { + final T e = pool.poll(); + return e != null ? e : factory.create(); + } + + public final void free(T e) { + pool.offer(e); + } + + @Override + public Iterator<T> iterator() { + return pool.iterator(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/CopyOnWriteHashMap.java b/vespajlib/src/main/java/com/yahoo/collections/CopyOnWriteHashMap.java new file mode 100644 index 00000000000..43f38c67e4d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/CopyOnWriteHashMap.java @@ -0,0 +1,154 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import com.google.common.annotations.Beta; + +import java.util.AbstractMap; +import java.util.AbstractSet; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +/** + * A hashmap wrapper which defers cloning of the enclosed map until it is written. + * Use this to make clones cheap in maps which are often not further modified. + * <p> + * As with regular maps, this can only be used safely if the content of the map is immutable. + * If not, the {@link #copyMap} method can be overridden to perform a deep clone. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +@Beta +public class CopyOnWriteHashMap<K,V> extends AbstractMap<K,V> implements Cloneable { + + private Map<K,V> map; + + /** True when this class is allowed to write to the map */ + private boolean writable = true; + + /** Lazily initialized view */ + private transient Set<Map.Entry<K,V>> entrySet = null; + + public CopyOnWriteHashMap() { + this.map = new HashMap<>(); + } + + public CopyOnWriteHashMap(int capacity) { + this.map = new HashMap<>(capacity); + } + + public CopyOnWriteHashMap(Map<K,V> map) { + this.map = new HashMap<>(map); + } + + private void makeReadOnly() { + writable = false; + } + + private void makeWritable() { + if (writable) return; + map = copyMap(map); + writable = true; + entrySet = null; + } + + /** + * Make a copy of the given map with the requisite deepness. + * This default implementation does return new HashMap<>(original); + */ + protected Map<K,V> copyMap(Map<K,V> original) { + return new HashMap<>(original); + } + + @SuppressWarnings("unchecked") + public CopyOnWriteHashMap<K,V> clone() { + try { + CopyOnWriteHashMap<K,V> clone = (CopyOnWriteHashMap<K,V>)super.clone(); + this.makeReadOnly(); + clone.makeReadOnly(); + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + + @Override + public Set<Entry<K, V>> entrySet() { + if (entrySet == null) + entrySet = new EntrySet(); + return entrySet; + } + + @Override + public V put(K key, V value) { + makeWritable(); + return map.put(key, value); + } + + /** Override to avoid using iterator.remove */ + @Override + public V remove(Object key) { + makeWritable(); + return map.remove(key); + } + + private final class EntrySet extends AbstractSet<Map.Entry<K,V>> { + + public Iterator<Map.Entry<K,V>> iterator() { + return new EntryIterator(); + } + + @SuppressWarnings("unchecked") + public boolean contains(Object o) { + if ( ! (o instanceof Map.Entry)) return false; + Map.Entry<K,V> entry = (Map.Entry<K,V>) o; + Object candidate = map.get(entry.getKey()); + if (candidate == null) return entry.getValue()==null; + return candidate.equals(entry.getValue()); + } + + public boolean remove(Object o) { + makeWritable(); + return map.remove(o) !=null; + } + + public int size() { + return map.size(); + } + + public void clear() { map.clear(); } + + } + + /** + * An entry iterator which does not allow removals if the map wasn't already modifiable + * There is no sane way to implement that given that the wrapped map changes mid iteration. + */ + private class EntryIterator implements Iterator<Map.Entry<K,V>> { + + /** Wrapped iterator */ + private Iterator<Map.Entry<K,V>> mapIterator; + + public EntryIterator() { + mapIterator = map.entrySet().iterator(); + } + + public final boolean hasNext() { + return mapIterator.hasNext(); + } + + public Entry<K,V> next() { + return mapIterator.next(); + } + + public void remove() { + if ( ! writable) + throw new UnsupportedOperationException("Cannot perform the copy-on-write operation during iteration"); + mapIterator.remove(); + } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/FreezableArrayList.java b/vespajlib/src/main/java/com/yahoo/collections/FreezableArrayList.java new file mode 100644 index 00000000000..e145a08be09 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/FreezableArrayList.java @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.Collection; +import java.util.List; + +/** + * An array list which can be frozen to disallow further edits. + * After freezing, edit operations will throw UnsupportedOperationException. + * Freezable lists may optionally allow new items to be added to the end of the list also after freeze. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.20 + */ +public class FreezableArrayList<ITEM> extends ListenableArrayList<ITEM> { + + private static final long serialVersionUID = 5900452593651895638L; + + private final boolean permitAddAfterFreeze; + private boolean frozen = false; + + /** Creates a freezable array list which does not permit adds after freeze */ + public FreezableArrayList() { + this(false); + } + + /** Creates a freezable array list which does not permit adds after freeze */ + public FreezableArrayList(int initialCapacity) { + this(false, initialCapacity); + } + + public FreezableArrayList(boolean permitAddAfterFreeze) { + this.permitAddAfterFreeze = permitAddAfterFreeze; + } + + public FreezableArrayList(boolean permitAddAfterFreeze, int initialCapacity) { + super(initialCapacity); + this.permitAddAfterFreeze = permitAddAfterFreeze; + } + + /** Irreversibly freezes the content of this */ + public void freeze() { + this.frozen = true; + } + + @Override + public boolean add(ITEM e) { + if ( ! permitAddAfterFreeze) throwIfFrozen(); + return super.add(e); + } + + @Override + public void add(int index, ITEM e) { + throwIfFrozen(); + super.add(index, e); + } + + @Override + public boolean addAll(Collection<? extends ITEM> a) { + if ( ! permitAddAfterFreeze) throwIfFrozen(); + return super.addAll(a); + } + + @Override + public boolean addAll(int index, Collection<? extends ITEM> a) { + throwIfFrozen(); + return super.addAll(index, a); + } + + @Override + public ITEM set(int index, ITEM e) { + throwIfFrozen(); + return super.set(index, e); + } + + @Override + public ITEM remove(int index) { + throwIfFrozen(); + return super.remove(index); + } + + @Override + public boolean remove(Object o) { + throwIfFrozen(); + return super.remove(o); + } + + @Override + public void clear() { + throwIfFrozen(); + super.clear(); + } + + @Override + protected void removeRange(int fromIndex, int toIndex) { + throwIfFrozen(); + super.removeRange(fromIndex, toIndex); + } + + @Override + public boolean removeAll(Collection<?> c) { + throwIfFrozen(); + return super.removeAll(c); + } + + @Override + public boolean retainAll(Collection<?> c) { + throwIfFrozen(); + return super.retainAll(c); + } + + private void throwIfFrozen() { + if ( frozen ) + throw new UnsupportedOperationException(this + " is frozen"); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/Hashlet.java b/vespajlib/src/main/java/com/yahoo/collections/Hashlet.java new file mode 100644 index 00000000000..86e82bb3241 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/Hashlet.java @@ -0,0 +1,226 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + + +/** + * Lightweight hash map from key to value with limited + * functionality. This class lets you build a map from key to + * value. The value for a key may be overwritten and the put and get + * methods have the same semantics as for normal Java Maps, but there + * is no remove operation. Also, there is no iterator support, but + * keys and values can be accessed directly by index. The access order + * of keys and values are defined by the insert order of the keys. The + * goal of this class is to reduce the amount of object that are + * allocated by packing everything into two internal arrays. The keys + * and values are packed in an Object array and the hash table and + * entries are packed in an int array. The internal arrays are not + * created until space is needed. The default initial capacity is 16 + * entries. If you know you need much more space than this, you can + * explicitly reserve more space before starting to insert values. The + * maximum load factor is 0.7 and drops slightly with increasing + * capacity. + * + * @author <a href="mailto:havardpe@yahoo-inc.com">Havard Pettersen</a> + **/ +public final class Hashlet<K, V> { + + private static final int[] emptyHash = new int[1]; + private int capacity = 0; + private int hashSize() { return (capacity + (capacity / 2) - 1); } + private int used = 0; + private Object[] store; + private int[] hash = emptyHash; + + /** + * Create an empty Hashlet. + **/ + public Hashlet() {} + + /** + * Create a Hashlet that is a shallow copy of another Hashlet. + * + * @param hashlet the Hashlet to copy. + **/ + public Hashlet(Hashlet<K, V> hashlet) { + if (hashlet.used > 0) { + capacity = hashlet.capacity; + used = hashlet.used; + store = new Object[hashlet.store.length]; + hash = new int[hashlet.hash.length]; + System.arraycopy(hashlet.store, 0, store, 0, store.length); + System.arraycopy(hashlet.hash, 0, hash, 0, hash.length); + } + } + + /** + * Reserve space for more key value pairs. This method is used by + * the put method to perform rehashing when needed. It can be + * invoked directly by the application to reduce the number of + * rehashes needed to insert a large number of entries. + * + * @param n the number of additional entries to reserve space for + **/ + public void reserve(int n) { + if (used + n > capacity) { + final int c = capacity; + if (capacity == 0) { + capacity = 16; + } + while (used + n > capacity) { + capacity *= 2; + } + final Object[] s = store; + store = new Object[capacity * 2]; + hash = new int[hashSize() + (capacity * 2)]; + if (c > 0) { + System.arraycopy(s, 0, store, 0, used); + System.arraycopy(s, c, store, capacity, used); + for (int i = 0; i < used; i++) { + int prev = Math.abs(s[i].hashCode() % hashSize()); + int entry = hash[prev]; + while (entry != 0) { + prev = entry + 1; + entry = hash[prev]; + } + final int insertIdx = (hashSize() + (i * 2)); + hash[prev] = insertIdx; + hash[insertIdx] = i; + } + } + } + } + + /** + * The current size. This is the number of key value pairs + * currently stored in this object. + * + * @return current size + **/ + public int size() { + return used; + } + + /** + * Obtain a key. Keys are accessed in the order they were first + * inserted. + * + * @return the requested key + * @param i the index of the key, must be in the range [0, size() - 1] + **/ + @SuppressWarnings("unchecked") + public K key(int i) { + return (K) store[i]; + } + + /** + * Obtain a value. Values are accessed in the order in which + * theirs keys were first inserted. + * + * @return the requested value + * @param i the index of the value, must be in the range [0, size() - 1] + **/ + @SuppressWarnings("unchecked") + public V value(int i) { + return (V) store[capacity + i]; + } + + /** + * This will replace the value at the index give. + * + * @param i the index of the value, must be in the range [0, size() - 1] + * @param value The new value you want to set for this index. + * @return previous value + */ + public V setValue(int i, V value) { + V prev = value(i); + store[capacity + i] = value; + return prev; + } + + /** + * Associate a value with a specific key. + * + * @return the old value for the key, if it was already present + * @param key the key + * @param value the value + **/ + public V put(K key, V value) { + reserve(1); + int prev = Math.abs(key.hashCode() % hashSize()); + int entry = hash[prev]; + while (entry != 0) { + final int idx = hash[entry]; + if (store[idx].equals(key)) { // found entry + @SuppressWarnings("unchecked") + final V ret = (V) store[capacity + idx]; + store[capacity + idx] = value; + return ret; + } + prev = entry + 1; + entry = hash[prev]; + } + final int insertIdx = (hashSize() + (used * 2)); + hash[prev] = insertIdx; + hash[insertIdx] = used; + store[used] = key; + store[capacity + (used++)] = value; + return null; + } + + /** + * Obtain the value for a specific key. + * + * @return the value for a key, or null if not found + * @param key the key + **/ + public V get(Object key) { + int index = getIndexOfKey(key); + return (index != -1) ? value(index) : null; + } + + /** + * Finds the index where the key,value pair is stored. + * @param key to look for + * @return the index where the key is found or -1 if it is not found + */ + public int getIndexOfKey(Object key) { + int entry = hash[Math.abs(key.hashCode() % hashSize())]; + while (entry != 0) { + final int idx = hash[entry]; + if (store[idx].equals(key)) { // found entry + return idx; + } + entry = hash[entry + 1]; + } + return -1; + } + + @Override + public int hashCode() { + int h = 0; + for (int i = 0; i < used; i++) { + h += key(i).hashCode(); + V v = value(i); + if (v != null) { + h += v.hashCode(); + } + } + return h; + } + + @Override + public boolean equals(Object o) { + if (! (o instanceof Hashlet) ) return false; + Hashlet<?, ?> rhs = (Hashlet<?, ?>) o; + if (used != rhs.used) return false; + for (int i = 0; i < used; i++) { + int bi = rhs.getIndexOfKey(key(i)); + if (bi == -1) return false; + Object a = value(i); + Object b = rhs.value(bi); + boolean equal = (a == null) ? b == null : a.equals(b); + if ( !equal ) return false; + } + return true; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/IntArrayComparator.java b/vespajlib/src/main/java/com/yahoo/collections/IntArrayComparator.java new file mode 100644 index 00000000000..21c6f514cbf --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/IntArrayComparator.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +/** + * Utility class which is useful when implementing <code>Comparable</code> and one needs to + * compare int arrays as instance variables. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class IntArrayComparator { + /** + * Compare the arguments. Shorter arrays are always considered + * smaller than longer arrays. For arrays of equal lengths, the elements + * are compared one-by-one. Whenever two corresponding elements in the + * two arrays are non-equal, the method returns. If all elements at + * corresponding positions in the two arrays are equal, the arrays + * are considered equal. + * + * @param first an int array to be compared + * @param second an int array to be compared + * @return 0 if the arguments are equal, -1 if the first argument is smaller, 1 if the second argument is smaller + * @throws NullPointerException if any of the arguments are null + */ + public static int compare(int[] first, int[] second) { + if (first.length < second.length) { + return -1; + } + if (first.length > second.length) { + return 1; + } + + //lengths are equal, compare contents + for (int i = 0; i < first.length; i++) { + if (first[i] < second[i]) { + return -1; + } else if (first[i] > second[i]) { + return 1; + } + //values at index i are equal, continue... + } + + //we haven't returned yet; contents must be equal: + return 0; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/LazyMap.java b/vespajlib/src/main/java/com/yahoo/collections/LazyMap.java new file mode 100644 index 00000000000..1e1a75402eb --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/LazyMap.java @@ -0,0 +1,271 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.AbstractMap; +import java.util.AbstractSet; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Set; + +/** + * @author <a href="mailto:simon@hult-thoresen.com">Simon Thoresen Hult</a> + */ +public abstract class LazyMap<K, V> implements Map<K, V> { + + private Map<K, V> delegate = newEmpty(); + + @Override + public final int size() { + return delegate.size(); + } + + @Override + public final boolean isEmpty() { + return delegate.isEmpty(); + } + + @Override + public final boolean containsKey(Object key) { + return delegate.containsKey(key); + } + + @Override + public final boolean containsValue(Object value) { + return delegate.containsValue(value); + } + + @Override + public final V get(Object key) { + return delegate.get(key); + } + + @Override + public final V put(K key, V value) { + return delegate.put(key, value); + } + + @Override + public final V remove(Object key) { + return delegate.remove(key); + } + + @Override + public final void putAll(Map<? extends K, ? extends V> m) { + delegate.putAll(m); + } + + @Override + public final void clear() { + delegate.clear(); + } + + @Override + public final Set<K> keySet() { + return delegate.keySet(); + } + + @Override + public final Collection<V> values() { + return delegate.values(); + } + + @Override + public final Set<Entry<K, V>> entrySet() { + return delegate.entrySet(); + } + + @Override + public final int hashCode() { + return delegate.hashCode(); + } + + @Override + public final boolean equals(Object obj) { + return obj == this || (obj instanceof Map && delegate.equals(obj)); + } + + private Map<K, V> newEmpty() { + return new EmptyMap(); + } + + private Map<K, V> newSingleton(K key, V value) { + return new SingletonMap(key, value); + } + + protected abstract Map<K, V> newDelegate(); + + final Map<K, V> getDelegate() { + return delegate; + } + + class EmptyMap extends AbstractMap<K, V> { + + @Override + public V put(K key, V value) { + delegate = newSingleton(key, value); + return null; + } + + @Override + public void putAll(Map<? extends K, ? extends V> m) { + switch (m.size()) { + case 0: + break; + case 1: + Entry<? extends K, ? extends V> entry = m.entrySet().iterator().next(); + put(entry.getKey(), entry.getValue()); + break; + default: + delegate = newDelegate(); + delegate.putAll(m); + break; + } + } + + @Override + public Set<Entry<K, V>> entrySet() { + return Collections.emptySet(); + } + } + + class SingletonMap extends AbstractMap<K, V> { + + final K key; + V value; + + SingletonMap(K key, V value) { + this.key = key; + this.value = value; + } + + @Override + public V put(K key, V value) { + if (containsKey(key)) { + V oldValue = this.value; + this.value = value; + return oldValue; + } else { + delegate = newDelegate(); + delegate.put(this.key, this.value); + return delegate.put(key, value); + } + } + + @Override + public void putAll(Map<? extends K, ? extends V> m) { + switch (m.size()) { + case 0: + break; + case 1: + Entry<? extends K, ? extends V> entry = m.entrySet().iterator().next(); + put(entry.getKey(), entry.getValue()); + break; + default: + delegate = newDelegate(); + delegate.put(this.key, this.value); + delegate.putAll(m); + break; + } + } + + @Override + public Set<Entry<K, V>> entrySet() { + return new AbstractSet<Entry<K, V>>() { + + @Override + public Iterator<Entry<K, V>> iterator() { + return new Iterator<Entry<K, V>>() { + + boolean hasNext = true; + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public Entry<K, V> next() { + if (hasNext) { + hasNext = false; + return new Entry<K, V>() { + + @Override + public K getKey() { + return key; + } + + @Override + public V getValue() { + return value; + } + + @Override + public V setValue(V value) { + V oldValue = SingletonMap.this.value; + SingletonMap.this.value = value; + return oldValue; + } + + @Override + public int hashCode() { + return Objects.hashCode(key) + Objects.hashCode(value) * 31; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof Entry)) { + return false; + } + @SuppressWarnings("unchecked") + Entry<K, V> rhs = (Entry<K, V>)obj; + if (!Objects.equals(key, rhs.getKey())) { + return false; + } + if (!Objects.equals(value, rhs.getValue())) { + return false; + } + return true; + } + }; + } else { + throw new NoSuchElementException(); + } + } + + @Override + public void remove() { + if (hasNext) { + throw new IllegalStateException(); + } else { + delegate = newEmpty(); + } + } + }; + } + + @Override + public int size() { + return 1; + } + }; + } + } + + public static <K, V> LazyMap<K, V> newHashMap() { + return new LazyMap<K, V>() { + + @Override + protected Map<K, V> newDelegate() { + return new HashMap<>(); + } + }; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/LazySet.java b/vespajlib/src/main/java/com/yahoo/collections/LazySet.java new file mode 100644 index 00000000000..356b194c51f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/LazySet.java @@ -0,0 +1,225 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.AbstractSet; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Set; + +/** + * @author <a href="mailto:simon@hult-thoresen.com">Simon Thoresen Hult</a> + */ +public abstract class LazySet<E> implements Set<E> { + + private Set<E> delegate = newEmpty(); + + @Override + public final int size() { + return delegate.size(); + } + + @Override + public final boolean isEmpty() { + return delegate.isEmpty(); + } + + @Override + public final boolean contains(Object o) { + return delegate.contains(o); + } + + @Override + public final Iterator<E> iterator() { + return delegate.iterator(); + } + + @Override + public final Object[] toArray() { + return delegate.toArray(); + } + + @Override + public final <T> T[] toArray(T[] a) { + // noinspection SuspiciousToArrayCall + return delegate.toArray(a); + } + + @Override + public final boolean add(E e) { + return delegate.add(e); + } + + @Override + public final boolean remove(Object o) { + return delegate.remove(o); + } + + @Override + public final boolean containsAll(Collection<?> c) { + return delegate.containsAll(c); + } + + @Override + public final boolean addAll(Collection<? extends E> c) { + return delegate.addAll(c); + } + + @Override + public final boolean retainAll(Collection<?> c) { + return delegate.retainAll(c); + } + + @Override + public final boolean removeAll(Collection<?> c) { + return delegate.removeAll(c); + } + + @Override + public final void clear() { + delegate.clear(); + } + + @Override + public final int hashCode() { + return delegate.hashCode(); + } + + @Override + public final boolean equals(Object obj) { + return obj == this || (obj instanceof Set && delegate.equals(obj)); + } + + private Set<E> newEmpty() { + return new EmptySet(); + } + + private Set<E> newSingleton(E e) { + return new SingletonSet(e); + } + + protected abstract Set<E> newDelegate(); + + final Set<E> getDelegate() { + return delegate; + } + + class EmptySet extends AbstractSet<E> { + + @Override + public Iterator<E> iterator() { + return Collections.emptyIterator(); + } + + @Override + public int size() { + return 0; + } + + @Override + public boolean add(E e) { + delegate = newSingleton(e); + return true; + } + + @Override + public boolean addAll(Collection<? extends E> c) { + switch (c.size()) { + case 0: + return false; + case 1: + add(c.iterator().next()); + return true; + default: + delegate = newDelegate(); + delegate.addAll(c); + return true; + } + } + } + + class SingletonSet extends AbstractSet<E> { + + final E element; + + SingletonSet(E e) { + this.element = e; + } + + @Override + public Iterator<E> iterator() { + return new Iterator<E>() { + + boolean hasNext = true; + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public E next() { + if (hasNext) { + hasNext = false; + return element; + } else { + throw new NoSuchElementException(); + } + } + + @Override + public void remove() { + if (hasNext) { + throw new IllegalStateException(); + } else { + delegate = newEmpty(); + } + } + }; + } + + @Override + public int size() { + return 1; + } + + @Override + public boolean add(E e) { + if (contains(e)) { + return false; + } else { + delegate = newDelegate(); + delegate.add(element); + delegate.add(e); + return true; + } + } + + @Override + public boolean addAll(Collection<? extends E> c) { + switch (c.size()) { + case 0: + return false; + case 1: + return add(c.iterator().next()); + default: + delegate = newDelegate(); + delegate.add(element); + delegate.addAll(c); + return true; + } + } + } + + public static <E> LazySet<E> newHashSet() { + return new LazySet<E>() { + + @Override + protected Set<E> newDelegate() { + return new HashSet<>(); + } + }; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/ListMap.java b/vespajlib/src/main/java/com/yahoo/collections/ListMap.java new file mode 100644 index 00000000000..ab2c97fda17 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/ListMap.java @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.commons.lang.builder.ToStringBuilder; + +import java.util.*; + +/** + * A map holding multiple items at each key (using ArrayList and HashMap). + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ListMap<K, V> { + + private boolean frozen = false; + + private Map<K, List<V>> map; + + public ListMap() { + this(HashMap.class); + } + + @SuppressWarnings("unchecked") + public ListMap(@SuppressWarnings("rawtypes") Class<? extends Map> implementation) { + try { + this.map = implementation.newInstance(); + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } + + /** Puts an element into this. Multiple elements at the same position are added to the list at this key */ + public void put(K key, V value) { + List<V> list = map.get(key); + if (list == null) { + list = new ArrayList<>(); + map.put(key, list); + } + list.add(value); + } + + public void removeAll(K key) { + map.remove(key); + } + + public boolean removeValue(K key, V value) { + List<V> list = map.get(key); + if (list != null) + return list.remove(value); + else + return false; + } + + /** + * Removes the value at the given index. + * + * @return the removed value + * @throws IndexOutOfBoundsException if there is no value at the given index for this key + */ + public V removeValue(K key, int index) { + List<V> list = map.get(key); + if (list != null) + return list.remove(index); + else + throw new IndexOutOfBoundsException("The list at '" + key + "' is empty"); + } + + /** + * Returns the List containing the elements with this key, or an empty list + * if there are no elements. The list returned is unmodifiable. + */ + public List<V> get(K key) { + List<V> list = map.get(key); + if (list == null) + return ImmutableList.of();; + return ImmutableList.copyOf(list); + } + + /** The same as get */ + public List<V> getList(K key) { + return get(key); + } + + /** Returns the entries of this. Entries will be unmodifiable if this is frozen. */ + public Set<Map.Entry<K,List<V>>> entrySet() { return map.entrySet(); } + + /** Returns the keys of this */ + public Set<K> keySet() { return map.keySet(); } + + /** Returns the list values of this */ + public Collection<List<V>> values() { return map.values(); } + + /** + * Irreversibly prevent changes to the content of this. + * If this is already frozen, this method does nothing. + */ + public void freeze() { + if (frozen) return; + + for (Map.Entry<K,List<V>> entry : map.entrySet()) + entry.setValue(ImmutableList.copyOf(entry.getValue())); + this.map = ImmutableMap.copyOf(this.map); + } + + /** Returns whether this allows changes */ + public boolean isFrozen() { return frozen; } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this); + } + + /** Returns the number of keys in this map */ + public int size() { return map.size(); } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/ListenableArrayList.java b/vespajlib/src/main/java/com/yahoo/collections/ListenableArrayList.java new file mode 100644 index 00000000000..1b77e97d159 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/ListenableArrayList.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * An array list which notifies listeners after one or more items are added + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.1.17 + */ +@SuppressWarnings("serial") +public class ListenableArrayList<ITEM> extends ArrayList<ITEM> { + + private List<Runnable> listeners = null; + + public ListenableArrayList() {} + + public ListenableArrayList(int initialCapacity) { + super(initialCapacity); + } + + @Override + public boolean add(ITEM e) { + boolean result = super.add(e); + notifyListeners(); + return result; + } + + @Override + public void add(int index, ITEM e) { + super.add(index, e); + notifyListeners(); + } + + @Override + public boolean addAll(Collection<? extends ITEM> a) { + boolean result = super.addAll(a); + notifyListeners(); + return result; + } + + @Override + public boolean addAll(int index, Collection<? extends ITEM> a) { + boolean result = super.addAll(index, a); + notifyListeners(); + return result; + } + + @Override + public ITEM set(int index, ITEM e) { + ITEM result = super.set(index, e); + notifyListeners(); + return result; + } + + /** + * Adds a listener which is invoked whenever elements are added to this. + * This may not be invoked once for each added element. + */ + public void addListener(Runnable listener) { + if (listeners == null) + listeners = new ArrayList<>(); + listeners.add(listener); + } + + private void notifyListeners() { + if (listeners == null) return; + for (Runnable listener : listeners) + listener.run(); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/MD5.java b/vespajlib/src/main/java/com/yahoo/collections/MD5.java new file mode 100644 index 00000000000..b80a823eff0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/MD5.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import com.yahoo.text.Utf8; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * Convenience class for hashing a String with MD5, and either returning + * an int with the 4 LSBytes, or the whole 12-byte MD5 hash. + * <p> + * Note that instantiating this class can be expensive, so re-using instances + * is a good idea. + * <p> + * This class is not thread safe. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class MD5 { + public static final ThreadLocal<MessageDigest> md5 = new MD5Factory(); + + private static class MD5Factory extends ThreadLocal<MessageDigest> { + + @Override + protected MessageDigest initialValue() { + return createMD5(); + } + } + private static MessageDigest createMD5() { + try { + return MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + final private MessageDigest digester; + public MD5() { + try { + digester = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new AssertionError("MD5 algorithm not found."); + } + } + + public int hash(String s) { + byte[] md5 = digester.digest(Utf8.toBytes(s)); + int hash = 0; + assert (md5.length == 16); + + //produce an int by using only the 32 lsb: + int byte1 = (((int) md5[12]) << 24) & 0xFF000000; + int byte2 = (((int) md5[13]) << 16) & 0x00FF0000; + int byte3 = (((int) md5[14]) << 8) & 0x0000FF00; + int byte4 = (((int) md5[15])) & 0x000000FF; + + hash |= byte1; + hash |= byte2; + hash |= byte3; + hash |= byte4; + return hash; + } + + public byte[] hashFull(String s) { + return digester.digest(Utf8.toBytes(s)); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/MethodCache.java b/vespajlib/src/main/java/com/yahoo/collections/MethodCache.java new file mode 100644 index 00000000000..5dd9f68e5cc --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/MethodCache.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import com.yahoo.concurrent.CopyOnWriteHashMap; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 6/12/13 + * Time: 9:03 AM + * To change this template use File | Settings | File Templates. + */ +public final class MethodCache { + private final String methodName; + private final CopyOnWriteHashMap<String, Method> cache = new CopyOnWriteHashMap<>(); + + public MethodCache(String methodName) { + this.methodName = methodName; + } + + public final Method get(Object object) { + Method m = cache.get(object.getClass().getName()); + if (m == null) { + m = lookupMethod(object); + if (m != null) { + cache.put(object.getClass().getName(), m); + } + } + return m; + } + private Method lookupMethod(Object object) { + try { + return object.getClass().getMethod(methodName); + } catch (NoSuchMethodException e) { + return null; + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/Pair.java b/vespajlib/src/main/java/com/yahoo/collections/Pair.java new file mode 100644 index 00000000000..8969e1b1021 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/Pair.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +/** + * An immutable pair of objects. This implements equals and hashCode by delegating to the + * pair objects. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class Pair<F, S> { + + /** The first member for the pair. May be null. */ + private final F first; + /** The second member for the pair. May be null. */ + private final S second; + + /** Creates a pair. Each member may be set to null. */ + public Pair(final F first, final S second) { + this.first = first; + this.second = second; + } + + /** Returns the first member. This may be null. */ + public F getFirst() { return first; } + + /** Returns the second member. This may be null. */ + public S getSecond() { return second; } + + @Override + public int hashCode() { + return ( first != null ? first.hashCode() : 0 ) + + ( second != null ? 17*second.hashCode() : 0) ; + } + + @Override + public boolean equals(final Object o) { + if (o == this) return true; + if (!(o instanceof Pair)) return false; + + @SuppressWarnings("rawtypes") + final Pair other = (Pair) o; + return equals(this.first, other.first) + && equals(this.second, other.second); + } + + private static boolean equals(final Object a, final Object b) { + if (a == null) return b == null; + return a.equals(b); + } + + @Override + public String toString() { + return "(" + first + "," + second + ")"; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/PredicateSplit.java b/vespajlib/src/main/java/com/yahoo/collections/PredicateSplit.java new file mode 100644 index 00000000000..68dd39dc283 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/PredicateSplit.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.List; +import java.util.ArrayList; +import java.util.function.Predicate; + +/** + * Class holding the result of a partition-by-predicate operation. + **/ +public class PredicateSplit<E> { + public final List<E> falseValues; /// list of values where the predicate returned false + public final List<E> trueValues; /// list of values where the predicate returned true + + private PredicateSplit() { + falseValues = new ArrayList<E>(); + trueValues = new ArrayList<E>(); + } + + /** + * Perform a partition-by-predicate operation. + * Each value in the input is tested by the predicate and + * added to either the falseValues list or the trueValues list. + * @param collection The input collection. + * @param predicate A test for selecting the target list. + * @return Two lists bundled in an object. + **/ + public static <V> PredicateSplit<V> partition(Iterable<V> collection, Predicate<? super V> predicate) + { + PredicateSplit<V> r = new PredicateSplit<V>(); + for (V value : collection) { + if (predicate.test(value)) { + r.trueValues.add(value); + } else { + r.falseValues.add(value); + } + } + return r; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/ResourceFactory.java b/vespajlib/src/main/java/com/yahoo/collections/ResourceFactory.java new file mode 100644 index 00000000000..44d99f78cfe --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/ResourceFactory.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public abstract class ResourceFactory<T> { + + public abstract T create(); +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/ResourcePool.java b/vespajlib/src/main/java/com/yahoo/collections/ResourcePool.java new file mode 100644 index 00000000000..dcf73425f6d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/ResourcePool.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; + +/** + * <p>This implements a simple stack based resource pool. If you are out of resources new are allocated from the + * factory.</p> + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public final class ResourcePool<T> implements Iterable<T> { + + private final Deque<T> pool = new ArrayDeque<>(); + private final ResourceFactory<T> factory; + + public ResourcePool(ResourceFactory<T> factory) { + this.factory = factory; + } + + public final T alloc() { + return pool.isEmpty() ? factory.create() : pool.pop(); + } + + public final void free(T e) { + pool.push(e); + } + + @Override + public Iterator<T> iterator() { + return pool.iterator(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java b/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java new file mode 100644 index 00000000000..177a4e6720b --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java @@ -0,0 +1,260 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Set; + +/** + * A Set implementation which only considers object identity. It should only be + * used for small number of objects, as it is implemented as scanning an + * ArrayList for identity matches. In other words: Performance will only be + * acceptable for <i>small</i> sets. + * + * <p> + * The rationale for this class is the high cost of the object identifier used + * in IdentityHashMap, where the key set is often used as an identity set. + * </p> + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @since 5.1.4 + * @see java.util.IdentityHashMap + * + * @param <E> + * the type contained in the Set + */ +public final class TinyIdentitySet<E> implements Set<E> { + private class ArrayIterator<T> implements Iterator<E> { + private int i = -1; + private boolean removed = false; + + @Override + public boolean hasNext() { + return i + 1 < size; + } + + @SuppressWarnings("unchecked") + @Override + public E next() { + if (!hasNext()) { + throw new NoSuchElementException("No more elements available"); + } + removed = false; + return (E) entries[++i]; + } + + @Override + public void remove() { + if (removed) { + throw new IllegalStateException( + "Trying to remove same element twice."); + } + if (i == -1) { + throw new IllegalStateException( + "Trying to remove before entering iterator."); + } + delete(i--); + removed = true; + } + + } + + private Object[] entries; + private int size = 0; + + /** + * Create a set with an initial capacity of initSize. The internal array + * will grow automatically with a linear growth rate if more elements than + * initSize are added. + * + * @param initSize + * initial size of internal element array + */ + public TinyIdentitySet(final int initSize) { + entries = new Object[initSize]; + } + + /** + * Expose the index in the internal array of a given object. -1 is returned + * if the object is not present in the internal array. + * + * @param e + * an object to check whether exists in this set + * @return the index of the argument e in the internal array, or -1 if the + * object is not present + */ + public int indexOf(final Object e) { + for (int i = 0; i < size; ++i) { + if (e == entries[i]) { + return i; + } + } + return -1; + } + + private void clean() { + int offset = 0; + for (int i = 0; i < size; ++i) { + if (entries[i] == null) { + ++offset; + } else { + entries[i - offset] = entries[i]; + } + } + size -= offset; + } + + private void grow() { + // linear growth, as we should always be working on small sets + entries = Arrays.copyOf(entries, entries.length + 10); + } + + private void append(final Object arg) { + if (size == entries.length) { + grow(); + } + entries[size++] = arg; + } + + @Override + public boolean add(final E arg) { + final int i = indexOf(arg); + if (i >= 0) { + return false; + } + append(arg); + return true; + } + + @Override + public boolean addAll(final Collection<? extends E> arg) { + boolean changed = false; + for (final E entry : arg) { + changed |= add(entry); + } + return changed; + } + + @Override + public void clear() { + size = 0; + } + + @Override + public boolean contains(final Object arg) { + return indexOf(arg) >= 0; + } + + /** + * This is an extremely expensive implementation of + * {@link Set#containsAll(Collection)}. It is implemented as O(n**2). + */ + @Override + public boolean containsAll(final Collection<?> arg) { + for (final Object entry : arg) { + if (indexOf(entry) < 0) { + return false; + } + } + return true; + } + + @Override + public boolean isEmpty() { + return size == 0; + } + + @Override + public Iterator<E> iterator() { + return new ArrayIterator<E>(); + } + + private void delete(int i) { + if (i < 0 || i >= size) { + return; + } + --size; + while (i < size) { + entries[i] = entries[i + 1]; + ++i; + } + entries[i] = null; + } + + @Override + public boolean remove(final Object arg) { + final int i = indexOf(arg); + if (i < 0) { + return false; + } + delete(i); + return true; + } + + /** + * This is an extremely expensive implementation of + * {@link Set#removeAll(Collection)}. It is implemented as O(n**2). + */ + @Override + public boolean removeAll(final Collection<?> arg) { + boolean changed = false; + for (final Object entry : arg) { + final int i = indexOf(entry); + if (i >= 0) { + entries[i] = null; + changed = true; + } + } + if (changed) { + clean(); + } + return changed; + } + + /** + * This is an extremely expensive implementation of + * {@link Set#retainAll(Collection)}. It is implemented as O(n**2). + */ + @Override + public boolean retainAll(final Collection<?> arg) { + boolean changed = false; + for (int i = 0; i < size; ++i) { + final Object entry = entries[i]; + boolean exists = false; + // cannot use Collection.contains(), as we want identity + for (final Object v : arg) { + if (v == entry) { + exists = true; + break; + } + } + if (!exists) { + entries[i] = null; + changed = true; + } + } + if (changed) { + clean(); + } + return changed; + } + + @Override + public int size() { + return size; + } + + @Override + public Object[] toArray() { + return Arrays.copyOf(entries, size); + } + + @SuppressWarnings("unchecked") + @Override + public <T> T[] toArray(final T[] arg) { + return Arrays.copyOf(entries, size, (Class<T[]>) arg.getClass()); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/Tuple2.java b/vespajlib/src/main/java/com/yahoo/collections/Tuple2.java new file mode 100644 index 00000000000..4a817381f9c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/Tuple2.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +/** + * A representation of a pair of values, typically of different types. + * + * <p> + * This class is to avoid littering a class with thin wrapper objects for + * passing around e.g. the state of an operation and the result value. Using + * this class may be correct, but it is a symptom that you may want to redesign + * your code. (Should you pass mutable objects to the method instead? Create a + * new class and do the work inside that class instead? Etc.) + * </p> + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class Tuple2<T1, T2> { + + public final T1 first; + public final T2 second; + + public Tuple2(final T1 first, final T2 second) { + this.first = first; + this.second = second; + } + + /** + * hashCode() will always throw UnsupportedOperationException. The reason is + * this class is not meant for being put in Container implementation or + * similar use where Java generics will lead to a type unsafe maintenance + * nightmare. + * + * @throws UnsupportedOperationException + * will always throw this when invoked + */ + @Override + public int hashCode() { + throw new UnsupportedOperationException( + "com.yahoo.collections.Tuple2<T1, T2> does not support equals(Object) by design. Refer to JavaDoc for details."); + } + + /** + * equals(Object) will always throw UnsupportedOperationException. The + * intention is always using the objects contained in the tuple directly. + * + * @param obj + * ignored + * @throws UnsupportedOperationException + * will always throw this when invoked + */ + @Override + public boolean equals(final Object obj) { + throw new UnsupportedOperationException( + "com.yahoo.collections.Tuple2<T1, T2> does not support equals(Object) by design. Refer to JavaDoc for details."); + } + + /** + * Human readable string representation which invokes the contained + * instances' toString() implementation. + */ + @Override + public String toString() { + return "Tuple2(" + first + ", " + second + ")"; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/package-info.java b/vespajlib/src/main/java/com/yahoo/collections/package-info.java new file mode 100644 index 00000000000..7f09f0dd3d7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.collections; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java b/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java new file mode 100644 index 00000000000..a689884db0a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +/** + * Compression type enum. + * + * @author bratseth + */ +public enum CompressionType { + + // Do not change the type->ordinal association. The gap is due to historic types no longer supported. + NONE((byte) 0), + INCOMPRESSIBLE((byte) 5), + LZ4((byte) 6); + + private byte code; + + CompressionType(byte code) { + this.code = code; + } + + public byte getCode() { + return code; + } + + /** + * Returns whether this type represent actually compressed data + */ + public boolean isCompressed() { + return this != NONE && this != INCOMPRESSIBLE; + } + + public static CompressionType valueOf(byte value) { + switch (value) { + case ((byte) 0): + return NONE; + case ((byte) 5): + return INCOMPRESSIBLE; + case ((byte) 6): + return LZ4; + default: + throw new IllegalArgumentException("Unknown compression type ordinal " + value); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java new file mode 100644 index 00000000000..664ceaea7dc --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java @@ -0,0 +1,154 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import java.util.Arrays; +import java.util.Optional; + +/** + * Compressor which can compress and decompress in various formats. + * This class is thread safe. Creating a reusable instance is faster than creating instances as needed. + * + * @author bratseth + */ +public class Compressor { + + private final CompressionType type; + private final int level; + private final double compressionThresholdFactor; + private final int compressMinSizeBytes; + + private final LZ4Factory factory = LZ4Factory.fastestInstance(); + + /** Creates a compressor with default settings. */ + public Compressor() { + this(CompressionType.LZ4); + } + + /** Creates a compressor with a default compression type. */ + public Compressor(CompressionType type) { + this(type, 9, 0.95, 0); + } + + /** + * Creates a compressor. + * + * @param type the type of compression to use to compress data + * @param level a number between 0 and 9 where a higher value means more compression + * @param compressionThresholdFactor the compression factor we need to achieve to return the compressed data + * instead of raw data + * @param compressMinSizeBytes the minimal input data size to perform compression + */ + public Compressor(CompressionType type, int level, double compressionThresholdFactor, int compressMinSizeBytes) { + this.type = type; + this.level = level; + this.compressionThresholdFactor = compressionThresholdFactor; + this.compressMinSizeBytes = compressMinSizeBytes; + } + + /** Returns the default compression type used by this */ + public CompressionType type() { return type; } + + /** Returns the compression level this will use - a number between 0 and 9 where higher means more compression */ + public int level() { return level; } + + /** Returns the compression factor we need to achieve to return compressed rather than raw data */ + public double compressionThresholdFactor() { return compressionThresholdFactor; } + + /** Returns the minimal data size required to perform compression */ + public int compressMinSizeBytes() { return compressMinSizeBytes; } + + /** + * Compresses some data + * + * @param requestedCompression the desired compression type, which will be used if the data is deemed suitable. + * Not all the existing types are actually supported. + * @param data the data to compress. This array is only read by this method. + * @param uncompressedSize uncompressedSize the size in bytes of the data array. If this is not present, it is + * assumed that the size is the same as the data array size, i.e that it is completely + * filled with uncompressed data. + * @return the compression result + * @throws IllegalArgumentException if the compression type is not supported + */ + public Compression compress(CompressionType requestedCompression, byte[] data, Optional<Integer> uncompressedSize) { + switch (requestedCompression) { + case NONE: + data = uncompressedSize.isPresent() ? Arrays.copyOf(data, uncompressedSize.get()) : data; + return new Compression(CompressionType.NONE, data); + case LZ4: + int dataSize = uncompressedSize.isPresent() ? uncompressedSize.get() : data.length; + if (dataSize < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, data); + LZ4Compressor compressor = level < 7 ? factory.fastCompressor() : factory.highCompressor(); + byte[] compressedData = compressor.compress(data); + if (compressedData.length + 8 >= dataSize * compressionThresholdFactor) + return new Compression(CompressionType.INCOMPRESSIBLE, data); + return new Compression(CompressionType.LZ4, compressedData); + default: + throw new IllegalArgumentException(requestedCompression + " is not supported"); + } + } + /** Compresses some data using the compression type of this compressor */ + public Compression compress(CompressionType requestedCompression, byte[] data) { return compress(type, data, Optional.empty()); } + /** Compresses some data using the compression type of this compressor */ + public Compression compress(byte[] data, int uncompressedSize) { return compress(type, data, Optional.of(uncompressedSize)); } + /** Compresses some data using the compression type of this compressor */ + public Compression compress(byte[] data) { return compress(type, data, Optional.empty()); } + + /** + * Decompresses some data + * + * @param compression the compression type used + * @param compressedData the compressed data. This array is only read by this method. + * @param compressedDataOffset the offset in the compressed data at which to start decompression + * @param expectedUncompressedSize the uncompressed size in bytes of this data + * @param expectedCompressedSize the expected compressed size of the data in bytes, optionally for validation with LZ4. + * @return the uncompressed data, of the given size + * @throws IllegalArgumentException if the compression type is not supported + * @throws IllegalStateException if the expected compressed size is non-empty and specifies a different size than the actual size + */ + public byte[] decompress(CompressionType compression, byte[] compressedData, int compressedDataOffset, + int expectedUncompressedSize, Optional<Integer> expectedCompressedSize) { + switch (compression) { + case NONE: case INCOMPRESSIBLE: // return a copy of the requested slize of the input buffer + int endPosition = expectedCompressedSize.isPresent() ? compressedDataOffset + expectedCompressedSize.get() : compressedData.length; + return Arrays.copyOfRange(compressedData, compressedDataOffset, endPosition); + case LZ4: + byte[] uncompressedLZ4Data = new byte[expectedUncompressedSize]; + int compressedSize = factory.fastDecompressor().decompress(compressedData, compressedDataOffset, + uncompressedLZ4Data, 0, expectedUncompressedSize); + if (expectedCompressedSize.isPresent() && compressedSize != expectedCompressedSize.get()) + throw new IllegalStateException("Compressed size mismatch. Expected " + compressedSize + ". Got " + expectedCompressedSize.get()); + return uncompressedLZ4Data; + default: + throw new IllegalArgumentException(compression + " is not supported"); + } + } + /** Decompress some data */ + public byte[] decompress(byte[] compressedData, CompressionType compression, int uncompressedSize) { + return decompress(compression, compressedData, 0, uncompressedSize, Optional.empty()); + } + + public static class Compression { + + private final CompressionType compressionType; + + private final byte[] data; + + public Compression(CompressionType compressionType, byte[] data) { + this.compressionType = compressionType; + this.data = data; + } + + /** + * Returns the compression type used to compress this data. + * This will be either the requested compression or INCOMPRESSIBLE. + */ + public CompressionType type() { return compressionType; } + + /** Returns the uncompressed data in a buffer which gets owned by the caller */ + public byte[] data() { return data; } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/compress/IntegerCompressor.java b/vespajlib/src/main/java/com/yahoo/compress/IntegerCompressor.java new file mode 100644 index 00000000000..cbf12bd3d94 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/compress/IntegerCompressor.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +import java.nio.ByteBuffer; + +/** + * TODO: balder + */ +public class IntegerCompressor { + + public static void putCompressedNumber(int n, ByteBuffer buf) { + int negative = n < 0 ? 0x80 : 0x0; + if (negative != 0) { + n = -n; + } + if (n < (0x1 << 5)) { + byte b = (byte)(n | negative); + buf.put(b); + } else if (n < (0x1 << 13)) { + n = n | 0x4000 | (negative << 8); + buf.putShort((short)n); + } else if ( n < (0x1 << 29)) { + n = n | 0x60000000 | (negative << 24); + buf.putInt(n); + } else { + throw new IllegalArgumentException("Number '" + ((negative != 0) ? -n : n) + "' too big, must extend encoding"); + } + } + + public static void putCompressedPositiveNumber(int n, ByteBuffer buf) { + if (n < 0) { + throw new IllegalArgumentException("Number '" + n + "' must be positive"); + } + if (n < (0x1 << 6)) { + buf.put((byte)n); + } else if (n < (0x1 << 14)) { + n = n | 0x8000; + buf.putShort((short)n); + } else if ( n < (0x1 << 30)) { + n = n | 0xc0000000; + buf.putInt(n); + } else { + throw new IllegalArgumentException("Number '" + n + "' too big, must extend encoding"); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/compress/package-info.java b/vespajlib/src/main/java/com/yahoo/compress/package-info.java new file mode 100644 index 00000000000..52529a290e4 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/compress/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.compress; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/CopyOnWriteHashMap.java b/vespajlib/src/main/java/com/yahoo/concurrent/CopyOnWriteHashMap.java new file mode 100644 index 00000000000..e15a3734094 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/CopyOnWriteHashMap.java @@ -0,0 +1,94 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * <p>This is a thread hash map for small collections that are stable once built. Until it is stable there will be a + * race among all threads missing something in the map. They will then clone the map add the missing stuff and then put + * it back as active again. Here are no locks, but the cost is that inserts will happen a lot more than necessary. The + * map reference is volatile, but on most multicpu machines that has no cost unless modified.</p> + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public class CopyOnWriteHashMap<K, V> implements Map<K, V> { + + private volatile HashMap<K, V> map = new HashMap<>(); + + @Override + public int size() { + return map.size(); + } + + @Override + public boolean isEmpty() { + return map.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return map.containsKey(key); + } + + @Override + public boolean containsValue(Object value) { + return map.containsValue(value); + } + + @Override + public V get(Object key) { + return map.get(key); + } + + @Override + public V put(K key, V value) { + HashMap<K, V> next = new HashMap<>(map); + V old = next.put(key, value); + map = next; + return old; + } + + @Override + @SuppressWarnings("SuspiciousMethodCalls") + public V remove(Object key) { + HashMap<K, V> prev = map; + if (!prev.containsKey(key)) { + return null; + } + HashMap<K, V> next = new HashMap<>(prev); + V old = next.remove(key); + map = next; + return old; + } + + @Override + public void putAll(Map<? extends K, ? extends V> m) { + HashMap<K, V> next = new HashMap<>(map); + next.putAll(m); + map = next; + } + + @Override + public void clear() { + map = new HashMap<>(); + } + + @Override + public Set<K> keySet() { + return map.keySet(); + } + + @Override + public Collection<V> values() { + return map.values(); + } + + @Override + public Set<Entry<K, V>> entrySet() { + return map.entrySet(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java b/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java new file mode 100644 index 00000000000..38c5bafc0d6 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; + +/** + * A simple thread factory that decorates <code>Executors.defaultThreadFactory()</code> + * and sets all created threads to be daemon threads. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DaemonThreadFactory implements ThreadFactory { + private ThreadFactory defaultThreadFactory = Executors.defaultThreadFactory(); + private String prefix = null; + + /** + * Creates a deamon thread factory that creates threads with the default names + * provided by <code>Executors.defaultThreadFactory()</code>. + */ + public DaemonThreadFactory() { + } + + /** + * Creates a deamon thread factory that creates threads with the default names + * provided by <code>Executors.defaultThreadFactory()</code> prepended by the + * specified prefix. + * + * @param prefix the thread name prefix to use + */ + public DaemonThreadFactory(String prefix) { + this.prefix = prefix; + } + + public String getPrefix() { + return prefix; + } + + @Override + public Thread newThread(Runnable runnable) { + Thread t = defaultThreadFactory.newThread(runnable); + t.setDaemon(true); + if (prefix != null) { + t.setName(prefix + t.getName()); + } + return t; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/EventBarrier.java b/vespajlib/src/main/java/com/yahoo/concurrent/EventBarrier.java new file mode 100644 index 00000000000..389fe8a85ea --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/EventBarrier.java @@ -0,0 +1,140 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent;
+
+
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Reference implementation of the 'Incremental Minimal Event Barrier'
+ * algorithm. An event in this context is defined to be something that
+ * happens during a time interval. An event barrier is a time interval
+ * for which events may start before or end after, but not both. The
+ * problem solved by the algorithm is to determine the minimal event
+ * barrier starting at a given time. In other words; wait for the
+ * currently active events to complete. The most natural use of this
+ * algorithm would be to make a thread wait for events happening in
+ * other threads to complete.
+ *
+ * @author <a href="mailto:havardpe@yahoo-inc.com">Haavard Pettersen</a>
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class EventBarrier {
+
+ private final List<Entry> queue = new LinkedList<>();
+ private int barrierToken = 0;
+ private int eventCount = 0;
+
+ /**
+ * At creation there are no active events and no pending barriers.
+ */
+ public EventBarrier() {
+ // empty
+ }
+
+ /**
+ * Obtain the current number of active events. This method is
+ * intended for testing and debugging.
+ *
+ * @return Number of active events.
+ */
+ int getNumEvents() {
+ int cnt = eventCount;
+ for (Entry entry : queue) {
+ cnt += entry.eventCount;
+ }
+ return cnt;
+ }
+
+ /**
+ * Obtain the current number of pending barriers. This method is
+ * intended for testing and debugging.
+ *
+ * @return Number of pending barriers.
+ */
+ int getNumBarriers() {
+ return queue.size();
+ }
+
+ /**
+ * Signal the start of an event. The value returned from this
+ * method must later be passed to the completeEvent method when
+ * signaling the completion of the event.
+ *
+ * @return Opaque token identifying the started event.
+ */
+ public int startEvent() {
+ ++eventCount;
+ return barrierToken;
+ }
+
+ /**
+ * Signal the completion of an event. The value passed to this
+ * method must be the same as the return value previously obtained
+ * from the startEvent method. This method will signal the
+ * completion of all pending barriers that were completed by the
+ * completion of this event.
+ *
+ * @param token Opaque token identifying the completed event.
+ */
+ public void completeEvent(int token) {
+ if (token == this.barrierToken) {
+ --eventCount;
+ return;
+ }
+ --queue.get(queue.size() - (this.barrierToken - token)).eventCount;
+ while (!queue.isEmpty() && queue.get(0).eventCount == 0) {
+ queue.remove(0).handler.completeBarrier();
+ }
+ }
+
+ /**
+ * Initiate the detection of the minimal event barrier starting
+ * now. If this method returns false it means that no events were
+ * currently active and the minimal event barrier was infinitely
+ * small. If this method returns false the handler will not be
+ * notified of the completion of the barrier. If this method
+ * returns true it means that the started barrier is pending and
+ * that the handler passed to this method will be notified of its
+ * completion at a later time.
+ *
+ * @param handler Handler notified of the completion of the barrier.
+ * @return True if a barrier was started, false if no events were active.
+ */
+ public boolean startBarrier(BarrierWaiter handler) {
+ if (eventCount == 0 && queue.isEmpty()) {
+ return false;
+ }
+ queue.add(new Entry(eventCount, handler));
+ ++barrierToken;
+ eventCount = 0;
+ return true;
+ }
+
+ /**
+ * Declares the interface required to wait for the detection of a
+ * minimal event barrier. An object that implements this is passed
+ * to the {@link EventBarrier#startBarrier(BarrierWaiter)}.
+ */
+ public interface BarrierWaiter {
+
+ /**
+ * Callback invoked by the thread that detected the minimal
+ * event barrier. Once this is called, all events taking place
+ * at or before the corresponding call to {@link
+ * EventBarrier#startBarrier(BarrierWaiter)} have ended.
+ */
+ public void completeBarrier();
+ }
+
+ private static class Entry {
+
+ int eventCount;
+ final BarrierWaiter handler;
+
+ Entry(int eventCount, BarrierWaiter handler) {
+ this.eventCount = eventCount;
+ this.handler = handler;
+ }
+ }
+}
diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/LocalInstance.java b/vespajlib/src/main/java/com/yahoo/concurrent/LocalInstance.java new file mode 100644 index 00000000000..c2d19831810 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/LocalInstance.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import com.yahoo.concurrent.ThreadLocalDirectory.ObservableUpdater; +import com.yahoo.concurrent.ThreadLocalDirectory.Updater; + +/** + * Only for use along with ThreadLocalDirectory. A thread local data container + * instance. The class is visible to avoid indirection through the internal + * {@link ThreadLocal} in ThreadLocalDirectory if possible, but has no user + * available methods. + * + * @param AGGREGATOR + * the structure to insert produced data into + * @param SAMPLE + * type of produced data to insert from each participating thread + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class LocalInstance<AGGREGATOR, SAMPLE> { + /** + * The current generation of data produced from a single thread, where + * generation is the period between two subsequent calls to + * ThreadLocalDirectory.fetch(). + */ + private AGGREGATOR current; + + // see comment on setRegistered(boolean) for locking explanation + private boolean isRegistered = false; + private final Object lock = new Object(); + + LocalInstance(Updater<AGGREGATOR, SAMPLE> updater) { + current = updater.createGenerationInstance(null); + } + + boolean update(SAMPLE x, Updater<AGGREGATOR, SAMPLE> updater) { + synchronized (lock) { + current = updater.update(current, x); + return isRegistered; + } + } + + AGGREGATOR getAndReset(Updater<AGGREGATOR, SAMPLE> updater) { + AGGREGATOR previous; + synchronized (lock) { + previous = current; + current = updater.createGenerationInstance(previous); + setRegistered(false); + } + return previous; + } + + AGGREGATOR copyCurrent(ObservableUpdater<AGGREGATOR, SAMPLE> updater) { + AGGREGATOR view; + synchronized (lock) { + view = updater.copy(current); + } + return view; + } + + // This is either set by the putting thread or the fetching thread. If + // it is set by the putting thread, then there is no memory barrier, + // because it is only _read_ in the putting thread. If it is set by the + // fetching thread, then the memory barrier is this.lock. This + // roundabout way is to avoid creating many-to-many memory barrier and + // locking relationships. + void setRegistered(boolean isRegistered) { + this.isRegistered = isRegistered; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/Receiver.java b/vespajlib/src/main/java/com/yahoo/concurrent/Receiver.java new file mode 100644 index 00000000000..339d8002c4f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/Receiver.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import com.yahoo.collections.Tuple2; + +/** + * A class for sending single messages between threads with timeout. Typical use + * would be + * + * <pre> + * Receiver<SomeMessage> receiver = new Receiver<SomeMessage>(); + * SomeRunnable runnable = new SomeRunnable(receiver); + * Thread worker = new Thread(runnable); + * worker.start(); + * Pair<Receiver.MessageState, SomeMessage> answer = receiver.get(500L); + * </pre> + * + * ... and in the worker thread simply + * + * <pre> + * receiver.put(new SomeMessage(...)) + * </pre> + * + * <p> + * Any number of threads may wait for the same message. Sending null references + * is supported. The object is intended for delivering only single message, + * there is no support for recycling it. + * </p> + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class Receiver<T> { + /** + * MessageState is the reason for returning from get(). If a message is + * received before timeout, the state will be VALID. If no message is + * received before timeout, state is TIMEOUT. + */ + public enum MessageState { + VALID, TIMEOUT; + }; + private final Object lock = new Object(); + private T message = null; + private boolean received = false; + + /** + * Make a message available for consumers. + * + * @param message the message to send + * @throws IllegalStateException if a message has already been received here + */ + public void put(T message) { + synchronized (lock) { + if (received) { + throw new IllegalStateException("Multiple puts on a single Receiver instance is not legal."); + } + this.message = message; + received = true; + lock.notifyAll(); + } + } + + /** + * Wait for up to "timeout" milliseconds for an incoming message. This hides + * spurious wakeup, but InterruptedException will be propagated. + * + * @param timeout + * maximum time to wait for message in milliseconds + * @return a Pair instance containing the reason for returning and the + * message possible received + * @throws InterruptedException if the waiting thread is interrupted + */ + public Tuple2<MessageState, T> get(long timeout) throws InterruptedException { + long barrier = System.currentTimeMillis() + timeout; + synchronized (lock) { + while (!received) { + long t = System.currentTimeMillis(); + if (t >= barrier) { + return new Tuple2<>(MessageState.TIMEOUT, null); + } + lock.wait(barrier - t); + } + return new Tuple2<>(MessageState.VALID, message); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java b/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java new file mode 100644 index 00000000000..5aa4990a86a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This is an implementation of {@link Timer} that is backed by an actual system timer.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public enum SystemTimer implements Timer {
+
+ INSTANCE;
+
+ private volatile long millis;
+
+ private SystemTimer() {
+ millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
+ Thread thread = new Thread() {
+
+ @Override
+ public void run() {
+ while (true) {
+ millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
+ try {
+ Thread.sleep(1);
+ } catch (InterruptedException e) {
+ break;
+ }
+ }
+ }
+ };
+ thread.setDaemon(true);
+ thread.start();
+ }
+
+ @Override
+ public long milliTime() {
+ return millis;
+ }
+}
diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadFactoryFactory.java b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadFactoryFactory.java new file mode 100644 index 00000000000..5be6da8c66d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadFactoryFactory.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 24.04.13 + * Time: 19:00 + * To change this template use File | Settings | File Templates. + */ +public class ThreadFactoryFactory { + static public synchronized ThreadFactory getThreadFactory(String name) { + PooledFactory p = factory.get(name); + if (p == null) { + p = new PooledFactory(name); + factory.put(name, p); + } + return p.getFactory(false); + } + static public synchronized ThreadFactory getDaemonThreadFactory(String name) { + PooledFactory p = factory.get(name); + if (p == null) { + p = new PooledFactory(name); + factory.put(name, p); + } + return p.getFactory(true); + } + private static class PooledFactory { + private static class Factory implements ThreadFactory { + final ThreadGroup group; + final AtomicInteger threadNumber = new AtomicInteger(1); + final String namePrefix; + final boolean isDaemon; + + Factory(final String name, boolean isDaemon) { + this.isDaemon = isDaemon; + final SecurityManager s = System.getSecurityManager(); + group = (s != null) + ? s.getThreadGroup() + : Thread.currentThread().getThreadGroup(); + namePrefix = name; + } + + @Override + public Thread newThread(final Runnable r) { + final Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0); + if (t.isDaemon() != isDaemon) { + t.setDaemon(isDaemon); + } + if (t.getPriority() != Thread.NORM_PRIORITY) { + t.setPriority(Thread.NORM_PRIORITY); + } + return t; + } + } + PooledFactory(String name) { + this.name = name; + } + ThreadFactory getFactory(boolean isDaemon) { + return new Factory(name + "-" + poolId.getAndIncrement() + "-thread-", isDaemon); + + } + private final String name; + private final AtomicInteger poolId = new AtomicInteger(1); + } + static private Map<String, PooledFactory> factory = new HashMap<>(); +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java new file mode 100644 index 00000000000..ef2273bdb25 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java @@ -0,0 +1,346 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import java.util.ArrayList; +import java.util.List; + +/** + * A class for multiple producers and potentially multiple consumers (usually + * only one). + * + * <p> + * The consuming threads always unregisters the data producers when doing + * fetch(). This is the reason for having to do update through the directory. + * The reason for this is otherwise, we would either get reference leaks from + * registered objects belonging to dead threads if we did not unregister + * instances, otherwise the sampling thread would have to unregister the + * instance, and then we would create a memory relationship between all + * producing threads, which is exactly what this class aims to avoid. + * </p> + * + * <p> + * A complete example from a test: + * </p> + * + * <pre> + * private static class SumUpdater implements ThreadLocalDirectory.Updater<Integer, Integer> { + * + * {@literal @}Override + * public Integer update(Integer current, Integer x) { + * return Integer.valueOf(current.intValue() + x.intValue()); + * } + * + * {@literal @}Override + * public Integer createGenerationInstance(Integer previous) { + * return Integer.valueOf(0); + * } + * } + * + * ... then the producers does (where r is in instance of + * ThreadLocalDirectory)... + * + * {@literal @}Override + * public void run() { + * LocalInstance<Integer, Integer> s = r.getLocalInstance(); + * for (int i = 0; i < 500; ++i) { + * r.update(Integer.valueOf(i), s); + * } + * } + * + * ... and the consumer... + * + * List<Integer> measurements = s.fetch() + * </pre> + * + * <p> + * Invoking r.fetch() will produce a list of integers from all the participating + * threads at any time. + * </p> + * + * <p> + * Refer to e.g. com.yahoo.search.statistics.PeakQpsSearcher for a production + * example. + * </p> + * + * @param AGGREGATOR + * the type input data is aggregated into + * @param SAMPLE + * the type of input data + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { + /** + * Factory interface to create the data container for each generation of + * samples, and putting data into it. + * + * <p> + * The method for actual insertion of a single sample into the current data + * generation exists separate from LocalInstance.AGGREGATOR to make it + * possible to use e.g. Integer and List as AGGREGATOR types. + * </p> + * + * <p> + * The allocation and sampling is placed in the same class, since you always + * need to implement both. + * </p> + * + * @param AGGREGATOR + * The type of the data container to produce + * @param SAMPLE + * The type of the incoming data to store in the container. + */ + public interface Updater<AGGREGATOR, SAMPLE> { + /** + * Create data container to receive produced data. This is invoked once + * on every instance every time ThreadLocalDirectory.fetch() is invoked. + * This might be an empty list, creating a new counter set to zero, or + * even copying the current state of LocalInstance.current. + * LocalInstance.current will be set to the value received from this + * factory after invokation this method. + * + * <p> + * The first time this method is invoked for a thread, previous will be + * null. + * </p> + * + * <p> + * If using mutable objects, an implementation should always create a + * new instance in this method, as the previous data generation will be + * transmitted to the consuming thread. This obviously does not matter + * if using immutable (value) objects. + * </p> + * + * <p> + * Examples: + * </p> + * + * <p> + * Using a mutable aggregator (a list of integers): + * </p> + * + * <pre> + * if (previous == null) { + * return new ArrayList<Integer>(); + * } else { + * return new ArrayList<Integer>(previous.size()); + * } + * </pre> + * + * <p> + * Using an immutable aggregator (an integer): + * </p> + * + * <pre> + * return Integer.valueOf(0); + * </pre> + * + * @return a fresh structure to receive data + */ + public AGGREGATOR createGenerationInstance(AGGREGATOR previous); + + /** + * Insert a data element of type S into the current generation of data + * carrier T. This could be e.g. adding to a list, putting into a local + * histogram or increasing a counter. + * + * <p> + * The method may or may not return a fresh instance of the current + * value for each invokation, if using a mutable aggregator the typical + * case will be returning the same instance for the new and old value of + * current, while if using an immutable aggregator, one is forced to + * return new instances. + * </p> + * + * <p> + * Examples: + * </p> + * + * <p> + * Using a mutable aggregator (a list of instances of type SAMPLE): + * </p> + * + * <pre> + * current.add(x); + * return current; + * </pre> + * + * <p> + * Using an immutable aggregator (Integer) while also using Integer as + * type for SAMPLE: + * </p> + * + * <pre> + * return Integer.valueOf(current.intValue() + x.intValue()); + * </pre> + * + * @param current + * the current generation's data container + * @param x + * the data to insert + * @return the new current value, may be the same as previous + */ + public AGGREGATOR update(AGGREGATOR current, SAMPLE x); + } + + /** + * Implement this interface to be able to view the contents of a + * ThreadLocalDirectory without resetting the local instances in each + * thread. + * + * @param <AGGREGATOR> + * as for {@link Updater} + * @param <SAMPLE> + * as for {@link Updater} + * @see ThreadLocalDirectory#view() + */ + public interface ObservableUpdater<AGGREGATOR, SAMPLE> extends + Updater<AGGREGATOR, SAMPLE> { + /** + * Create an application specific copy of the AGGREGATOR for a thread. + * + * @param current + * the AGGREGATOR instance to copy + * @return a copy of the incoming parameter + */ + public AGGREGATOR copy(AGGREGATOR current); + } + + private final ThreadLocal<LocalInstance<AGGREGATOR, SAMPLE>> local = new ThreadLocal<>(); + private final Object directoryLock = new Object(); + private List<LocalInstance<AGGREGATOR, SAMPLE>> directory = new ArrayList<>(); + private final Updater<AGGREGATOR, SAMPLE> updater; + private final ObservableUpdater<AGGREGATOR, SAMPLE> observableUpdater; + + public ThreadLocalDirectory(Updater<AGGREGATOR, SAMPLE> updater) { + this.updater = updater; + if (updater instanceof ObservableUpdater) { + observableUpdater = (ObservableUpdater<AGGREGATOR, SAMPLE>) updater; + } else { + observableUpdater = null; + } + } + + private void put(LocalInstance<AGGREGATOR, SAMPLE> q) { + // Has to set registered before adding to the list. Otherwise, the + // instance might be removed from the list, set as unregistered, and + // then the local thread might happily remove that information. The Java + // memory model is a guarantuee for the minimum amount of visibility, + // not a definition of the actual amount. + q.setRegistered(true); + synchronized (directoryLock) { + directory.add(q); + } + } + + /** + * Fetch the current set of sampled data, and reset state of all thread + * local instances. The producer threads will not alter data in the list + * returned from this method. + * + * @return a list of data from all producer threads + */ + public List<AGGREGATOR> fetch() { + List<AGGREGATOR> contained; + List<LocalInstance<AGGREGATOR, SAMPLE>> previous; + int previousIntervalSize; + + synchronized (directoryLock) { + previousIntervalSize = directory.size(); + previous = directory; + directory = new ArrayList<>( + previousIntervalSize); + } + contained = new ArrayList<>(previousIntervalSize); + // Yes, this is an inconsistence about when the registered state is + // reset and when the thread local is removed from the list. + // LocalInstance.isRegistered tells whether the data is available to + // some consumer, not whether the LocalInstance is a member of the + // directory. + for (LocalInstance<AGGREGATOR, SAMPLE> x : previous) { + contained.add(x.getAndReset(updater)); + } + return contained; + } + + /** + * Get a view of the current data. This requires this ThreadLocalDirectory + * to have been instantiated with an updater implementing ObservableUpdater. + * + * @return a list of a copy of the current data in all producer threads + * @throws IllegalStateException + * if the updater does not implement {@link ObservableUpdater} + */ + public List<AGGREGATOR> view() { + if (observableUpdater == null) { + throw new IllegalStateException("Does not use observable updaters."); + } + List<LocalInstance<AGGREGATOR, SAMPLE>> current; + List<AGGREGATOR> view; + synchronized (directoryLock) { + current = new ArrayList<>( + directory); + } + view = new ArrayList<>(current.size()); + for (LocalInstance<AGGREGATOR, SAMPLE> x : current) { + view.add(x.copyCurrent(observableUpdater)); + } + return view; + } + + private LocalInstance<AGGREGATOR, SAMPLE> getOrCreateLocal() { + LocalInstance<AGGREGATOR, SAMPLE> current = local.get(); + if (current == null) { + current = new LocalInstance<>(updater); + local.set(current); + } + return current; + } + + /** + * Expose the thread local for the running thread, for use in conjunction + * with update(SAMPLE, LocalInstance<AGGREGATOR, SAMPLE>). + * + * @return the current thread's local instance + */ + public LocalInstance<AGGREGATOR, SAMPLE> getLocalInstance() { + return getOrCreateLocal(); + } + + /** + * Input data from a producer thread. + * + * @param x + * the data to insert + */ + public void update(SAMPLE x) { + update(x, getOrCreateLocal()); + } + + /** + * Update a value with a given thread local instance. + * + * <p> + * If a producer thread is to insert a series of data, it is desirable to + * limit the number of memory transactions to the theoretical minimum. Since + * reading a thread local is the memory equivalence of reading a volatile, + * it is then useful to avoid re-reading the running threads' input + * instance. For this scenario, fetch the running thread's instance with + * getLocalInstance(), and then insert the produced data with the multiple + * calls necessary to update(SAMPLE, LocalInstance<AGGREGATOR, SAMPLE>). + * </p> + * + * @param x + * the data to insert + * @param localInstance + * the local data insertion instance + */ + public void update(SAMPLE x, LocalInstance<AGGREGATOR, SAMPLE> localInstance) { + boolean isRegistered; + isRegistered = localInstance.update(x, updater); + if (!isRegistered) { + put(localInstance); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadRobustList.java b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadRobustList.java new file mode 100644 index 00000000000..8a79db6a6eb --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadRobustList.java @@ -0,0 +1,151 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * A list which tolerates concurrent adds from one other thread while it is + * read. More precisely: <i>This list is guaranteed to provide a self-consistent + * read view regardless of the internal order in which the primitive mutating + * operations on it are observed from the reading thread.</i> + * <p> + * This is useful for traced information as there may be timed out threads + * working on the structure after it is returned upwards for consumption. + * + * @since 4.2 + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ThreadRobustList<T> implements Iterable<T> { + + private Object[] items; + + /** Index of the next item */ + private int next = 0; + + public ThreadRobustList() { + this(10); + } + + public ThreadRobustList(final int initialCapacity) { + items = new Object[initialCapacity]; + } + + public void add(final T item) { + Object[] workItems = items; + if (next >= items.length) { + final int newLength = 20 + items.length * 2; + workItems = Arrays.copyOf(workItems, newLength); + workItems[next++] = item; + items = workItems; + } else { + workItems[next++] = item; + } + } + + /** + * Returns an iterator over the elements of this. This iterator does not + * support remove. + */ + @Override + public Iterator<T> iterator() { + return new ThreadRobustIterator(items); + } + + /** + * Returns an iterator over the elements of this, starting at the last + * element and working backwards. This iterator does not support remove. + */ + public Iterator<T> reverseIterator() { + return new ThreadRobustReverseIterator(items); + } + + public boolean isEmpty() { + return next == 0; + } + + private class ThreadRobustIterator implements Iterator<T> { + + private final Object[] items; + + private int nextIndex = 0; + + public ThreadRobustIterator(final Object[] items) { + this.items = items; + } + + public @Override + void remove() { + throw new UnsupportedOperationException( + "remove() is not supported on thread robust list iterators"); + } + + @SuppressWarnings("unchecked") + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException("No more elements"); + } + + return (T) items[nextIndex++]; + } + + @Override + public boolean hasNext() { + if (nextIndex >= items.length) { + return false; + } + if (items[nextIndex] == null) { + return false; + } + return true; + } + + } + + private class ThreadRobustReverseIterator implements Iterator<T> { + + private final Object[] items; + + private int nextIndex; + + public ThreadRobustReverseIterator(final Object[] items) { + this.items = items; + nextIndex = findLastAssignedIndex(items); + } + + private int findLastAssignedIndex(final Object[] items) { + for (int i = items.length - 1; i >= 0; i--) { + if (items[i] != null) { + return i; + } + } + return -1; + } + + public @Override + void remove() { + throw new UnsupportedOperationException( + "remove() is not supported on thread robust list iterators"); + } + + @SuppressWarnings("unchecked") + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException("No more elements"); + } + + return (T) items[nextIndex--]; + } + + @Override + public boolean hasNext() { + return nextIndex >= 0; + } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java b/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java new file mode 100644 index 00000000000..aefbfafb7b1 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent;
+
+/**
+ * This interface wraps access to some timer that can be used to measure elapsed time, in milliseconds. This
+ * abstraction allows for unit testing the behavior of time-based constructs.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public interface Timer {
+
+ /**
+ * Returns the current value of some arbitrary timer, in milliseconds. This method can only be used to measure
+ * elapsed time and is not related to any other notion of system or wall-clock time.
+ *
+ * @return The current value of the timer, in milliseconds.
+ */
+ public long milliTime();
+}
diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/package-info.java b/vespajlib/src/main/java/com/yahoo/concurrent/package-info.java new file mode 100644 index 00000000000..dd0d639166d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.concurrent; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/data/access/ArrayTraverser.java b/vespajlib/src/main/java/com/yahoo/data/access/ArrayTraverser.java new file mode 100644 index 00000000000..99d79dd8eb5 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/ArrayTraverser.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access; + +/** + * Callback interface for traversing arrays. + * Implement this and call Inspector.traverse() + * and you will get one callback for each array entry. + **/ +public interface ArrayTraverser { + /** + * Callback function to implement. + * @param idx array index for the current array entry. + * @param inspector accessor for the current array entry's value. + **/ + public void entry(int idx, Inspector inspector); +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/Inspectable.java b/vespajlib/src/main/java/com/yahoo/data/access/Inspectable.java new file mode 100644 index 00000000000..bf3344a0fe9 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/Inspectable.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access; + +/** + * Minimal API to implement for objects containing or exposing + * structured, generic, schemaless data. Use this when it's + * impractical to implement the Inspector interface directly. + **/ +public interface Inspectable { + /** get an Inspector exposing this object's structured data. */ + public Inspector inspect(); +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/Inspector.java b/vespajlib/src/main/java/com/yahoo/data/access/Inspector.java new file mode 100644 index 00000000000..0b0061792bd --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/Inspector.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access; + + +import java.util.Map; + +/** + * This is a generic API for accessing structured, generic, schemaless data. + * An inspector is a handle to a value that has one of 8 specific types: + * EMPTY, the 5 scalar types BOOL, LONG, DOUBLE, STRING, or DATA, the + * simple list-like ARRAY container and the struct-like OBJECT container. + * Instrospection methods are available, but you can also use accessors + * with a default value if you expect a certain type and just want your + * default value if some field doesn't exist or was of the wrong type. + **/ +public interface Inspector extends Inspectable { + + /** + * Check if the inspector is valid. + * If you try to access a field or array entry that does not exist, + * you will get an invalid Inspector returned. + */ + public boolean valid(); + + /** Get the type of an inspector */ + public Type type(); + + /** Get the number of entries in an ARRAY (always returns 0 for non-arrays) */ + public int entryCount(); + + /** Get the number of fields in an OBJECT (always returns 0 for non-objects) */ + public int fieldCount(); + + /** Access the inspector's value if it's a BOOLEAN; otherwise throws exception */ + public boolean asBool(); + + /** Access the inspector's value if it's a LONG (or DOUBLE); otherwise throws exception */ + public long asLong(); + + /** Access the inspector's value if it's a DOUBLE (or LONG); otherwise throws exception */ + public double asDouble(); + + /** Access the inspector's value if it's a STRING; otherwise throws exception */ + public String asString(); + + /** + * Access the inspector's value (in utf-8 representation) if it's + * a STRING; otherwise throws exception + **/ + public byte[] asUtf8(); + + /** Access the inspector's value if it's DATA; otherwise throws exception */ + public byte[] asData(); + + /** Get the inspector's value (or the supplied default), never throws */ + public boolean asBool(boolean defaultValue); + + /** Get the inspector's value (or the supplied default), never throws */ + public long asLong(long defaultValue); + + /** Get the inspector's value (or the supplied default), never throws */ + public double asDouble(double defaultValue); + + /** Get the inspector's value (or the supplied default), never throws */ + public String asString(String defaultValue); + + /** Get the inspector's value (or the supplied default), never throws */ + public byte[] asUtf8(byte[] defaultValue); + + /** Get the inspector's value (or the supplied default), never throws */ + public byte[] asData(byte[] defaultValue); + + /** + * Traverse an array value, performing callbacks for each entry. + * + * If the current Inspector is connected to an array value, + * perform callbacks to the given traverser for each entry + * contained in the array. Otherwise a no-op. + * @param at traverser callback object. + **/ + @SuppressWarnings("overloads") + public void traverse(ArrayTraverser at); + + /** + * Traverse an object value, performing callbacks for each field. + * + * If the current Inspector is connected to an object value, + * perform callbacks to the given traverser for each field + * contained in the object. Otherwise a no-op. + * @param ot traverser callback object. + **/ + @SuppressWarnings("overloads") + public void traverse(ObjectTraverser ot); + + /** + * Access an array entry. + * + * If the current Inspector doesn't connect to an array value, + * or the given array index is out of bounds, the returned + * Inspector will be invalid. + * @param idx array index. + * @return a new Inspector for the entry value. + **/ + public Inspector entry(int idx); + + /** + * Access an field in an object. + * + * If the current Inspector doesn't connect to an object value, or + * the object value does not contain a field with the given symbol + * name, the returned Inspector will be invalid. + * @param name symbol name. + * @return a new Inspector for the field value. + **/ + public Inspector field(String name); + + /** + * Convert an array to an iterable list. Other types will just + * return an empty list. + **/ + public Iterable<Inspector> entries(); + + /** + * Convert an object to an iterable list of (name, value) pairs. + * Other types will just return an empty list. + **/ + public Iterable<Map.Entry<String,Inspector>> fields(); +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/ObjectTraverser.java b/vespajlib/src/main/java/com/yahoo/data/access/ObjectTraverser.java new file mode 100644 index 00000000000..90ff360d63d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/ObjectTraverser.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access; + +/** + * Callback interface for traversing objects. + * Implement this and call Inspector.traverse() + * and you will get one callback for each field in an object. + **/ +public interface ObjectTraverser { + /** + * Callback function to implement. + * @param name the name of the current field. + * @param inspector accessor for the current field's value. + **/ + public void field(String name, Inspector inspector); +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/Type.java b/vespajlib/src/main/java/com/yahoo/data/access/Type.java new file mode 100644 index 00000000000..e09b0ba9adc --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/Type.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access; + +/** + * Enumeration of all possible types accessed by the Inspector API. + * Note that: + * - the EMPTY type is used as a placeholder where data is missing. + * - all integers are put into LONGs; the encoding takes care of + * packing small integers compactly so this is also efficient. + * - likeweise DOUBLE is the only floating-point type, but "simple" + * numbers (like 0.0 or 1.0) are packed compactly anyway. + * - DATA can be used anything for wrapping anything else serialized + * as an array of bytes. + * - maps should be represented as an ARRAY of OBJECTs where each + * object has the fields "key" and "value". + **/ +public enum Type { + EMPTY, BOOL, LONG, DOUBLE, STRING, DATA, ARRAY, OBJECT; +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/package-info.java b/vespajlib/src/main/java/com/yahoo/data/access/package-info.java new file mode 100644 index 00000000000..904686c5d78 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.data.access; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/data/access/simple/JsonRender.java b/vespajlib/src/main/java/com/yahoo/data/access/simple/JsonRender.java new file mode 100644 index 00000000000..6acc43c2198 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/simple/JsonRender.java @@ -0,0 +1,166 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access.simple; + +import com.yahoo.text.DoubleFormatter; +import com.yahoo.data.access.*; + +/** + * Encodes json from an inspectable object. + * + * @author arnej27959 + */ +public final class JsonRender +{ + public static StringBuilder render(Inspectable value, + StringBuilder target, + boolean compact) + { + StringEncoder enc = new StringEncoder(target, compact); + enc.encode(value.inspect()); + return target; + } + + public static final class StringEncoder implements ArrayTraverser, ObjectTraverser + { + private final StringBuilder out; + private boolean head = true; + private boolean compact; + private int level = 0; + + public StringEncoder(StringBuilder out, boolean compact) { + this.out = out; + this.compact = compact; + } + + public void encode(Inspector top) { + encodeValue(top); + if (!compact) { + out.append('\n'); + } + } + + private void encodeEMPTY() { + out.append("null"); + } + + private void encodeBOOL(boolean value) { + out.append(value ? "true" : "false"); + } + + private void encodeLONG(long value) { + out.append(String.valueOf(value)); + } + + private void encodeDOUBLE(double value) { + if (Double.isNaN(value) || Double.isInfinite(value)) { + out.append("null"); + } else { + out.append(DoubleFormatter.stringValue(value)); + } + } + + static final char[] hex = "0123456789ABCDEF".toCharArray(); + + private void encodeSTRING(String value) { + out.append('"'); + for (char c : value.toCharArray()) { + switch (c) { + case '"': out.append('\\').append('"'); break; + case '\\': out.append('\\').append('\\'); break; + case '\b': out.append('\\').append('b'); break; + case '\f': out.append('\\').append('f'); break; + case '\n': out.append('\\').append('n'); break; + case '\r': out.append('\\').append('r'); break; + case '\t': out.append('\\').append('t'); break; + default: + if (c > 0x1f && c < 127) { + out.append(c); + } else { // requires escaping according to RFC 4627 + out.append('\\').append('u'); + out.append(hex[(c >> 12) & 0xf]); + out.append(hex[(c >> 8) & 0xf]); + out.append(hex[(c >> 4) & 0xf]); + out.append(hex[c & 0xf]); + } + } + } + out.append('"'); + } + + private void encodeDATA(byte[] value) { + out.append('"'); + out.append("0x"); + for (int pos = 0; pos < value.length; pos++) { + out.append(hex[(value[pos] >> 4) & 0xf]); + out.append(hex[value[pos] & 0xf]); + } + out.append('"'); + } + + private void encodeARRAY(Inspector inspector) { + openScope("["); + ArrayTraverser at = this; + inspector.traverse(at); + closeScope("]"); + } + + private void encodeOBJECT(Inspector inspector) { + openScope("{"); + ObjectTraverser ot = this; + inspector.traverse(ot); + closeScope("}"); + } + + private void openScope(String opener) { + out.append(opener); + level++; + head = true; + } + + private void closeScope(String closer) { + level--; + separate(false); + out.append(closer); + } + + private void encodeValue(Inspector inspector) { + switch(inspector.type()) { + case EMPTY: encodeEMPTY(); return; + case BOOL: encodeBOOL(inspector.asBool()); return; + case LONG: encodeLONG(inspector.asLong()); return; + case DOUBLE: encodeDOUBLE(inspector.asDouble()); return; + case STRING: encodeSTRING(inspector.asString()); return; + case DATA: encodeDATA(inspector.asData()); return; + case ARRAY: encodeARRAY(inspector); return; + case OBJECT: encodeOBJECT(inspector); return; + } + assert false : "Should not be reached"; + } + + private void separate(boolean useComma) { + if (!head && useComma) { + out.append(','); + } else { + head = false; + } + if (!compact) { + out.append("\n"); + for (int lvl = 0; lvl < level; lvl++) { out.append(" "); } + } + } + + public void entry(int idx, Inspector inspector) { + separate(true); + encodeValue(inspector); + } + + public void field(String name, Inspector inspector) { + separate(true); + encodeSTRING(name); + out.append(':'); + if (!compact) + out.append(' '); + encodeValue(inspector); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/simple/Value.java b/vespajlib/src/main/java/com/yahoo/data/access/simple/Value.java new file mode 100644 index 00000000000..baf5ce3cc54 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/simple/Value.java @@ -0,0 +1,215 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access.simple; + + +import com.yahoo.data.access.*; +import java.util.Collections; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.ArrayList; +import java.nio.charset.StandardCharsets; + + +public class Value implements Inspector { + private static Value empty = new EmptyValue(); + private static Value invalid = new Value(); + private static byte[] empty_array = new byte[0]; + public static Inspector empty() { return empty; } + public static Inspector invalid() { return invalid; } + public Inspector inspect() { return this; } + public boolean valid() { return false; } + public Type type() { return Type.EMPTY; } + public int entryCount() { return 0; } + public int fieldCount() { return 0; } + public boolean asBool() { throw new IllegalStateException("invalid data access!"); } + public long asLong() { throw new IllegalStateException("invalid data access!"); } + public double asDouble() { throw new IllegalStateException("invalid data access!"); } + public java.lang.String asString() { throw new IllegalStateException("invalid data access!"); } + public byte[] asUtf8() { throw new IllegalStateException("invalid data access!"); } + public byte[] asData() { throw new IllegalStateException("invalid data access!"); } + public boolean asBool(boolean defaultValue) { return defaultValue; } + public long asLong(long defaultValue) { return defaultValue; } + public double asDouble(double defaultValue) { return defaultValue; } + public java.lang.String asString(java.lang.String defaultValue) { return defaultValue; } + public byte[] asUtf8(byte[] defaultValue) { return defaultValue; } + public byte[] asData(byte[] defaultValue) { return defaultValue; } + public void traverse(ArrayTraverser at) {} + public void traverse(ObjectTraverser ot) {} + public Inspector entry(int idx) { return invalid; } + public Inspector field(java.lang.String name) { return invalid; } + public Iterable<Inspector> entries() { return Collections.emptyList(); } + public Iterable<Map.Entry<java.lang.String,Inspector>> fields() { return Collections.emptyList(); } + public StringBuilder writeJson(StringBuilder target) { + return JsonRender.render(this, target, true); + } + public String toJson() { return writeJson(new StringBuilder()).toString(); } + public String toString() { return toJson(); } + static public class EmptyValue extends Value { + public boolean valid() { return true; } + public boolean asBool() { return false; } + public long asLong() { return 0L; } + public double asDouble() { return 0.0; } + public java.lang.String asString() { return ""; } + public byte[] asUtf8() { return empty_array; } + public byte[] asData() { return empty_array; } + } + static public class BoolValue extends Value { + private boolean value; + public BoolValue(boolean v) { value = v; } + public boolean valid() { return true; } + public Type type() { return Type.BOOL; } + public boolean asBool() { return value; } + public boolean asBool(boolean x) { return value; } + } + static public class LongValue extends Value { + private long value; + public LongValue(long v) { value = v; } + public boolean valid() { return true; } + public Type type() { return Type.LONG; } + public long asLong() { return value; } + public double asDouble() { return (double)value; } + public long asLong(long x) { return value; } + public double asDouble(double x) { return (double)value; } + } + static public class DoubleValue extends Value { + private double value; + public DoubleValue(double v) { value = v; } + public boolean valid() { return true; } + public Type type() { return Type.DOUBLE; } + public double asDouble() { return value; } + public long asLong() { return (long)value; } + public double asDouble(double x) { return value; } + public long asLong(long x) { return (long)value; } + } + static public class StringValue extends Value { + private java.lang.String string_value = null; + private byte[] utf8_value = null; + private void handle_null() { + if (string_value == null && utf8_value == null) { + string_value = ""; + utf8_value = empty_array; + } + } + public StringValue(java.lang.String v) { + string_value = v; + handle_null(); + } + public StringValue(byte[] v) { + utf8_value = v; + handle_null(); + } + public boolean valid() { return true; } + public Type type() { return Type.STRING; } + public java.lang.String asString() { + if (string_value == null) { + string_value = new java.lang.String(utf8_value, StandardCharsets.UTF_8); + } + return string_value; + } + public java.lang.String asString(java.lang.String x) { return asString(); } + public byte[] asUtf8() { + if (utf8_value == null) { + utf8_value = string_value.getBytes(StandardCharsets.UTF_8); + } + return utf8_value; + } + public byte[] asUtf8(byte[] x) { return asUtf8(); } + } + static public class DataValue extends Value { + private byte[] value; + public DataValue(byte[] v) { + value = v; + if (v == null) { + value = empty_array; + } + } + public boolean valid() { return true; } + public Type type() { return Type.DATA; } + public byte[] asData() { return value; } + public byte[] asData(byte[] x) { return value; } + } + static public class ArrayValue extends Value { + private List<Inspector> values = new ArrayList<>(); + public boolean valid() { return true; } + public Type type() { return Type.ARRAY; } + public int entryCount() { return values.size(); } + public Inspector entry(int idx) { + if (idx < 0 || idx >= values.size()) { + return invalid; + } + return values.get(idx); + } + public void traverse(ArrayTraverser at) { + int idx = 0; + for (Inspector i: values) { + at.entry(idx++, i); + } + } + public Iterable<Inspector> entries() { + return Collections.unmodifiableList(values); + } + public ArrayValue add(Inspector v) { + if (v == null || !v.valid()) { + throw new IllegalArgumentException("tried to add an invalid value to an array"); + } + values.add(v); + return this; + } + public ArrayValue add(java.lang.String value) { + return add(new Value.StringValue(value)); + } + public ArrayValue add(long value) { + return add(new Value.LongValue(value)); + } + public ArrayValue add(int value) { + return add(new Value.LongValue(value)); + } + public ArrayValue add(double value) { + return add(new Value.DoubleValue(value)); + } + } + static public class ObjectValue extends Value { + private Map<java.lang.String,Inspector> values = new LinkedHashMap<>(); + public boolean valid() { return true; } + public Type type() { return Type.OBJECT; } + public int fieldCount() { return values.size(); } + public Inspector field(java.lang.String name) { + Inspector v = values.get(name); + if (v == null) { + return invalid; + } + return v; + } + public void traverse(ObjectTraverser ot) { + for (Map.Entry<java.lang.String,Inspector> i: values.entrySet()) { + ot.field(i.getKey(), i.getValue()); + } + } + public Iterable<Map.Entry<java.lang.String,Inspector>> fields() { + return Collections.<java.lang.String,Inspector>unmodifiableMap(values).entrySet(); + } + public ObjectValue put(java.lang.String name, Inspector v) { + if (name == null) { + throw new IllegalArgumentException("field name was <null>"); + } + if (v == null || !v.valid()) { + throw new IllegalArgumentException("tried to put an invalid value into an object"); + } + values.put(name, v); + return this; + } + public ObjectValue put(java.lang.String name, java.lang.String value) { + return put(name, new Value.StringValue(value)); + } + public ObjectValue put(java.lang.String name, long value) { + return put(name, new Value.LongValue(value)); + } + public ObjectValue put(java.lang.String name, int value) { + return put(name, new Value.LongValue(value)); + } + public ObjectValue put(java.lang.String name, double value) { + return put(name, new Value.DoubleValue(value)); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/simple/package-info.java b/vespajlib/src/main/java/com/yahoo/data/access/simple/package-info.java new file mode 100644 index 00000000000..730dc508b21 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/simple/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.data.access.simple; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/data/access/slime/SlimeAdapter.java b/vespajlib/src/main/java/com/yahoo/data/access/slime/SlimeAdapter.java new file mode 100644 index 00000000000..adfadfe8bb8 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/slime/SlimeAdapter.java @@ -0,0 +1,156 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access.slime; + + +import java.util.Map; +import java.util.AbstractMap; +import java.util.List; +import java.util.ArrayList; + + +public final class SlimeAdapter implements com.yahoo.data.access.Inspector { + private com.yahoo.slime.Inspector inspector; + public SlimeAdapter(com.yahoo.slime.Inspector inspector) { this.inspector = inspector; } + @Override public boolean equals(Object rhs) { + if (!(rhs instanceof SlimeAdapter)) { + return false; + } + return inspector.equals(((SlimeAdapter)rhs).inspector); + } + @Override public int hashCode() { return inspector.hashCode(); } + @Override public String toString() { return inspector.toString(); } + public com.yahoo.data.access.Inspector inspect() { return this; } + public boolean valid() { return inspector.valid(); } + public com.yahoo.data.access.Type type() { + switch(inspector.type()) { + case NIX: return com.yahoo.data.access.Type.EMPTY; + case BOOL: return com.yahoo.data.access.Type.BOOL; + case LONG: return com.yahoo.data.access.Type.LONG; + case DOUBLE: return com.yahoo.data.access.Type.DOUBLE; + case STRING: return com.yahoo.data.access.Type.STRING; + case DATA: return com.yahoo.data.access.Type.DATA; + case ARRAY: return com.yahoo.data.access.Type.ARRAY; + case OBJECT: return com.yahoo.data.access.Type.OBJECT; + } + return com.yahoo.data.access.Type.EMPTY; + } + private boolean verify(com.yahoo.slime.Type ok_type_a) { + com.yahoo.slime.Type my_type = inspector.type(); + return (valid() && (my_type == ok_type_a)); + } + private boolean verify(com.yahoo.slime.Type ok_type_a, + com.yahoo.slime.Type ok_type_b) + { + com.yahoo.slime.Type my_type = inspector.type(); + return (valid() && (my_type == ok_type_a || my_type == ok_type_b)); + } + private boolean verify(com.yahoo.slime.Type ok_type_a, + com.yahoo.slime.Type ok_type_b, + com.yahoo.slime.Type ok_type_c) + { + com.yahoo.slime.Type my_type = inspector.type(); + return (valid() && (my_type == ok_type_a || my_type == ok_type_b || my_type == ok_type_c)); + } + public int entryCount() { return inspector.entries(); } + public int fieldCount() { return inspector.fields(); } + public boolean asBool() { + if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.BOOL)) { + throw new IllegalStateException("invalid data extraction!"); + } + return inspector.asBool(); + } + public long asLong() { + if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.LONG, com.yahoo.slime.Type.DOUBLE)) { + throw new IllegalStateException("invalid data extraction!"); + } + return inspector.asLong(); + } + public double asDouble() { + if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.DOUBLE, com.yahoo.slime.Type.LONG)) { + throw new IllegalStateException("invalid data extraction!"); + } + return inspector.asDouble(); + } + public String asString() { + if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.STRING)) { + throw new IllegalStateException("invalid data extraction!"); + } + return inspector.asString(); + } + public byte[] asUtf8() { + if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.STRING)) { + throw new IllegalStateException("invalid data extraction!"); + } + return inspector.asUtf8(); + } + public byte[] asData() { + if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.DATA)) { + throw new IllegalStateException("invalid data extraction!"); + } + return inspector.asData(); + } + public boolean asBool(boolean defaultValue) { + if (!verify(com.yahoo.slime.Type.BOOL)) { + return defaultValue; + } + return inspector.asBool(); + } + public long asLong(long defaultValue) { + if (!verify(com.yahoo.slime.Type.LONG, com.yahoo.slime.Type.DOUBLE)) { + return defaultValue; + } + return inspector.asLong(); + } + public double asDouble(double defaultValue) { + if (!verify(com.yahoo.slime.Type.DOUBLE, com.yahoo.slime.Type.LONG)) { + return defaultValue; + } + return inspector.asDouble(); + } + public String asString(String defaultValue) { + if (!verify(com.yahoo.slime.Type.STRING)) { + return defaultValue; + } + return inspector.asString(); + } + public byte[] asUtf8(byte[] defaultValue) { + if (!verify(com.yahoo.slime.Type.STRING)) { + return defaultValue; + } + return inspector.asUtf8(); + } + public byte[] asData(byte[] defaultValue) { + if (!verify(com.yahoo.slime.Type.DATA)) { + return defaultValue; + } + return inspector.asData(); + } + public void traverse(final com.yahoo.data.access.ArrayTraverser at) { + inspector.traverse(new com.yahoo.slime.ArrayTraverser() { + public void entry(int idx, com.yahoo.slime.Inspector inspector) { at.entry(idx, new SlimeAdapter(inspector)); } + }); + } + public void traverse(final com.yahoo.data.access.ObjectTraverser ot) { + inspector.traverse(new com.yahoo.slime.ObjectTraverser() { + public void field(String name, com.yahoo.slime.Inspector inspector) { ot.field(name, new SlimeAdapter(inspector)); } + }); + } + public com.yahoo.data.access.Inspector entry(int idx) { return new SlimeAdapter(inspector.entry(idx)); } + public com.yahoo.data.access.Inspector field(String name) { return new SlimeAdapter(inspector.field(name)); } + public Iterable<com.yahoo.data.access.Inspector> entries() { + final List<com.yahoo.data.access.Inspector> list = new ArrayList<>(); + inspector.traverse(new com.yahoo.slime.ArrayTraverser() { + public void entry(int idx, com.yahoo.slime.Inspector inspector) { list.add(new SlimeAdapter(inspector)); } + }); + return list; + } + public Iterable<Map.Entry<String,com.yahoo.data.access.Inspector>> fields() { + final List<Map.Entry<String,com.yahoo.data.access.Inspector>> list = new ArrayList<>(); + inspector.traverse(new com.yahoo.slime.ObjectTraverser() { + public void field(String name, com.yahoo.slime.Inspector inspector) { + list.add(new AbstractMap.SimpleImmutableEntry<String,com.yahoo.data.access.Inspector>(name, new SlimeAdapter(inspector))); + } + }); + return list; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/slime/package-info.java b/vespajlib/src/main/java/com/yahoo/data/access/slime/package-info.java new file mode 100644 index 00000000000..bf6ae26baee --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/slime/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.data.access.slime; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/data/inspect/slime/.gitignore b/vespajlib/src/main/java/com/yahoo/data/inspect/slime/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/inspect/slime/.gitignore diff --git a/vespajlib/src/main/java/com/yahoo/errorhandling/Results.java b/vespajlib/src/main/java/com/yahoo/errorhandling/Results.java new file mode 100644 index 00000000000..310f679c883 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/errorhandling/Results.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.errorhandling; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * @author tonytv + */ +public class Results<DATA, ERROR> { + private final List<DATA> data; + private final List<ERROR> errors; + + public Results(List<DATA> data, List<ERROR> errors) { + this.data = ImmutableList.copyOf(data); + this.errors = ImmutableList.copyOf(errors); + } + + public boolean hasErrors() { + return !errors.isEmpty(); + } + + public List<DATA> data() { + return data; + } + + public List<ERROR> errors() { + return errors; + } + + public static class Builder<DATA, ERROR> { + private final List<DATA> data = new ArrayList<>(); + private final List<ERROR> errors = new ArrayList<>(); + + public void addData(DATA d) { + data.add(d); + } + + public void addAllData(Collection<? extends DATA> d) { + data.addAll(d); + } + + public void addError(ERROR e) { + errors.add(e); + } + + public void addAllErrors(Collection<? extends ERROR> e) { + errors.addAll(e); + } + + public Results<DATA, ERROR> build() { + return new Results<>(data, errors); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/errorhandling/package-info.java b/vespajlib/src/main/java/com/yahoo/errorhandling/package-info.java new file mode 100644 index 00000000000..4d07d20053d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/errorhandling/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.errorhandling; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/geo/BoundingBoxParser.java b/vespajlib/src/main/java/com/yahoo/geo/BoundingBoxParser.java new file mode 100644 index 00000000000..001386cd4b0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/geo/BoundingBoxParser.java @@ -0,0 +1,147 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.geo; + +import com.yahoo.text.DoubleParser; + + +/** + * Class for parsing a bounding box in text format: + * "n=37.44899,s=37.3323,e=-121.98241,w=-122.06566" + * + * <pre> + * Input from: + * http://gws.maps.yahoo.com/findlocation?q=sunnyvale,ca&amp;flags=X + * which gives this format: + * <boundingbox> + * <north>37.44899</north><south>37.3323</south><east>-121.98241</east><west>-122.06566</west> + * </boundingbox> + * it's also easy to use the geoplanet bounding box + * <boundingBox> + * <southWest> + * <latitude>40.183868</latitude> + * <longitude>-74.819519</longitude> + * </southWest> + * <northEast> + * <latitude>40.248291</latitude> + * <longitude>-74.728798</longitude> + * </northEast> + * </boundingBox> + * can be input as: + * s=40.183868,w=-74.819519,n=40.248291,e=-74.728798 + * </pre> + * + * @author Arne J + */ +public class BoundingBoxParser { + + // return variables + public double n = 0.0; + public double s = 0.0; + public double e = 0.0; + public double w = 0.0; + + /** + * parse the given string as a bounding box and return a parser object with parsed coordinates in member variables + * @throws IllegalArgumentException if the input is malformed in any way + **/ + public BoundingBoxParser(String bb) { + this.parseString = bb; + this.len = bb.length(); + parse(); + } + + private final String parseString; + private final int len; + private int pos = 0; + + private char getNextChar() throws IllegalArgumentException { + if (pos == len) { + pos++; + return 0; + } else if (pos > len) { + throw new IllegalArgumentException("position after end of string"); + } else { + return parseString.charAt(pos++); + } + } + + private boolean isCompassDirection(char ch) { + return (ch == 'N' || ch == 'S' || ch == 'E' || ch == 'W' || + ch == 'n' || ch == 's' || ch == 'e' || ch == 'w'); + } + + private int lastNumStartPos = 0; + + private char nsew = 0; + private boolean doneN = false; + private boolean doneS = false; + private boolean doneE = false; + private boolean doneW = false; + + private void parse() { + do { + char ch = getNextChar(); + if (isCompassDirection(ch) && nsew == 0) { + if (ch == 'n' || ch =='N') { + nsew = 'n'; + } else if (ch == 's' || ch == 'S') { + nsew = 's'; + } else if (ch == 'e' || ch == 'E') { + nsew = 'e'; + } else if (ch == 'w' || ch == 'W') { + nsew = 'w'; + } + lastNumStartPos = 0; + } + if (ch == '=' || ch == ':') { + if (nsew != 0) { + lastNumStartPos = pos; + } + } + if (ch == ',' || ch == 0 || ch == ' ') { + if (nsew != 0 && lastNumStartPos > 0) { + String sub = parseString.substring(lastNumStartPos, pos-1); + try { + double v = DoubleParser.parse(sub); + if (nsew == 'n') { + if (doneN) { + throw new IllegalArgumentException("multiple limits for 'n' boundary"); + } + n = v; + doneN = true; + } else if (nsew == 's') { + if (doneS) { + throw new IllegalArgumentException("multiple limits for 's' boundary"); + } + s = v; + doneS = true; + } else if (nsew == 'e') { + if (doneE) { + throw new IllegalArgumentException("multiple limits for 'e' boundary"); + } + e = v; + doneE = true; + } else if (nsew == 'w') { + if (doneW) { + throw new IllegalArgumentException("multiple limits for 'w' boundary"); + } + w = v; + doneW = true; + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Could not parse "+nsew+" limit '"+sub+"' as a number"); + } + nsew = 0; + } + } + } while (pos <= len); + + if (doneN && doneS && doneE && doneW) { + return; + } else { + throw new IllegalArgumentException("Missing bounding box limits, n="+doneN+" s="+doneS+" e="+doneE+" w="+doneW); + } + } + +} + diff --git a/vespajlib/src/main/java/com/yahoo/geo/DegreesParser.java b/vespajlib/src/main/java/com/yahoo/geo/DegreesParser.java new file mode 100644 index 00000000000..40398a2b1a0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/geo/DegreesParser.java @@ -0,0 +1,284 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.geo; + +/** + * utility for parsing geographical coordinates + * + * @author arnej27959 + **/ +public class DegreesParser { + /** + * the parsed latitude (degrees north if positive) + **/ + public double latitude = 0; + /** + * the parsed longitude (degrees east if positive) + **/ + public double longitude = 0; + + private boolean isDigit(char ch) { + return (ch >= '0' && ch <= '9'); + } + private boolean isCompassDirection(char ch) { + return (ch == 'N' || ch == 'S' || ch == 'E' || ch == 'W'); + } + + private String parseString = null; + private int len = 0; + private int pos = 0; + + private char getNextChar() throws IllegalArgumentException { + if (pos == len) { + pos++; + return 0; + } else if (pos > len) { + throw new IllegalArgumentException("position after end of string"); + } else { + return parseString.charAt(pos++); + } + } + + /** + * Parse the given string. + * + * The string must contain both a latitude and a longitude, + * separated by a semicolon, in any order. A latitude must + * contain "N" or "S" and a number signifying degrees north or + * south. A longitude must contain "E" or "W" and a number + * signifying degrees east or west. No signs or spaces are + * allowed. + * <br> + * Fractional degrees are recommended as the main input format, + * but degrees plus fractional minutes may be used for testing. + * You can use the degree sign (U+00B0 as seen in unicode at + * http://www.unicode.org/charts/PDF/U0080.pdf) to separate + * degrees from minutes, put the direction (NSEW) between as a + * separator, or use a small letter 'o' as a replacement for the + * degrees sign. + * <br> + * Some valid input formats: <br> + * "N37.416383;W122.024683" → Sunnyvale <br> + * "37N24.983;122W01.481" → same <br> + * "N37\u00B024.983;W122\u00B001.481" → same <br> + * "N63.418417;E10.433033" → Trondheim <br> + * "N63o25.105;E10o25.982" → same <br> + * "E10o25.982;N63o25.105" → same <br> + * "N63.418417;E10.433033" → same <br> + * "63N25.105;10E25.982" → same <br> + * @param latandlong Latitude and longitude separated by semicolon. + * + **/ + public DegreesParser(String latandlong) throws IllegalArgumentException { + this.parseString = latandlong; + this.len = parseString.length(); + + char ch = getNextChar(); + + boolean latSet = false; + boolean longSet = false; + + double degrees = 0.0; + double minutes = 0.0; + double seconds = 0.0; + boolean degSet = false; + boolean minSet = false; + boolean secSet = false; + boolean dirSet = false; + boolean foundDot = false; + boolean foundDigits = false; + + boolean findingLatitude = false; + boolean findingLongitude = false; + + double sign = 0.0; + + int lastpos = -1; + + do { + boolean valid = false; + if (pos == lastpos) { + throw new RuntimeException("internal logic error at '"+parseString+"' pos:"+pos); + } else { + lastpos = pos; + } + + // first, see if we can find some number + double accum = 0.0; + + if (isDigit(ch) || ch == '.') { + valid = true; + if (foundDigits) { + throw new IllegalArgumentException("found digits after not consuming previous digits"); + } + double divider = 1.0; + foundDot = false; + while (isDigit(ch)) { + foundDigits = true; + accum *= 10; + accum += (ch - '0'); + ch = getNextChar(); + } + if (ch == '.') { + foundDot = true; + ch = getNextChar(); + while (isDigit(ch)) { + foundDigits = true; + accum *= 10; + accum += (ch - '0'); + divider *= 10; + ch = getNextChar(); + } + } + if (!foundDigits) { + throw new IllegalArgumentException("just a . is not a valid number"); + } + accum /= divider; + } + + // next, did we find a separator after the number? + // degree sign is a separator after degrees, before minutes + if (ch == '\u00B0' || ch == 'o') { + valid = true; + if (degSet) { + throw new IllegalArgumentException("degrees sign only valid just after degrees"); + } + if (!foundDigits) { + throw new IllegalArgumentException("must have number before degrees sign"); + } + if (foundDot) { + throw new IllegalArgumentException("cannot have fractional degrees before degrees sign"); + } + ch = getNextChar(); + } + // apostrophe is a separator after minutes, before seconds + if (ch == '\'') { + if (minSet || !degSet || !foundDigits) { + throw new IllegalArgumentException("minutes sign only valid just after minutes"); + } + if (foundDot) { + throw new IllegalArgumentException("cannot have fractional minutes before minutes sign"); + } + ch = getNextChar(); + } + + // if we found some number, assign it into the next unset variable + if (foundDigits) { + valid = true; + if (degSet) { + if (minSet) { + if (secSet) { + throw new IllegalArgumentException("extra number after full field"); + } else { + seconds = accum; + secSet = true; + } + } else { + minutes = accum; + minSet = true; + if (foundDot) { + secSet = true; + } + } + } else { + degrees = accum; + degSet = true; + if (foundDot) { + minSet = true; + secSet = true; + } + } + foundDot = false; + foundDigits = false; + } + + // there needs to be a direction (NSEW) somewhere, too + if (isCompassDirection(ch)) { + valid = true; + if (dirSet) { + throw new IllegalArgumentException("already set direction once, cannot add direction: "+ch); + } + dirSet = true; + if (ch == 'S' || ch == 'W') { + sign = -1; + } else { + sign = 1; + } + if (ch == 'E' || ch == 'W') { + findingLongitude = true; + } else { + findingLatitude = true; + } + ch = getNextChar(); + } + + // lastly, did we find the end-of-string or a separator between lat and long? + if (ch == 0 || ch == ';' || ch == ' ') { + valid = true; + + if (!dirSet) { + throw new IllegalArgumentException("end of field without any compass direction seen"); + } + if (!degSet) { + throw new IllegalArgumentException("end of field without any number seen"); + } + degrees += minutes / 60.0; + degrees += seconds / 3600.0; + degrees *= sign; + + if (findingLatitude) { + if (latSet) { + throw new IllegalArgumentException("found latitude (N or S) twice"); + } + if (degrees < -90.0 || degrees > 90.0) { + throw new IllegalArgumentException("out of range [-90,+90]: "+degrees); + } + latitude = degrees; + latSet = true; + } else if (findingLongitude) { + if (longSet) { + throw new IllegalArgumentException("found longitude (E or W) twice"); + } + if (degrees < -180.0 || degrees > 180.0) { + throw new IllegalArgumentException("out of range [-180,+180]: "+degrees); + } + longitude = degrees; + longSet = true; + } else { + throw new IllegalArgumentException("no direction found"); + } + // reset + degrees = 0.0; + minutes = 0.0; + seconds = 0.0; + degSet = false; + minSet = false; + secSet = false; + dirSet = false; + foundDot = false; + foundDigits = false; + findingLatitude = false; + findingLongitude = false; + sign = 0.0; + + if (ch == 0) { + break; + } else { + ch = getNextChar(); + } + } + + if (!valid) { + throw new IllegalArgumentException("invalid character: "+ch); + } + + } while (ch != 0); + + if (!latSet) { + throw new IllegalArgumentException("missing latitude"); + } + if (!longSet) { + throw new IllegalArgumentException("missing longitude"); + } + // everything parsed OK + } +} diff --git a/vespajlib/src/main/java/com/yahoo/geo/ZCurve.java b/vespajlib/src/main/java/com/yahoo/geo/ZCurve.java new file mode 100644 index 00000000000..3e24316363e --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/geo/ZCurve.java @@ -0,0 +1,201 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.geo; + +/** + * Contains utility methods for a Z-curve (Morton-order) encoder and + * decoder. + * + * @author gjoranv + */ +public class ZCurve { + /** + * Encode two 32 bit integers by bit-interleaving them into one 64 bit + * integer value. The x-direction owns the least significant bit (bit + * 0). Both x and y can have negative values. + * + * <p> + * This is a time-efficient implementation. In the first step, the input + * value is split in two blocks, one containing the most significant bits, and + * the other containing the least significant bits. The most significant block + * is then shifted left for as many bits it contains. For each following step + * every block from the previous step is split in the same manner, with a + * least and most significant block, and the most significant blocks are shifted + * left for as many bits they contain (half the number from the previous step). + * This continues until each block has only one bit. + * + * <p> + * This algorithm works by placing the LSB of all blocks in the correct position + * after the bit-shifting is done in each step. This algorithm is quite similar + * to computing the Hamming Weight (or population count) of a bit + * string, see http://en.wikipedia.org/wiki/Hamming_weight. + * + * <p> + * Efficiency considerations: The encoding operations in this method + * should require 42 cpu operations, of which many can be executed + * in parallell. Practical experiments show that one call takes ~15 ns + * on a 64-bit Intel Xeon processor @2.33GHz, or 35 cycles. This gives + * an efficiency gain of just ~17% due to the CPUs ability to process + * parallell instructions, compared to ~50% for the slow method. + * But still it is 5 times faster. + * + * @param x x value + * @param y y value + * @return The bit-interleaved long containing x and y. + */ + public static long encode(int x, int y) { + long xl = (long)x; + long yl = (long)y; + + long rx = ((xl & 0x00000000ffff0000L) << 16) | (xl & 0x000000000000ffffL); + long ry = ((yl & 0x00000000ffff0000L) << 16) | (yl & 0x000000000000ffffL); + + rx = ((rx & 0xff00ff00ff00ff00L) << 8) | (rx & 0x00ff00ff00ff00ffL); + ry = ((ry & 0xff00ff00ff00ff00L) << 8) | (ry & 0x00ff00ff00ff00ffL); + + rx = ((rx & 0xf0f0f0f0f0f0f0f0L) << 4) | (rx & 0x0f0f0f0f0f0f0f0fL); + ry = ((ry & 0xf0f0f0f0f0f0f0f0L) << 4) | (ry & 0x0f0f0f0f0f0f0f0fL); + + rx = ((rx & 0xccccccccccccccccL) << 2) | (rx & 0x3333333333333333L); + ry = ((ry & 0xccccccccccccccccL) << 2) | (ry & 0x3333333333333333L); + + rx = ((rx & 0xaaaaaaaaaaaaaaaaL) << 1) | (rx & 0x5555555555555555L); + ry = ((ry & 0xaaaaaaaaaaaaaaaaL) << 1) | (ry & 0x5555555555555555L); + + return (rx | (ry << 1)); + } + + + /** + * Decode a z-value into the original two integers. Returns an + * array of two Integers, x and y in indices 0 and 1 respectively. + * + * @param z The bit-interleaved long containing x and y. + * @return Array of two Integers, x and y. + */ + public static int[] decode(long z) { + int[] xy = new int[2]; + + long xl = z & 0x5555555555555555L; + long yl = z & 0xaaaaaaaaaaaaaaaaL; + + xl = ((xl & 0xccccccccccccccccL) >> 1) | (xl & 0x3333333333333333L); + yl = ((yl & 0xccccccccccccccccL) >> 1) | (yl & 0x3333333333333333L); + + xl = ((xl & 0xf0f0f0f0f0f0f0f0L) >> 2) | (xl & 0x0f0f0f0f0f0f0f0fL); + yl = ((yl & 0xf0f0f0f0f0f0f0f0L) >> 2) | (yl & 0x0f0f0f0f0f0f0f0fL); + + xl = ((xl & 0xff00ff00ff00ff00L) >> 4) | (xl & 0x00ff00ff00ff00ffL); + yl = ((yl & 0xff00ff00ff00ff00L) >> 4) | (yl & 0x00ff00ff00ff00ffL); + + xl = ((xl & 0xffff0000ffff0000L) >> 8) | (xl & 0x0000ffff0000ffffL); + yl = ((yl & 0xffff0000ffff0000L) >> 8) | (yl & 0x0000ffff0000ffffL); + + xl = ((xl & 0xffffffff00000000L) >> 16) | (xl & 0x00000000ffffffffL); + yl = ((yl & 0xffffffff00000000L) >> 16) | (yl & 0x00000000ffffffffL); + + xy[0] = (int)xl; + xy[1] = (int)(yl >> 1); + return xy; + } + + + + /** + * Encode two integers by bit-interleaving them into one Long + * value. The x-direction owns the least significant bit (bit + * 0). Both x and y can have negative values. + * <br> + * Efficiency considerations: If Java compiles and runs this code + * as efficiently as would be the case with a good c-compiler, it + * should require 5 cpu operations per bit with optimal usage of + * the CPUs registers on a 64 bit processor(2 bit-shifts, 1 OR, 1 + * AND, and 1 conditional jump for the for-loop). This would + * correspond to 320+ cycles with no parallell execution. + * Practical experiments show that one call takes ~75 ns on a + * 64-bit Intel Xeon processor @2.33GHz, or 175 cycles. This gives + * an efficiency gain of ~50% due to the CPUs ability to perform + * several instructions in one clock-cycle. Here, it is probably the + * bit-shifts that can be done independently of the AND an OR + * operations, which must be done in sequence. + * + * @param x x value + * @param y y value + * @return The bit-interleaved long containing x and y. + */ + public static long encode_slow(int x, int y) { + long z = 0L; + long xl = (long)x; + long yl = (long)y; + + long mask = 1L; + for (int i=0; i<32; i++) { + long bit = (xl << i) & mask; + z |= bit; + //System.out.println("xs "+ i + ": " + toFullBinaryString(xl << i)); + //System.out.println("m "+ i + ": " + toFullBinaryString(mask)); + //System.out.println("bit "+ i + ": " + toFullBinaryString(bit)); + //System.out.println("z "+ i + ": " + toFullBinaryString(z)); + mask = mask << 2; + } + + mask = 2L; + for (int i=1; i<=32; i++) { + long bit = (yl << i) & mask; + z |= bit; + mask = mask << 2; + } + return z; + } + + /** + * Decode a z-value into the original two integers. Returns an + * array of two Integers, x and y in indices 0 and 1 respectively. + * + * @param z The bit-interleaved long containing x and y. + * @return Array of two Integers, x and y. + */ + public static int[] decode_slow(long z) { + int[] xy = new int[2]; + long xl = 0L; + long yl = 0L; + + long mask = 1L; + for (int i=0; i<32; i++) { + long bit = (z >> i) & mask; + xl |= bit; + //System.out.println("bits : m lm lm lm lm lm lm lm l"); + //System.out.println("zs "+ i + ": " + toFullBinaryString(z >> i)); + //System.out.println("m "+ i + ": " + toFullBinaryString(mask)); + //System.out.println("bit "+ i + ": " + toFullBinaryString(bit)); + //System.out.println("xl "+ i + ": " + toFullBinaryString(xl)); + mask = mask << 1; + } + + mask = 1L; + for (int i=1; i<=32; i++) { + long bit = (z >> i) & mask; + yl |= bit; + mask = mask << 1; + } + xy[0] = (int)xl; + xy[1] = (int)yl; + return xy; + } + + /** + * Debugging utility that returns a long value as binary string + * including the leading zeroes. + */ + public static String toFullBinaryString(long l) { + StringBuilder s = new StringBuilder(64); + for (int i=0; i<Long.numberOfLeadingZeros(l); i++) { + s.append('0'); + } + if (l == 0) { + s.deleteCharAt(0); + } + s.append(Long.toBinaryString(l)); + return s.toString(); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/geo/package-info.java b/vespajlib/src/main/java/com/yahoo/geo/package-info.java new file mode 100644 index 00000000000..2e515809012 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/geo/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.geo; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/io/AbstractByteWriter.java b/vespajlib/src/main/java/com/yahoo/io/AbstractByteWriter.java new file mode 100644 index 00000000000..65016ff5384 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/AbstractByteWriter.java @@ -0,0 +1,129 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import com.yahoo.text.GenericWriter; +import com.yahoo.text.AbstractUtf8Array; +import com.yahoo.text.Utf8; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; + +/** + * Base class for writers needing to accept binary data. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public abstract class AbstractByteWriter extends GenericWriter implements + WritableByteTransmitter { + + protected final CharsetEncoder encoder; + protected final BufferChain buffer; + protected final CharBuffer charBuffer = CharBuffer.allocate(2); + + protected AbstractByteWriter(final CharsetEncoder encoder) { + this.encoder = encoder; + buffer = new BufferChain(this); + } + + /** Returns the charset this encodes its output in */ + public Charset getEncoding() { + return encoder.charset(); + } + + @Override + public GenericWriter write(AbstractUtf8Array v) throws java.io.IOException { + buffer.append(v); + return this; + } + + @Override + public GenericWriter write(long v) throws java.io.IOException { + buffer.append(Utf8.toAsciiBytes(v)); + return this; + } + + /** + * Do note, if writing the first character of a surrogate pair, the next + * character written must be the second part of the pair. If this is not the + * case, the surrogate will be omitted from output. + */ + @Override + public void write(int v) throws java.io.IOException { + char c = (char) v; + if (Character.isSurrogate(c)) { + charBuffer.append(c); + if (!charBuffer.hasRemaining()) { + charBuffer.flip(); + buffer.append(charBuffer, encoder); + charBuffer.clear(); + } + } else { + charBuffer.clear(); // to nuke misplaced singleton surrogates + charBuffer.append((char) v); + charBuffer.flip(); + buffer.append(charBuffer, encoder); + charBuffer.clear(); + } + } + + @Override + public GenericWriter write(double v) throws java.io.IOException { + buffer.append(Utf8.toBytes(String.valueOf(v))); + return this; + } + @Override + public GenericWriter write(float v) throws java.io.IOException { + buffer.append(Utf8.toBytes(String.valueOf(v))); + return this; + } + + @Override + public GenericWriter write(short v) throws java.io.IOException { + buffer.append(Utf8.toAsciiBytes(v)); + return this; + } + @Override + public GenericWriter write(boolean v) throws java.io.IOException { + buffer.append(Utf8.toAsciiBytes(v)); + return this; + } + + @Override + public void write(final char[] cbuf, final int offset, final int len) + throws java.io.IOException { + final CharBuffer in = CharBuffer.wrap(cbuf, offset, len); + buffer.append(in, encoder); + } + + public void append(final ByteBuffer alreadyEncoded) + throws java.io.IOException { + buffer.append(alreadyEncoded); + } + + public void append(final byte alreadyEncoded) throws java.io.IOException { + buffer.append(alreadyEncoded); + } + + public void append(final byte[] alreadyEncoded) throws java.io.IOException { + buffer.append(alreadyEncoded); + } + + public void append(final byte[] alreadyEncoded, final int offset, + final int length) throws java.io.IOException { + buffer.append(alreadyEncoded, offset, length); + } + + /** + * Return the number of bytes this writer will produce for the underlying + * layer. That is, it sums the length of the raw bytes received and the + * number of bytes in the written strings after encoding. + * + * @return the number of bytes appended to this writer + */ + public long appended() { + return buffer.appended(); + } +}
\ No newline at end of file diff --git a/vespajlib/src/main/java/com/yahoo/io/Acceptor.java b/vespajlib/src/main/java/com/yahoo/io/Acceptor.java new file mode 100644 index 00000000000..62f19c186f6 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/Acceptor.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.io.IOException; +import java.nio.channels.SocketChannel; +import java.nio.channels.ServerSocketChannel; +import java.util.logging.Logger; +import java.util.logging.Level; +import java.net.InetSocketAddress; + + +/** + * Class for accepting new connections in separate thread. + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * + */ +public class Acceptor extends Thread { + private static Logger log = Logger.getLogger(Acceptor.class.getName()); + + private int port; + ServerSocketChannel socket; + private Listener listener; + private boolean initialized = false; + private ConnectionFactory factory; + private FatalErrorHandler fatalErrorHandler; + + public Acceptor(Listener listener, ConnectionFactory factory, int port) { + super("Acceptor-" + listener.getName() + "-" + port); + this.listener = listener; + this.factory = factory; + this.port = port; + } + + public Acceptor listen() throws IOException { + socket = ServerSocketChannel.open(); + socket.configureBlocking(true); + socket.socket().setReuseAddress(true); + socket.socket().bind(new InetSocketAddress(port)); + initialized = true; + return this; + } + + /** + * Register a handler for fatal errors. + * + * @param f The FatalErrorHandler instance to be registered + */ + public synchronized void setFatalErrorHandler(FatalErrorHandler f) { + fatalErrorHandler = f; + } + + public void run() { + try { + log.fine("Acceptor thread started"); + if (!initialized) { + log.severe("Acceptor was not initialized. aborting"); + return; + } + + while (!isInterrupted()) { + SocketChannel c = null; // hush jikes + + try { + c = socket.accept(); + c.configureBlocking(false); + listener.addNewConnection(factory.newConnection(c, listener)); + } catch (java.nio.channels.IllegalBlockingModeException e) { + log.log(Level.SEVERE, "Unable to set nonblocking", e); + try { + if (c != null) { + c.close(); + } + } catch (IOException ee) {} + } catch (IOException e) { + log.log(Level.WARNING, + "Error accepting connection on port=" + port, e); + try { + if (c != null) { + c.close(); + } + } catch (IOException ee) {} + } + } + } catch (Throwable t) { + if (fatalErrorHandler != null) { + fatalErrorHandler.handle(t, null); + } + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/Blob.java b/vespajlib/src/main/java/com/yahoo/io/Blob.java new file mode 100644 index 00000000000..808371e7b58 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/Blob.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.ByteBuffer; + +/** + * A Blob contains opaque data in the form of a byte array. + **/ +public class Blob { + + /** + * Shared empty array. + **/ + private static byte[] empty = new byte[0]; + + /** + * Internal data, will never be 'null'. + **/ + private byte[] data; + + /** + * Create a Blob containing an empty byte array. + **/ + public Blob() { + data = empty; + } + + /** + * Create a Blob containg a copy of a subset of the given byte + * array. + **/ + public Blob(byte[] src, int offset, int length) { + data = new byte[length]; + System.arraycopy(src, offset, data, 0, length); + } + + /** + * Create a Blob containing a copy of the given byte array. + **/ + public Blob(byte[] src) { + this(src, 0, src.length); + } + + /** + * Create a Blob containing a copy of the data held by the given + * blob. + **/ + public Blob(Blob src) { + this(src.data); + } + + /** + * Create a Blob containing a number of bytes read from a byte + * buffer. + **/ + public Blob(ByteBuffer src, int length) { + data = new byte[length]; + src.get(data); + } + + /** + * Create a Blob containing all bytes that could be read from a + * byte buffer. + **/ + public Blob(ByteBuffer src) { + this(src, src.remaining()); + } + + /** + * Obtain the internal data held by this object. + * + * @return internal data + **/ + public byte[] get() { + return data; + } + + /** + * Write the data held by this object to the given byte buffer. + * + * @param dst where to write the contained data + **/ + public void write(ByteBuffer dst) { + dst.put(data); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/BufferChain.java b/vespajlib/src/main/java/com/yahoo/io/BufferChain.java new file mode 100644 index 00000000000..fc9fadc64ca --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/BufferChain.java @@ -0,0 +1,147 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import com.yahoo.text.AbstractUtf8Array; + +import java.io.IOException; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.util.ArrayList; +import java.util.List; + +/** + * Data store for AbstractByteWriter. Tested in unit tests for ByteWriter. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class BufferChain { + // refer to the revision history of ByteWriter for more information about + // the reasons behind the sizing of BUFFERSIZE, WATERMARK and MAXBUFFERS + static final int BUFFERSIZE = 4096; + static final int WATERMARK = 1024; + static final int MAXBUFFERS = 50; + static { + //noinspection ConstantConditions + assert BUFFERSIZE > WATERMARK; + } + private final List<ByteBuffer> buffers = new ArrayList<>(); + private final WritableByteTransmitter endpoint; + private ByteBuffer current = ByteBuffer.allocate(BUFFERSIZE); + private long appended = 0L; + + public BufferChain(final WritableByteTransmitter endpoint) { + this.endpoint = endpoint; + } + + public void append(final byte b) throws IOException { + makeRoom(1); + current.put(b); + } + private final boolean shouldCopy(int length) { + return (length < WATERMARK); + } + private final void makeRoom(int length) throws IOException { + if (current.remaining() < length) { + scratch(); + } + } + public void append(AbstractUtf8Array v) throws IOException { + final int length = v.getByteLength(); + if (shouldCopy(length)) { + makeRoom(length); + v.writeTo(current); + } else { + append(v.wrap()); + } + } + public void append(final byte[] alreadyEncoded) throws java.io.IOException { + if (alreadyEncoded.length > 0) { + append(alreadyEncoded, 0, alreadyEncoded.length); + } + } + + public void append(final byte[] alreadyEncoded, final int offset, final int length) throws java.io.IOException { + if (shouldCopy(length)) { + makeRoom(length); + current.put(alreadyEncoded, offset, length); + } else { + append(ByteBuffer.wrap(alreadyEncoded, offset, length)); + } + } + + public void append(final ByteBuffer alreadyEncoded) throws java.io.IOException { + if (alreadyEncoded.remaining() == 0) { + return; + } + final int length = alreadyEncoded.limit() - alreadyEncoded.position(); + if (shouldCopy(length)) { + makeRoom(length); + current.put(alreadyEncoded); + } else { + scratch(); + add(alreadyEncoded); + } + } + private final void add(final ByteBuffer buf) { + buffers.add(buf); + appended += buf.limit(); + } + + public void append(final CharBuffer toEncode, final CharsetEncoder encoder) + throws java.io.IOException { + CoderResult overflow; + do { + overflow = encoder.encode(toEncode, current, true); + if (overflow.isOverflow()) { + scratch(); + } else if (overflow.isError()) { + try { + toEncode.get(); + } catch (final BufferUnderflowException e) { + // Give up if we can't discard some presumptively malformed + // or unmappable data + break; + } + } + } while (!overflow.isUnderflow()); + } + + private void scratch() throws java.io.IOException { + if (!possibleFlush() && current.position() != 0) { + current.flip(); + add(current); + current = ByteBuffer.allocate(BUFFERSIZE); + } + } + + private boolean possibleFlush() throws java.io.IOException { + if (buffers.size() > MAXBUFFERS) { + flush(); + return true; + } + return false; + } + + public void flush() throws IOException { + for (final ByteBuffer b : buffers) { + endpoint.send(b); + } + buffers.clear(); + if (current.position() > 0) { + current.flip(); + appended += current.limit(); + endpoint.send(current); + current = ByteBuffer.allocate(BUFFERSIZE); + } + } + + /** + * @return number of bytes written to this buffer + */ + public long appended() { + return appended + current.position(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java b/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java new file mode 100644 index 00000000000..8345f97f291 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import com.yahoo.text.Utf8; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.CharsetEncoder; + +/** + * A buffered writer which accepts byte arrays in addition to character arrays. + * + * @author <a href="mailt:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ByteWriter extends AbstractByteWriter { + private final OutputStream stream; + + public ByteWriter(final OutputStream stream, final CharsetEncoder encoder) { + super(encoder); + this.stream = stream; + } + public ByteWriter(final OutputStream stream) { + super(Utf8.getNewEncoder()); + this.stream = stream; + } + + @Override + public void send(final ByteBuffer b) throws IOException { + // we know from how BufferChain works we have a backing array + stream.write(b.array(), b.position() + b.arrayOffset(), b.limit() - b.position()); + } + + @Override + public void close() throws java.io.IOException { + buffer.flush(); + // Unit tests in prelude depends on the stream _not_ being flushed, it + // is necessary for Jetty to write content length headers, it seems. + // stream.flush(); + stream.close(); + } + + @Override + public void flush() throws IOException { + buffer.flush(); + // Unit tests in prelude depends on the stream _not_ being flushed, it + // is necessary for Jetty to write content length headers, it seems. + // stream.flush(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/Connection.java b/vespajlib/src/main/java/com/yahoo/io/Connection.java new file mode 100644 index 00000000000..18b91ff3b42 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/Connection.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.channels.SocketChannel; +import java.io.IOException; + + +/** + * Connection interface is the abstraction for an operating + * asynchronous NIO connection. One is created for each + * "accept" on the channel. + * + * @author <a href="mailto:travisb@yahoo-inc.com">Bob Travis</a> + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * + */ +public interface Connection { + + /** + * called when the channel can accept a write, and is + * enabled for writing + */ + public void write() throws IOException; + + /** + * Called when the channel can accept a read, and is + * enabled for reading + */ + public void read() throws IOException; + + /** + * Called when the channel should be closed. + */ + public void close() throws IOException; + + /** + * Called when a socket has completed connecting to its + * destination. (Asynchronous connect) + */ + public void connect() throws IOException; + + /** + * called to get the correct initial SelectionKey operation + * flags for the next Select cycle, for this channel + */ + public int selectOps(); + + /** + * Called to get the SocketChannel for this Connection. + * + * @return Returns the SocketChannel representing this connection + */ + public SocketChannel socketChannel(); +} + diff --git a/vespajlib/src/main/java/com/yahoo/io/ConnectionFactory.java b/vespajlib/src/main/java/com/yahoo/io/ConnectionFactory.java new file mode 100644 index 00000000000..a0bdedfba7d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/ConnectionFactory.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00F8rn Borud</a> + */ + +import java.nio.channels.SocketChannel; + + +/** + * A factory interface used for associating SocketChannel and Listener + * information with the application's Connection object. + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author <a href="mailto:travisb@yahoo-inc.com">Bob Travis</a> + */ +public interface ConnectionFactory { + public Connection newConnection(SocketChannel channel, Listener listener); +} diff --git a/vespajlib/src/main/java/com/yahoo/io/FatalErrorHandler.java b/vespajlib/src/main/java/com/yahoo/io/FatalErrorHandler.java new file mode 100644 index 00000000000..11b7d1cbc66 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/FatalErrorHandler.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* -*- c-basic-offset: 4 -*- + * + * $Id$ + * + */ +package com.yahoo.io; + + +import java.util.logging.Logger; +import java.util.logging.Level; + + +/** + * What to do if a fatal condition happens in an IO component. + * + * <P> + * TODO: We need to re-think this design a bit. First off, we + * probably need to make the interface an abstract class + * or a pure interface type. Second we provide a few + * default implementations which are named after what policy + * they implement -- like SystemExitOnError etc. Also, + * runnables that have fatal error handling capability should + * probably implement a standard interface for get/set etc. + * Also, we should encourage application authors to provide + * their own, application specific error handlers rather than + * relying on the default. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ + +public class FatalErrorHandler { + protected static final Logger log = Logger.getLogger(FatalErrorHandler.class.getName()); + + /** + * Do something reasonable when a an Error occurs. + * + * Override this to change behavior. Default behavior is to log + * the error, then exit. + * + * @param t The Throwable causing the handler to be activated. + * @param context The object calling the handler. + */ + public void handle(Throwable t, Object context) { + try { + log.log(Level.SEVERE, "Exiting due to error", t); + } finally { + Runtime.getRuntime().halt(1); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java b/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java new file mode 100644 index 00000000000..85b249432d4 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java @@ -0,0 +1,186 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.channels.WritableByteChannel; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Stack; +import java.util.LinkedList; +import java.util.Iterator; +import java.nio.ByteBuffer; + + +/** + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class GrowableBufferOutputStream extends OutputStream { +// private static final int MINIMUM_BUFFERSIZE = (64 * 1024); + private ByteBuffer lastBuffer; + private ByteBuffer directBuffer; + private LinkedList<ByteBuffer> bufferList = new LinkedList<>(); + private Stack<ByteBuffer> recycledBuffers = new Stack<>(); + + private int bufferSize; + private int maxBuffers; + + public GrowableBufferOutputStream(int bufferSize, int maxBuffers) { + this.bufferSize = bufferSize; + this.maxBuffers = maxBuffers; + lastBuffer = ByteBuffer.allocate(bufferSize); + directBuffer = ByteBuffer.allocateDirect(bufferSize); + } + + @Override + public void write(byte[] cbuf, int off, int len) throws IOException { + if (lastBuffer.remaining() >= len) { + lastBuffer.put(cbuf, off, len); + return; + } + + int residue = len; + + while (residue > 0) { + int newOffset = len - residue; + int toWrite = Math.min(lastBuffer.remaining(), residue); + + lastBuffer.put(cbuf, newOffset, toWrite); + residue -= toWrite; + if (residue != 0) { + extend(); + } + } + } + + @Override + public void write(byte[] b) throws IOException { + write(b,0,b.length); + } + + @Override + public String toString() { + return "GrowableBufferOutputStream, writable size " + writableSize() + + " bytes, " + numWritableBuffers() + " buffers, last buffer" + + " position " + lastBuffer.position() + ", last buffer limit " + + lastBuffer.limit(); + } + + public void write(int b) { + if (lastBuffer.remaining() == 0) { + extend(); + } + lastBuffer.put((byte) b); + } + + @Override + public void flush() { + // if the last buffer is untouched we do not need to do anything; if + // it has been touched we call extend(), which enqueues the buffer + // and allocates or recycles a buffer for us + if (lastBuffer.position() > 0) { + extend(); + } + } + + @Override + public void close() { + flush(); + } + + public int channelWrite(WritableByteChannel channel) throws IOException { + ByteBuffer buffer; + int totalWritten = 0; + + while (!bufferList.isEmpty()) { + buffer = bufferList.getFirst(); + int written = 0; + + synchronized (directBuffer) { + directBuffer.clear(); + directBuffer.put(buffer); + directBuffer.flip(); + written = channel.write(directBuffer); + int left = directBuffer.remaining(); + + if (left > 0) { + int oldpos = buffer.position(); + + buffer.position(oldpos - left); + } + totalWritten += written; + } + + // if we've completed writing this buffer we can dispose of it + if (buffer.remaining() == 0) { + bufferList.removeFirst(); + recycleBuffer(buffer); + } + + // if we didn't write any bytes we terminate + if (written == 0) { + break; + } + } + + return totalWritten; + } + + public int numWritableBuffers() { + return bufferList.size(); + } + + public void clear() { + flush(); + bufferList.clear(); + } + + public void clearCache() { + recycledBuffers.clear(); + } + + public void clearAll() { + clear(); + clearCache(); + } + + public int writableSize() { + Iterator<ByteBuffer> it = bufferList.iterator(); + int size = 0; + + while (it.hasNext()) { + size += (it.next()).remaining(); + } + + return size; + } + + public ByteBuffer[] getWritableBuffers() { + flush(); + ByteBuffer[] result = new ByteBuffer[numWritableBuffers()]; + return bufferList.toArray(result); + } + + private void extend() { + enqueueBuffer(lastBuffer); + + if (recycledBuffers.empty()) { + lastBuffer = ByteBuffer.allocate(bufferSize); + } else { + lastBuffer = recycledBuffers.pop(); + lastBuffer.clear(); + } + } + + private void enqueueBuffer(ByteBuffer buffer) { + buffer.flip(); + bufferList.addLast(buffer); + } + + private void recycleBuffer(ByteBuffer buffer) { + if (recycledBuffers.size() >= maxBuffers) { + return; + } + recycledBuffers.push(buffer); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/io/GrowableByteBuffer.java b/vespajlib/src/main/java/com/yahoo/io/GrowableByteBuffer.java new file mode 100644 index 00000000000..c33882052b4 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/GrowableByteBuffer.java @@ -0,0 +1,746 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.*; + +/** + * GrowableByteBuffer encapsulates a ByteBuffer and grows it as needed. + * The implementation is safe and simple (and certainly a bit inefficient) + * - when growing the buffer a new buffer + * is allocated, the old contents are copied into the new buffer, + * and the new buffer's position is set to the position of the old + * buffer. + * It is possible to set a growth factor. The default is 2.0, meaning that + * the buffer will double its size when growing. + * + * Note that NO methods are re-implemented (except growing the buffer, + * of course), all are delegated to the encapsulated ByteBuffer. + * This also includes toString(), hashCode(), equals() and compareTo(). + * + * No methods except getByteBuffer() expose the encapsulated + * ByteBuffer, which is intentional. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class GrowableByteBuffer implements Comparable<GrowableByteBuffer> { + public static final int DEFAULT_BASE_SIZE = 64*1024; + public static final float DEFAULT_GROW_FACTOR = 2.0f; + private ByteBuffer buffer; + private float growFactor; + private int mark = -1; + + //NOTE: It might have been better to subclass HeapByteBuffer, + //but that class is package-private. Subclassing ByteBuffer would involve + //implementing a lot of abstract methods, which would mean reinventing + //some (too many) wheels. + + //CONSTRUCTORS: + + public GrowableByteBuffer() { + this(DEFAULT_BASE_SIZE, DEFAULT_GROW_FACTOR); + } + + public GrowableByteBuffer(int baseSize, float growFactor) { + setGrowFactor(growFactor); + //NOTE: We MUST NEVER have a base size of 0, since checkAndGrow() will go into an infinite loop then + if (baseSize < 16) baseSize = 16; + buffer = ByteBuffer.allocate(baseSize); + } + + public GrowableByteBuffer(int baseSize) { + this(baseSize, DEFAULT_GROW_FACTOR); + } + + public GrowableByteBuffer(ByteBuffer buffer) { + this(buffer, DEFAULT_GROW_FACTOR); + } + + public GrowableByteBuffer(ByteBuffer buffer, float growFactor) { + this.buffer = buffer; + setGrowFactor(growFactor); + } + + + //ACCESSORS: + + public float getGrowFactor() { + return growFactor; + } + + public void setGrowFactor(float growFactor) { + if (growFactor <= 1.00f) { + throw new IllegalArgumentException("Growth factor must be greater than 1.00f, otherwise buffer will never grow!"); + } + this.growFactor = growFactor; + } + + public ByteBuffer getByteBuffer() { + return buffer; + } + + //PRIVATE GROWTH METHODS + + //TODO: Implement more efficient buffer growth + //Allocating a new buffer and copying the old buffer into the new one + //is a simple and uncomplicated strategy. + //For performance, it would be much better to have a linked list of + //ByteBuffers and keep track of global position etc., much like + //GrowableBufferOutputStream does it. + + protected void grow(int newSize) { + //create new buffer: + ByteBuffer newByteBuf; + if (buffer.isDirect()) { + newByteBuf = ByteBuffer.allocateDirect(newSize); + } else { + newByteBuf = ByteBuffer.allocate(newSize); + } + //set same byte order: + newByteBuf.order(buffer.order()); + + //copy old contents and set correct position: + int oldPos = buffer.position(); + newByteBuf.position(0); + buffer.position(0); + newByteBuf.put(buffer); + newByteBuf.position(oldPos); + + //set same mark: + if (mark >= 0) { + newByteBuf.position(mark); + newByteBuf.mark(); + newByteBuf.position(oldPos); + } + + //NOTE: No need to preserve "read-only" property, + //since a read-only buffer cannot grow and will never + //reach this point anyway + + //NOTE: No need to preserve "limit" property, it would be + //pointless to grow then... + + //set new buffer to be our buffer: + buffer = newByteBuf; + } + + private void accomodate(int putSize) { + int bufPos = buffer.position(); + int bufSize = buffer.capacity(); + int bufRem = bufSize - bufPos; + + if (bufRem >= putSize) return; + + while (bufRem < putSize) { + bufSize = (int) ((((float) bufSize) * growFactor) + 100.0); + bufRem = bufSize - bufPos; + } + + grow(bufSize); + } + + //VESPA-ENCODED INTEGERS: + + /** + * Writes a 62-bit positive integer to the buffer, using 2, 4, or 8 bytes. + * + * @param number the integer to write + */ + public void putInt2_4_8Bytes(long number) { + if (number < 0L) { + throw new IllegalArgumentException("Cannot encode negative number."); + } else if (number > 0x3FFFFFFFFFFFFFFFL) { + throw new IllegalArgumentException("Cannot encode number larger than 2^62."); + } + + if (number < 0x8000L) { + //length 2 bytes + putShort((short) number); + } else if (number < 0x40000000L) { + //length 4 bytes + putInt(((int) number) | 0x80000000); + } else { + //length 8 bytes + putLong(number | 0xC000000000000000L); + } + } + + /** + * Writes a 32 bit positive integer (or 31 bit unsigned) to the buffer, + * using 4 bytes. + * + * @param number the integer to write + */ + public void putInt2_4_8BytesAs4(long number) { + if (number < 0L) { + throw new IllegalArgumentException("Cannot encode negative number."); + } else if (number > 0x7FFFFFFFL) { + throw new IllegalArgumentException("Cannot encode number larger than 2^31-1."); + } + putInt(((int) number) | 0x80000000); + } + + /** + * Reads a 62-bit positive integer from the buffer, which was written using 2, 4, or 8 bytes. + * + * @return the integer read + */ + public long getInt2_4_8Bytes() { + byte flagByte = get(); + position(position() - 1); + + if ((flagByte & 0x80) != 0) { + if ((flagByte & 0x40) != 0) { + //length 8 bytes + return getLong() & 0x3FFFFFFFFFFFFFFFL; + } else { + //length 4 bytes + return getInt() & 0x3FFFFFFF; + } + } else { + //length 2 bytes + return getShort(); + } + } + + /** + * Computes the size used for storing the given integer using 2, 4 or 8 bytes. + * + * @param number the integer to check length of + * @return the number of bytes used to store it; 2, 4 or 8 + */ + public static int getSerializedSize2_4_8Bytes(long number) { + if (number < 0L) { + throw new IllegalArgumentException("Cannot encode negative number."); + } else if (number > 0x3FFFFFFFFFFFFFFFL) { + throw new IllegalArgumentException("Cannot encode number larger than 2^62."); + } + + if (number < 0x8000L) { + //length 2 bytes + return 2; + } else if (number < 0x40000000L) { + //length 4 bytes + return 4; + } else { + //length 8 bytes + return 8; + } + } + + /** + * Writes a 30-bit positive integer to the buffer, using 1, 2, or 4 bytes. + * + * @param number the integer to write + */ + public void putInt1_2_4Bytes(int number) { + if (number < 0) { + throw new IllegalArgumentException("Cannot encode negative number"); + } else if (number > 0x3FFFFFFF) { + throw new IllegalArgumentException("Cannot encode number larger than 2^30."); + } + + if (number < 0x80) { + //length 1 byte + put((byte) number); + } else if (number < 0x4000) { + //length 2 bytes + putShort((short) (((short)number) | ((short) 0x8000))); + } else { + //length 4 bytes + putInt(number | 0xC0000000); + } + } + + /** + * Writes a 30-bit positive integer to the buffer, using 4 bytes. + * + * @param number the integer to write + */ + public void putInt1_2_4BytesAs4(int number) { + if (number < 0) { + throw new IllegalArgumentException("Cannot encode negative number"); + } else if (number > 0x3FFFFFFF) { + throw new IllegalArgumentException("Cannot encode number larger than 2^30."); + } + putInt(number | 0xC0000000); + } + + /** + * Reads a 30-bit positive integer from the buffer, which was written using 1, 2, or 4 bytes. + * + * @return the integer read + */ + public int getInt1_2_4Bytes() { + byte flagByte = get(); + position(position() - 1); + + if ((flagByte & 0x80) != 0) { + if ((flagByte & 0x40) != 0) { + //length 4 bytes + return getInt() & 0x3FFFFFFF; + } else { + //length 2 bytes + return getShort() & 0x3FFF; + } + } else { + //length 1 byte + return get(); + } + } + + /** + * Computes the size used for storing the given integer using 1, 2 or 4 bytes. + * + * @param number the integer to check length of + * @return the number of bytes used to store it; 1, 2 or 4 + */ + public static int getSerializedSize1_2_4Bytes(int number) { + if (number < 0) { + throw new IllegalArgumentException("Cannot encode negative number"); + } else if (number > 0x3FFFFFFF) { + throw new IllegalArgumentException("Cannot encode number larger than 2^30."); + } + + if (number < 0x80) { + //length 1 byte + return 1; + } else if (number < 0x4000) { + //length 2 bytes + return 2; + } else { + //length 4 bytes + return 4; + } + } + + /** + * Writes a 31-bit positive integer to the buffer, using 1 or 4 bytes. + * + * @param number the integer to write + */ + public void putInt1_4Bytes(int number) { + if (number < 0) { + throw new IllegalArgumentException("Cannot encode negative number"); + } + //no need to check upper boundary, since INT_MAX == 2^31 + + if (number < 0x80) { + //length 1 byte + put((byte) number); + } else { + //length 4 bytes + putInt(number | 0x80000000); + } + } + + /** + * Writes a 31-bit positive integer to the buffer, using 4 bytes. + * + * @param number the integer to write + */ + public void putInt1_4BytesAs4(int number) { + if (number < 0) { + throw new IllegalArgumentException("Cannot encode negative number"); + } + //no need to check upper boundary, since INT_MAX == 2^31 + putInt(number | 0x80000000); + } + + /** + * Reads a 31-bit positive integer from the buffer, which was written using 1 or 4 bytes. + * + * @return the integer read + */ + public int getInt1_4Bytes() { + byte flagByte = get(); + position(position() - 1); + + if ((flagByte & 0x80) != 0) { + //length 4 bytes + return getInt() & 0x7FFFFFFF; + } else { + //length 1 byte + return get(); + } + } + + /** + * Computes the size used for storing the given integer using 1 or 4 bytes. + * + * @param number the integer to check length of + * @return the number of bytes used to store it; 1 or 4 + */ + public static int getSerializedSize1_4Bytes(int number) { + if (number < 0) { + throw new IllegalArgumentException("Cannot encode negative number"); + } + //no need to check upper boundary, since INT_MAX == 2^31 + + if (number < 0x80) { + //length 1 byte + return 1; + } else { + //length 4 bytes + return 4; + } + } + + //METHODS OF ENCAPSULATED BYTEBUFFER: + public static GrowableByteBuffer allocate(int capacity) { + return new GrowableByteBuffer(ByteBuffer.allocate(capacity)); + } + public static GrowableByteBuffer allocate(int capacity, float growFactor) { + return new GrowableByteBuffer(ByteBuffer.allocate(capacity), growFactor); + } + public static GrowableByteBuffer allocateDirect(int capacity) { + return new GrowableByteBuffer(ByteBuffer.allocateDirect(capacity)); + } + public static GrowableByteBuffer allocateDirect(int capacity, float growFactor) { + return new GrowableByteBuffer(ByteBuffer.allocateDirect(capacity), growFactor); + } + public final byte[] array() { + return buffer.array(); + } + public final int arrayOffset() { + return buffer.arrayOffset(); + } + public CharBuffer asCharBuffer() { + return buffer.asCharBuffer(); + } + public DoubleBuffer asDoubleBuffer() { + return buffer.asDoubleBuffer(); + } + public FloatBuffer asFloatBuffer() { + return buffer.asFloatBuffer(); + } + public IntBuffer asIntBuffer() { + return buffer.asIntBuffer(); + } + public LongBuffer asLongBuffer() { + return buffer.asLongBuffer(); + } + public GrowableByteBuffer asReadOnlyBuffer() { + return new GrowableByteBuffer(buffer.asReadOnlyBuffer(), growFactor); + } + public ShortBuffer asShortBuffer() { + return buffer.asShortBuffer(); + } + public GrowableByteBuffer compact() { + buffer.compact(); + return this; + } + public int compareTo(GrowableByteBuffer that) { + return buffer.compareTo(that.buffer); + } + public GrowableByteBuffer duplicate() { + return new GrowableByteBuffer(buffer.duplicate(), growFactor); + } + public boolean equals(Object obj) { + if (!(obj instanceof GrowableByteBuffer)) { + return false; + } + GrowableByteBuffer rhs = (GrowableByteBuffer)obj; + if (!buffer.equals(rhs.buffer)) { + return false; + } + return true; + } + public byte get() { + return buffer.get(); + } + public GrowableByteBuffer get(byte[] dst) { + buffer.get(dst); + return this; + } + public GrowableByteBuffer get(byte[] dst, int offset, int length) { + buffer.get(dst, offset, length); + return this; + } + public byte get(int index) { + return buffer.get(index); + } + public char getChar() { + return buffer.getChar(); + } + public char getChar(int index) { + return buffer.getChar(index); + } + public double getDouble() { + return buffer.getDouble(); + } + public double getDouble(int index) { + return buffer.getDouble(index); + } + public float getFloat() { + return buffer.getFloat(); + } + public float getFloat(int index) { + return buffer.getFloat(index); + } + public int getInt() { + return buffer.getInt(); + } + public int getInt(int index) { + return buffer.getInt(index); + } + public long getLong() { + return buffer.getLong(); + } + public long getLong(int index) { + return buffer.getLong(index); + } + public short getShort() { + return buffer.getShort(); + } + public short getShort(int index) { + return buffer.getShort(index); + } + public boolean hasArray() { + return buffer.hasArray(); + } + public int hashCode() { + return buffer.hashCode(); + } + public boolean isDirect() { + return buffer.isDirect(); + } + public ByteOrder order() { + return buffer.order(); + } + public GrowableByteBuffer order(ByteOrder bo) { + buffer.order(bo); + return this; + } + + public GrowableByteBuffer put(byte b) { + try { + buffer.put(b); + } catch (BufferOverflowException e) { + accomodate(1); + buffer.put(b); + } + return this; + } + public GrowableByteBuffer put(byte[] src) { + + accomodate(src.length); + buffer.put(src); + return this; + } + public GrowableByteBuffer put(byte[] src, int offset, int length) { + + accomodate(length); + buffer.put(src, offset, length); + return this; + } + public GrowableByteBuffer put(ByteBuffer src) { + accomodate(src.remaining()); + buffer.put(src); + return this; + } + public GrowableByteBuffer put(GrowableByteBuffer src) { + + accomodate(src.remaining()); + buffer.put(src.buffer); + return this; + } + // XXX: the put{Type}(index, value) methods do not handle index > position + public GrowableByteBuffer put(int index, byte b) { + try { + buffer.put(index, b); + } catch (IndexOutOfBoundsException e) { + accomodate(1); + buffer.put(index, b); + } + return this; + } + public GrowableByteBuffer putChar(char value) { + try { + buffer.putChar(value); + } catch (BufferOverflowException e) { + accomodate(2); + buffer.putChar(value); + } + return this; + } + public GrowableByteBuffer putChar(int index, char value) { + try { + buffer.putChar(index, value); + } catch (IndexOutOfBoundsException e) { + accomodate(2); + buffer.putChar(index, value); + } + return this; + } + public GrowableByteBuffer putDouble(double value) { + try { + buffer.putDouble(value); + } catch (BufferOverflowException e) { + accomodate(8); + buffer.putDouble(value); + } + return this; + } + public GrowableByteBuffer putDouble(int index, double value) { + try { + buffer.putDouble(index, value); + } catch (IndexOutOfBoundsException e) { + accomodate(8); + buffer.putDouble(index, value); + } + return this; + } + public GrowableByteBuffer putFloat(float value) { + try { + buffer.putFloat(value); + } catch (BufferOverflowException e) { + accomodate(4); + buffer.putFloat(value); + } + return this; + } + public GrowableByteBuffer putFloat(int index, float value) { + try { + buffer.putFloat(index, value); + } catch (IndexOutOfBoundsException e) { + accomodate(4); + buffer.putFloat(index, value); + } + return this; + } + public GrowableByteBuffer putInt(int value) { + try { + buffer.putInt(value); + } catch (BufferOverflowException e) { + accomodate(4); + buffer.putInt(value); + } + return this; + } + public GrowableByteBuffer putInt(int index, int value) { + try { + buffer.putInt(index, value); + } catch (IndexOutOfBoundsException e) { + accomodate(4); + buffer.putInt(index, value); + } + return this; + } + public GrowableByteBuffer putLong(int index, long value) { + try { + buffer.putLong(index, value); + } catch (IndexOutOfBoundsException e) { + accomodate(8); + buffer.putLong(index, value); + } + return this; + } + public GrowableByteBuffer putLong(long value) { + try { + buffer.putLong(value); + } catch (BufferOverflowException e) { + accomodate(8); + buffer.putLong(value); + } + return this; + } + public GrowableByteBuffer putShort(int index, short value) { + try { + buffer.putShort(index, value); + } catch (IndexOutOfBoundsException e) { + accomodate(2); + buffer.putShort(index, value); + } + return this; + } + public GrowableByteBuffer putShort(short value) { + try { + buffer.putShort(value); + } catch (BufferOverflowException e) { + accomodate(2); + buffer.putShort(value); + } + return this; + } + + /** + * Behaves as ByteBuffer slicing, but the internal buffer will no longer be + * shared if one of the buffers is forced to grow. + * + * @return a new buffer with shared contents + * @see ByteBuffer#slice() + */ + public GrowableByteBuffer slice() { + ByteBuffer b = buffer.slice(); + return new GrowableByteBuffer(b, growFactor); + } + + public String toString() { + return "GrowableByteBuffer" + + "[pos="+ position() + + " lim=" + limit() + + " cap=" + capacity() + + " grow=" + growFactor + + "]"; + } + public static GrowableByteBuffer wrap(byte[] array) { + return new GrowableByteBuffer(ByteBuffer.wrap(array)); + } + public static GrowableByteBuffer wrap(byte[] array, float growFactor) { + return new GrowableByteBuffer(ByteBuffer.wrap(array), growFactor); + } + public static GrowableByteBuffer wrap(byte[] array, int offset, int length) { + return new GrowableByteBuffer(ByteBuffer.wrap(array, offset, length)); + } + public static GrowableByteBuffer wrap(byte[] array, int offset, int length, float growFactor) { + return new GrowableByteBuffer(ByteBuffer.wrap(array, offset, length), growFactor); + } + + //METHODS FROM ENCAPSULATED BUFFER: + + public final int capacity() { + return buffer.capacity(); + } + public final void clear() { + buffer.clear(); + mark = -1; + } + public final void flip() { + buffer.flip(); + mark = -1; + } + public final boolean hasRemaining() { + return buffer.hasRemaining(); + } + public final boolean isReadOnly() { + return buffer.isReadOnly(); + } + public final int limit() { + return buffer.limit(); + } + public final void limit(int newLimit) { + buffer.limit(newLimit); + if (mark > newLimit) mark = -1; + } + public final void mark() { + buffer.mark(); + mark = position(); + } + public final int position() { + return buffer.position(); + } + public final void position(int newPosition) { + buffer.position(newPosition); + if (mark > newPosition) mark = -1; + } + public final int remaining() { + return buffer.remaining(); + } + public final void reset() { + buffer.reset(); + } + public final void rewind() { + buffer.rewind(); + mark = -1; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/HexDump.java b/vespajlib/src/main/java/com/yahoo/io/HexDump.java new file mode 100644 index 00000000000..65a6f8d2b5a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/HexDump.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +/** + * @author bratseth + */ +public class HexDump { + + private static final String HEX_CHARS = "0123456789ABCDEF"; + + public static String toHexString(byte[] buf) { + if (buf == null) { + return null; + } + StringBuilder sb = new StringBuilder(); + for (byte b : buf) { + int x = b; + if (x < 0) { + x += 256; + } + sb.append(HEX_CHARS.charAt(x / 16)); + sb.append(HEX_CHARS.charAt(x % 16)); + } + return sb.toString(); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java new file mode 100644 index 00000000000..61687f92659 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java @@ -0,0 +1,441 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.util.List; +import java.nio.charset.Charset; +import java.nio.ByteBuffer; + + +/** + * <p>Some static io convenience methods.</p> + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public abstract class IOUtils { + static private final Charset utf8Charset = Charset.forName("utf-8"); + + /** Closes a writer, or does nothing if the writer is null */ + public static void closeWriter(Writer writer) { + if (writer == null) return; + try { writer.close(); } catch (IOException e) {} + } + + /** Closes a reader, or does nothing if the reader is null */ + public static void closeReader(Reader reader) { + if (reader == null) return; + try { reader.close(); } catch (IOException e) {} + } + + /** Closes an input stream, or does nothing if the stream is null */ + public static void closeInputStream(InputStream stream) { + if (stream == null) return; + try { stream.close(); } catch (IOException e) {} + } + + /** Closes an output stream, or does nothing if the stream is null */ + public static void closeOutputStream(OutputStream stream) { + if (stream == null) return; + try { stream.close(); } catch (IOException e) {} + } + + /** + * Creates a buffered reader + * + * @param filename the name or path of the file + * @param encoding the encoding of the file, for instance "UTF-8" + */ + public static BufferedReader createReader(File filename, String encoding) throws IOException { + return new BufferedReader(new InputStreamReader(new FileInputStream(filename), encoding)); + } + + /** + * Creates a buffered reader + * + * @param filename the name or path of the file + * @param encoding the encoding of the file, for instance "UTF-8" + */ + public static BufferedReader createReader(String filename, String encoding) throws IOException { + return new BufferedReader(new InputStreamReader(new FileInputStream(filename), encoding)); + } + + /** Creates a buffered reader in the default encoding */ + public static BufferedReader createReader(String filename) throws IOException { + return new BufferedReader(new FileReader(filename)); + } + + /** + * Creates a buffered writer, + * and the directories to contain it if they do not exist + * + * @param filename the name or path of the file + * @param encoding the encoding to use, for instance "UTF-8" + * @param append whether to append to the files if it exists + */ + public static BufferedWriter createWriter(String filename, String encoding, boolean append) throws IOException { + createDirectory(filename); + return new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename, append), encoding)); + } + + /** + * Creates a buffered writer, + * and the directories to contain it if they do not exist + * + * @param file the file to write to + * @param encoding the encoding to use, for instance "UTF-8" + * @param append whether to append to the files if it exists + */ + public static BufferedWriter createWriter(File file, String encoding, boolean append) throws IOException { + createDirectory(file.getAbsolutePath()); + return new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file, append),encoding)); + } + + /** + * Creates a buffered writer in the default encoding + * + * @param filename the name or path of the file + * @param append whether to append to the files if it exists + */ + public static BufferedWriter createWriter(String filename, boolean append) throws IOException { + createDirectory(filename); + return new BufferedWriter(new FileWriter(filename, append)); + } + + /** + * Creates a buffered writer in the default encoding + * + * @param file the file to write to + * @param append whether to append to the files if it exists + */ + public static BufferedWriter createWriter(File file, boolean append) throws IOException { + createDirectory(file.getAbsolutePath()); + return new BufferedWriter(new FileWriter(file, append)); + } + + /** Creates the directory path of this file if it does not exist */ + public static void createDirectory(String filename) { + File directory = new File(filename).getParentFile(); + + if (directory != null) + directory.mkdirs(); + } + + /** + * Copies the n first lines of a file to another file. + * If the out file exists it will be overwritten + * + * @throws IOException if copying fails + */ + public static void copy(String inFile, String outFile, int lineCount) throws IOException { + BufferedReader reader = null; + BufferedWriter writer = null; + + try { + reader = createReader(inFile); + writer = createWriter(outFile, false); + int c; + + int newLines = 0; + while (-1 != (c=reader.read()) && newLines<lineCount) { + writer.write(c); + if (c=='\n') + newLines++; + } + } finally { + closeReader(reader); + closeWriter(writer); + } + } + + /** + * Copies a file to another file. + * If the out file exists it will be overwritten. + * NOTE: Not an optimal implementation currently. + * + * @throws IOException if copying fails + */ + public static void copy(String inFile, String outFile) throws IOException { + BufferedReader reader=null; + BufferedWriter writer=null; + + try { + reader = createReader(inFile); + writer = createWriter(outFile, false); + int c; + while (-1 != (c = reader.read()) ) + writer.write(c); + } finally { + closeReader(reader); + closeWriter(writer); + } + } + + /** + * Copies a file to another file. + * If the out file exists it will be overwritten. + * NOTE: Not an optimal implementation currently. + */ + public static void copy(File inFile, File outFile) throws IOException { + copy(inFile.toString(),outFile.toString()); + } + + /** + * Copies all files and subdirectories in a directory to another. + * Any existing files are overwritten. + * + * @param sourceLocation the source directory + * @param targetLocation the target directory + * @param maxRecurseLevel if this is 1, only files immediately in sourceLocation are copied, + * if it is 2, then files contained in immediate subdirectories are copied, etc. + * If it is 0, sourceLocation will only be copied if it is a file, not a directory. + * If it is negative, recursion is infinite. + * @throws IOException if copying any file fails. This will typically result in some files being copied and + * others not, i.e this method is not exception safe + */ + public static void copyDirectory(File sourceLocation , File targetLocation, int maxRecurseLevel) throws IOException { + copyDirectory(sourceLocation, targetLocation, maxRecurseLevel, new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return true; + } + }); + } + + /** + * Copies all files and subdirectories in a directory to another. + * Any existing files are overwritten. + * + * @param sourceLocation the source directory + * @param targetLocation the target directory + * @param maxRecurseLevel if this is 1, only files immediately in sourceLocation are copied, + * if it is 2, then files contained in immediate subdirectories are copied, etc. + * If it is 0, sourceLocation will only be copied if it is a file, not a directory. + * If it is negative, recursion is infinite. + * @param filter Only copy files passing through filter. + * @throws IOException if copying any file fails. This will typically result in some files being copied and + * others not, i.e this method is not exception safe + */ + public static void copyDirectory(File sourceLocation , File targetLocation, int maxRecurseLevel, FilenameFilter filter) throws IOException { + if ( ! sourceLocation.isDirectory()) { // copy file + InputStream in=null; + OutputStream out=null; + try { + in = new FileInputStream(sourceLocation); + out = new FileOutputStream(targetLocation); + // Copy the bits from instream to outstream + byte[] buf = new byte[1024]; + int len; + while ((len = in.read(buf)) > 0) { + out.write(buf, 0, len); + } + } + finally { + closeInputStream(in); + closeOutputStream(out); + } + } + else if (maxRecurseLevel!=0) { // copy directory if allowed + if (!targetLocation.exists()) + targetLocation.mkdirs(); + + String[] children = sourceLocation.list(filter); + for (int i=0; i<children.length; i++) + copyDirectory(new File(sourceLocation, children[i]), + new File(targetLocation, children[i]), + maxRecurseLevel-1); + } + } + + /** + * Copies all files and subdirectories (infinitely recursively) in a directory to another. + * Any existing files are overwritten. + * + * @param sourceLocation the source directory + * @param targetLocation the target directory + * @throws IOException if copying any file fails. This will typically result in some files being copied and + * others not, i.e this method is not exception safe + */ + public static void copyDirectory(File sourceLocation , File targetLocation) throws IOException { + copyDirectory(sourceLocation, targetLocation, -1); + } + + /** + * Copies the whole source directory (infinitely recursively) into the target directory. + * @throws IOException if copying any file fails. This will typically result in some files being copied and + * others not, i.e this method is not exception safe + */ + public static void copyDirectoryInto(File sourceLocation, File targetLocation) throws IOException { + File destination = new File(targetLocation, sourceLocation.getAbsoluteFile().getName()); + copyDirectory(sourceLocation, destination); + } + + /** + * Returns the number of line in a file. + * If the files does not exists, 0 is returned + */ + public static int countLines(String file) { + BufferedReader reader = null; + int lineCount = 0; + + try { + reader = createReader(file,"utf8"); + while (reader.readLine() != null) + lineCount++; + return lineCount; + } catch (IOException e) { + return lineCount; + } finally { + closeReader(reader); + } + + } + + /** + * Returns a list containing the lines in the given file as strings + * + * @return a list of Strings for the lines of the file, in order + * @throws IOException if the file could not be read + */ + public static List<String> getLines(String fileName) throws IOException { + BufferedReader reader = null; + + try { + List<String> lines = new java.util.ArrayList<>(); + + reader = createReader(fileName,"utf8"); + String line; + + while (null != (line = reader.readLine())) + lines.add(line); + return lines; + } finally { + closeReader(reader); + } + } + + /** + * Recursive deletion of directories + */ + public static boolean recursiveDeleteDir(File dir) { + if (dir.isDirectory()) { + String[] children = dir.list(); + + for (String child : children) { + boolean success = recursiveDeleteDir(new File(dir, child)); + + if (!success) return false; + } + } + + // The directory is now empty so delete it + return dir.delete(); + } + + /** + * Encodes string as UTF-8 into ByteBuffer + */ + public static ByteBuffer utf8ByteBuffer(String s) { + return utf8Charset.encode(s); + } + + /** + * Reads the contents of a UTF-8 text file into a String. + * + * @param file the file to read, or null + * @return the file content as a string, or null if the input file was null + */ + public static String readFile(File file) throws IOException { + try { + if (file == null) return null; + return new String(Files.readAllBytes(file.toPath()), "utf-8"); + } + catch (NoSuchFileException e) { + throw new NoSuchFileException("Could not find file '" + file.getAbsolutePath() + "'"); + } + } + + /** + * Reads all the content of the given array, in chunks of at max chunkSize + */ + public static byte[] readBytes(InputStream stream, int chunkSize) throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + + int nRead; + byte[] data = new byte[chunkSize]; + while ((nRead = stream.read(data, 0, data.length)) != -1) + buffer.write(data, 0, nRead); + buffer.flush(); + return buffer.toByteArray(); + } + + /** + * Reads the content of a file into a byte array + */ + public static byte[] readFileBytes(File file) throws IOException { + long lengthL = file.length(); + if (lengthL>Integer.MAX_VALUE) + throw new IllegalArgumentException("File too big for byte array: "+file.getCanonicalPath()); + + InputStream in = null; + try { + in = new FileInputStream(file); + int length = (int)lengthL; + byte[] array = new byte[length]; + int offset = 0; + int count=0; + while (offset < length && (count = in.read(array, offset, (length - offset)))>=0) + offset += count; + return array; + } + finally { + if (in != null) + in.close(); + } + } + + /** + * Reads all data from a reader into a string. Uses a buffer to speed up reading. + */ + public static String readAll(Reader reader) throws IOException { + StringBuilder ret=new StringBuilder(); + BufferedReader buffered = new BufferedReader(reader); + int c; + while ((c=buffered.read())!=-1) + ret.appendCodePoint(c); + buffered.close(); + return ret.toString(); + } + + /** Convenience method for closing a list of readers. Does nothing if the given reader list is null. */ + public static void closeAll(List<Reader> readers) { + if (readers==null) return; + for (Reader reader : readers) + closeReader(reader); + } + + /** + * Writes the given string to the file + */ + public static void writeFile(File file, String text, boolean append) throws IOException { + BufferedWriter out = null; + try { + out = createWriter(file, append); + out.write(text); + } + finally { + closeWriter(out); + } + } + + /** + * Writes the given string to the file + */ + public static void writeFile(String file, String text, boolean append) throws IOException { + writeFile(new File(file), text, append); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/io/Listener.java b/vespajlib/src/main/java/com/yahoo/io/Listener.java new file mode 100644 index 00000000000..134cf828c60 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/Listener.java @@ -0,0 +1,564 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + + +import java.io.IOException; +import java.nio.channels.ClosedChannelException; +import java.net.InetSocketAddress; + +import java.util.Iterator; +import java.util.Map; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.ArrayList; + +import java.util.logging.Logger; +import java.util.logging.Level; + +import java.nio.channels.SocketChannel; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.Selector; +import java.nio.channels.SelectionKey; + + +/** + * A basic Reactor implementation using NIO. + * + * @author <a href="mailto:travisb@yahoo-inc.com">Bob Travis</a> + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * + */ +public class Listener extends Thread { + private static Logger log = Logger.getLogger(Listener.class.getName()); + private Selector selector; + Map<Integer, Acceptor> acceptors = new HashMap<>(); + Map<ServerSocketChannel, ConnectionFactory> factories = new IdentityHashMap<>(); + + private FatalErrorHandler fatalErrorHandler; + + private List<SelectLoopHook> selectLoopPreHooks; + private List<SelectLoopHook> selectLoopPostHooks; + + final private LinkedList<Connection> newConnections = new LinkedList<>(); + + // queue of SelectionKeys that need to be updated + final private LinkedList<UpdateInterest> modifyInterestOpsQueue = new LinkedList<>(); + + public Listener(String name) { + super("Listener-" + name); + + try { + selector = Selector.open(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + log.fine(name + " listener created " + this); + } + + /** + * Register a handler for fatal errors. + * + * @param f The FatalErrorHandler instance to be registered + */ + public synchronized void setFatalErrorHandler(FatalErrorHandler f) { + fatalErrorHandler = f; + } + + /** + * Add pre-select loop hook. Not threadsafe so please do this + * during initial setup before you start the listener. + */ + public void addSelectLoopPreHook(SelectLoopHook hook) { + if (selectLoopPreHooks == null) { + selectLoopPreHooks = new ArrayList<>(5); + } + selectLoopPreHooks.add(hook); + } + + /** + * Add pre-select loop hook. Not threadsafe so please do this + * during initial setup before you start the listener. + */ + public void addSelectLoopPostHook(SelectLoopHook hook) { + if (selectLoopPostHooks == null) { + selectLoopPostHooks = new ArrayList<>(5); + } + selectLoopPostHooks.add(hook); + } + + /** + * Run all the select loop pre hooks + */ + private void runSelectLoopPreHooks() { + if (selectLoopPreHooks == null) { + return; + } + + for (SelectLoopHook hook : selectLoopPreHooks) { + hook.selectLoopHook(true); + } + } + + /** + * Run all the select loop post hooks + */ + private void runSelectLoopPostHooks() { + if (selectLoopPostHooks == null) { + return; + } + + for (SelectLoopHook hook : selectLoopPostHooks) { + hook.selectLoopHook(false); + } + } + + /** + * Add a listening port and create an Acceptor thread which accepts + * new connections on this port. + * + * @param factory The connection factory for new connections + * on this port + * @param port The port we are going to listen to. + */ + public synchronized void listen(ConnectionFactory factory, int port) + throws IOException { + // make sure we have only one acceptor per listen port + if (acceptors.containsKey(port)) { + log.warning("Already listening to port=" + port); + return; + } + + Acceptor a = new Acceptor(this, factory, port); + + // inherit the fatal error handling of listener + if (fatalErrorHandler != null) { + a.setFatalErrorHandler(fatalErrorHandler); + } + + a.listen().start(); + acceptors.put(port, a); + } + + /** + * Add a listening port without creating a separate acceptor + * thread. + * + * @param factory The connection factory for new connections + * on this port + * @param port The port we are going to listen to. + */ + public synchronized void listenNoAcceptor(ConnectionFactory factory, int port) + throws IOException { + ServerSocketChannel s = ServerSocketChannel.open(); + + s.configureBlocking(false); + s.socket().setReuseAddress(true); + s.socket().bind(new InetSocketAddress(port)); // use non-specific IP + String host = s.socket().getInetAddress().getHostName(); + + factories.put(s, factory); + s.register(selector, SelectionKey.OP_ACCEPT); + log.fine("listener " + host + ":" + port); + } + + // ================================================================== + // ================================================================== + // ================================================================== + + + /** + * This is the preferred way of modifying interest ops, giving a + * Connection rather than a SelectionKey as input. This way the + * we can look it up and ensure the correct SelectionKey is always + * used. + * + * @return Returns a <code>this</code> reference for chaining + */ + public Listener modifyInterestOps(Connection connection, + int op, boolean set) { + return modifyInterestOps(connection.socketChannel().keyFor(selector), op, + set); + } + + /** + * Batch version of modifyInterestOps(). + * + * @return Returns a <code>this</code> reference for chaining + */ + public Listener modifyInterestOpsBatch(Connection connection, + int op, boolean set) { + return modifyInterestOpsBatch( + connection.socketChannel().keyFor(selector), op, set); + } + + /** + * Enqueue change to interest set of SelectionKey. This is a workaround + * for an NIO design error that makes it impossible to update interest + * sets for a SelectionKey while a select is in progress -- and sometimes + * you actually want to do this from other threads, which will then + * block. Hence, we make it possible to enqueue requests for + * SelectionKey modification in the thread where select runs. + * + * @return Returns a <code>this</code> reference for chaining + */ + public Listener modifyInterestOps(SelectionKey key, int op, boolean set) { + synchronized (modifyInterestOpsQueue) { + modifyInterestOpsQueue.addLast(new UpdateInterest(key, op, set)); + } + selector.wakeup(); + return this; + } + + /** + * Does the same as modifyInterestOps(), but does not call + * wakeup on the selector. Allows adding more modifications + * before we wake up the selector. + * + * @return Returns a <code>this</code> reference for chaining + */ + public Listener modifyInterestOpsBatch(SelectionKey key, + int op, + boolean set) { + synchronized (modifyInterestOpsQueue) { + modifyInterestOpsQueue.addLast(new UpdateInterest(key, op, set)); + } + return this; + } + + /** + * Signal that a batch update of SelectionKey is done and the + * selector should be awoken. Also see modifyInterestOps(). + * + * @return Returns a <code>this</code> reference for chaining + */ + public Listener modifyInterestOpsDone() { + selector.wakeup(); + return this; + } + + /** + * Process enqueued changes to SelectionKeys. Also see + * modifyInterestOps(). + */ + private void processModifyInterestOps() { + synchronized (modifyInterestOpsQueue) { + while (!modifyInterestOpsQueue.isEmpty()) { + UpdateInterest u = modifyInterestOpsQueue.removeFirst(); + + u.doUpdate(); + } + } + } + + // ================================================================== + // ================================================================== + // ================================================================== + + + /** + * Thread entry point + */ + public void run() { + log.fine("Started listener"); + try { + selectLoop(); + } catch (Throwable t) { + if (fatalErrorHandler != null) { + fatalErrorHandler.handle(t, null); + } + } + } + + /** + * Check channels for readiness and deal with channels that have + * pending operations. + */ + private void selectLoop() { + while (!Thread.currentThread().isInterrupted()) { + processNewConnections(); + processModifyInterestOps(); + + try { + int n = selector.select(); + + if (0 == n) { + continue; + } + } catch (java.io.IOException e) { + log.log(Level.WARNING, "error during select", e); + return; + } + + runSelectLoopPreHooks(); + + Iterator<SelectionKey> i = selector.selectedKeys().iterator(); + + while (i.hasNext()) { + SelectionKey key = i.next(); + + i.remove(); + + if (!key.isValid()) { + continue; + } + + if (key.isReadable()) { + performRead(key); + if (!key.isValid()) { + continue; + } + } + + if (key.isWritable()) { + performWrite(key); + if (!key.isValid()) { + continue; + } + } + + if (key.isConnectable()) { + performConnect(key); + if (!key.isValid()) { + continue; + } + } + + if (key.isAcceptable()) { + performAccept(key); + } + } + + runSelectLoopPostHooks(); + } + } + + /** + * This method is used by the Acceptor to hand off newly accepted + * connections to the Listener. Note that this is run in the + * context of the Acceptor thread, so doing things here versus + * doing them in the acceptNewConnections(), which runs in the context + * of the Listener thread, is a tradeoff that may need to be + * re-evaluated + * + */ + public Connection addNewConnection(Connection newConn) { + + // ensure nonblocking and handle possible errors + // if setting nonblocking fails. this code is really redundant + // but necessary because the older version of this method set + // the connection nonblocking, and clients might still expect + // this behavior. + // + SocketChannel channel = newConn.socketChannel(); + + if (channel.isBlocking()) { + try { + channel.configureBlocking(false); + } catch (java.nio.channels.IllegalBlockingModeException e) { + log.log(Level.SEVERE, "Unable to set nonblocking", e); + try { + channel.close(); + } catch (java.io.IOException ee) { + log.log(Level.WARNING, "channel close failed", ee); + } + return newConn; + } catch (java.io.IOException e) { + log.log(Level.SEVERE, "Unable to set nonblocking", e); + return newConn; + } + } + + synchronized (newConnections) { + newConnections.addLast(newConn); + } + selector.wakeup(); + return newConn; + } + + /** + * This method is called from the selectLoop() method in order to + * process new incoming connections. + */ + private synchronized void processNewConnections() { + synchronized (newConnections) { + while (!newConnections.isEmpty()) { + Connection conn = newConnections.removeFirst(); + + try { + conn.socketChannel().register(selector, conn.selectOps(), + conn); + } catch (ClosedChannelException e) { + log.log(Level.WARNING, "register channel failed", e); + return; + } + } + } + } + + /** + * Accept new connection. This will loop over accept() until + * there are no more new connections to accept. If any error + * occurs after a successful accept, the socket in question will + * be discarded, but we will continue to try to accept new + * connections if available. + * + */ + private void performAccept(SelectionKey key) { + SocketChannel channel; + ServerSocketChannel ssChannel; + + if (Thread.currentThread().isInterrupted()) { + return; + } + + while (true) { + try { + ssChannel = (ServerSocketChannel) key.channel(); + channel = ssChannel.accept(); + + // if for some reason there was no connection we just + // ignore it. + if (null == channel) { + return; + } + } catch (java.io.IOException e) { + log.log(Level.WARNING, "accept failed", e); + return; + } + + // set nonblocking and handle possible errors + try { + channel.configureBlocking(false); + } catch (java.nio.channels.IllegalBlockingModeException e) { + log.log(Level.SEVERE, "Unable to set nonblocking", e); + try { + channel.close(); + } catch (java.io.IOException ee) { + log.log(Level.WARNING, "channel close failed", ee); + continue; + } + continue; + } catch (java.io.IOException e) { + log.log(Level.WARNING, "IO error occurred", e); + try { + channel.close(); + } catch (java.io.IOException ee) { + log.log(Level.WARNING, "channel close failed", ee); + continue; + } + continue; + } + + ConnectionFactory factory = factories.get(ssChannel); + Connection conn = factory.newConnection(channel, this); + + try { + channel.register(selector, conn.selectOps(), conn); + } catch (java.nio.channels.ClosedChannelException e) { + log.log(Level.WARNING, "register channel failed", e); + } + } + } + + /** + * Complete asynchronous connect operation. <em>Note that + * asynchronous connect does not work properly in 1.4, + * so you should not use this if you run anything older + * than 1.5/5.0</em>. + * + */ + private void performConnect(SelectionKey key) { + if (Thread.currentThread().isInterrupted()) { + return; + } + + Connection c = (Connection) key.attachment(); + + try { + c.connect(); + } catch (IOException e) { + log.log(Level.FINE, "connect failed", e); + try { + c.close(); + } catch (IOException e2) { + log.log(Level.FINE, "close failed", e); + } + } + } + + /** + * Perform read operation on channel which is now ready for reading + */ + private void performRead(SelectionKey key) { + if (Thread.currentThread().isInterrupted()) { + return; + } + + Connection c = (Connection) key.attachment(); + + try { + c.read(); + } catch (IOException e) { + log.log(Level.FINE, "read failed", e); + try { + c.close(); + } catch (IOException e2) { + log.log(Level.FINE, "close failed", e); + } + } + } + + /** + * Perform write operation(s) on channel which is now ready for + * writing + */ + private void performWrite(SelectionKey key) { + if (Thread.currentThread().isInterrupted()) { + return; + } + + Connection c = (Connection) key.attachment(); + + try { + c.write(); + } catch (IOException e) { + log.log(Level.FINE, " write failed", e); + try { + c.close(); + } catch (IOException e2) {// ignore + } + } + } + + // ============================================================ + // ==== connections made outside listener + // ============================================================ + + /** + * Register a connection that was set up outside the listener. + * Typically what we do when we actively reach out and connect + * somewhere. + */ + public void registerConnection(Connection connection) { + synchronized (newConnections) { + newConnections.addLast(connection); + } + selector.wakeup(); + } + + /** + * Perform clean shutdown of Listener. + * + * TODO: implement + */ + public void shutdown() {// make writing impossible + // make listening on new ports impossible + // close all listening connections (kill all listener threads) + // flush outbound data if the connection wants it + // close all connections + // have some sort of grace-period before forcibly shutting down + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/ReadLine.java b/vespajlib/src/main/java/com/yahoo/io/ReadLine.java new file mode 100644 index 00000000000..aba67c0c8ca --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/ReadLine.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.charset.Charset; +import java.nio.ByteBuffer; + + +/** + * Conventient utility for reading lines from ByteBuffers. Please + * read the method documentation for readLine() carefully. The NIO + * ByteBuffer abstraction is somewhat clumsy and thus usage of this + * code requires that you understand the semantics clearly. + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * + */ +public class ReadLine { + static private Charset charset = Charset.forName("latin1"); + + /** + * Extract next line from a byte buffer. Looks for EOL characters + * between start and limit, and returns a string between start and + * the EOL charachers. It skips ahead past any remaining EOL + * characters and sets position to the first non-EOL character. + * + * If it doesn't find an EOL characher between start and limit + */ + public static String readLine(ByteBuffer buffer) { + int start = buffer.position(); + + for (int i = start; i < buffer.limit(); i++) { + + if (isEolChar(buffer.get(i))) { + + // detect and skip EOL at beginning. Also, update + // position so we compact the buffer if we exit the + // for loop without having found a proper string + if (i == start) { + for (; (i < buffer.limit()) && isEolChar(buffer.get(i)); i++) { + ; + } + start = i; + buffer.position(i); + continue; + } + + // extract string between start and i. limit() returns + // a buffer so we have to up-cast again + String line = charset.decode((ByteBuffer) buffer.slice().limit(i - start)).toString(); + + // skip remaining + for (; (i < buffer.limit()) && isEolChar(buffer.get(i)); i++) { + ; + } + + buffer.position(i); + return line; + } + } + + // if we get here we didn't find any string. this may be + // because the buffer has no more content, ie. limit == position. + // if that is the case we clear the buffer. + // + // if we have content, but no more EOL characters we compact the + // buffer. + // + if (buffer.hasRemaining()) { + buffer.compact(); + } else { + buffer.clear(); + } + + return null; + } + + static boolean isEolChar(byte b) { + return ((10 == b) || (13 == b)); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/SelectLoopHook.java b/vespajlib/src/main/java/com/yahoo/io/SelectLoopHook.java new file mode 100644 index 00000000000..bcc3c0f3e1d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/SelectLoopHook.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + + +/** + * This interface defines a callback hook which applications can + * use to get work done before or after the select loop finishes + * its tasks. + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * + */ +public interface SelectLoopHook { + + /** + * Callback which can be called before or after + * select loop has done its work, depending on + * how you register the hook. + * + * @param before is <code>true</code> if the hook + * was called before the channels in the ready + * set have been processed, and <code>false</code> + * if called after. + */ + public void selectLoopHook(boolean before); +} diff --git a/vespajlib/src/main/java/com/yahoo/io/SlowInflate.java b/vespajlib/src/main/java/com/yahoo/io/SlowInflate.java new file mode 100644 index 00000000000..b25591aa5b7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/SlowInflate.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + + +import java.util.zip.Inflater; + + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class SlowInflate { + private Inflater inflater = new Inflater(); + + public byte[] unpack(byte[] compressed, int inflatedLen) { + byte[] decompressed = new byte[inflatedLen]; + + inflater.reset(); + inflater.setInput(compressed); + inflater.finished(); + try { + inflater.inflate(decompressed); + } catch (java.util.zip.DataFormatException e) { + throw new RuntimeException("Decompression failure: " + e); + } + return decompressed; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/UpdateInterest.java b/vespajlib/src/main/java/com/yahoo/io/UpdateInterest.java new file mode 100644 index 00000000000..bb718218aeb --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/UpdateInterest.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.channels.SelectionKey; + + +/** + * Command object to perform interest set updates. Workaround for NIO + * design flaw which makes it impossible to update the interest set of + * a SelectionKey while select() is in progress. There should be a + * more elegant way around this, but if it turns out to be performant + * enough we leave it like this. + * + * <P> + * Of course, the ideal would be to have NIO fixed. + * + * @author <a href="mailto:travisb@yahoo-inc.com">Bob Travis</a> + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class UpdateInterest { + private SelectionKey key; + private int operation; + private boolean set; + + /** + * Make sure this can't be run + */ + @SuppressWarnings("unused") + private UpdateInterest() {} + + /** + * Create an object for encapsulating a interest set change + * request. + * + * @param key The key we wish to update + * @param operation The operation we wish to set or remove + * @param set Whether we want to set (true) or clear (false) the + * operation in the interest set + */ + public UpdateInterest(SelectionKey key, int operation, boolean set) { + this.key = key; + this.operation = operation; + this.set = set; + } + + /** + * This method is used for actually applying the updates to the + * SelectionKey in question at a time when it is safe to do so. + * If the SelectionKey has been invalidated in the meanwhile we + * do nothing. + */ + public void doUpdate() { + // bail if this key isn't valid anymore + if ((key == null) || (!key.isValid())) { + return; + } + + if (set) { + key.interestOps(key.interestOps() | operation); + } else { + key.interestOps(key.interestOps() & (~operation)); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/WritableByteTransmitter.java b/vespajlib/src/main/java/com/yahoo/io/WritableByteTransmitter.java new file mode 100644 index 00000000000..a7a6a9a1410 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/WritableByteTransmitter.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Marker interface for use with the BufferChain data store. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public interface WritableByteTransmitter { + public void send(ByteBuffer src) throws IOException; +} diff --git a/vespajlib/src/main/java/com/yahoo/io/package-info.java b/vespajlib/src/main/java/com/yahoo/io/package-info.java new file mode 100644 index 00000000000..db0caeb29d7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.io; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java b/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java new file mode 100644 index 00000000000..d0d52a8c619 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io.reader; + +import com.google.common.annotations.Beta; + +import java.io.IOException; +import java.io.Reader; +import java.util.List; + +/** + * A reader with a name. All reader methods are delegated to the wrapped reader. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +@Beta +public class NamedReader extends Reader { + + private final String name; + private final Reader reader; + + public NamedReader(String name, Reader reader) { + this.name = name; + this.reader = reader; + } + + public String getName() { return name; } + + public Reader getReader() { return reader; } + + /** Returns the name */ + public @Override String toString() { + return name; + } + + // The rest is reader method implementations which delegates to the wrapped reader + public @Override int read(java.nio.CharBuffer charBuffer) throws java.io.IOException { return reader.read(charBuffer); } + public @Override int read() throws java.io.IOException { return reader.read(); } + public @Override int read(char[] chars) throws java.io.IOException { return reader.read(chars); } + public @Override int read(char[] chars, int i, int i1) throws java.io.IOException { return reader.read(chars,i,i1); } + public @Override long skip(long l) throws java.io.IOException { return reader.skip(l); } + public @Override boolean ready() throws java.io.IOException { return reader.ready(); } + public @Override boolean markSupported() { return reader.markSupported(); } + public @Override void mark(int i) throws java.io.IOException { reader.mark(i); } + public @Override void reset() throws java.io.IOException { reader.reset(); } + public @Override void close() throws java.io.IOException { reader.close(); } + + /** Convenience method for closing a list of readers. Does nothing if the given reader list is null. */ + public static void closeAll(List<NamedReader> readers) { + if (readers==null) return; + for (Reader reader : readers) { + try { + reader.close(); + } + catch (IOException e) { + // Nothing to do about it + } + } + } + +}
\ No newline at end of file diff --git a/vespajlib/src/main/java/com/yahoo/io/reader/package-info.java b/vespajlib/src/main/java/com/yahoo/io/reader/package-info.java new file mode 100644 index 00000000000..34f57b61a55 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/reader/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * The classes in this package are not intended for external use. + */ +@PublicApi +@ExportPackage +package com.yahoo.io.reader; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/java7compat/Util.java b/vespajlib/src/main/java/com/yahoo/java7compat/Util.java new file mode 100644 index 00000000000..8a838308fbb --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/java7compat/Util.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.java7compat; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ + +public class Util { + private static final int javaVersion = Integer.valueOf(System.getProperty("java.version").substring(2,3)); + public static boolean isJava7Compatible() { return javaVersion >= 7; } + /** + * Takes the double value and prints it in a way that is compliant with the way java7 prints them. + * This is due to java7 finally fixing the trailing zero problem + * @param d the double value + * @return string representation of the double value + */ + public static String toJava7String(double d) { + String s = String.valueOf(d); + if ( ! isJava7Compatible() ) { + s = nonJava7CompatibleString(s); + } + return s; + } + + static String nonJava7CompatibleString(String s) { + if ((s.length() >= 3) && s.contains(".")) { + int l = s.length(); + for(; l > 2 && (s.charAt(l-1) == '0') && (s.charAt(l-2) >= '0') && (s.charAt(l-1) <= '9'); l--); + if (l != s.length()) { + s = s.substring(0, l); + } + } + return s; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/javacc/FastCharStream.java b/vespajlib/src/main/java/com/yahoo/javacc/FastCharStream.java new file mode 100644 index 00000000000..892240ce253 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/javacc/FastCharStream.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.javacc; + +import java.io.IOException; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class FastCharStream { + + private static final String JAVACC_EXCEPTION_FORMAT = "line -1, column "; + private static final IOException EOF = new IOException(); + private final String inputStr; + private final char[] inputArr; + private int tokenPos = 0; + private int readPos = 0; + + public FastCharStream(String input) { + this.inputStr = input; + this.inputArr = input.toCharArray(); + } + + public char readChar() throws IOException { + if (readPos >= inputArr.length) { + throw EOF; + } + return inputArr[readPos++]; + } + + @SuppressWarnings("deprecation") + public int getColumn() { + return getEndColumn(); + } + + @SuppressWarnings("deprecation") + public int getLine() { + return getEndLine(); + } + + public int getEndColumn() { + return readPos + 1; + } + + public int getEndLine() { + return -1; // indicate unset + } + + public int getBeginColumn() { + return tokenPos + 1; + } + + public int getBeginLine() { + return -1; // indicate unset + } + + public void backup(int amount) { + readPos -= amount; + } + + public char BeginToken() throws IOException { + tokenPos = readPos; + return readChar(); + } + + public String GetImage() { + return inputStr.substring(tokenPos, readPos); + } + + @SuppressWarnings("UnusedParameters") + public char[] GetSuffix(int len) { + throw new UnsupportedOperationException(); + } + + public void Done() { + + } + + public String formatException(String parseException) { + int errPos = findErrPos(parseException); + if (errPos < 0 || errPos > inputArr.length + 1) { + return parseException; + } + int errLine = 0; + int errColumn = 0; + for (int i = 0; i < errPos - 1; ++i) { + if (inputStr.charAt(i) == '\n') { + ++errLine; + errColumn = 0; + } else { + ++errColumn; + } + } + StringBuilder out = new StringBuilder(); + out.append(parseException.replace(JAVACC_EXCEPTION_FORMAT + errPos, + "line " + (errLine + 1) + ", column " + (errColumn + 1))); + out.append("\nAt position:\n"); + appendErrorPosition(errLine, out); + for (int i = 0; i < errColumn; ++i) { + out.append(" "); + } + out.append("^"); + return out.toString(); + } + + private void appendErrorPosition(int errLine, StringBuilder out) { + String[] inputStrLines = inputStr.split("\n"); + if (inputStrLines.length<errLine+1) { + out.append("EOF\n"); + } else { + out.append(inputStrLines[errLine]).append("\n"); + } + } + + private static int findErrPos(String str) { + int from = str.indexOf(JAVACC_EXCEPTION_FORMAT); + if (from < 0) { + return -1; + } + from = from + JAVACC_EXCEPTION_FORMAT.length(); + + int to = from; + while (to < str.length() && Character.isDigit(str.charAt(to))) { + ++to; + } + if (to == from) { + return -1; + } + + return Integer.valueOf(str.substring(from, to)); + } +}
\ No newline at end of file diff --git a/vespajlib/src/main/java/com/yahoo/javacc/UnicodeUtilities.java b/vespajlib/src/main/java/com/yahoo/javacc/UnicodeUtilities.java new file mode 100644 index 00000000000..45099a6855e --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/javacc/UnicodeUtilities.java @@ -0,0 +1,181 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.javacc; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class UnicodeUtilities { + + /** + * Adds a leading and trailing double quotation mark to the given string. This will escape whatever content is + * within the string literal. + * + * @param str The string to quote. + * @param quote The quote character. + * @return The quoted string. + */ + public static String quote(String str, char quote) { + StringBuilder ret = new StringBuilder(); + ret.append(quote); + for (int i = 0; i < str.length(); ++i) { + char c = str.charAt(i); + if (c == quote) { + ret.append("\\").append(c); + } else { + ret.append(escape(c)); + } + } + ret.append(quote); + return ret.toString(); + } + + /** + * Removes leading and trailing quotation mark from the given string. This method will properly unescape whatever + * content is withing the string literal as well. + * + * @param str The string to unquote. + * @return The unquoted string. + */ + public static String unquote(String str) { + if (str.length() == 0) { + return str; + } + char quote = str.charAt(0); + if (quote != '"' && quote != '\'') { + return str; + } + if (str.charAt(str.length() - 1) != quote) { + return str; + } + StringBuilder ret = new StringBuilder(); + for (int i = 1; i < str.length() - 1; ++i) { + char c = str.charAt(i); + if (c == '\\') { + if (++i == str.length() - 1) { + break; // done + } + c = str.charAt(i); + if (c == 'f') { + ret.append("\f"); + } else if (c == 'n') { + ret.append("\n"); + } else if (c == 'r') { + ret.append("\r"); + } else if (c == 't') { + ret.append("\t"); + } else if (c == 'u') { + if (++i > str.length() - 4) { + break; // done + } + try { + ret.append((char)Integer.parseInt(str.substring(i, i + 4), 16)); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(e); + } + i += 3; + } else { + ret.append(c); + } + } else if (c == quote) { + throw new IllegalArgumentException(); + } else { + ret.append(c); + } + } + return ret.toString(); + } + + private static String escape(char c) { + switch (c) { + case '\b': + return "\\b"; + case '\t': + return "\\t"; + case '\n': + return "\\n"; + case '\f': + return "\\f"; + case '\r': + return "\\r"; + case '\\': + return "\\\\"; + } + if (c < 0x20 || c > 0x7e) { + String unicode = Integer.toString(c, 16); + return "\\u" + "0000".substring(0, 4 - unicode.length()) + unicode + ""; + } + return "" + c; + } + + public static String generateToken(Predicate predicate) { + TokenBuilder builder = new TokenBuilder(); + for (int c = 0; c <= 0xffff; ++c) { + if (!predicate.accepts((char)c)) { + continue; + } + builder.add(c); + } + return builder.build(); + } + + public static interface Predicate { + + public boolean accepts(char c); + } + + private static class TokenBuilder { + + final StringBuilder token = new StringBuilder(); + int prevC = -1; + int fromC = 0; + int charCnt = 0; + + void add(int c) { + if (prevC + 1 == c) { + // in range + } else { + flushRange(); + fromC = c; + } + prevC = c; + } + + void flushRange() { + if (fromC > prevC) { + return; // handle initial condition + } + append(fromC); + if (fromC < prevC) { + token.append('-'); + append(prevC); + ++charCnt; + } + token.append(','); + if (++charCnt > 16) { + token.append('\n'); + charCnt = 0; + } + } + + void append(int c) { + token.append("\""); + if (c == '\n') { + token.append("\\n"); + } else if (c == '\r') { + token.append("\\r"); + } else if (c == '"') { + token.append("\\\""); + } else if (c == '\\') { + token.append("\\\\"); + } else { + token.append("\\u").append(String.format("%04x", c & 0xffff)); + } + token.append("\""); + } + + String build() { + flushRange(); + return token.toString(); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/javacc/package-info.java b/vespajlib/src/main/java/com/yahoo/javacc/package-info.java new file mode 100644 index 00000000000..c80e05df51f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/javacc/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.javacc; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/lang/MutableInteger.java b/vespajlib/src/main/java/com/yahoo/lang/MutableInteger.java new file mode 100644 index 00000000000..1dfe8bf5e88 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/lang/MutableInteger.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.lang; + +/** + * A mutable integer + * + * @author bratseth + */ +public class MutableInteger { + + private int value; + + public MutableInteger(int value) { + this.value = value; + } + + public int get() { return value; } + + public void set(int value) { this.value = value; } + + /** Adds the increment to the current value and returns the resulting value */ + public int add(int increment) { + value += increment; + return value; + } + + /** Adds the increment to the current value and returns the resulting value */ + public int subtract(int increment) { + value -= increment; + return value; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/lang/package-info.java b/vespajlib/src/main/java/com/yahoo/lang/package-info.java new file mode 100644 index 00000000000..08c86572029 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/lang/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.lang; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/net/HostName.java b/vespajlib/src/main/java/com/yahoo/net/HostName.java new file mode 100644 index 00000000000..3fb1fe49efd --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/HostName.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import java.io.BufferedReader; +import java.io.InputStreamReader; + +/** + * Utilities for getting the hostname on a system running with the JVM. This is moved here from the old + * HostSystem#getHostName in config-model. + * + * @author lulf + */ +public class HostName { + + private static String myHost = null; + + /** + * Static method that returns the name of localhost using shell + * command "hostname". + * + * @return the name of localhost. + * @throws RuntimeException if executing the command 'hostname' fails. + */ + public static synchronized String getLocalhost() { + if (myHost == null) { + try { + Process p = Runtime.getRuntime().exec("hostname"); + BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream())); + myHost = in.readLine(); + p.waitFor(); + if (p.exitValue() != 0) { + throw new RuntimeException("Command 'hostname' failed: exit("+p.exitValue()+")"); + } + } catch (Exception e) { + throw new RuntimeException("Failed when executing command 'hostname'", e); + } + } + return myHost; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java b/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java new file mode 100644 index 00000000000..540f8300f95 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import java.net.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Utilities for returning localhost addresses on Linux. + * See + * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4665037 + * on why this is necessary. + * + * @author bratseth + */ +public class LinuxInetAddress { + + private static Logger log = Logger.getLogger(LinuxInetAddress.class.getName()); + + /** + * Returns an InetAddress representing the address of the localhost. + * A non-loopback address is preferred if available. + * IPv4 is preferred over IPv6 if available. + * + * @return a localhost address + * @throws UnknownHostException if an address could not be determined + */ + public static InetAddress getLocalHost() throws UnknownHostException { + InetAddress localAddress; + try { + localAddress = InetAddress.getLocalHost(); + } catch (UnknownHostException e) { + return InetAddress.getLoopbackAddress(); + } + + if ( ! localAddress.isLoopbackAddress()) return localAddress; + + List<InetAddress> nonLoopbackAddresses = + getAllLocalFromNetwork().stream().filter(a -> ! a.isLoopbackAddress()).collect(Collectors.toList()); + if (nonLoopbackAddresses.isEmpty()) return localAddress; + + List<InetAddress> ipV4NonLoopbackAddresses = + nonLoopbackAddresses.stream().filter(a -> a instanceof Inet4Address).collect(Collectors.toList()); + if ( ! ipV4NonLoopbackAddresses.isEmpty()) return ipV4NonLoopbackAddresses.get(0); + + return nonLoopbackAddresses.get(0); + } + + /** + * Returns all local addresses of this host. + * + * @return an array of the addresses of this + * @throws UnknownHostException if we cannot access the network + */ + public static InetAddress[] getAllLocal() throws UnknownHostException { + InetAddress[] localInetAddresses = InetAddress.getAllByName("127.0.0.1"); + if ( ! localInetAddresses[0].isLoopbackAddress()) return localInetAddresses; + return getAllLocalFromNetwork().toArray(new InetAddress[0]); + } + + /** + * Returns all local addresses of this host. + * + * @return a list of the addresses of this + * @throws UnknownHostException if we cannot access the network + */ + private static List<InetAddress> getAllLocalFromNetwork() throws UnknownHostException { + try { + List<InetAddress> addresses = new ArrayList<>(); + for (NetworkInterface networkInterface : Collections.list(NetworkInterface.getNetworkInterfaces())) + addresses.addAll(Collections.list(networkInterface.getInetAddresses())); + return addresses; + } + catch (SocketException ex) { + throw new UnknownHostException("127.0.0.1"); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/net/URI.java b/vespajlib/src/main/java/com/yahoo/net/URI.java new file mode 100644 index 00000000000..1f9baa36c06 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/URI.java @@ -0,0 +1,819 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +import static com.yahoo.text.Lowercase.toLowerCase; + +/** + * <p>An URI. This is a pure (immutable) value object.</p> + * + * <p>This does more normalization of hierarchical URIs (URLs) than + * described in the RFC and allows hosts with underscores.</p> + * + * @author <a href="mailto:bratseth@fast.no">Jon S Bratseth</a> + */ +public class URI implements Cloneable, java.io.Serializable, Comparable<URI> { + + /** + * + */ + private static final long serialVersionUID = 2271558213498856909L; + + /** The uri string */ + private String uri; + + /** The scheme of the uri */ + private String scheme = null; + + /** The host part of the uri */ + private String host = null; + + /** The port number of the uri, or -1 if no port is explicitly given */ + private int port = -1; + + /** The part of the uri following the host (host and port) */ + private String rest = null; + + private static final Pattern tokenizePattern = Pattern.compile("[^\\w\\-]"); + + private boolean parsedDomain = false; + private String domain = null; + + private boolean parsedMainTld = false; + private String mainTld = null; + + private boolean parsedPath = false; + private String path = null; + + private boolean parsedParams = false; + private String params = null; + + private boolean parsedFilename = false; + private String filename = null; + + private boolean parsedExtension = false; + private String extension = null; + + private boolean parsedQuery = false; + private String query = null; + + private boolean parsedFragment = false; + private String fragment = null; + + + /** The explanation of why this uri is invalid, or null if it is valid */ + private String invalidExplanation = null; + + /** True if this uri is opaque, false if it is hierarchical */ + private boolean opaque = true; + + /** + * <p>Creates an URI without keeping the fragment (the part starting by #). + * If the uri is hierarchical, it is normalized and incorrect hierarchical uris + * which looks like urls are attempted repaired.</p> + * + * <p>Relative uris are not supported.</p> + * + * @param uriString the uri string + * @throws NullPointerException if the given uriString is null + */ + public URI(String uriString) { + this(uriString, false); + } + + /** + * Creates an URI, optionaly keeping the fragment (the part starting by #). + * If the uri is hierarchical, it is normalized and incorrect hierarchical uris + * which looks like urls are attempted repaired. + * + * <p>Relative uris are not supported.</p> + * + * @param uriString the uri string + * @param keepFragment true to keep the fragment + * @throws NullPointerException if the given uriString is null + */ + public URI(String uriString, boolean keepFragment) { + this(uriString, keepFragment, false); + } + + /** + * Creates an URI, optionaly keeping the fragment (the part starting by #). + * If the uri is hierarchical, it is normalized and incorrect hierarchical uris + * which looks like urls are attempted repaired. + * + * <p>Relative uris are not supported.</p> + * + * @param uriString the uri string + * @param keepFragment true to keep the fragment + * @param hierarchicalOnly will force any uri string given to be parsed as + * a hierarchical one, causing the uri to be invalid if it isn't + * @throws NullPointerException if the given uriString is null + */ + public URI(String uriString, boolean keepFragment, boolean hierarchicalOnly) { + if (uriString == null) { + throw new NullPointerException("Can not create an uri from null"); + } + + if (!keepFragment) { + int fragmentIndex = uriString.indexOf("#"); + + if (fragmentIndex >= 0) { + uriString = uriString.substring(0, fragmentIndex); + } + } + + try { + this.uri = uriString.trim(); + opaque = isOpaque(uri); + + // No further parsing of opaque uris + if (isOpaque() && !hierarchicalOnly) { + return; + } + opaque = false; + normalizeHierarchical(); + } catch (IllegalArgumentException e) { + if (e.getMessage() != null) { + invalidExplanation = e.getMessage(); + } else { + Throwable t = e.getCause(); + if (t != null && t.getMessage() != null) { + invalidExplanation = t.getMessage(); + } else { + invalidExplanation = "Invalid uri: " + e; + } + } + } + } + + /** Creates an url type uri */ + public URI(String scheme, String host, int port, String rest) { + this.scheme = scheme; + this.host = host; + this.port = port; + this.rest = rest; + recombine(); + normalizeHierarchical(); + opaque = false; + } + + /** Returns whether an url is opaque or hierarchical */ + private boolean isOpaque(String uri) { + int colonIndex = uri.indexOf(":"); + + if (colonIndex < 0) { + return true; + } else { + return !(uri.length() > colonIndex + 1 + && uri.charAt(colonIndex + 1) == '/'); + } + } + + /** + * Returns whether this is a valid URI (after normalizing). + * All non-hierarchical uri's containing a scheme is valid. + */ + public boolean isValid() { + return invalidExplanation == null; + } + + /** + * Normalizes this hierarchical uri according to FRC 2396 and the Overture + * standard. Before normalizing, some simple heuritics are use to make + * the uri complete if needed. After normalizing, the scheme, + * host, port and rest of this uri is set if defined. + * + * @throws IllegalArgumentException if this uri can not be normalized into a legal uri + */ + private void normalizeHierarchical() { + complete(); + escapeNonAscii(); + unescapeHtmlEntities(); + decompose(); + lowCaseHost(); + removeDefaultPortNumber(); + removeTrailingHostDot(); + makeDoubleSlashesSingle(); + recombine(); + } + + /** Applies simple heuristics to complete this uri if needed */ + private void complete() { + if (uri.startsWith("www.")) { + uri = "http://" + uri; + } else if (uri.startsWith("WWW")) { + uri = "http://" + uri; + } else if (uri.startsWith("/http:")) { + uri = uri.substring(1); + } else if (isFileURIShortHand(uri)) { + uri = "file://" + uri; + } + } + + private boolean isFileURIShortHand(String uri) { + if (uri.indexOf(":\\") == 1) { + return true; + } + if (uri.indexOf("c:/") == 0) { + return true; + } + if (uri.indexOf("d:/") == 0) { + return true; + } + return false; + } + + /** + * Decomposes this uri into scheme, host, port and rest. + */ + private void decompose() { + java.net.URI neturi = java.net.URI.create(uri).normalize(); + + scheme = neturi.getScheme(); + + host = neturi.getHost(); + boolean portAlreadyParsed = false; + + // No host if the host contains underscores + if (host == null) { + host = neturi.getAuthority(); + if (host != null) { + int colonPos = host.lastIndexOf(":"); + if (!scheme.equals("file") && colonPos > -1) { + //we probably have an (illegal) URI of type http://under_score.com:5000/ + try { + port = Integer.parseInt(host.substring(colonPos + 1, host.length())); + host = host.substring(0, colonPos); + portAlreadyParsed = true; + } catch (NumberFormatException nfe) { + //empty + } + } + } + } + + if ("file".equalsIgnoreCase(scheme)) { + if (host == null) { + host = "localhost"; + } else { + host = repairWindowsDrive(host, uri); + } + } + if (host == null) { + throw new IllegalArgumentException( + "A complete uri must specify a host"); + } + if (!portAlreadyParsed) { + port = neturi.getPort(); + } + rest = (neturi.getRawPath() != null ? neturi.getRawPath() : "") + + (neturi.getRawQuery() != null + ? ("?" + neturi.getRawQuery()) + : "") + + (neturi.getRawFragment() != null + ? ("#" + neturi.getRawFragment()) + : ""); + } + + /** c: turns to c when interpreted by URI. Repair it */ + private String repairWindowsDrive(String host, String uri) { + if (host.length() != 1) { + return host; + } + int driveIndex = uri.indexOf(host + ":"); + + if (driveIndex == 5 || driveIndex == 7) { // file:<drive> or file://<drive> + return host + ":"; + } else { + return host; + } + } + + /** "http://a/\u00E6" → "http://a/%E6;" */ + private void escapeNonAscii() { + char[] uriChars = uri.toCharArray(); + StringBuilder result = new StringBuilder(uri.length()); + + for (char uriChar : uriChars) { + if (uriChar >= 0x80 || uriChar == 0x22) { + result.append("%"); + result.append(Integer.toHexString(uriChar)); + result.append(";"); + } else { + result.append(uriChar); + } + } + uri = result.toString(); + } + + /** "http://a/&amp;" → "http://a/&" Currently ampersand only */ + private void unescapeHtmlEntities() { + int ampIndex = uri.indexOf("&"); + + if (ampIndex < 0) { + return; + } + + StringBuilder result = new StringBuilder(uri.substring(0, ampIndex)); + + while (ampIndex >= 0) { + result.append("&"); + int nextAmpIndex = uri.indexOf("&", ampIndex + 5); + + result.append( + uri.substring(ampIndex + 5, + nextAmpIndex > 0 ? nextAmpIndex : uri.length())); + ampIndex = nextAmpIndex; + } + uri = result.toString(); + } + + /** "HTTP://a" → "http://a" */ + private void lowCaseHost() { + host = toLowerCase(host); + } + + /** "http://a:80" → "http://a" and "https://a:443" → https//a */ + private void removeDefaultPortNumber() { + if (port == 80 && scheme.equals("http")) { + port = -1; + } else if (port == 443 && scheme.equals("https")) { + port = -1; + } + } + + /** "http://a./b" → "http://a/b" */ + private void removeTrailingHostDot() { + if (host.endsWith(".")) { + host = host.substring(0, host.length() - 1); + } + } + + /** "http://a//b" → "http://a/b" */ + private void makeDoubleSlashesSingle() { + StringBuilder result = new StringBuilder(rest.length()); + char[] restChars = rest.toCharArray(); + + for (int i = 0; i < restChars.length; i++) { + if (!(i + 1 < restChars.length && restChars[i] == '/' + && restChars[i + 1] == '/')) { + result.append(restChars[i]); + } + } + rest = result.toString(); + } + + /** Recombines the uri from the scheme, host, port and rest */ + private void recombine() { + StringBuilder recombined = new StringBuilder(100); + + recombined.append(scheme); + recombined.append("://"); + recombined.append(host); + if (port > -1) { + recombined.append(":").append(port); + } + if (rest != null) { + if (!rest.startsWith("/")) { + recombined.append("/"); + } + recombined.append(rest); + } else { + recombined.append("/"); // RFC 2396 violation, as required by search + } + uri = recombined.toString(); + } + + /** + * Returns the normalized scheme of this URI. + * + * @return the normalized scheme (protocol), or null if there is none, + * which may only be the case with non-hierarchical URIs + */ + public String getScheme() { + return scheme; + } + + /** + * Returns whether this URI is hierarchical or opaque. + * A typical example of an hierarchical URI is an URL, + * while URI's are mailto, news and such. + * + * @return true if the url is opaque, false if it is hierarchical + */ + public boolean isOpaque() { + return opaque; + } + + /** + * Returns the normalized host of this URI. + * + * @return the normalized host, or null if there is none, which may + * only be the case if this is a non-hierarchical uri + */ + public String getHost() { + return host; + } + + /** Returns the port number of this scheme if set explicitly, or -1 otherwise */ + public int getPort() { + return port; + } + + /** + * Returns the <i>rest</i> of this uri, that is what is following the host or port. + * This is path, query and fragment as defined in RFC 2396. Returns an empty string + * if this uri has no rest. + */ + public String getRest() { + if (rest == null) { + return null; + } else if (rest.equals("/")) { + return ""; + } else { + return rest; + } + } + + public String getDomain() { + if (parsedDomain) { + return domain; + } + String host = getHost(); + if (host == null) return null; + + int firstDotPos = host.indexOf("."); + int lastDotPos = host.lastIndexOf("."); + + String domain; + if (firstDotPos < 0) { + // "." was not found at all + domain = host; + } else if (firstDotPos == lastDotPos) { + //there is only one "." in the host + domain = host; + } else { + //for www.host.com return host.com + //TODO: Must be corrected when implementing tldlist + domain = host.substring(firstDotPos + 1, host.length()); + } + + this.parsedDomain = true; + this.domain = domain; + return domain; + } + + public String getMainTld() { + if (parsedMainTld) { + return mainTld; + } + String host = getHost(); + if (host == null) return null; + + int lastDotPos = host.lastIndexOf("."); + + String mainTld; + if (lastDotPos < 0) { + //no ".", no TLD + mainTld = null; + } else if (lastDotPos == host.length() - 1) { + //the "." is the last character + mainTld = null; + } else { + //for www.yahoo.co.uk return uk + //TODO: Implement list of TLDs from config? + mainTld = host.substring(lastDotPos + 1, host.length()); + } + this.parsedMainTld = true; + this.mainTld = mainTld; + return mainTld; + } + + public String getPath() { + if (parsedPath) { + return path; + } + String rest = this.rest; + if (rest == null) return null; + + rest = removeFragment(rest); + + int queryPos = rest.lastIndexOf("?"); + if (queryPos > -1) { + rest = rest.substring(0, queryPos); + } + this.parsedPath = true; + this.path = rest; + return this.path; + } + + private String removeFragment(String path) { + int fragmentPos = path.lastIndexOf("#"); + return (fragmentPos > -1) ? path.substring(0, fragmentPos) : path; + } + + public String getFilename() { + if (parsedFilename) { + return filename; + } + String path = getPath(); + if (path == null) return null; + + path = removeParams(path); + + int lastSlash = path.lastIndexOf("/"); + + String filename; + if (lastSlash < 0) { + //there is no slash, return the path, excluding params + filename = path; + } else if (lastSlash == path.length() - 1) { + //the slash is the last character, there is no filename here + filename = ""; + } else { + filename = path.substring(lastSlash + 1, path.length()); + } + this.parsedFilename = true; + this.filename = filename; + return filename; + } + + private String removeParams(String filename) { + int firstSemicolon = filename.indexOf(";"); + + if (firstSemicolon < 0) { + //there are no params + return filename; + } + return filename.substring(0, firstSemicolon); + } + + public String getExtension() { + if (parsedExtension) { + return extension; + } + String filename = getFilename(); + if (filename == null) return null; + + int lastDotPos = filename.lastIndexOf("."); + + String extension; + if (lastDotPos < 0) { + //there is no ".", there is no extension + extension = null; + } else if (lastDotPos == filename.length() - 1) { + //the "." is the last character, there is no extension + extension = null; + } else { + extension = filename.substring(lastDotPos + 1, filename.length()); + } + this.parsedExtension = true; + this.extension = extension; + return extension; + } + + public String getQuery() { + if (parsedQuery) { + return query; + } + String rest = this.rest; + if (rest == null) return null; + + rest = removeFragment(rest); + + int queryPos = rest.lastIndexOf("?"); + String query = null; + if (queryPos > -1) { + //we have a query + query = rest.substring(queryPos+1, rest.length()); + } + this.parsedQuery = true; + this.query = query; + return query; + } + + public String getFragment() { + if (parsedFragment) { + return fragment; + } + String path = this.rest; + if (path == null) return null; + + int fragmentPos = path.lastIndexOf("#"); + String fragment = null; + if (fragmentPos > -1) { + //we have a fragment + fragment = path.substring(fragmentPos+1, path.length()); + } + this.parsedFragment = true; + this.fragment = fragment; + return fragment; + } + + public String getParams() { + if (parsedParams) { + return params; + } + String path = getPath(); + if (path == null) return null; + + int semicolonPos = path.indexOf(";"); + String params; + if (semicolonPos < 0) { + //there is no semicolon, there are no params here + params = null; + } else if (semicolonPos == path.length() - 1) { + //the semicolon is the last character, there are no params here + params = null; + } else { + params = path.substring(semicolonPos + 1, path.length()); + } + this.parsedParams = true; + this.params = params; + return params; + } + + public static String[] tokenize(String item) { + return tokenizePattern.split(item); + } + + public List<Token> tokenize() { + List<Token> tokens = new ArrayList<>(); + + tokens.addAll(tokenize(URLContext.URL_SCHEME, getScheme())); + tokens.addAll(tokenize(URLContext.URL_HOST, getHost())); + tokens.addAll(tokenize(URLContext.URL_PORT, getPort() > -1 ? "" + getPort() : null)); + tokens.addAll(tokenize(URLContext.URL_PATH, getPath())); + tokens.addAll(tokenize(URLContext.URL_QUERY, getQuery())); + tokens.addAll(tokenize(URLContext.URL_FRAGMENT, getFragment())); + + return tokens; + } + + private List<Token> tokenize(URLContext context, String item) { + if (item == null) { + return new ArrayList<>(0); + } + String[] tokenStrings = tokenize(item); + List<Token> tokens = new ArrayList<>(tokenStrings.length); + for (String tokenString : tokenStrings) { + if (tokenString.length() > 0) { + tokens.add(new Token(context, tokenString)); + } + } + return tokens; + } + + /** Returns an explanation of why this uri is invalid, or null if it is valid */ + public String getInvalidExplanation() { + return invalidExplanation; + } + + public int hashCode() { + return uri.hashCode(); + } + + public boolean equals(Object object) { + if (!(object instanceof URI)) { + return false; + } + return (toString().equals(object.toString())); + } + + public int compareTo(URI object) { + return toString().compareTo(object.toString()); + } + + public Object clone() { + try { + return super.clone(); + } catch (CloneNotSupportedException e) { + throw new RuntimeException("Someone made me unclonable!", e); + } + } + + /** Returns a new URI with a changed scheme */ + public URI setScheme(String scheme) { + return new URI(scheme, host, port, rest); + } + + /** Returns a new URI with a changed host (or authority) */ + public URI setHost(String host) { + return new URI(scheme, host, port, rest); + } + + /** Returns a new URI with a changed port */ + public URI setPort(int port) { + return new URI(scheme, host, port, rest); + } + + /** Returns a new URI with a changed rest */ + public URI setRest(String rest) { + return new URI(scheme, host, port, rest); + } + + /** Returns a new uri with the an additional parameter */ + public URI addParameter(String name, String value) { + String newRest = rest; + + if (newRest == null) { + newRest = ""; + } + if (newRest.indexOf("?") < 0) { + newRest += "?"; + } else { + newRest += "&"; + } + newRest += name + "=" + value; + return new URI(scheme, host, port, newRest); + } + + /** Returns this uri as a string */ + public String stringValue() { + return uri; + } + + /** Returns this URI as a string */ + public String toString() { + return uri; + } + + /** + * Returns the depth of this uri. + * The depth of an hierarchical uri equals the number of slashes + * which are not separating the protocol and the host, and not at the end. + * + * @return the depth of this uri if it is hierarchical, or 0 if it is opaque + */ + public int getDepth() { + int colonIndex = uri.indexOf(':'); + + // count number of slashes in the Uri + int currentIndex = colonIndex; + int depth = 0; + + while (currentIndex != -1) { + currentIndex = uri.indexOf('/', currentIndex); + if (currentIndex != -1) { + depth++; + currentIndex++; + } + } + + if (uri.charAt(colonIndex + 1) == '/') { + depth--; + } + if (uri.charAt(colonIndex + 2) == '/') { + depth--; + } + if ((uri.charAt(uri.length() - 1) == '/') + && ((uri.length() - 1) > (colonIndex + 2))) { + depth--; + } + return depth; + } + + + public static class Token { + private final URLContext context; + private final String token; + + private Token(URLContext context, String token) { + this.context = context; + this.token = token; + } + + public URLContext getContext() { + return context; + } + + public String getToken() { + return token; + } + } + + public static enum URLContext { + URL_SCHEME(0, "scheme"), + URL_HOST(1, "host"), + URL_DOMAIN(2, "domain"), + URL_MAINTLD(3, "maintld"), + URL_PORT(4, "port"), + URL_PATH(5, "path"), + URL_FILENAME(6, "filename"), + URL_EXTENSION(7, "extension"), + URL_PARAMS(8, "params"), + URL_QUERY(9, "query"), + URL_FRAGMENT(10, "fragment"); + + public final int id; + public final String name; + + private URLContext(int id, String name) { + this.id = id; + this.name = name; + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/net/UriTools.java b/vespajlib/src/main/java/com/yahoo/net/UriTools.java new file mode 100644 index 00000000000..34d88713274 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/UriTools.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import java.net.URI; + +/** + * Utility methods for working with URIs. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class UriTools { + private UriTools() { + } + + /** + * Build a string representation of the normalized form of the given URI, + * containg the path and optionally query and fragment parts. The query part + * will be delimeted from the preceding data with "?" and the fragment with + * "#". + * + * @param uri + * source for path, query and fragment in returned data + * @return a string containing path, and optionally query and fragment, + * delimited by question mark and hash + */ + public static String rawRequest(final URI uri) { + final String rawQuery = uri.getRawQuery(); + final String rawFragment = uri.getRawFragment(); + final StringBuilder rawRequest = new StringBuilder(); + + rawRequest.append(uri.getRawPath()); + if (rawQuery != null) { + rawRequest.append("?").append(rawQuery); + } + + if (rawFragment != null) { + rawRequest.append("#").append(rawFragment); + } + + return rawRequest.toString(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/net/Url.java b/vespajlib/src/main/java/com/yahoo/net/Url.java new file mode 100644 index 00000000000..33571f9eb34 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/Url.java @@ -0,0 +1,253 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Url { + + private static final Pattern pattern = Pattern.compile( + //12 3 456 7 8 9ab c d e f g h i j + // 2 1 6 87 5 c b ed a4 f hg ji + "^(([^:/?#]+):)?(//((([^:@/?#]+)(:([^@/?#]+))?@))?(((\\[([^\\]]+)\\]|[^:/?#]+)(:([^/?#]+))?)))?([^?#]+)?(\\?([^#]*))?(#(.*))?"); + private final String image; + private final int schemeBegin; + private final int schemeEnd; + private final int userInfoBegin; + private final int userInfoEnd; + private final int passwordBegin; + private final int passwordEnd; + private final int hostBegin; + private final int hostEnd; + private final int portBegin; + private final int portEnd; + private final int pathBegin; + private final int pathEnd; + private final int queryBegin; + private final int queryEnd; + private final int fragmentBegin; + private final int fragmentEnd; + + public Url(String scheme, String user, String password, String host, Integer port, String path, String query, + String fragment) + { + StringBuilder image = new StringBuilder(); + schemeBegin = image.length(); + if (scheme != null) { + image.append(scheme); + schemeEnd = image.length(); + image.append(':'); + } else { + schemeEnd = schemeBegin; + } + if (host != null) { + image.append("//"); + } + userInfoBegin = image.length(); + if (user != null) { + image.append(user); + userInfoEnd = image.length(); + } else { + userInfoEnd = userInfoBegin; + } + if (password != null) { + image.append(':'); + passwordBegin = image.length(); + image.append(password); + passwordEnd = image.length(); + } else { + passwordBegin = image.length(); + passwordEnd = passwordBegin; + } + if (user != null || password != null) { + image.append('@'); + } + if (host != null) { + boolean esc = host.indexOf(':') >= 0; + if (esc) { + image.append('['); + } + hostBegin = image.length(); + image.append(host); + hostEnd = image.length(); + if (esc) { + image.append(']'); + } + } else { + hostBegin = image.length(); + hostEnd = hostBegin; + } + if (port != null) { + image.append(':'); + portBegin = image.length(); + image.append(port); + portEnd = image.length(); + } else { + portBegin = image.length(); + portEnd = portBegin; + } + pathBegin = image.length(); + if (path != null) { + image.append(path); + pathEnd = image.length(); + } else { + pathEnd = pathBegin; + } + if (query != null) { + image.append('?'); + queryBegin = image.length(); + image.append(query); + queryEnd = image.length(); + } else { + queryBegin = image.length(); + queryEnd = queryBegin; + } + if (fragment != null) { + image.append("#"); + fragmentBegin = image.length(); + image.append(fragment); + fragmentEnd = image.length(); + } else { + fragmentBegin = image.length(); + fragmentEnd = fragmentBegin; + } + this.image = image.toString(); + } + + public static Url fromString(String image) { + Matcher matcher = pattern.matcher(image); + if (!matcher.matches()) { + throw new IllegalArgumentException("Malformed URL."); + } + String host = matcher.group(12); + if (host == null) { + host = matcher.group(11); + } + if (host == null) { + host = matcher.group(9); + } + String port = matcher.group(14); + return new Url(matcher.group(2), matcher.group(6), matcher.group(8), host, + port != null ? Integer.valueOf(port) : null, matcher.group(15), matcher.group(17), + matcher.group(19)); + } + + public int getSchemeBegin() { + return schemeBegin; + } + + public int getSchemeEnd() { + return schemeEnd; + } + + public int getUserInfoBegin() { + return userInfoBegin; + } + + public int getUserInfoEnd() { + return userInfoEnd; + } + + public int getPasswordBegin() { + return passwordBegin; + } + + public int getPasswordEnd() { + return passwordEnd; + } + + public int getHostBegin() { + return hostBegin; + } + + public int getHostEnd() { + return hostEnd; + } + + public int getPortBegin() { + return portBegin; + } + + public int getPortEnd() { + return portEnd; + } + + public int getPathBegin() { + return pathBegin; + } + + public int getPathEnd() { + return pathEnd; + } + + public int getQueryBegin() { + return queryBegin; + } + + public int getQueryEnd() { + return queryEnd; + } + + public int getFragmentBegin() { + return fragmentBegin; + } + + public int getFragmentEnd() { + return fragmentEnd; + } + + public String getScheme() { + return schemeBegin < schemeEnd ? image.substring(schemeBegin, schemeEnd) : null; + } + + public String getUserInfo() { + return userInfoBegin < userInfoEnd ? image.substring(userInfoBegin, userInfoEnd) : null; + } + + public String getPassword() { + return passwordBegin < passwordEnd ? image.substring(passwordBegin, passwordEnd) : null; + } + + public String getHost() { + return hostBegin < hostEnd ? image.substring(hostBegin, hostEnd) : null; + } + + public Integer getPort() { + String str = getPortString(); + return str != null ? Integer.valueOf(str) : null; + } + + public String getPortString() { + return portBegin < portEnd ? image.substring(portBegin, portEnd) : null; + } + + public String getPath() { + return pathBegin < pathEnd ? image.substring(pathBegin, pathEnd) : null; + } + + public String getQuery() { + return queryBegin < queryEnd ? image.substring(queryBegin, queryEnd) : null; + } + + public String getFragment() { + return fragmentBegin < fragmentEnd ? image.substring(fragmentBegin, fragmentEnd) : null; + } + + @Override + public int hashCode() { + return image.hashCode(); + } + + @Override + public boolean equals(Object obj) { + return (obj instanceof Url) && image.equals(((Url)obj).image); + } + + @Override + public String toString() { + return image; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/net/UrlToken.java b/vespajlib/src/main/java/com/yahoo/net/UrlToken.java new file mode 100644 index 00000000000..785c3b1fe43 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/UrlToken.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class UrlToken { + + public enum Type { + SCHEME, + USERINFO, + PASSWORD, + HOST, + PORT, + PATH, + QUERY, + FRAGMENT + } + + private final Type type; + private final int offset; + private final String orig; + private final String term; + + public UrlToken(Type type, int offset, String orig, String term) { + if (type == null) { + throw new NullPointerException(); + } + this.type = type; + this.offset = offset; + this.orig = orig; + this.term = term; + } + + public Type getType() { + return type; + } + + public int getOffset() { + return offset; + } + + public int getLength() { + return orig != null ? orig.length() : 0; + } + + public String getOrig() { + return orig; + } + + public String getTerm() { + return term; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof UrlToken)) { + return false; + } + UrlToken rhs = (UrlToken)obj; + if (offset != rhs.offset) { + return false; + } + if (orig != null ? !orig.equals(rhs.orig) : rhs.orig != null) { + return false; + } + if (term != null ? !term.equals(rhs.term) : rhs.term != null) { + return false; + } + if (type != rhs.type) { + return false; + } + return true; + } + + @Override + public int hashCode() { + int result = type != null ? type.hashCode() : 0; + result = 31 * result + offset; + result = 31 * result + (orig != null ? orig.hashCode() : 0); + result = 31 * result + (term != null ? term.hashCode() : 0); + return result; + } + + @Override + public String toString() { + StringBuilder ret = new StringBuilder("UrlToken("); + ret.append("type=").append(type).append(", "); + ret.append("offset=").append(offset).append(", "); + if (orig != null) { + ret.append("orig='").append(orig).append("', "); + } + if (term != null) { + ret.append("term='").append(term).append("', "); + } + ret.setLength(ret.length() - 2); + ret.append(")"); + return ret.toString(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/net/UrlTokenizer.java b/vespajlib/src/main/java/com/yahoo/net/UrlTokenizer.java new file mode 100644 index 00000000000..ec617607b8a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/UrlTokenizer.java @@ -0,0 +1,178 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class UrlTokenizer { + + public static final String TERM_STARTHOST = "StArThOsT"; + public static final String TERM_ENDHOST = "EnDhOsT"; + + private static final Map<String, String> schemeToPort = new HashMap<>(); + private static final Map<String, String> portToScheme = new HashMap<>(); + private static final char TO_LOWER = (char)('A' - 'a'); + private final Url url; + + static { + registerScheme("ftp", 21); + registerScheme("gopher", 70); + registerScheme("http", 80); + registerScheme("https", 443); + registerScheme("imap", 143); + registerScheme("mailto", 25); + registerScheme("news", 119); + registerScheme("nntp", 119); + registerScheme("pop", 110); + registerScheme("rsync", 873); + registerScheme("rtsp", 554); + registerScheme("sftp", 22); + registerScheme("shttp", 443); + registerScheme("sip", 5060); + registerScheme("sips", 5061); + registerScheme("snmp", 161); + registerScheme("ssh", 22); + registerScheme("telnet", 23); + registerScheme("tftp", 69); + } + + public UrlTokenizer(String url) { + this(Url.fromString(url)); + } + + public UrlTokenizer(Url url) { + this.url = url; + } + + private String guessScheme(String port) { + String scheme = portToScheme.get(port); + if (scheme != null) { + return scheme; + } + return "http"; + } + + private String guessPort(String scheme) { + String port = schemeToPort.get(scheme); + if (port != null) { + return port; + } + return null; + } + + public List<UrlToken> tokenize() { + List<UrlToken> lst = new LinkedList<>(); + + int offset = 0; + String port = url.getPortString(); + String scheme = url.getScheme(); + if (scheme == null) { + scheme = guessScheme(port); + addTokens(lst, UrlToken.Type.SCHEME, offset, scheme, false); + } else { + addTokens(lst, UrlToken.Type.SCHEME, url.getSchemeBegin(), scheme, true); + offset = url.getSchemeEnd(); + } + + String userInfo = url.getUserInfo(); + if (userInfo != null) { + addTokens(lst, UrlToken.Type.USERINFO, url.getUserInfoBegin(), userInfo, true); + offset = url.getUserInfoEnd(); + } + + String password = url.getPassword(); + if (password != null) { + addTokens(lst, UrlToken.Type.PASSWORD, url.getPasswordBegin(), password, true); + offset = url.getPasswordEnd(); + } + + String host = url.getHost(); + if (host == null || host.isEmpty()) { + if (host != null) { + offset = url.getHostBegin(); + } + if ("file".equalsIgnoreCase(scheme)) { + addHostTokens(lst, offset, offset, "localhost", false); + } + } else { + addHostTokens(lst, url.getHostBegin(), url.getHostEnd(), host, true); + offset = url.getHostEnd(); + } + + port = url.getPortString(); + if (port == null) { + if ((port = guessPort(scheme)) != null) { + addTokens(lst, UrlToken.Type.PORT, offset, port, false); + } + } else { + addTokens(lst, UrlToken.Type.PORT, url.getPortBegin(), port, true); + } + + String path = url.getPath(); + if (path != null) { + addTokens(lst, UrlToken.Type.PATH, url.getPathBegin(), path, true); + } + + String query = url.getQuery(); + if (query != null) { + addTokens(lst, UrlToken.Type.QUERY, url.getQueryBegin(), query, true); + } + + String fragment = url.getFragment(); + if (fragment != null) { + addTokens(lst, UrlToken.Type.FRAGMENT, url.getFragmentBegin(), fragment, true); + } + + return lst; + } + + public static void addTokens(List<UrlToken> lst, UrlToken.Type type, int offset, String image, boolean orig) { + StringBuilder term = new StringBuilder(); + int prev = 0; + for (int skip, next = 0, len = image.length(); next < len; next += skip) { + char c = image.charAt(next); + if (c == '%') { + c = (char)Integer.parseInt(image.substring(next + 1, next + 3), 16); + skip = 3; + } else { + skip = 1; + } + if ((c >= '0' && c <= '9') || + (c >= 'a' && c <= 'z') || + (c == '-' || c == '_')) + { + term.append(c); + } else if (c >= 'A' && c <= 'Z') { + term.append((char)(c - TO_LOWER)); + } else { + if (prev < next) { + lst.add(new UrlToken(type, offset + (orig ? prev : 0), orig ? image.substring(prev, next) : null, + term.toString())); + term = new StringBuilder(); + } + prev = next + skip; + } + } + if (term.length() > 0) { + lst.add(new UrlToken(type, offset + (orig ? prev : 0), orig ? image.substring(prev) : null, + term.toString())); + } + } + + private static void addHostTokens(List<UrlToken> lst, int begin, int end, String image, boolean orig) { + lst.add(new UrlToken(UrlToken.Type.HOST, begin, null, TERM_STARTHOST)); + addTokens(lst, UrlToken.Type.HOST, begin, image, orig); + lst.add(new UrlToken(UrlToken.Type.HOST, end, null, TERM_ENDHOST)); + } + + private static void registerScheme(String scheme, int port) { + String str = String.valueOf(port); + schemeToPort.put(scheme, str); + portToScheme.put(str, scheme); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/net/package-info.java b/vespajlib/src/main/java/com/yahoo/net/package-info.java new file mode 100644 index 00000000000..ab474304da2 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.net; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/path/Path.java b/vespajlib/src/main/java/com/yahoo/path/Path.java new file mode 100644 index 00000000000..a15ebacc4cf --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/path/Path.java @@ -0,0 +1,211 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.path; + +import com.google.common.annotations.Beta; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +// TODO: Remove and replace usage by java.nio.file.Path + +/** + * Represents a path represented by a list of elements. Immutable + * + * @author lulf + * @since 5.1 + */ +@Beta +public final class Path { + + private final String delimiter; + private final List<String> elements = new ArrayList<>(); + + /** + * Create an empty path. + */ + private Path(String delimiter) { + this(new ArrayList<>(), delimiter); + } + + /** + * Create a new path as a copy of the provided path. + * @param rhs the path to copy. + */ + private Path(Path rhs) { + this(rhs.elements, rhs.delimiter); + } + + /** + * Create path with given elements. + * @param elements a list of path elements + */ + private Path(List<String> elements, String delimiter) { + this.elements.addAll(elements); + this.delimiter = delimiter; + } + + /** Returns whether this path is an immediate child of the given path */ + public boolean isChildOf(Path parent) { + return toString().startsWith(parent.toString()) && this.elements.size() -1 == parent.elements.size(); + } + + /** + * Add path elements by splitting based on delimiter and appending to elements. + */ + private void addElementsFromString(String path) { + String[] pathElements = path.split(delimiter); + if (pathElements != null) { + for (String elem : pathElements) { + if (!"".equals(elem)) { + elements.add(elem); + } + } + } + } + + /** + * Append an element to the path. Returns a new path with this element appended. + * @param name name of element to append. + * @return this, for chaining + */ + public Path append(String name) { + Path path = new Path(this); + path.addElementsFromString(name); + return path; + } + + /** + * Appends a path to another path, thereby creating a new path with the provided path + * appended to this. + * @param path The path to append. + * @return a new path with argument appended to it. + */ + public Path append(Path path) { + Path newPath = new Path(this); + newPath.elements.addAll(path.elements); + return newPath; + } + + /** + * Get the name of this path element, typically the last element in the path string. + * @return the name + */ + public String getName() { + if (elements.isEmpty()) { + return ""; + } + return elements.get(elements.size() - 1); + } + + /** + * Get a string representation of the path represented by this. + * @return a path string. + */ + public String getRelative() { + if (elements.isEmpty()) { + return ""; + } + StringBuilder sb = new StringBuilder(); + sb.append(elements.get(0)); + for (int i = 1; i < elements.size(); i++) { + sb.append(delimiter); + sb.append(elements.get(i)); + } + return sb.toString(); + } + + /** + * Get the parent path (all elements except last). + * @return the parent path. + */ + public Path getParentPath() { + ArrayList<String> parentElements = new ArrayList<>(); + if (elements.size() > 1) { + for (int i = 0; i < elements.size() - 1; i++) { + parentElements.add(elements.get(i)); + } + } + return new Path(parentElements, delimiter); + } + + /** + * Get string representation of path represented from the root node. + * @return string representation of path + */ + public String getAbsolute() { + return delimiter + getRelative(); + } + + public boolean isRoot() { + return elements.isEmpty(); + } + + public Iterator<String> iterator() { return elements.iterator(); } + + /** + * Convert to string. + * + * @return string representation of relative path + */ + @Override + public String toString() { + // TODO: This and the relative/absolute thing is wrong. The Path either *is* relative or absolute + // and should return accordingly here. getAbsolute/relative should be replaced by an asRelative/absolute + // returning another Path + return getRelative(); + } + + /** + * Create a path from a string. The string is treated as a relative path, and all redundant '/'-characters are + * stripped. + * @param path the relative path that this path should represent. + * @return a path object that may be used with the application package. + */ + public static Path fromString(String path) { + return fromString(path, "/"); + } + + /** + * Create a path from a string. The string is treated as a relative path, and all redundant delimiter-characters are + * stripped. + * @param path the relative path that this path should represent. + * @return a path object that may be used with the application package. + */ + public static Path fromString(String path, String delimiter) { + Path pathObj = new Path(delimiter); + pathObj.addElementsFromString(path); + return pathObj; + } + + /** + * Create an empty root path with '/' delimiter. + * + * @return an empty root path that can be appended + */ + public static Path createRoot() { + return createRoot("/"); + } + + /** + * Create an empty root path with delimiter. + * + * @return an empty root path that can be appended + */ + public static Path createRoot(String delimiter) { + return new Path(delimiter); + } + + @Override + public int hashCode() { + return elements.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other instanceof Path) { + return getRelative().equals(((Path) other).getRelative()); + } + return false; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/path/package-info.java b/vespajlib/src/main/java/com/yahoo/path/package-info.java new file mode 100644 index 00000000000..675e6b64ec2 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/path/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@PublicApi // Mainly because it's imported by config-model-fat +@ExportPackage +package com.yahoo.path; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/protect/ClassValidator.java b/vespajlib/src/main/java/com/yahoo/protect/ClassValidator.java new file mode 100644 index 00000000000..79e9d49c9f5 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/protect/ClassValidator.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.protect; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.List; + +/** + * Static utility methods to validate class properties. + * + * <p> + * Do note, this class will not be a reliable guarantee for correctness if you + * have a forest of methods only differing by return type (as + * contradistinguished from name and argument types), the current implementation + * is minimal. + * </p> + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class ClassValidator { + + /** + * Check all protected, public and package private declared methods of + * maskedClass is implemented in testClass. Note, this will by definition + * blow up on final methods in maskedClass. + * + * @param testClass + * class which wraps or masks another class + * @param maskedClass + * class which is masked or wrapped + * @return the methods which seem to miss from testClass to be complete + */ + public static List<Method> unmaskedMethods(Class<?> testClass, + Class<?> maskedClass) { + List<Method> unmasked = new ArrayList<>(); + Method[] methodsToMask = maskedClass.getDeclaredMethods(); + for (Method m : methodsToMask) { + int modifiers = m.getModifiers(); + if (Modifier.isPrivate(modifiers)) { + continue; + } + try { + testClass.getDeclaredMethod(m.getName(), m.getParameterTypes()); + } catch (NoSuchMethodException e) { + unmasked.add(m); + } + } + return unmasked; + } + + /** + * Check testClass overrides all protected, public and package private + * methods of its immediate super class. See unmaskedMethods(). + * + * @param testClass + * the class to check whether completely masks its super class + * @return the methods missing from testClass to completely override its + * immediate super class + */ + public static List<Method> unmaskedMethodsFromSuperclass(Class<?> testClass) { + return unmaskedMethods(testClass, testClass.getSuperclass()); + } + +}
\ No newline at end of file diff --git a/vespajlib/src/main/java/com/yahoo/protect/ErrorMessage.java b/vespajlib/src/main/java/com/yahoo/protect/ErrorMessage.java new file mode 100644 index 00000000000..c0cdc6017a0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/protect/ErrorMessage.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.protect; + + +/** + * An error message with a code. + * This class should be treated as immutable. + * + * @author bratseth + */ +public class ErrorMessage { + + /** An error code */ + protected int code; + + /** The short message of this error, always set */ + protected String message; + + /** The detailed instance message of this error, not always set */ + protected String detailedMessage = null; + + /** The cause of this error, or null if none is recorded */ + protected Throwable cause = null; + + /** + * Create an invalid instance for a subclass to initialize. + */ + public ErrorMessage() { + } + + public ErrorMessage(int code, String message) { + this.code = code; + this.message = message; + } + + /** + * Create an application specific error message with an application + * specific code + */ + public ErrorMessage(int code, String message, String detailedMessage) { + this(code, message); + this.detailedMessage = detailedMessage; + } + + /** Create an application specific error message with an application specific code */ + public ErrorMessage(int code, String message, String detailedMessage, Throwable cause) { + this(code, message, detailedMessage); + this.cause = cause; + } + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + /** Returns the detailed message, or null if there is no detailed message */ + public String getDetailedMessage() { + return detailedMessage; + } + /** + * Sets the cause of this. This should be set on errors which likely have their origin in plugin component code, + * not on others. + */ + public void setCause(Throwable cause) { this.cause=cause; } + + /** Returns the cause of this, or null if none is set */ + public Throwable getCause() { return cause; } + + public int hashCode() { + return code * 7 + message.hashCode() + (detailedMessage == null ? 0 : 17 * detailedMessage.hashCode()); + } + + /** + * Two error messages are equal if they have the same code and message. + * The cause is ignored in the comparison. + */ + @Override + public boolean equals(Object o) { + if (!(o instanceof ErrorMessage)) return false; + + ErrorMessage other = (ErrorMessage) o; + + if (this.code != other.code) return false; + + if (!this.message.equals(other.message)) return false; + + if (this.detailedMessage==null) return other.detailedMessage==null; + if (other.detailedMessage==null) return false; + + return this.detailedMessage.equals(other.detailedMessage); + } + + @Override + public String toString() { + String details = ""; + + if (detailedMessage != null) { + details = detailedMessage; + } + if (cause !=null) { + if (details.length()>0) + details+=": "; + details+= com.yahoo.yolean.Exceptions.toMessageString(cause); + } + if (details.length()>0) + details=" (" + details + ")"; + + return "error : " + message + details; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/protect/Process.java b/vespajlib/src/main/java/com/yahoo/protect/Process.java new file mode 100644 index 00000000000..6f381b40cd7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/protect/Process.java @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.protect; + + +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + + +/** + * A class for interacting with the global state of the running VM. + * + * @author Steinar Knutsen + */ +public final class Process { + + private static final Logger log = Logger.getLogger(Process.class.getName()); + + /** Die with a message, without dumping thread state */ + public static void logAndDie(String message) { + logAndDie(message, null); + } + + /** Die with a message, optionally dumping thread state */ + public static void logAndDie(String message, boolean dumpThreads) { + logAndDie(message, null, dumpThreads); + } + + /** Die with a message containing an exception, without dumping thread state */ + public static void logAndDie(String message, Throwable thrown) { + logAndDie(message, thrown, false); + } + + /** + * Log message as severe error, then forcibly exit runtime, without running + * exit handlers or otherwise waiting for cleanup. + * + * @param message message to log before exit + * @param thrown the throwable that caused the application to exit. + * @param dumpThreads if true the stack trace of all threads is dumped to the + * log with level info before shutting down + */ + public static void logAndDie(String message, Throwable thrown, boolean dumpThreads) { + try { + if (dumpThreads) + dumpThreads(); + if (thrown != null) + log.log(Level.SEVERE, message, thrown); + else + log.log(Level.SEVERE, message); + } finally { + try { + Runtime.getRuntime().halt(1); + } + catch (Throwable t) { + log.log(Level.SEVERE, "Runtime.halt rejected. Throwing an error."); + throw new ShutdownError("Shutdown requested, but failed to shut down"); + } + } + } + + + private static void dumpThreads() { + try { + log.log(Level.INFO, "About to shut down. Commencing full thread dump for diagnosis."); + Map<Thread, StackTraceElement[]> allStackTraces = Thread.getAllStackTraces(); + for (Map.Entry<Thread, StackTraceElement[]> e : allStackTraces.entrySet()) { + Thread t = e.getKey(); + StackTraceElement[] stack = e.getValue(); + StringBuilder forOneThread = new StringBuilder(); + int initLen; + forOneThread.append("Stack for thread: ").append(t.getName()).append(": "); + initLen = forOneThread.length(); + for (StackTraceElement s : stack) { + if (forOneThread.length() > initLen) { + forOneThread.append(" "); + } + forOneThread.append(s.toString()); + } + log.log(Level.INFO, forOneThread.toString()); + } + log.log(Level.INFO, "End of diagnostic thread dump."); + } catch (Exception e) { + // just give up... + } + } + + @SuppressWarnings("serial") + public static class ShutdownError extends Error { + + public ShutdownError(String message) { + super(message); + } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/protect/Validator.java b/vespajlib/src/main/java/com/yahoo/protect/Validator.java new file mode 100644 index 00000000000..9572fcb2ae4 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/protect/Validator.java @@ -0,0 +1,133 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.protect; + + +/** + * <p>Static utility methods for validating input.</p> + * + * @author bratseth + */ +public abstract class Validator { + + /** Throws NullPointerException if the argument is null */ + public static void ensureNotNull(String argumentDescription, Object argument) { + if (argument == null) + throw new NullPointerException(argumentDescription + " can not be null"); + } + + /** + * Throws an IllegalStateException if the given field value + * is initialized (not null) + */ + public static void ensureNotInitialized(String fieldDescription, Object fieldOwner, Object fieldValue) { + if (fieldValue != null) { + throw new IllegalStateException( + fieldDescription + " of " + fieldOwner + + " cannot be changed, it is already set " + "to " + + fieldValue); + } + } + + /** + * Throws an IllegalArgumentException if the given argument is not + * in the given range + * + * @param argumentDescription a description of the argument + * @param from the range start, inclusive + * @param to the range end, inclusive + * @param argument the argument value to check + */ + public static void ensureInRange(String argumentDescription, int from, int to, int argument) { + if (argument < from || argument > to) { + throw new IllegalArgumentException( + argumentDescription + " is " + argument + + " but must be between " + from + " and " + to); + } + } + + /** + * Throws an IllegalArgumentException if the first argument is not strictly + * smaller than the second argument + * + * @param smallDescription description of the smallest argument + * @param small the smallest argument + * @param largeDescription description of the lergest argument + * @param large the largest argument + */ + public static void ensureSmaller(String smallDescription, int small, String largeDescription, int large) { + if (small >= large) { + throw new IllegalArgumentException( + smallDescription + " is " + small + " but should be " + + "less than " + largeDescription + " " + large); + } + } + + /** + * Throws an IllegalArgumentException if the first argument is not strictly + * smaller than the second argument + * + * @param smallDescription + * description of the smallest argument + * @param small + * the smallest argument + * @param largeDescription + * description of the largest argument + * @param large + * the largest argument + */ + @SuppressWarnings({ "rawtypes", "unchecked" }) + public static void ensureSmaller(String smallDescription, Comparable small, String largeDescription, Comparable large) { + if (small.compareTo(large) >= 0) { + throw new IllegalArgumentException(smallDescription + " is " + + small + " but should be " + "less than " + + largeDescription + " " + large); + } + } + + /** + * Ensures that the given argument is true + * + * @param description of what is the case if the condition is false + * @param condition the condition to ensure is true + * @throws IllegalArgumentException if the given condition was false + */ + public static void ensure(String description, boolean condition) { + if (!condition) { + throw new IllegalArgumentException(description); + } + } + + /** + * Ensure the given argument is true, if not throw IllegalArgumentException + * concatenating the String representation of the description arguments. + */ + public static void ensure(boolean condition, Object... description) { + if (!condition) { + StringBuilder msg = new StringBuilder(); + for (Object part : description) { + msg.append(part.toString()); + } + throw new IllegalArgumentException(msg.toString()); + } + } + + /** + * Ensures that an item is of a particular class + * + * @param description + * a description of the item to be checked + * @param item + * the item to check the type of + * @param type + * the type the given item should be instanceof + * @throws IllegalArgumentException + * if the given item is not of the correct type + */ + public static void ensureInstanceOf(String description, Object item, Class<?> type) { + if (!type.isAssignableFrom(item.getClass())) { + throw new IllegalArgumentException(description + ", " + item + + " should " + "have been an instance of " + type); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/protect/package-info.java b/vespajlib/src/main/java/com/yahoo/protect/package-info.java new file mode 100644 index 00000000000..9260cd2e1e0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/protect/package-info.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Input validators, integrity checkers, error messages + * and similar classes. + */ +@ExportPackage +@PublicApi +package com.yahoo.protect; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/reflection/Casting.java b/vespajlib/src/main/java/com/yahoo/reflection/Casting.java new file mode 100644 index 00000000000..1f5c00bee59 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/reflection/Casting.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.reflection; + +import java.util.Optional; + +/** + * Utility methods for doing casting + * @author tonytv + */ +public class Casting { + /** + * Returns the casted instance if it is assignment-compatible with targetClass, + * or empty otherwise. + * @see Class#isInstance(Object) + */ + public static <T> Optional<T> cast(Class<T> targetClass, Object instance) { + return targetClass.isInstance(instance)? + Optional.of(targetClass.cast(instance)): + Optional.empty(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/reflection/package-info.java b/vespajlib/src/main/java/com/yahoo/reflection/package-info.java new file mode 100644 index 00000000000..43eda3cec94 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/reflection/package-info.java @@ -0,0 +1,9 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Package for reflection utility methods. + * @author tonytv + */ +@ExportPackage +package com.yahoo.reflection; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/rmi/.gitignore b/vespajlib/src/main/java/com/yahoo/rmi/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/rmi/.gitignore diff --git a/vespajlib/src/main/java/com/yahoo/slime/ArrayInserter.java b/vespajlib/src/main/java/com/yahoo/slime/ArrayInserter.java new file mode 100644 index 00000000000..ab1ae28d885 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/ArrayInserter.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Helper class for inserting values into an ArrayValue. + * For justification read Inserter documentation. + **/ +final class ArrayInserter implements Inserter { + private Cursor target; + public final ArrayInserter adjust(Cursor c) { + target = c; + return this; + } + public final Cursor insertNIX() { return target.addNix(); } + public final Cursor insertBOOL(boolean value) { return target.addBool(value); } + public final Cursor insertLONG(long value) { return target.addLong(value); } + public final Cursor insertDOUBLE(double value) { return target.addDouble(value); } + public final Cursor insertSTRING(String value) { return target.addString(value); } + public final Cursor insertSTRING(byte[] utf8) { return target.addString(utf8); } + public final Cursor insertDATA(byte[] value) { return target.addData(value); } + public final Cursor insertARRAY() { return target.addArray(); } + public final Cursor insertOBJECT() { return target.addObject(); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/ArrayTraverser.java b/vespajlib/src/main/java/com/yahoo/slime/ArrayTraverser.java new file mode 100644 index 00000000000..4cd24f15028 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/ArrayTraverser.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Callback interface for traversing arrays. + * Implement this and call Inspector.traverse() + * and you will get one callback for each array entry. + **/ +public interface ArrayTraverser +{ + /** + * Callback function to implement. + * @param idx array index for the current array entry. + * @param inspector accessor for the current array entry's value. + **/ + public void entry(int idx, Inspector inspector); +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/ArrayValue.java b/vespajlib/src/main/java/com/yahoo/slime/ArrayValue.java new file mode 100644 index 00000000000..4e520cde3c4 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/ArrayValue.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + + +final class ArrayValue extends Value { + + private int capacity = 16; + private int used = 0; + private Value[] values = new Value[capacity]; + private final SymbolTable names; + + public ArrayValue(SymbolTable names) { this.names = names; } + public final Type type() { return Type.ARRAY; } + public final int children() { return used; } + public final int entries() { return used; } + public final Value entry(int index) { + return (index < used) ? values[index] : NixValue.invalid(); + } + + public final void accept(Visitor v) { v.visitArray(this); } + + public final void traverse(ArrayTraverser at) { + for (int i = 0; i < used; i++) { + at.entry(i, values[i]); + } + } + + private void grow() { + Value[] v = values; + capacity = (capacity << 1); + values = new Value[capacity]; + System.arraycopy(v, 0, values, 0, used); + } + + protected final Value addLeaf(Value value) { + if (used == capacity) { + grow(); + } + values[used++] = value; + return value; + } + + public final Value addArray() { return addLeaf(new ArrayValue(names)); } + public final Value addObject() { return addLeaf(new ObjectValue(names)); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java new file mode 100644 index 00000000000..70e6892ce9f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import static com.yahoo.slime.BinaryFormat.*; + +final class BinaryDecoder { + BufferedInput in; + + private final SlimeInserter slimeInserter = new SlimeInserter(); + private final ArrayInserter arrayInserter = new ArrayInserter(); + private final ObjectInserter objectInserter = new ObjectInserter(); + + public BinaryDecoder() {} + + public Slime decode(byte[] bytes) { + return decode(bytes, 0, bytes.length); + } + public Slime decode(byte[] bytes, int offset, int length) { + Slime slime = new Slime(); + in = new BufferedInput(bytes, offset, length); + decodeSymbolTable(slime); + decodeValue(slimeInserter.adjust(slime)); + if (in.failed()) { + slime.wrap("partial_result"); + slime.get().setData("offending_input", in.getOffending()); + slime.get().setString("error_message", in.getErrorMessage()); + } + return slime; + } + + long read_cmpr_long() { + long next = in.getByte(); + long value = (next & 0x7f); + int shift = 7; + while ((next & 0x80) != 0) { + next = in.getByte(); + value |= ((next & 0x7f) << shift); + shift += 7; + } + return value; + } + + long read_size(int meta) { + return (meta == 0) ? read_cmpr_long() : (meta - 1); + } + + long read_bytes_le(int bytes) { + long value = 0; + int shift = 0; + for (int i = 0; i < bytes; ++i) { + long b = in.getByte(); + value |= (b & 0xff) << shift; + shift += 8; + } + return value; + } + + long read_bytes_be(int bytes) { + long value = 0; + int shift = 56; + for (int i = 0; i < bytes; ++i) { + long b = in.getByte(); + value |= (b & 0xff) << shift; + shift -= 8; + } + return value; + } + + Cursor decodeNIX(Inserter inserter) { + return inserter.insertNIX(); + } + + Cursor decodeBOOL(Inserter inserter, int meta) { + return inserter.insertBOOL(meta != 0); + } + + Cursor decodeLONG(Inserter inserter, int meta) { + long encoded = read_bytes_le(meta); + return inserter.insertLONG(decode_zigzag(encoded)); + } + + Cursor decodeDOUBLE(Inserter inserter, int meta) { + long encoded = read_bytes_be(meta); + return inserter.insertDOUBLE(decode_double(encoded)); + } + + Cursor decodeSTRING(Inserter inserter, int meta) { + long size = read_size(meta); + int sz = (int)size; // XXX + byte[] image = in.getBytes(sz); + return inserter.insertSTRING(image); + } + + Cursor decodeDATA(Inserter inserter, int meta) { + long size = read_size(meta); + int sz = (int)size; // XXX + byte[] image = in.getBytes(sz); + return inserter.insertDATA(image); + } + + Cursor decodeARRAY(Inserter inserter, int meta) { + Cursor cursor = inserter.insertARRAY(); + long size = read_size(meta); + for (int i = 0; i < size; ++i) { + decodeValue(arrayInserter.adjust(cursor)); + } + return cursor; + } + + Cursor decodeOBJECT(Inserter inserter, int meta) { + Cursor cursor = inserter.insertOBJECT(); + long size = read_size(meta); + for (int i = 0; i < size; ++i) { + long l = read_cmpr_long(); + int symbol = (int)l; // check for overflow? + decodeValue(objectInserter.adjust(cursor, symbol)); + } + return cursor; + } + + Cursor decodeValue(Inserter inserter, Type type, int meta) { + switch (type) { + case NIX: return decodeNIX(inserter); + case BOOL: return decodeBOOL(inserter, meta); + case LONG: return decodeLONG(inserter, meta); + case DOUBLE: return decodeDOUBLE(inserter, meta); + case STRING: return decodeSTRING(inserter, meta); + case DATA: return decodeDATA(inserter, meta); + case ARRAY: return decodeARRAY(inserter, meta); + case OBJECT: return decodeOBJECT(inserter, meta); + } + assert false : "should not be reached"; + return null; + } + + void decodeValue(Inserter inserter) { + byte b = in.getByte(); + Cursor cursor = decodeValue(inserter, + decode_type(b), + decode_meta(b)); + if (!cursor.valid()) { + in.fail("failed to decode value"); + } + } + + void decodeSymbolTable(Slime slime) { + long numSymbols = read_cmpr_long(); + final byte [] backing = in.getBacking(); + for (int i = 0; i < numSymbols; ++i) { + long size = read_cmpr_long(); + int sz = (int)size; // XXX + int offset = in.getPosition(); + in.skip(sz); + int symbol = slime.insert(Utf8Codec.decode(backing, offset, sz)); + if (symbol != i) { + in.fail("duplicate symbols in symbol table"); + return; + } + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java new file mode 100644 index 00000000000..daa926ec45b --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java @@ -0,0 +1,144 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import static com.yahoo.slime.BinaryFormat.*; + +final class BinaryEncoder implements +ArrayTraverser, ObjectSymbolTraverser +{ + BufferedOutput out; + + public BinaryEncoder(int capacity) { + out = new BufferedOutput(capacity); + } + + public BinaryEncoder() { + out = new BufferedOutput(); + } + + public byte[] encode(Slime slime) { + out.reset(); + encodeSymbolTable(slime); + encodeValue(slime.get()); + return out.toArray(); + } + + void encode_cmpr_long(long value) { + byte next = (byte)(value & 0x7f); + value >>>= 7; // unsigned shift + while (value != 0) { + next |= 0x80; + out.put(next); + next = (byte)(value & 0x7f); + value >>>= 7; + } + out.put(next); + } + + void write_type_and_size(int type, long size) { + if (size <= 30) { + out.put(encode_type_and_meta(type, (int)(size + 1))); + } else { + out.put(encode_type_and_meta(type, 0)); + encode_cmpr_long(size); + } + } + + void write_type_and_bytes_le(int type, long bits) { + int pos = out.position(); + byte val = 0; + out.put(val); + while (bits != 0) { + val = (byte)(bits & 0xff); + bits >>>= 8; + out.put(val); + } + val = encode_type_and_meta(type, out.position() - pos - 1); + out.absolutePut(pos, val); + } + + void write_type_and_bytes_be(int type, long bits) { + int pos = out.position(); + byte val = 0; + out.put(val); + while (bits != 0) { + val = (byte)(bits >> 56); + bits <<= 8; + out.put(val); + } + val = encode_type_and_meta(type, out.position() - pos - 1); + out.absolutePut(pos, val); + } + + void encodeNIX() { + out.put(Type.NIX.ID); + } + + void encodeBOOL(boolean value) { + out.put(encode_type_and_meta(Type.BOOL.ID, value ? 1 : 0)); + } + + void encodeLONG(long value) { + write_type_and_bytes_le(Type.LONG.ID, encode_zigzag(value)); + } + + void encodeDOUBLE(double value) { + write_type_and_bytes_be(Type.DOUBLE.ID, encode_double(value)); + } + + void encodeSTRING(byte[] value) { + write_type_and_size(Type.STRING.ID, value.length); + out.put(value); + } + + void encodeDATA(byte[] value) { + write_type_and_size(Type.DATA.ID, value.length); + out.put(value); + } + + void encodeARRAY(Inspector inspector) { + write_type_and_size(Type.ARRAY.ID, inspector.children()); + ArrayTraverser at = this; + inspector.traverse(at); + } + + void encodeOBJECT(Inspector inspector) { + write_type_and_size(Type.OBJECT.ID, inspector.children()); + ObjectSymbolTraverser ot = this; + inspector.traverse(ot); + } + + void encodeValue(Inspector inspector) { + switch(inspector.type()) { + case NIX: encodeNIX(); return; + case BOOL: encodeBOOL(inspector.asBool()); return; + case LONG: encodeLONG(inspector.asLong()); return; + case DOUBLE: encodeDOUBLE(inspector.asDouble()); return; + case STRING: encodeSTRING(inspector.asUtf8()); return; + case DATA: encodeDATA(inspector.asData()); return; + case ARRAY: encodeARRAY(inspector); return; + case OBJECT: encodeOBJECT(inspector); return; + } + assert false : "Should not be reached"; + } + + void encodeSymbolTable(Slime slime) { + int numSymbols = slime.symbols(); + encode_cmpr_long(numSymbols); + for (int i = 0 ; i < numSymbols; ++i) { + String name = slime.inspect(i); + byte[] bytes = Utf8Codec.encode(name); + encode_cmpr_long(bytes.length); + out.put(bytes); + } + } + + public void entry(int idx, Inspector inspector) { + encodeValue(inspector); + } + + public void field(int symbol, Inspector inspector) { + encode_cmpr_long(symbol); + encodeValue(inspector); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryFormat.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryFormat.java new file mode 100644 index 00000000000..4a126932f1e --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryFormat.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Class for serializing Slime data into binary format, or deserializing + * the binary format into a Slime object. + **/ +public class BinaryFormat { + static long encode_zigzag(long x) { + return ((x << 1) ^ (x >> 63)); // note ASR + } + + static long decode_zigzag(long x) { + return ((x >>> 1) ^ (-(x & 0x1))); // note LSR + } + + static long encode_double(double x) { + return Double.doubleToRawLongBits(x); + } + + static double decode_double(long x) { + return Double.longBitsToDouble(x); + } + + static byte encode_type_and_meta(int type, int meta) { + return (byte) ((meta << 3) | (type & 0x7)); + } + + static Type decode_type(byte type_and_meta) { + return Type.asType(type_and_meta & 0x7); + } + + static int decode_meta(byte type_and_meta) { + return ((type_and_meta & 0xff) >>> 3); + } + + /** + * Take a Slime object and serialize it into binary format. + * @param slime the object which is to be serialized. + * @return a new byte array with just the encoded slime. + **/ + public static byte[] encode(Slime slime) { + BinaryEncoder encoder = new BinaryEncoder(); + return encoder.encode(slime); + } + + /** + * Take binary data and deserialize it into a Slime object. + * The data is assumed to be the binary representation + * as if obtained by a call to the @ref encode() method. + * + * If the binary data can't be deserialized without problems + * the returned Slime object will instead only contain the + * three fields "partial_result" (contains anything successfully + * decoded before encountering problems), "offending_input" + * (containing any data that could not be deserialized) and + * "error_message" (a string describing the problem encountered). + * + * @param data the data to be deserialized. + * @return a new Slime object constructed from the data. + **/ + public static Slime decode(byte[] data) { + BinaryDecoder decoder = new BinaryDecoder(); + return decoder.decode(data); + } + + /** + * Take binary data and deserialize it into a Slime object. + * The data is assumed to be the binary representation + * as if obtained by a call to the @ref encode() method. + * + * If the binary data can't be deserialized without problems + * the returned Slime object will instead only contain the + * three fields "partial_result" (contains anything successfully + * decoded before encountering problems), "offending_input" + * (containing any data that could not be deserialized) and + * "error_message" (a string describing the problem encountered). + * + * @param data array containing the data to be deserialized. + * @param offset where in the array to start deserializing. + * @param length how many bytes the deserializer is allowed to consume. + * @return a new Slime object constructed from the data. + **/ + public static Slime decode(byte[] data, int offset, int length) { + BinaryDecoder decoder = new BinaryDecoder(); + return decoder.decode(data, offset, length); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/BoolValue.java b/vespajlib/src/main/java/com/yahoo/slime/BoolValue.java new file mode 100644 index 00000000000..95b22d47d4f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/BoolValue.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +final class BoolValue extends Value { + private static final BoolValue trueValue = new BoolValue(true); + private static final BoolValue falseValue = new BoolValue(false); + private final boolean value; + private BoolValue(boolean value) { this.value = value; } + final public Type type() { return Type.BOOL; } + final public boolean asBool() { return this.value; } + public final void accept(Visitor v) { v.visitBool(value); } + public static BoolValue instance(boolean bit) { return (bit ? trueValue : falseValue); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/BufferedInput.java b/vespajlib/src/main/java/com/yahoo/slime/BufferedInput.java new file mode 100644 index 00000000000..8eecc0d50f2 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/BufferedInput.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +final class BufferedInput { + + private final byte[] source; + private final int end; + private final int start; + private int position; + private String failReason; + private int failPos; + + void fail(String reason) { + if (failed()) { + return; + } + failReason = reason; + failPos = position; + position = end; + } + + public BufferedInput(byte[] bytes) { + this(bytes, 0, bytes.length); + } + + public BufferedInput(byte[] bytes, int offset, int length) { + this.source = bytes; + this.start = offset; + position = offset; + this.end = offset + length; + } + public final byte getByte() { + if (position == end) { + fail("underflow"); + return 0; + } + return source[position++]; + } + + public boolean failed() { + return failReason != null; + } + + public boolean eof() { + return this.position == this.end; + } + + public String getErrorMessage() { + return failReason; + } + + public int getConsumedSize() { + return failed() ? 0 : position - start; + } + + public byte[] getOffending() { + byte[] ret = new byte[failPos-start]; + System.arraycopy(source, start, ret, 0, failPos-start); + return ret; + } + + public final byte [] getBacking() { return source; } + public final int getPosition() { return position; } + public final void skip(int size) { + if (position + size > end) { + fail("underflow"); + } else { + position += size; + } + } + + public final byte[] getBytes(int size) { + if (position + size > end) { + fail("underflow"); + return new byte[0]; + } + byte[] ret = new byte[size]; + for (int i = 0; i < size; i++) { + ret[i] = source[position++]; + } + return ret; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java b/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java new file mode 100644 index 00000000000..ad7d2191130 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +final class BufferedOutput { + + private byte[] buf; + private int capacity; + private int pos; + + public BufferedOutput(int cap) { + capacity = (cap < 64) ? 64 : cap; + buf = new byte[capacity]; + } + + public BufferedOutput() { + this(4096); + } + + public void reset() { + pos = 0; + } + + private void reserve(int bytes) { + if (pos + bytes > capacity) { + while (pos + bytes > capacity) { + capacity = capacity * 2; + } + byte[] tmp = new byte[capacity]; + System.arraycopy(buf, 0, tmp, 0, pos); + buf = tmp; + } + } + + public int position() { return pos; } + + final void put(byte b) { + reserve(1); + buf[pos++] = b; + } + + final void absolutePut(int position, byte b) { + buf[position] = b; + } + + final void put(byte[] bytes) { + reserve(bytes.length); + for (byte b : bytes) { + buf[pos++] = b; + } + } + + public byte[] toArray() { + byte[] ret = new byte[pos]; + System.arraycopy(buf, 0, ret, 0, pos); + return ret; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Cursor.java b/vespajlib/src/main/java/com/yahoo/slime/Cursor.java new file mode 100644 index 00000000000..18d225c97be --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Cursor.java @@ -0,0 +1,285 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Interface for read-write access to any value or object that is part + * of a Slime. All accessors (including meta-data) are inherited from + * the Inspector interface. The navigational methods also work the + * same, except that they return a new Cursor for contained values and + * sub-structures, to permit writes to embedded values. + * + * The write operations are adding a new entry (to arrays), or setting + * a field value (for objects). If adding an entry or setting a field + * cannot be performed for any reason, an invalid Cursor is returned. + * + * This could happen because the current cursor is invalid, or it's + * not connected to an array value (for add methods), or it's not + * connected to an object (for set methods). Also note that you can + * only set() a field once; you cannot overwrite the field in any way. + **/ +public interface Cursor extends Inspector { + + /** + * Access an array entry. + * + * If the current Cursor doesn't connect to an array value, + * or the given array index is out of bounds, the returned + * Cursor will be invalid. + * @param idx array index. + * @return a new Cursor for the entry value. + **/ + @Override + public Cursor entry(int idx); + + /** + * Access an field in an object by symbol id. + * + * If the current Cursor doesn't connect to an object value, or + * the object value does not contain a field with the given symbol + * id, the returned Cursor will be invalid. + * @param sym symbol id. + * @return a new Cursor for the field value. + **/ + @Override + public Cursor field(int sym); + + /** + * Access an field in an object by symbol name. + * + * If the current Cursor doesn't connect to an object value, or + * the object value does not contain a field with the given symbol + * name, the returned Cursor will be invalid. + * @param name symbol name. + * @return a new Cursor for the field value. + **/ + @Override + public Cursor field(String name); + + /** + * Append an array entry containing a new value of NIX type. + * Returns an invalid Cursor if unsuccessful. + * @return a valid Cursor referencing the new entry value if successful. + **/ + public Cursor addNix(); + + /** + * Append an array entry containing a new value of BOOL type. + * Returns an invalid Cursor if unsuccessful. + * @param bit the actual boolean value for initializing a new BoolValue. + * @return a valid Cursor referencing the new entry value if successful. + **/ + public Cursor addBool(boolean bit); + + /** add a new entry of LONG type to an array */ + public Cursor addLong(long l); + + /** add a new entry of DOUBLE type to an array */ + public Cursor addDouble(double d); + + /** add a new entry of STRING type to an array */ + public Cursor addString(String str); + + /** add a new entry of STRING type to an array */ + public Cursor addString(byte[] utf8); + + /** add a new entry of DATA type to an array */ + public Cursor addData(byte[] data); + + /** + * Append an array entry containing a new value of ARRAY type. + * Returns a valid Cursor (thay may again be used for adding new + * sub-array entries) referencing the new entry value if + * successful; otherwise returns an invalid Cursor. + * @return new Cursor for the new entry value + **/ + public Cursor addArray(); + + /** + * Append an array entry containing a new value of OBJECT type. + * Returns a valid Cursor (thay may again be used for setting + * sub-fields inside the new object) referencing the new entry + * value if successful; otherwise returns an invalid Cursor. + * @return new Cursor for the new entry value + **/ + public Cursor addObject(); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of NIX type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @return new Cursor for the new field value + **/ + public Cursor setNix(int sym); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of BOOL type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @param bit the actual boolean value for the new field + * @return new Cursor for the new field value + **/ + public Cursor setBool(int sym, boolean bit); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of BOOL type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @param l the actual long value for the new field + * @return new Cursor for the new field value + **/ + public Cursor setLong(int sym, long l); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of BOOL type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @param d the actual double value for the new field + * @return new Cursor for the new field value + **/ + public Cursor setDouble(int sym, double d); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of BOOL type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @param str the actual string for the new field + * @return new Cursor for the new field value + **/ + public Cursor setString(int sym, String str); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of BOOL type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @param utf8 the actual string (encoded as UTF-8 data) for the new field + * @return new Cursor for the new field value + **/ + public Cursor setString(int sym, byte[] utf8); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of BOOL type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @param data the actual data to be put into the new field + * @return new Cursor for the new field value + **/ + public Cursor setData(int sym, byte[] data); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of ARRAY type. Returns a valid Cursor (thay may again be + * used for adding new array entries) referencing the new field + * value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @return new Cursor for the new field value + **/ + public Cursor setArray(int sym); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of OBJECT type. Returns a valid Cursor (thay may again + * be used for setting sub-fields inside the new object) + * referencing the new field value if successful; otherwise + * returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @return new Cursor for the new field value + **/ + public Cursor setObject(int sym); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of NIX type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @return new Cursor for the new field value + **/ + public Cursor setNix(String name); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of BOOL type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @param bit the actual boolean value for the new field + * @return new Cursor for the new field value + **/ + public Cursor setBool(String name, boolean bit); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of LONG type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @param l the actual long value for the new field + * @return new Cursor for the new field value + **/ + public Cursor setLong(String name, long l); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of DOUBLE type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @param d the actual double value for the new field + * @return new Cursor for the new field value + **/ + public Cursor setDouble(String name, double d); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of STRING type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @param str the actual string for the new field + * @return new Cursor for the new field value + **/ + public Cursor setString(String name, String str); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of STRING type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @param utf8 the actual string (encoded as UTF-8 data) for the new field + * @return new Cursor for the new field value + **/ + public Cursor setString(String name, byte[] utf8); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of DATA type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @param data the actual data to be put into the new field + * @return new Cursor for the new field value + **/ + public Cursor setData(String name, byte[] data); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of ARRAY type. Returns a valid Cursor (thay may again be + * used for adding new array entries) referencing the new field + * value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @return new Cursor for the new field value + **/ + public Cursor setArray(String name); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of OBJECT type. Returns a valid Cursor (thay may again + * be used for setting sub-fields inside the new object) + * referencing the new field value if successful; otherwise + * returns an invalid Cursor. + * @param name symbol name for the field to be set + * @return new Cursor for the new field value + **/ + public Cursor setObject(String name); +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/DataValue.java b/vespajlib/src/main/java/com/yahoo/slime/DataValue.java new file mode 100644 index 00000000000..abaeac52245 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/DataValue.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +final class DataValue extends Value { + private final byte[] value; + public DataValue(byte[] value) { this.value = value; } + public final Type type() { return Type.DATA; } + public final byte[] asData() { return this.value; } + public final void accept(Visitor v) { v.visitData(value); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/DoubleValue.java b/vespajlib/src/main/java/com/yahoo/slime/DoubleValue.java new file mode 100644 index 00000000000..75b5acafb9b --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/DoubleValue.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +final class DoubleValue extends Value { + private final double value; + public DoubleValue(double value) { this.value = value; } + public final Type type() { return Type.DOUBLE; } + public final long asLong() { return (long)this.value; } + public final double asDouble() { return this.value; } + public final void accept(Visitor v) { v.visitDouble(value); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Inserter.java b/vespajlib/src/main/java/com/yahoo/slime/Inserter.java new file mode 100644 index 00000000000..8319efeb4f0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Inserter.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Helper interface for inserting values into any of the container + * classes (ArrayValue, ObjectValue, or Slime). May be useful for + * deserializers where you can use it to decouple the actual value + * decoding from the container where the value should be inserted. + **/ +interface Inserter { + Cursor insertNIX(); + Cursor insertBOOL(boolean value); + Cursor insertLONG(long value); + Cursor insertDOUBLE(double value); + Cursor insertSTRING(String value); + Cursor insertSTRING(byte[] utf8); + Cursor insertDATA(byte[] value); + Cursor insertARRAY(); + Cursor insertOBJECT(); +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Inspector.java b/vespajlib/src/main/java/com/yahoo/slime/Inspector.java new file mode 100644 index 00000000000..c4a98d98627 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Inspector.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Interface for read-only access to any value or object that is part + * of a Slime. You can access meta-data such as validity and actual + * type. You can always convert to any basic type by calling the + * various "as" accessor methods; these return a default value if the + * current Inspector is invalid or the type doesn't match your + * accessor type. If you want to do something exceptional instead + * when the types don't match, you must check using type() first. + **/ +public interface Inspector { + + /** check if this inspector is valid */ + public boolean valid(); + + /** return an enum describing value type */ + public Type type(); + + /** + * Check how many entries or fields are contained in the current value. + * Useful for arrays and objects; anything else always returns 0. + * @return number of entries/fields contained. + **/ + public int children(); + + /** + * Check how many entries are contained in the current value. + * Useful for arrays; anything else always returns 0. + * @return number of entries contained. + **/ + public int entries(); + + /** + * Check how many fields are contained in the current value. + * Useful for objects; anything else always returns 0. + * @return number of fields contained. + **/ + public int fields(); + + /** the current value (for booleans); default: false */ + public boolean asBool(); + + /** the current value (for integers); default: 0 */ + public long asLong(); + + /** the current value (for floating-point values); default: 0.0 */ + public double asDouble(); + + /** the current value (for string values); default: empty string */ + public String asString(); + + /** the current value encoded into UTF-8 (for string values); default: empty array */ + public byte[] asUtf8(); + + /** the current value (for data values); default: empty array */ + public byte[] asData(); + + /** + * Use the visitor pattern to resolve the underlying type of this value. + * @param v the visitor + **/ + public void accept(Visitor v); + + /** + * Traverse an array value, performing callbacks for each entry. + * + * If the current Inspector is connected to an array value, + * perform callbacks to the given traverser for each entry + * contained in the array. + * @param at traverser callback object. + **/ + @SuppressWarnings("overloads") + public void traverse(ArrayTraverser at); + + /** + * Traverse an object value, performing callbacks for each field. + * + * If the current Inspector is connected to an object value, + * perform callbacks to the given traverser for each field + * contained in the object. + * @param ot traverser callback object. + **/ + @SuppressWarnings("overloads") + public void traverse(ObjectSymbolTraverser ot); + + /** + * Traverse an object value, performing callbacks for each field. + * + * If the current Inspector is connected to an object value, + * perform callbacks to the given traverser for each field + * contained in the object. + * @param ot traverser callback object. + **/ + @SuppressWarnings("overloads") + public void traverse(ObjectTraverser ot); + + /** + * Access an array entry. + * + * If the current Inspector doesn't connect to an array value, + * or the given array index is out of bounds, the returned + * Inspector will be invalid. + * @param idx array index. + * @return a new Inspector for the entry value. + **/ + public Inspector entry(int idx); + + /** + * Access an field in an object by symbol id. + * + * If the current Inspector doesn't connect to an object value, or + * the object value does not contain a field with the given symbol + * id, the returned Inspector will be invalid. + * @param sym symbol id. + * @return a new Inspector for the field value. + **/ + public Inspector field(int sym); + + /** + * Access an field in an object by symbol name. + * + * If the current Inspector doesn't connect to an object value, or + * the object value does not contain a field with the given symbol + * name, the returned Inspector will be invalid. + * @param name symbol name. + * @return a new Inspector for the field value. + **/ + public Inspector field(String name); +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java b/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java new file mode 100644 index 00000000000..72837dc3354 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java @@ -0,0 +1,305 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import com.yahoo.text.Utf8; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * A port of the C++ json decoder intended to be fast. + * + * @author lulf + * @since 5.1.21 + */ +public class JsonDecoder { + private BufferedInput in; + private byte c; + + private final SlimeInserter slimeInserter = new SlimeInserter(); + private final ArrayInserter arrayInserter = new ArrayInserter(); + private final JsonObjectInserter objectInserter = new JsonObjectInserter(); + private final ByteArrayOutputStream buf = new ByteArrayOutputStream(); + + private static final byte[] TRUE = {'t', 'r', 'u', 'e'}; + private static final byte[] FALSE = {'f', 'a', 'l', 's', 'e'}; + private static final byte[] NULL = {'n', 'u', 'l', 'l'}; + private static final byte [] SQUARE_BRACKET_OPEN = { '[' }; + private static final byte [] SQUARE_BRACKET_CLOSE = { ']' }; + private static final byte [] CURLY_BRACE_OPEN = { '{' }; + private static final byte [] CURLY_BRACE_CLOSE = { '}' }; + private static final byte [] COLON = { ':' }; + private static final byte COMMA = ','; + + public JsonDecoder() {} + + public Slime decode(Slime slime, byte[] bytes) { + in = new BufferedInput(bytes); + next(); + decodeValue(slimeInserter.adjust(slime)); + if (in.failed()) { + slime.wrap("partial_result"); + slime.get().setData("offending_input", in.getOffending()); + slime.get().setString("error_message", in.getErrorMessage()); + } + return slime; + } + + private void decodeValue(Inserter inserter) { + skipWhiteSpace(); + switch (c) { + case '"': case '\'': decodeString(inserter); return; + case '{': decodeObject(inserter); return; + case '[': decodeArray(inserter); return; + case 't': expect(TRUE); inserter.insertBOOL(true); return; + case 'f': expect(FALSE); inserter.insertBOOL(false); return; + case 'n': expect(NULL); inserter.insertNIX(); return; + case '-': case '0': case '1': case '2': case '3': case '4': case '5': + case '6': case '7': case '8': case '9': decodeNumber(inserter); return; + } + in.fail("invalid initial character for value"); + } + + @SuppressWarnings("fallthrough") + private void decodeNumber(Inserter inserter) { + buf.reset(); + boolean likelyFloatingPoint=false; + for (;;) { + switch (c) { + case '.': case 'e': case 'E': + likelyFloatingPoint = true; + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + case '+': case '-': + buf.write(c); + next(); + break; + default: + if (likelyFloatingPoint) { + double num = Double.parseDouble(Utf8.toString(buf.toByteArray())); + inserter.insertDOUBLE(num); + } else { + long num = Long.parseLong(Utf8.toString(buf.toByteArray())); + inserter.insertLONG(num); + } + return; + } + } + } + + private void expect(byte [] expected) { + int i; + for (i = 0; i < expected.length && skip(expected[i]); i++) + ; + if (i != expected.length) { + in.fail("unexpected character"); + } + + } + + private void decodeArray(Inserter inserter) { + Cursor cursor = inserter.insertARRAY(); + expect(SQUARE_BRACKET_OPEN); + skipWhiteSpace(); + if (c != ']') { + do { + arrayInserter.adjust(cursor); + decodeValue(arrayInserter); + skipWhiteSpace(); + } while (skip(COMMA)); + } + expect(SQUARE_BRACKET_CLOSE); + } + + private void decodeObject(Inserter inserter) { + Cursor cursor = inserter.insertOBJECT(); + expect(CURLY_BRACE_OPEN); + skipWhiteSpace(); + if (c != '}') { + do { + skipWhiteSpace(); + String key = readKey(); + skipWhiteSpace(); + expect(COLON); + objectInserter.adjust(cursor, key); + decodeValue(objectInserter); + skipWhiteSpace(); + } while (skip(COMMA)); + } + expect(CURLY_BRACE_CLOSE); + } + + private String readKey() { + buf.reset(); + switch (c) { + case '"': case '\'': return readString(); + default: + for (;;) { + switch (c) { + case ':': case ' ': case '\t': case '\n': case '\r': case '\0': return Utf8.toString(buf.toByteArray()); + default: + buf.write(c); + next(); + break; + } + } + } + } + + private void decodeString(Inserter inserter) { + String value = readString(); + inserter.insertSTRING(value); + } + + private String readString() { + buf.reset(); + byte quote = c; + assert(quote == '"' || quote == '\''); + next(); + for (;;) { + switch (c) { + case '\\': + next(); + switch (c) { + case '"': case '\\': case '/': case '\'': + buf.write(c); + break; + case 'b': buf.write((byte) '\b'); break; + case 'f': buf.write((byte) '\f'); break; + case 'n': buf.write((byte) '\n'); break; + case 'r': buf.write((byte) '\r'); break; + case 't': buf.write((byte) '\t'); break; + case 'u': writeUtf8(dequoteUtf16(), buf, 0xffffff80); continue; + default: + in.fail("invalid quoted char(" + c + ")"); + break; + } + next(); + break; + case '"': case '\'': + if (c == quote) { + next(); + return Utf8.toString(buf.toByteArray()); + } else { + buf.write(c); + next(); + } + break; + case '\0': + in.fail("unterminated string"); + return Utf8.toString(buf.toByteArray()); + default: + buf.write(c); + next(); + break; + } + } + } + + private static void writeUtf8(long codepoint, ByteArrayOutputStream buf, long mask) { + if ((codepoint & mask) == 0) { + buf.write((byte) ((mask << 1) | codepoint)); + } else { + writeUtf8(codepoint >> 6, buf, mask >> (2 - ((mask >> 6) & 0x1))); + buf.write((byte) (0x80 | (codepoint & 0x3f))); + } + + } + + private static byte[] unicodeStart = {'\\', 'u'}; + private long dequoteUtf16() { + long codepoint = readHexValue(4); + if (codepoint >= 0xd800) { + if (codepoint < 0xdc00) { // high + expect(unicodeStart); + long low = readHexValue(4); + if (low >= 0xdc00 && low < 0xe000) { + codepoint = 0x10000 + ((codepoint - 0xd800) << 10) + (low - 0xdc00); + } else { + in.fail("missing low surrogate"); + } + } else if (codepoint < 0xe000) { // low + in.fail("unexpected low surrogate"); + } + } + return codepoint; + } + + private long readHexValue(int numBytes) { + long ret = 0; + for (long i = 0; i < numBytes; ++i) { + switch (c) { + case '0': ret = (ret << 4); break; + case '1': ret = (ret << 4) | 1; break; + case '2': ret = (ret << 4) | 2; break; + case '3': ret = (ret << 4) | 3; break; + case '4': ret = (ret << 4) | 4; break; + case '5': ret = (ret << 4) | 5; break; + case '6': ret = (ret << 4) | 6; break; + case '7': ret = (ret << 4) | 7; break; + case '8': ret = (ret << 4) | 8; break; + case '9': ret = (ret << 4) | 9; break; + case 'a': case 'A': ret = (ret << 4) | 0xa; break; + case 'b': case 'B': ret = (ret << 4) | 0xb; break; + case 'c': case 'C': ret = (ret << 4) | 0xc; break; + case 'd': case 'D': ret = (ret << 4) | 0xd; break; + case 'e': case 'E': ret = (ret << 4) | 0xe; break; + case 'f': case 'F': ret = (ret << 4) | 0xf; break; + default: + in.fail("invalid hex character"); + return 0; + } + next(); + } + return ret; + } + + + private void next() { + if (!in.eof()) { + c = in.getByte(); + } else { + c = 0; + } + } + + private boolean skip(byte x) { + if (c != x) { + return false; + } + next(); + return true; + } + + private void skipWhiteSpace() { + for (;;) { + switch (c) { + case ' ': case '\t': case '\n': case '\r': + next(); + break; + default: return; + } + } + } + + private static final class JsonObjectInserter implements Inserter { + private Cursor target; + private String key; + public final JsonObjectInserter adjust(Cursor c, String key) { + target = c; + this.key = key; + return this; + } + public final Cursor insertNIX() { return target.setNix(key); } + public final Cursor insertBOOL(boolean value) { return target.setBool(key, value); } + public final Cursor insertLONG(long value) { return target.setLong(key, value); } + public final Cursor insertDOUBLE(double value) { return target.setDouble(key, value); } + public final Cursor insertSTRING(String value) { return target.setString(key, value); } + public final Cursor insertSTRING(byte[] utf8) { return target.setString(key, utf8); } + public final Cursor insertDATA(byte[] value) { return target.setData(key, value); } + public final Cursor insertARRAY() { return target.setArray(key); } + public final Cursor insertOBJECT() { return target.setObject(key); } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java new file mode 100644 index 00000000000..28879311372 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java @@ -0,0 +1,220 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import com.yahoo.io.AbstractByteWriter; +import com.yahoo.io.ByteWriter; +import com.yahoo.text.AbstractUtf8Array; +import com.yahoo.text.Utf8; +import com.yahoo.text.Utf8String; +import com.yahoo.text.DoubleFormatter; + +import java.io.*; + +/** + * Encodes json from a slime object. + * + * @author lulf + */ +public final class JsonFormat implements SlimeFormat +{ + private final static byte [] HEX = Utf8.toBytes("0123456789ABCDEF"); + private final boolean compact; + public JsonFormat(boolean compact) { + this.compact = compact; + } + + @Override + public void encode(OutputStream os, Slime slime) throws IOException { + new Encoder(slime.get(), os, compact).encode(); + } + + public void encode(OutputStream os, Inspector value) throws IOException { + new Encoder(value, os, compact).encode(); + } + + public void encode(AbstractByteWriter os, Slime slime) throws IOException { + new Encoder(slime.get(), os, compact).encode(); + } + + public void encode(AbstractByteWriter os, Inspector value) throws IOException { + new Encoder(value, os, compact).encode(); + } + + @Override + public void decode(InputStream is, Slime slime) throws IOException { + throw new UnsupportedOperationException("Not implemented"); + } + + public static final class Encoder implements ArrayTraverser, ObjectTraverser + { + private final Inspector top; + private final AbstractByteWriter out; + private boolean head = true; + private boolean compact; + private int level = 0; + final static AbstractUtf8Array NULL=new Utf8String("null"); + final static AbstractUtf8Array FALSE=new Utf8String("false"); + final static AbstractUtf8Array TRUE=new Utf8String("true"); + + public Encoder(Inspector value, OutputStream out, boolean compact) { + this.top = value; + this.out = new ByteWriter(out); + this.compact = compact; + } + + public Encoder(Inspector value, AbstractByteWriter out, boolean compact) { + this.top = value; + this.out = out; + this.compact = compact; + } + + public void encode() throws IOException { + encodeValue(top); + if (!compact) { + out.append((byte) '\n'); + } + out.flush(); + } + + private void encodeNIX() throws IOException { + out.write(NULL); + } + + private void encodeBOOL(boolean value) throws IOException { + out.write(value ? TRUE : FALSE); + } + + private void encodeLONG(long value) throws IOException { + out.write(value); + } + + private void encodeDOUBLE(double value) throws IOException { + if (Double.isNaN(value) || Double.isInfinite(value)) { + out.write(NULL); + } else { + out.write(DoubleFormatter.stringValue(value)); + } + } + + private void encodeSTRING(byte[] value) throws IOException { + + byte [] data = new byte[value.length * 6 + 2]; + int len = 2; + int p = 0; + data[p++] = '"'; + for (int pos = 0; pos < value.length; pos++) { + byte c = value[pos]; + switch (c) { + case '"': data[p++] = '\\'; data[p++] = '"'; len += 2; break; + case '\\': data[p++] = '\\'; data[p++] = '\\'; len += 2; break; + case '\b': data[p++] = '\\'; data[p++] = 'b'; len += 2; break; + case '\f': data[p++] = '\\'; data[p++] = 'f'; len += 2; break; + case '\n': data[p++] = '\\'; data[p++] = 'n'; len += 2; break; + case '\r': data[p++] = '\\'; data[p++] = 'r'; len += 2; break; + case '\t': data[p++] = '\\'; data[p++] = 't'; len += 2; break; + default: + if (c > 0x1f || c < 0) { + data[p++] = c; + len++; + } else { // requires escaping according to RFC 4627 + data[p++] = '\\'; data[p++] = 'u'; data[p++] = '0'; data[p++] = '0'; + data[p++] = HEX[(c >> 4) & 0xf]; data[p++] = HEX[c & 0xf]; + len += 6; + } + } + } + data[p] = '"'; + out.append(data, 0, len); + } + + private void encodeDATA(byte[] value) throws IOException { + int len = value.length * 2 + 4; + byte [] data = new byte[len]; + int p = 0; + + data[p++] = '"'; data[p++] = '0'; data[p++] = 'x'; + for (int pos = 0; pos < value.length; pos++) { + data[p++] = HEX[(value[pos] >> 4) & 0xf]; data[p++] = HEX[value[pos] & 0xf]; + } + data[p] = '"'; + out.append(data, 0, len); + } + + private void encodeARRAY(Inspector inspector) throws IOException { + openScope((byte)'['); + ArrayTraverser at = this; + inspector.traverse(at); + closeScope((byte)']'); + } + + private void encodeOBJECT(Inspector inspector) throws IOException { + openScope((byte)'{'); + ObjectTraverser ot = this; + inspector.traverse(ot); + closeScope((byte) '}'); + } + + private void openScope(byte opener) throws IOException { + out.append(opener); + level++; + head = true; + } + + private void closeScope(byte closer) throws IOException { + level--; + separate(false); + out.append(closer); + } + + private void encodeValue(Inspector inspector) throws IOException { + switch(inspector.type()) { + case NIX: encodeNIX(); return; + case BOOL: encodeBOOL(inspector.asBool()); return; + case LONG: encodeLONG(inspector.asLong()); return; + case DOUBLE: encodeDOUBLE(inspector.asDouble()); return; + case STRING: encodeSTRING(inspector.asUtf8()); return; + case DATA: encodeDATA(inspector.asData()); return; + case ARRAY: encodeARRAY(inspector); return; + case OBJECT: encodeOBJECT(inspector); return; + } + assert false : "Should not be reached"; + } + + private void separate(boolean useComma) throws IOException { + if (!head && useComma) { + out.append((byte)','); + } else { + head = false; + } + if (!compact) { + out.append((byte)'\n'); + for (int lvl = 0; lvl < level; lvl++) { out.append((byte)' '); } + } + } + + public void entry(int idx, Inspector inspector) { + try { + separate(true); + encodeValue(inspector); + } catch (Exception e) { + // FIXME: Should we fix ArrayTraverser/ObjectTraverser API or do something more fancy here? + e.printStackTrace(); + } + } + + public void field(String name, Inspector inspector) { + try { + separate(true); + encodeSTRING(Utf8Codec.encode(name)); + out.append((byte)':'); + if (!compact) + out.append((byte)' '); + encodeValue(inspector); + } catch (Exception e) { + // FIXME: Should we fix ArrayTraverser/ObjectTraverser API or do something more fancy here? + e.printStackTrace(); + } + } + } +} + diff --git a/vespajlib/src/main/java/com/yahoo/slime/LongValue.java b/vespajlib/src/main/java/com/yahoo/slime/LongValue.java new file mode 100644 index 00000000000..fd423e178ec --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/LongValue.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +final class LongValue extends Value { + private final long value; + public LongValue(long value) { this.value = value; } + public final Type type() { return Type.LONG; } + public final long asLong() { return this.value; } + public final double asDouble() { return (double)this.value; } + public final void accept(Visitor v) { v.visitLong(value); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/NixValue.java b/vespajlib/src/main/java/com/yahoo/slime/NixValue.java new file mode 100644 index 00000000000..524fd391cdd --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/NixValue.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +final class NixValue extends Value { + private static final NixValue invalidNix = new NixValue(); + private static final NixValue validNix = new NixValue(); + private NixValue() {} + public final Type type() { return Type.NIX; } + public final void accept(Visitor v) { + if (valid()) { + v.visitNix(); + } else { + v.visitInvalid(); + } + } + public static NixValue invalid() { return invalidNix; } + public static NixValue instance() { return validNix; } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/ObjectInserter.java b/vespajlib/src/main/java/com/yahoo/slime/ObjectInserter.java new file mode 100644 index 00000000000..e8651d53702 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/ObjectInserter.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Helper class for inserting values into an ObjectValue. + * For justification read Inserter documentation. + **/ +final class ObjectInserter implements Inserter { + private Cursor target; + private int symbol; + public final ObjectInserter adjust(Cursor c, int sym) { + target = c; + symbol = sym; + return this; + } + public final Cursor insertNIX() { return target.setNix(symbol); } + public final Cursor insertBOOL(boolean value) { return target.setBool(symbol, value); } + public final Cursor insertLONG(long value) { return target.setLong(symbol, value); } + public final Cursor insertDOUBLE(double value) { return target.setDouble(symbol, value); } + public final Cursor insertSTRING(String value) { return target.setString(symbol, value); } + public final Cursor insertSTRING(byte[] utf8) { return target.setString(symbol, utf8); } + public final Cursor insertDATA(byte[] value) { return target.setData(symbol, value); } + public final Cursor insertARRAY() { return target.setArray(symbol); } + public final Cursor insertOBJECT() { return target.setObject(symbol); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/ObjectSymbolTraverser.java b/vespajlib/src/main/java/com/yahoo/slime/ObjectSymbolTraverser.java new file mode 100644 index 00000000000..fe939d15969 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/ObjectSymbolTraverser.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Callback interface for traversing objects. + * Implement this and call Inspector.traverse() + * and you will get one callback for each field in an object. + **/ +public interface ObjectSymbolTraverser +{ + /** + * Callback function to implement. + * @param sym symbol id for the current field. + * @param inspector accessor for the current field's value. + **/ + public void field(int sym, Inspector inspector); +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/ObjectTraverser.java b/vespajlib/src/main/java/com/yahoo/slime/ObjectTraverser.java new file mode 100644 index 00000000000..9d933670363 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/ObjectTraverser.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Callback interface for traversing objects. + * Implement this and call Inspector.traverse() + * and you will get one callback for each field in an object. + **/ +public interface ObjectTraverser +{ + /** + * Callback function to implement. + * @param name symbol name for the current field. + * @param inspector accessor for the current field's value. + **/ + public void field(String name, Inspector inspector); +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/ObjectValue.java b/vespajlib/src/main/java/com/yahoo/slime/ObjectValue.java new file mode 100644 index 00000000000..3d8b54ed294 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/ObjectValue.java @@ -0,0 +1,108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * A Value holding a slime "Object", a dynamic collection of named + * value fields. Fields can be inspected or traversed using the + * {@link Inspector} interface, and you can add new fields by using the + * various "set" methods in the @ref Cursor interface. + **/ +final class ObjectValue extends Value { + + private int capacity = 16; + private int hashSize() { return (capacity + (capacity >> 1) - 1); } + private int used = 0; + private Value[] values = new Value[capacity]; + private int[] hash = new int[capacity + hashSize() + (capacity << 1)]; + private final SymbolTable names; + + private final void rehash() { + capacity = (capacity << 1); + Value[] v = values; + values = new Value[capacity]; + System.arraycopy(v, 0, values, 0, used); + int[] h = hash; + hash = new int[capacity + hashSize() + (capacity << 1)]; + System.arraycopy(h, 0, hash, 0, used); + for (int i = 0; i < used; i++) { + int prev = (capacity + (hash[i] % hashSize())); + int entry = hash[prev]; + while (entry != 0) { + prev = entry + 1; + entry = hash[prev]; + } + final int insertIdx = (capacity + hashSize() + (i << 1)); + hash[prev] = insertIdx; + hash[insertIdx] = i; + } + } + + private final Value put(int sym, Value value) { + if (used == capacity) { + rehash(); + } + int prev = (capacity + (sym % hashSize())); + int entry = hash[prev]; + while (entry != 0) { + final int idx = hash[entry]; + if (hash[idx] == sym) { // found entry + return NixValue.invalid(); + } + prev = entry + 1; + entry = hash[prev]; + } + final int insertIdx = (capacity + hashSize() + (used << 1)); + hash[prev] = insertIdx; + hash[insertIdx] = used; + hash[used] = sym; + values[used++] = value; + return value; + } + + private final Value get(int sym) { + int entry = hash[capacity + (sym % hashSize())]; + while (entry != 0) { + final int idx = hash[entry]; + if (hash[idx] == sym) { // found entry + return values[idx]; + } + entry = hash[entry + 1]; + } + return NixValue.invalid(); + } + + public ObjectValue(SymbolTable names) { this.names = names; } + public ObjectValue(SymbolTable names, int sym, Value value) { + this.names = names; + put(sym, value); + } + + public final Type type() { return Type.OBJECT; } + public final int children() { return used; } + public final int fields() { return used; } + + public final Value field(int sym) { return get(sym); } + public final Value field(String name) { return get(names.lookup(name)); } + + public final void accept(Visitor v) { v.visitObject(this); } + + public final void traverse(ObjectSymbolTraverser ot) { + for (int i = 0; i < used; ++i) { + ot.field(hash[i], values[i]); + } + } + + public final void traverse(ObjectTraverser ot) { + for (int i = 0; i < used; ++i) { + ot.field(names.inspect(hash[i]), values[i]); + } + } + + protected final Cursor setLeaf(int sym, Value value) { return put(sym, value); } + public final Cursor setArray(int sym) { return put(sym, new ArrayValue(names)); } + public final Cursor setObject(int sym) { return put(sym, new ObjectValue(names)); } + + protected final Cursor setLeaf(String name, Value value) { return put(names.insert(name), value); } + public final Cursor setArray(String name) { return put(names.insert(name), new ArrayValue(names)); } + public final Cursor setObject(String name) { return put(names.insert(name), new ObjectValue(names)); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Slime.java b/vespajlib/src/main/java/com/yahoo/slime/Slime.java new file mode 100644 index 00000000000..387a81a7655 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Slime.java @@ -0,0 +1,150 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Top-level value class that contains one Value data object and a + * symbol table (shared between all directly or indirectly contained + * ObjectValue data objects). + **/ +public final class Slime +{ + private final SymbolTable names = new SymbolTable(); + private Value root = NixValue.instance(); + + /** + * Construct an empty Slime with an empty top-level value. + **/ + public Slime() {} + + /** return count of names in the symbol table. */ + public int symbols() { + return names.symbols(); + } + + /** + * Return the symbol name associated with an id. + * @param symbol the id, must be in range [0, symbols()-1] + **/ + public String inspect(int symbol) { + return names.inspect(symbol); + } + + /** + * Add a name to the symbol table; if the name is already + * in the symbol table just returns the id it already had. + * @param name the name to insert + * @return the id now associated with the name + **/ + public int insert(String name) { + return names.insert(name); + } + + /** + * Find the id associated with a symbol name; if the + * name was not in the symbol table returns the + * constant Integer.MAX_VALUE instead. + **/ + public int lookup(String name) { + return names.lookup(name); + } + + /** Get a Cursor connected to the top-level data object. */ + public Cursor get() { return root; } + + /** + * Create a new empty value and make it the new top-level data object. + **/ + public Cursor setNix() { + root = NixValue.instance(); + return root; + } + + /** + * Create a new boolean value and make it the new top-level data object. + * @param bit the actual boolean value for the new value + **/ + public Cursor setBool(boolean bit) { + root = BoolValue.instance(bit); + return root; + } + + /** + * Create a new double value and make it the new top-level data object. + * @param l the actual long value for the new value + **/ + public Cursor setLong(long l) { + root = new LongValue(l); + return root; + } + + /** + * Create a new double value and make it the new top-level data object. + * @param d the actual double value for the new value + **/ + public Cursor setDouble(double d) { + root = new DoubleValue(d); + return root; + } + + /** + * Create a new string value and make it the new top-level data object. + * @param str the actual string for the new value + **/ + public Cursor setString(String str) { + root = new StringValue(str); + return root; + } + + /** + * Create a new string value and make it the new top-level data object. + * @param utf8 the actual string (encoded as UTF-8 data) for the new value + **/ + public Cursor setString(byte[] utf8) { + root = new Utf8Value(utf8); + return root; + } + + /** + * Create a new data value and make it the new top-level data object. + * @param data the actual data to be put into the new value. + **/ + public Cursor setData(byte[] data) { + root = new DataValue(data); + return root; + } + + /** + * Create a new array value and make it the new top-level data object. + **/ + public Cursor setArray() { + root = new ArrayValue(names); + return root; + } + + /** + * Create a new object value and make it the new top-level data object. + **/ + public Cursor setObject() { + root = new ObjectValue(names); + return root; + } + + /** + * Take the current top-level data object and make it a field in a + * new ObjectValue with the given symbol id as field id; the new + * ObjectValue will also become the new top-level data object. + **/ + public Cursor wrap(int sym) { + root = new ObjectValue(names, sym, root); + return root; + } + + /** + * Take the current top-level data object and make it a field in a + * new ObjectValue with the given symbol name as field name; the new + * ObjectValue will also become the new top-level data object. + **/ + public Cursor wrap(String name) { + return wrap(names.insert(name)); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeFormat.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeFormat.java new file mode 100644 index 00000000000..142514c45a8 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeFormat.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * @author lulf + * @since 5.1 + */ +public interface SlimeFormat { + /** + * Encode a slime object into the provided output stream + * @param os The outputstream to write to. + * @param slime The slime object to encode. + */ + public void encode(OutputStream os, Slime slime) throws IOException; + + /** + * Encode a slime object into the provided output stream + * @param is The input stream to read from. + * @param slime The slime object to decode into. + */ + public void decode(InputStream is, Slime slime) throws IOException; +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeInserter.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeInserter.java new file mode 100644 index 00000000000..d6e78873ec7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeInserter.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Helper class for inserting values into a Slime object. + * For justification read Inserter documentation. + **/ +final class SlimeInserter implements Inserter { + private Slime target; + public final SlimeInserter adjust(Slime slime) { + target = slime; + return this; + } + public final Cursor insertNIX() { return target.setNix(); } + public final Cursor insertBOOL(boolean value) { return target.setBool(value); } + public final Cursor insertLONG(long value) { return target.setLong(value); } + public final Cursor insertDOUBLE(double value) { return target.setDouble(value); } + public final Cursor insertSTRING(String value) { return target.setString(value); } + public final Cursor insertSTRING(byte[] utf8) { return target.setString(utf8); } + public final Cursor insertDATA(byte[] value) { return target.setData(value); } + public final Cursor insertARRAY() { return target.setArray(); } + public final Cursor insertOBJECT() { return target.setObject(); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/StringValue.java b/vespajlib/src/main/java/com/yahoo/slime/StringValue.java new file mode 100644 index 00000000000..a5b72578d5d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/StringValue.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * A value holding a String in Java native format. + * See also @ref Utf8Value (for lazy decoding). + **/ +final class StringValue extends Value { + private final String value; + private byte[] utf8; + public StringValue(String value) { this.value = value; } + public final Type type() { return Type.STRING; } + public final String asString() { return this.value; } + public final byte[] asUtf8() { + if (utf8 == null) { + utf8 = Utf8Codec.encode(value); + } + return utf8; + } + public final void accept(Visitor v) { v.visitString(value); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/SymbolTable.java b/vespajlib/src/main/java/com/yahoo/slime/SymbolTable.java new file mode 100644 index 00000000000..133cbd1ba8e --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/SymbolTable.java @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * A mapping from an arbitrary set of unique strings to a range of + * integers. Slime users normally won't need to use this class + * directly. + **/ +final class SymbolTable { + + public static final int INVALID = Integer.MAX_VALUE; + + private static final int[] emptyHash = new int[1]; + + private int capacity = 0; + private int hashSize() { return (capacity + (capacity >> 1) - 1); } + private int used = 0; + private String[] names; + private int[] hash = emptyHash; + + private final void rehash() { + if (capacity == 0) { + capacity = 32; + names = new String[capacity]; + hash = new int[hashSize() + (capacity << 1)]; + return; + } + capacity = (capacity << 1); + String[] n = names; + names = new String[capacity]; + System.arraycopy(n, 0, names, 0, used); + hash = new int[hashSize() + (capacity << 1)]; + for (int i = 0; i < used; i++) { + int prev = Math.abs(names[i].hashCode() % hashSize()); + int entry = hash[prev]; + while (entry != 0) { + prev = entry + 1; + entry = hash[prev]; + } + final int insertIdx = (hashSize() + (i << 1)); + hash[prev] = insertIdx; + hash[insertIdx] = i; + } + } + + /** Return count of contained symbol names. */ + final int symbols() { return used; } + + /** + * Return the symbol name associated with an id. + * @param symbol the id, must be in range [0, symbols()-1] + **/ + final String inspect(int symbol) { return names[symbol]; } + + /** + * Add a name to the symbol table; if the name is already + * in the symbol table just returns the id it already had. + * @param name the name to insert + * @return the id now associated with the name + **/ + final int insert(String name) { + if (used == capacity) { + rehash(); + } + int prev = Math.abs(name.hashCode() % hashSize()); + int entry = hash[prev]; + while (entry != 0) { + final int sym = hash[entry]; + if (names[sym].equals(name)) { // found entry + return sym; + } + prev = entry + 1; + entry = hash[prev]; + } + final int insertIdx = (hashSize() + (used << 1)); + hash[prev] = insertIdx; + hash[insertIdx] = used; + names[used++] = name; + return (used - 1); + } + + /** + * Find the id associated with a symbol name; if the + * name was not in the symbol table returns the + * INVALID constant instead. + **/ + final int lookup(String name) { + int entry = hash[Math.abs(name.hashCode() % hashSize())]; + while (entry != 0) { + final int sym = hash[entry]; + if (names[sym].equals(name)) { // found entry + return sym; + } + entry = hash[entry + 1]; + } + return INVALID; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Type.java b/vespajlib/src/main/java/com/yahoo/slime/Type.java new file mode 100644 index 00000000000..036d577e106 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Type.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Enumeration of all possibly Slime data types. + **/ +public enum Type { + NIX(0), + BOOL(1), + LONG(2), + DOUBLE(3), + STRING(4), + DATA(5), + ARRAY(6), + OBJECT(7); + + public final byte ID; + private Type(int id) { this.ID = (byte)id; } + + private static final Type[] types = values(); + static Type asType(int id) { return types[id]; } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Utf8Codec.java b/vespajlib/src/main/java/com/yahoo/slime/Utf8Codec.java new file mode 100644 index 00000000000..c9e86b73073 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Utf8Codec.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import com.yahoo.text.Utf8; + +/** + * Helper class for conversion between String and UTF-8 representations. + **/ +class Utf8Codec { + public static String decode(byte[] data, int pos, int len) { + return Utf8.toString(data, pos, len); + } + public static byte[] encode(String str) { + return Utf8.toBytes(str); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Utf8Value.java b/vespajlib/src/main/java/com/yahoo/slime/Utf8Value.java new file mode 100644 index 00000000000..6aa95310b86 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Utf8Value.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * A value type encapsulating a String in its UTF-8 representation. + * Useful for lazy decoding; if the data is just passed through in + * UTF-8 it will never be converted at all. + **/ +final class Utf8Value extends Value { + private final byte[] value; + private String string; + public Utf8Value(byte[] value) { this.value = value; } + public final Type type() { return Type.STRING; } + public final String asString() { + if (string == null) { + string = Utf8Codec.decode(value, 0, value.length); + } + return string; + } + public final byte[] asUtf8() { return value; } + public final void accept(Visitor v) { v.visitString(value); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Value.java b/vespajlib/src/main/java/com/yahoo/slime/Value.java new file mode 100644 index 00000000000..d86bf8607bd --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Value.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + + +import java.io.ByteArrayOutputStream; + +/** + * Common implementation for all value types. + * All default behavior is here, so specific types only + * need override their actually useful parts. + **/ + +abstract class Value implements Cursor { + + private static final String emptyString = ""; + private static final byte[] emptyData = new byte[0]; + + public final boolean valid() { return this != NixValue.invalid(); } + public int children() { return 0; } + public int entries() { return 0; } + public int fields() { return 0; } + + public boolean asBool() { return false; } + public long asLong() { return 0; } + public double asDouble() { return 0.0; } + public String asString() { return emptyString; } + public byte[] asUtf8() { return emptyData; } + public byte[] asData() { return emptyData; } + + public void traverse(ArrayTraverser at) {} + public void traverse(ObjectSymbolTraverser ot) {} + public void traverse(ObjectTraverser ot) {} + + public Value entry(int idx) { return NixValue.invalid(); } + public Value field(String name) { return NixValue.invalid(); } + public Value field(int sym) { return NixValue.invalid(); } + + protected Cursor addLeaf(Value value) { return NixValue.invalid(); } + public Cursor addArray() { return NixValue.invalid(); } + public Cursor addObject() { return NixValue.invalid(); } + + public final Cursor addNix() { return addLeaf(NixValue.instance()); } + public final Cursor addBool(boolean bit) { return addLeaf(BoolValue.instance(bit)); } + public final Cursor addLong(long l) { return addLeaf(new LongValue(l)); } + public final Cursor addDouble(double d) { return addLeaf(new DoubleValue(d)); } + public final Cursor addString(String str) { return addLeaf(new StringValue(str)); } + public final Cursor addString(byte[] utf8) { return addLeaf(new Utf8Value(utf8)); } + public final Cursor addData(byte[] data) { return addLeaf(new DataValue(data)); } + + protected Cursor setLeaf(int sym, Value value) { return NixValue.invalid(); } + public Cursor setArray(int sym) { return NixValue.invalid(); } + public Cursor setObject(int sym) { return NixValue.invalid(); } + + public final Cursor setNix(int sym) { return setLeaf(sym, NixValue.instance()); } + public final Cursor setBool(int sym, boolean bit) { return setLeaf(sym, BoolValue.instance(bit)); } + public final Cursor setLong(int sym, long l) { return setLeaf(sym, new LongValue(l)); } + public final Cursor setDouble(int sym, double d) { return setLeaf(sym, new DoubleValue(d)); } + public final Cursor setString(int sym, String str) { return setLeaf(sym, new StringValue(str)); } + public final Cursor setString(int sym, byte[] utf8) { return setLeaf(sym, new Utf8Value(utf8)); } + public final Cursor setData(int sym, byte[] data) { return setLeaf(sym, new DataValue(data)); } + + protected Cursor setLeaf(String name, Value value) { return NixValue.invalid(); } + public Cursor setArray(String name) { return NixValue.invalid(); } + public Cursor setObject(String name) { return NixValue.invalid(); } + + public final Cursor setNix(String name) { return setLeaf(name, NixValue.instance()); } + public final Cursor setBool(String name, boolean bit) { return setLeaf(name, BoolValue.instance(bit)); } + public final Cursor setLong(String name, long l) { return setLeaf(name, new LongValue(l)); } + public final Cursor setDouble(String name, double d) { return setLeaf(name, new DoubleValue(d)); } + public final Cursor setString(String name, String str) { return setLeaf(name, new StringValue(str)); } + public final Cursor setString(String name, byte[] utf8) { return setLeaf(name, new Utf8Value(utf8)); } + public final Cursor setData(String name, byte[] data) { return setLeaf(name, new DataValue(data)); } + + public final String toString() { + try { + // should produce non-compact json, but we need compact + // json for slime summaries until we have a more generic + // json rendering pipeline in place. + ByteArrayOutputStream a = new ByteArrayOutputStream(); + new JsonFormat(true).encode(a, this); + byte[] utf8 = a.toByteArray(); + return Utf8Codec.decode(utf8, 0, utf8.length); + } catch (Exception e) { + return "null"; + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Visitor.java b/vespajlib/src/main/java/com/yahoo/slime/Visitor.java new file mode 100644 index 00000000000..d36a7da9078 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Visitor.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Visitor interface used to resolve the underlying type of a value + * represented by an Inspector. + **/ +public interface Visitor { + /** + * Called when the visited Inspector is not valid. + **/ + public void visitInvalid(); + public void visitNix(); + public void visitBool(boolean bit); + public void visitLong(long l); + public void visitDouble(double d); + public void visitString(String str); + public void visitString(byte[] utf8); + public void visitData(byte[] data); + public void visitArray(Inspector arr); + public void visitObject(Inspector obj); +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/package-info.java b/vespajlib/src/main/java/com/yahoo/slime/package-info.java new file mode 100644 index 00000000000..92a37a2ca37 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/package-info.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * SLIME: 'Schema-Less Interface/Model/Exchange'. Slime is a way to + * handle schema-less structured data to be used as part of interfaces + * between components (RPC signatures), internal models + * (config/parameters) and data exchange between components + * (documents). The goal for Slime is to be flexible and lightweight + * and at the same time limit the extra overhead in space and time + * compared to schema-oriented approaches like protocol buffers and + * avro. The data model is inspired by JSON and associative arrays + * typically used in programming languages with dynamic typing. + **/ +@ExportPackage +package com.yahoo.slime; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/system/CatchSigTerm.java b/vespajlib/src/main/java/com/yahoo/system/CatchSigTerm.java new file mode 100644 index 00000000000..f58c161941a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/system/CatchSigTerm.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +import java.lang.reflect.*; + +// import sun.misc.Signal; +// import sun.misc.SignalHandler; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class CatchSigTerm { + /** + * Sets up a signal handler for SIGTERM, where a given AtomicBoolean + * gets a true value when the TERM signal is caught. + * + * Callers basically have two options for acting on the TERM signal: + * + * They may choose to synchronize and wait() on this variable, + * and they will be notified when it changes state to true. To avoid + * problems with spurious wakeups, use a while loop and wait() + * again if the state is still false. As soon as the caller has been + * woken up and the state is true, the application should exit as + * soon as possible. + * + * They may also choose to poll the state of this variable. As soon + * as its state becomes true, the signal has been received, and the + * application should exit as soon as possible. + * + * @param signalCaught set to false initially, will be set to true when SIGTERM is caught. + */ + @SuppressWarnings("rawtypes") + public static void setup(final AtomicBoolean signalCaught) { + signalCaught.set(false); + try { + Class shc = Class.forName("sun.misc.SignalHandler"); + Class ssc = Class.forName("sun.misc.Signal"); + + InvocationHandler ihandler = new InvocationHandler() { + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + synchronized (signalCaught) { + signalCaught.set(true); + signalCaught.notifyAll(); + } + return null; + } + }; + Object shandler = Proxy.newProxyInstance(CatchSigTerm.class.getClassLoader(), + new Class[] { shc }, + ihandler); + Constructor[] c = ssc.getDeclaredConstructors(); + assert c.length == 1; + Object sigterm = c[0].newInstance("TERM"); + Method m = findMethod(ssc, "handle"); + assert m != null; // "NoSuchMethodException" + m.invoke(null, sigterm, shandler); + } catch (ClassNotFoundException | InvocationTargetException | InstantiationException | IllegalAccessException e) { + System.err.println("FAILED setting up signal catching: "+e); + } + } + + @SuppressWarnings("rawtypes") + private static Method findMethod(Class c, String name) { + for (Method m : c.getDeclaredMethods()) { + if (m.getName().equals(name)) { + return m; + } + } + return null; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/system/CommandLineParser.java b/vespajlib/src/main/java/com/yahoo/system/CommandLineParser.java new file mode 100644 index 00000000000..9f6922e5084 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/system/CommandLineParser.java @@ -0,0 +1,216 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +import java.util.*; + +/** + * Simple command line parser, handling multiple arguments and multiple unary and binary switches starting with -. + * + * Terms used: + * + * progname -binaryswitch foo -unaryswitch argument1 argument2 + * + * @author vegardh + * + */ +public class CommandLineParser { + private List<String> inputStrings = new ArrayList<>(); + private Map<String, String> legalUnarySwitches = new HashMap<>(); + private Map<String, String> legalBinarySwitches = new HashMap<>(); + private List<String> unarySwitches = new ArrayList<>(); + private Map<String, String> binarySwitches = new HashMap<>(); + private List<String> arguments = new ArrayList<>(); + private Map<String, String> requiredUnarySwitches = new HashMap<>(); + private Map<String, String> requiredBinarySwitches = new HashMap<>(); + private String progname = "progname"; + private String argumentExplanation; + private int minArguments=0; + private int maxArguments=Integer.MAX_VALUE; + private String helpText; + private static HashSet<String> helpSwitches = new HashSet<>(); + private boolean helpSwitchUsed = false; + + static { + helpSwitches.add("-h"); + helpSwitches.add("-help"); + helpSwitches.add("--help"); + helpSwitches.add("-?"); + } + + public CommandLineParser(String[] cmds) { + inputStrings = Arrays.asList(cmds); + } + + public CommandLineParser(String progname, String[] cmds) { + this.progname=progname; + inputStrings = Arrays.asList(cmds); + } + + /** + * Parses the command line + * @throws IllegalArgumentException if a parse error occured + */ + public void parse() { + for (Iterator<String> it = inputStrings.iterator() ; it.hasNext() ; ) { + String i = it.next(); + if (isHelpSwitch(i)) { + helpSwitchUsed = true; + usageAndThrow(); + } + if (i.startsWith("-")) { + if (!isLegalSwitch(i)) { + usageAndThrow(); + } else if (legalUnarySwitches.keySet().contains(i)) { + unarySwitches.add(i); + } else if (legalBinarySwitches.keySet().contains(i)) { + if (!it.hasNext()) { + throw new IllegalArgumentException(i+ " requires value"); + } else { + String val = it.next(); + binarySwitches.put(i, val); + } + } + } else { + arguments.add(i); + } + } + if (!requiredUnarySwitches.isEmpty() && !getUnarySwitches().containsAll(requiredUnarySwitches.keySet())) { + usageAndThrow(); + } + if (!requiredBinarySwitches.isEmpty() && !getBinarySwitches().keySet().containsAll(requiredBinarySwitches.keySet())) { + usageAndThrow(); + } + if (getArguments().size()<minArguments || getArguments().size()>maxArguments) { + usageAndThrow(); + } + } + + private boolean isHelpSwitch(String i) { + return helpSwitches.contains(i); + } + + void usageAndThrow() { + StringBuffer error_sb = new StringBuffer(); + error_sb.append("\nusage: ").append(progname).append(" "); + if (argumentExplanation!=null) { + error_sb.append(argumentExplanation); + } + if (!legalUnarySwitches.isEmpty()) error_sb.append("\nSwitches:\n"); + error_sb.append("-h This help text\n"); + for (Map.Entry<String, String> e : legalUnarySwitches.entrySet()) { + error_sb.append(e.getKey()).append(" ").append(e.getValue()).append("\n"); + } + for (Map.Entry<String, String> e : legalBinarySwitches.entrySet()) { + error_sb.append(e.getKey()).append(" <").append(e.getValue()).append(">\n"); + } + if (helpText!=null) { + error_sb.append("\n").append(helpText).append("\n"); + } + throw new IllegalArgumentException(error_sb.toString()); + } + + private boolean isLegalSwitch(String s) { + return (legalUnarySwitches.containsKey(s) || legalBinarySwitches.containsKey(s)); + } + + /** + * Add a legal unary switch such as "-d" + */ + public void addLegalUnarySwitch(String s, String explanation) { + if (legalBinarySwitches.containsKey(s)) { + throw new IllegalArgumentException(s +" already added as a binary switch"); + } + legalUnarySwitches.put(s, explanation); + } + + public void addLegalUnarySwitch(String s) { + addLegalUnarySwitch(s, null); + } + + /** + * Adds a required switch, such as -p + */ + public void addRequiredUnarySwitch(String s, String explanation) { + addLegalUnarySwitch(s, explanation); + requiredUnarySwitches.put(s, explanation); + } + + /** + * Add a legal binary switch such as "-f /foo/bar" + */ + public void addLegalBinarySwitch(String s, String explanation) { + if (legalUnarySwitches.containsKey(s)) { + throw new IllegalArgumentException(s +" already added as a unary switch"); + } + legalBinarySwitches.put(s, explanation); + } + + /** + * Adds a legal binary switch without explanation + */ + public void addLegalBinarySwitch(String s) { + addLegalBinarySwitch(s, null); + } + + /** + * Adds a required binary switch + */ + public void addRequiredBinarySwitch(String s, String explanation) { + addLegalBinarySwitch(s, explanation); + requiredBinarySwitches.put(s, explanation); + } + + /** + * The unary switches that were given on the command line + */ + public List<String> getUnarySwitches() { + return unarySwitches; + } + + /** + * The binary switches that were given on the command line + */ + public Map<String, String> getBinarySwitches() { + return binarySwitches; + } + + /** + * All non-switch strings that were given on the command line + */ + public List<String> getArguments() { + return arguments; + } + + /** + * Sets the argument explanation used in printing method, i.e. "names,..." + */ + public void setArgumentExplanation(String argumentExplanation) { + this.argumentExplanation = argumentExplanation; + } + + public void setExtendedHelpText(String text) { + this.helpText=text; + } + + public String getHelpText() { + return helpText; + } + + /** + * Sets minimum number of required arguments + */ + public void setMinArguments(int minArguments) { + this.minArguments = minArguments; + } + + /** + * Sets the maximum number of allowed arguments + */ + public void setMaxArguments(int maxArguments) { + this.maxArguments = maxArguments; + } + + public boolean helpSwitchUsed() { + return helpSwitchUsed; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/system/ForceLoad.java b/vespajlib/src/main/java/com/yahoo/system/ForceLoad.java new file mode 100644 index 00000000000..f924740321f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/system/ForceLoad.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +/** + * Utility class used to force the loading of other classes. + **/ +public class ForceLoad { + + /** + * Force the loading of the given classes. If any of the named + * classes can not be loaded, an error will be thrown. + * + * @param packageName the name of the package for which + * we want to forceload classes. + * @param classNames array of names of classes (without package prefix) + * to force load. + **/ + public static void forceLoad(String packageName, String[] classNames) + throws ForceLoadError + { + String fullClassName = ""; + try { + for (String className : classNames) { + fullClassName = packageName + "." + className; + Class.forName(fullClassName); + } + } catch (Exception e) { + throw new ForceLoadError(fullClassName, e); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/system/ForceLoadError.java b/vespajlib/src/main/java/com/yahoo/system/ForceLoadError.java new file mode 100644 index 00000000000..376374a510d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/system/ForceLoadError.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +/** + * Special error to be propagated when force-loading a class fails. + **/ +@SuppressWarnings("serial") +public class ForceLoadError extends java.lang.Error { + + /** + * Create a new force load error + * + * @param className full name of offending class + * @param cause what caused the failure + **/ + public ForceLoadError(String className, Throwable cause) { + super("Force loading class '" + className + "' failed", cause); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java b/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java new file mode 100644 index 00000000000..bb2909b346a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.StringTokenizer; + +import com.yahoo.collections.Pair; + +/** + * Executes a system command synchronously. + * + * @author <a href="mailto:bratseth@yahoo-inc">Jon S Bratseth</a> + */ +public class ProcessExecuter { + + /** + * Executes the given command synchronously without timeout. + * @return Retcode and stdout/stderr merged + */ + public Pair<Integer, String> exec(String command) throws IOException { + StringTokenizer tok = new StringTokenizer(command); + List<String> tokens = new ArrayList<>(); + while (tok.hasMoreElements()) tokens.add(tok.nextToken()); + return exec(tokens.toArray(new String[0])); + } + + /** + * Executes the given command synchronously without timeout. + * @param command tokens + * @return Retcode and stdout/stderr merged + */ + public Pair<Integer, String> exec(String[] command) throws IOException { + ProcessBuilder pb = new ProcessBuilder(command); + StringBuilder ret = new StringBuilder(); + pb.environment().remove("VESPA_LOG_TARGET"); + pb.redirectErrorStream(true); + Process p = pb.start(); + InputStream is = p.getInputStream(); + while (true) { + int b = is.read(); + if (b==-1) break; + ret.append((char)b); + } + int rc=0; + try { + rc = p.waitFor(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return new Pair<>(rc, ret.toString()); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/system/package-info.java b/vespajlib/src/main/java/com/yahoo/system/package-info.java new file mode 100644 index 00000000000..397f0f5d791 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/system/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.system; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MapTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MapTensor.java new file mode 100644 index 00000000000..3bda4159ca6 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/MapTensor.java @@ -0,0 +1,136 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.annotations.Beta; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.UnaryOperator; + +/** + * A sparse implementation of a tensor backed by a Map of cells to values. + * + * @author bratseth + */ +@Beta +public class MapTensor implements Tensor { + + private final ImmutableSet<String> dimensions; + + private final ImmutableMap<TensorAddress, Double> cells; + + /** Creates a sparse tensor where the dimensions are determined by the cells */ + public MapTensor(Map<TensorAddress, Double> cells) { + this(dimensionsOf(cells.keySet()), cells); + } + + /** Creates a sparse tensor */ + MapTensor(Set<String> dimensions, Map<TensorAddress, Double> cells) { + ensureValidDimensions(cells, dimensions); + this.dimensions = ImmutableSet.copyOf(dimensions); + this.cells = ImmutableMap.copyOf(cells); + } + + private void ensureValidDimensions(Map<TensorAddress, Double> cells, Set<String> dimensions) { + for (TensorAddress address : cells.keySet()) + if ( ! dimensions.containsAll(address.dimensions())) + throw new IllegalArgumentException("Cell address " + address + " is outside this tensors dimensions " + + dimensions); + } + + /** + * Creates a tensor from the string form returned by the {@link #toString} of this. + * + * @param s the tensor string + * @throws IllegalArgumentException if the string is not in the correct format + */ + public static MapTensor from(String s) { + s = s.trim(); + if ( s.startsWith("(")) + return fromTensorWithEmptyDimensions(s); + else if ( s.startsWith("{")) + return fromTensor(s, Collections.emptySet()); + else + throw new IllegalArgumentException("Excepted a string starting by { or (, got '" + s + "'"); + } + + private static MapTensor fromTensorWithEmptyDimensions(String s) { + s = s.substring(1).trim(); + int multiplier = s.indexOf("*"); + if (multiplier < 0 || ! s.endsWith(")")) + throw new IllegalArgumentException("Expected a tensor on the form ({dimension:-,...}*{{cells}}), got '" + s + "'"); + MapTensor dimensionTensor = fromTensor(s.substring(0, multiplier).trim(), Collections.emptySet()); + return fromTensor(s.substring(multiplier + 1, s.length() - 1), dimensionTensor.dimensions()); + } + + private static MapTensor fromTensor(String s, Set<String> additionalDimensions) { + s = s.trim().substring(1).trim(); + ImmutableMap.Builder<TensorAddress, Double> cells = new ImmutableMap.Builder<>(); + while (s.length() > 1) { + int keyEnd = s.indexOf('}'); + TensorAddress address = TensorAddress.from(s.substring(0, keyEnd+1)); + s = s.substring(keyEnd + 1).trim(); + if ( ! s.startsWith(":")) + throw new IllegalArgumentException("Expecting a ':' after " + address + ", got '" + s + "'"); + int valueEnd = s.indexOf(','); + if (valueEnd < 0) { // last value + valueEnd = s.indexOf("}"); + if (valueEnd < 0) + throw new IllegalArgumentException("A tensor string must end by '}'"); + } + Double value = asDouble(address, s.substring(1, valueEnd).trim()); + cells.put(address, value); + s = s.substring(valueEnd+1).trim(); + } + + ImmutableMap<TensorAddress, Double> cellMap = cells.build(); + Set<String> dimensions = dimensionsOf(cellMap.keySet()); + dimensions.addAll(additionalDimensions); + return new MapTensor(dimensions, cellMap); + } + + private static Double asDouble(TensorAddress address, String s) { + try { + return Double.valueOf(s); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("At " + address + ": Expected a floating point number, got '" + s + "'"); + } + } + + private static Set<String> dimensionsOf(Set<TensorAddress> addresses) { + Set<String> dimensions = new HashSet<>(); + for (TensorAddress address : addresses) + for (TensorAddress.Element element : address.elements()) + dimensions.add(element.dimension()); + return dimensions; + } + + @Override + public Set<String> dimensions() { return dimensions; } + + @Override + public Map<TensorAddress, Double> cells() { return cells; } + + @Override + public double get(TensorAddress address) { return cells.getOrDefault(address, Double.NaN); } + + @Override + public int hashCode() { return cells.hashCode(); } + + @Override + public String toString() { return Tensor.toStandardString(this); } + + @Override + public boolean equals(Object o) { + if ( ! (o instanceof Tensor)) return false; + return Tensor.equals(this, (Tensor)o); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MapTensorBuilder.java b/vespajlib/src/main/java/com/yahoo/tensor/MapTensorBuilder.java new file mode 100644 index 00000000000..f46f000d1ee --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/MapTensorBuilder.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.annotations.Beta; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Builder class for a MapTensor. + * + * The set of dimensions of the resulting tensor is the union of + * the dimensions specified explicitly and the ones specified in the + * tensor cell addresses. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +@Beta +public class MapTensorBuilder { + + private final Set<String> dimensions = new HashSet<>(); + private final Map<TensorAddress, Double> cells = new HashMap<>(); + + public class CellBuilder { + + private final TensorAddress.Builder addressBuilder = new TensorAddress.Builder(); + + public CellBuilder label(String dimension, String label) { + dimensions.add(dimension); + addressBuilder.add(dimension, label); + return this; + } + public MapTensorBuilder value(double cellValue) { + cells.put(addressBuilder.build(), cellValue); + return MapTensorBuilder.this; + } + } + + public MapTensorBuilder() { + } + + public MapTensorBuilder dimension(String dimension) { + dimensions.add(dimension); + return this; + } + + public CellBuilder cell() { + return new CellBuilder(); + } + + public Tensor build() { + return new MapTensor(dimensions, cells); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MatchProduct.java b/vespajlib/src/main/java/com/yahoo/tensor/MatchProduct.java new file mode 100644 index 00000000000..074742acee1 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/MatchProduct.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; +import java.util.Set; + +/** + * Computes a <i>match product</i>, see {@link Tensor#match} + * + * @author bratseth + */ +class MatchProduct { + + private final Set<String> dimensions; + private final ImmutableMap.Builder<TensorAddress, Double> cells = new ImmutableMap.Builder<>(); + + public MatchProduct(Tensor a, Tensor b) { + this.dimensions = TensorOperations.combineDimensions(a, b); + for (Map.Entry<TensorAddress, Double> aCell : a.cells().entrySet()) { + Double sameValueInB = b.cells().get(aCell.getKey()); + if (sameValueInB != null) + cells.put(aCell.getKey(), aCell.getValue() * sameValueInB); + } + } + + /** Returns the result of taking this product */ + public MapTensor result() { + return new MapTensor(dimensions, cells.build()); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java new file mode 100644 index 00000000000..41f4d6c0b3d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -0,0 +1,247 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.annotations.Beta; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.UnaryOperator; + +/** +/** + * A multidimensional array which can be used in computations. + * <p> + * A tensor consists of a set of <i>dimension</i> names and a set of <i>cells</i> containing scalar <i>values</i>. + * Each cell is is identified by its <i>address</i>, which consists of a set of dimension-label pairs which defines + * the location of that cell. Both dimensions and labels are string on the form of an identifier or integer. + * Any dimension in an address may be assigned the special label "undefined", represented in string form as "-". + * <p> + * The size of the set of dimensions of a tensor is called its <i>order</i>. + * <p> + * In contrast to regular mathematical formulations of tensors, this definition of a tensor allows <i>sparseness</i> + * as there is no built-in notion of a contiguous space, and even in cases where a space is implied (such as when + * address labels are integers), there is no requirement that every implied cell has a defined value. + * Undefined values have no define representation as they are never observed. + * <p> + * Tensors can be read and serialized to and from a string form documented in the {@link #toString} method. + * + * @author bratseth + */ +@Beta +public interface Tensor { + + /** + * Returns the immutable set of dimensions of this tensor. + * The size of this set is the tensor's <i>order</i>. + */ + Set<String> dimensions(); + + /** Returns an immutable map of the cells of this */ + Map<TensorAddress, Double> cells(); + + /** Returns the value of a cell, or NaN if this cell does not exist/have no value */ + double get(TensorAddress address); + + /** + * Returns the <i>sparse tensor product</i> of this tensor and the argument tensor. + * This is the all-to-all combinations of cells in the argument tenors, except the combinations + * which have conflicting labels for the same dimension. The value of each combination is the product + * of the values of the two input cells. The dimensions of the tensor product is the set union of the + * dimensions of the argument tensors. + * <p> + * If there are no overlapping dimensions this is the regular tensor product. + * If the two tensors have exactly the same dimensions this is the Hadamard product. + * <p> + * The sparse tensor product is associative and commutative. + * + * @param argument the tensor to multiply by this + * @return the resulting tensor. + */ + default Tensor multiply(Tensor argument) { + return new TensorProduct(this, argument).result(); + } + + /** + * Returns the <i>match product</i> of two tensors. + * This returns a tensor which contains the <i>matching</i> cells in the two tensors, with their + * values multiplied. + * <p> + * Two cells are matching if they have the same labels for all dimensions shared between the two argument tensors, + * and have the value undefined for any non-shared dimension. + * <p> + * The dimensions of the resulting tensor is the set intersection of the two argument tensors. + * <p> + * If the two tensors have exactly the same dimensions, this is the Hadamard product. + */ + default Tensor match(Tensor argument) { + return new MatchProduct(this, argument).result(); + } + + /** + * Returns a tensor which contains the cells of both argument tensors, where the value for + * any <i>matching</i> cell is the min of the two possible values. + * <p> + * Two cells are matching if they have the same labels for all dimensions shared between the two argument tensors, + * and have the value undefined for any non-shared dimension. + */ + default Tensor min(Tensor argument) { + return new TensorMin(this, argument).result(); + } + + /** + * Returns a tensor which contains the cells of both argument tensors, where the value for + * any <i>matching</i> cell is the max of the two possible values. + * <p> + * Two cells are matching if they have the same labels for all dimensions shared between the two argument tensors, + * and have the value undefined for any non-shared dimension. + */ + default Tensor max(Tensor argument) { + return new TensorMax(this, argument).result(); + } + + /** + * Returns a tensor which contains the cells of both argument tensors, where the value for + * any <i>matching</i> cell is the sum of the two possible values. + * <p> + * Two cells are matching if they have the same labels for all dimensions shared between the two argument tensors, + * and have the value undefined for any non-shared dimension. + */ + default Tensor add(Tensor argument) { + return new TensorSum(this, argument).result(); + } + + /** + * Returns a tensor which contains the cells of both argument tensors, where the value for + * any <i>matching</i> cell is the difference of the two possible values. + * <p> + * Two cells are matching if they have the same labels for all dimensions shared between the two argument tensors, + * and have the value undefined for any non-shared dimension. + */ + default Tensor subtract(Tensor argument) { + return new TensorDifference(this, argument).result(); + } + + /** + * Returns a tensor with the same cells as this and the given function is applied to all its cell values. + * + * @param function the function to apply to all cells + * @return the tensor with the function applied to all the cells of this + */ + default Tensor apply(UnaryOperator<Double> function) { + return new TensorFunction(this, function).result(); + } + + /** + * Returns a tensor with the given dimension removed and cells which contains the sum of the values + * in the removed dimension. + */ + default Tensor sum(String dimension) { + return new TensorDimensionSum(dimension, this).result(); + } + + /** + * Returns the sum of all the cells of this tensor. + */ + default double sum() { + double sum = 0; + for (Map.Entry<TensorAddress, Double> cell : cells().entrySet()) + sum += cell.getValue(); + return sum; + } + + /** + * Returns true if the given tensor is mathematically equal to this: + * Both are of type Tensor and have the same content. + */ + @Override + boolean equals(Object o); + + /** Returns true if the two given tensors are mathematically equivalent, that is whether both have the same content */ + static boolean equals(Tensor a, Tensor b) { + if (a == b) return true; + if ( ! a.dimensions().equals(b.dimensions())) return false; + if ( ! a.cells().equals(b.cells())) return false; + return true; + } + + /** + * Returns this tensor on the form + * <code>{address1:value1,address2:value2,...}</code> + * where each address is on the form <code>{dimension1:label1,dimension2:label2,...}</code>, + * and values are numbers. + * <p> + * Cells are listed in the natural order of tensor addresses: Increasing size primarily + * and by element lexical order secondarily. + * <p> + * Note that while this is suggestive of JSON, it is not JSON. + */ + @Override + String toString(); + + /** Returns a tensor instance containing the given data on the standard string format returned by toString */ + static Tensor from(String tensorString) { + return MapTensor.from(tensorString); + } + + /** + * Returns a tensor instance containing the given data on the standard string format returned by toString + * + * @param tensorType the type of the tensor to return, as a string on the tensor type format, given in + * {@link TensorType#fromSpec} + * @param tensorString the tensor on the standard tensor string format + */ + static Tensor from(String tensorType, String tensorString) { + TensorType.fromSpec(tensorType); // Just validate type spec for now, as we only have one, generic implementation + return MapTensor.from(tensorString); + } + + /** + * Call this from toString in implementations to return the standard string format. + * (toString cannot be a default method because default methods cannot override super methods). + * + * @param tensor the tensor to return the standard string format of + * @return the tensor on the standard string format + */ + static String toStandardString(Tensor tensor) { + Set<String> emptyDimensions = emptyDimensions(tensor); + if (emptyDimensions.size() > 0) // explicitly list empty dimensions + return "( " + unitTensorWithDimensions(emptyDimensions) + " * " + contentToString(tensor) + " )"; + else + return contentToString(tensor); + } + + static String contentToString(Tensor tensor) { + List<Map.Entry<TensorAddress, Double>> cellEntries = new ArrayList<>(tensor.cells().entrySet()); + Collections.sort(cellEntries, Map.Entry.<TensorAddress, Double>comparingByKey()); + + StringBuilder b = new StringBuilder("{"); + for (Map.Entry<TensorAddress, Double> cell : cellEntries) { + b.append(cell.getKey()).append(":").append(cell.getValue()); + b.append(","); + } + if (b.length() > 1) + b.setLength(b.length() - 1); + b.append("}"); + return b.toString(); + } + + /** + * Returns the dimensions of this which have no values. + * This is a possibly empty subset of the dimensions of this tensor. + */ + static Set<String> emptyDimensions(Tensor tensor) { + Set<String> emptyDimensions = new HashSet<>(tensor.dimensions()); + for (TensorAddress address : tensor.cells().keySet()) + emptyDimensions.removeAll(address.dimensions()); + return emptyDimensions; + } + + static String unitTensorWithDimensions(Set<String> dimensions) { + return new MapTensor(Collections.singletonMap(TensorAddress.emptyWithDimensions(dimensions), 1.0)).toString(); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java new file mode 100644 index 00000000000..11c6a5f6685 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -0,0 +1,207 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.annotations.Beta; +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * An immutable address to a tensor cell. + * This is sparse: Only dimensions which have a different label than "undefined" are + * explicitly included. + * <p> + * Tensor addresses are ordered by increasing size primarily, and by the natural order of the elements in sorted + * order secondarily. + * + * @author bratseth + */ +@Beta +public final class TensorAddress implements Comparable<TensorAddress> { + + public static final TensorAddress empty = new TensorAddress.Builder().build(); + + private final ImmutableList<Element> elements; + + /** Note that the elements list MUST be sorted before calling this */ + private TensorAddress(List<Element> elements) { + this.elements = ImmutableList.copyOf(elements); + } + + public static TensorAddress fromSorted(List<Element> elements) { + return new TensorAddress(elements); + } + + /** + * Creates a tensor address from an unsorted list of elements. + * This call assigns ownership of the elements list to this class. + */ + public static TensorAddress fromUnsorted(List<Element> elements) { + Collections.sort(elements); + return new TensorAddress(elements); + } + + /** Creates a tenor address from a string on the form {dimension1:label1,dimension2:label2,...} */ + public static TensorAddress from(String address) { + address = address.trim(); + if ( ! (address.startsWith("{") && address.endsWith("}"))) + throw new IllegalArgumentException("Expecting a tensor address to be enclosed in {}, got '" + address + "'"); + + String addressBody = address.substring(1, address.length() - 1).trim(); + if (addressBody.isEmpty()) return TensorAddress.empty; + + List<Element> elements = new ArrayList<>(); + for (String elementString : addressBody.split(",")) { + String[] pair = elementString.split(":"); + if (pair.length != 2) + throw new IllegalArgumentException("Expecting argument elements to be on the form dimension:label, " + + "got '" + elementString + "'"); + elements.add(new Element(pair[0].trim(), pair[1].trim())); + } + Collections.sort(elements); + return TensorAddress.fromSorted(elements); + } + + /** Creates an empty address with a set of dimensions */ + public static TensorAddress emptyWithDimensions(Set<String> dimensions) { + List<Element> elements = new ArrayList<>(dimensions.size()); + for (String dimension : dimensions) + elements.add(new Element(dimension, Element.undefinedLabel)); + return TensorAddress.fromUnsorted(elements); + } + + /** Returns an immutable list of the elements of this address in sorted order */ + public List<Element> elements() { return elements; } + + /** Returns true if this address has a value (other than implicit "undefined") for the given dimension */ + public boolean hasDimension(String dimension) { + for (TensorAddress.Element element : elements) + if (element.dimension().equals(dimension)) + return true; + return false; + } + + /** Returns a possibly immutable set of the dimensions of this */ + public Set<String> dimensions() { + Set<String> dimensions = new HashSet<>(); + for (Element e : elements) + dimensions.add(e.dimension()); + return dimensions; + } + + @Override + public int compareTo(TensorAddress other) { + int sizeComparison = Integer.compare(this.elements.size(), other.elements.size()); + if (sizeComparison != 0) return sizeComparison; + + for (int i = 0; i < elements.size(); i++) { + int elementComparison = this.elements.get(i).compareTo(other.elements.get(i)); + if (elementComparison != 0) return elementComparison; + } + + return 0; + } + + @Override + public int hashCode() { + return elements.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof TensorAddress)) return false; + return ((TensorAddress)other).elements.equals(this.elements); + } + + /** Returns this on the form {dimension1:label1,dimension2:label2,... */ + @Override + public String toString() { + StringBuilder b = new StringBuilder("{"); + for (TensorAddress.Element element : elements) { + //if (element.label() == Element.undefinedLabel) continue; + b.append(element.toString()); + b.append(","); + } + if (b.length() > 1) + b.setLength(b.length() - 1); + b.append("}"); + return b.toString(); + } + + /** A tensor address element. Elements have the lexical order of the dimensions as natural order. */ + public static class Element implements Comparable<Element> { + + static final String undefinedLabel = "-"; + + private final String dimension; + private final String label; + private final int hashCode; + + public Element(String dimension, String label) { + this.dimension = dimension; + if (label.equals(undefinedLabel)) + this.label = undefinedLabel; + else + this.label = label; + this.hashCode = dimension.hashCode() + label.hashCode(); + } + + public String dimension() { return dimension; } + + public String label() { return label; } + + @Override + public int compareTo(Element other) { + return this.dimension.compareTo(other.dimension); + } + + @Override + public int hashCode() { return hashCode; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof Element)) return false; + Element other = (Element)o; + if ( ! other.dimension.equals(this.dimension)) return false; + if ( ! other.label.equals(this.label)) return false; + return true; + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append(dimension).append(":").append(label); + return b.toString(); + } + + } + + /** Supports building of a tensor address */ + public static class Builder { + + private final List<Element> elements = new ArrayList<>(); + + /** + * Adds a label in a dimension to this. + * + * @return this for convenience + */ + public Builder add(String dimension, String label) { + elements.add(new Element(dimension, label)); + return this; + } + + public TensorAddress build() { + Collections.sort(elements); // Consistent order to get a consistent hash + return TensorAddress.fromSorted(elements); + } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorDifference.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorDifference.java new file mode 100644 index 00000000000..ceb003b1615 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorDifference.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Takes the difference between two tensors, see {@link Tensor#subtract} + * + * @author bratseth + */ +class TensorDifference { + + private final Set<String> dimensions; + private final Map<TensorAddress, Double> cells = new HashMap<>(); + + public TensorDifference(Tensor a, Tensor b) { + this.dimensions = TensorOperations.combineDimensions(a, b); + cells.putAll(a.cells()); + for (Map.Entry<TensorAddress, Double> bCell : b.cells().entrySet()) + cells.put(bCell.getKey(), a.cells().getOrDefault(bCell.getKey(), 0d) - bCell.getValue()); + } + + /** Returns the result of taking this sum */ + public Tensor result() { + return new MapTensor(dimensions, cells); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorDimensionSum.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorDimensionSum.java new file mode 100644 index 00000000000..3cd791fc60e --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorDimensionSum.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Returns a tensor with the given dimension removed and the cell values in that dimension summed + * + * @author bratseth + */ +class TensorDimensionSum { + + private final Set<String> dimensions; + private final Map<TensorAddress, Double> cells = new HashMap<>(); + + public TensorDimensionSum(String dimension, Tensor t) { + dimensions = new HashSet<>(t.dimensions()); + dimensions.remove(dimension); + + for (Map.Entry<TensorAddress, Double> cell : t.cells().entrySet()) { + TensorAddress reducedAddress = removeDimension(dimension, cell.getKey()); + Double newValue = cell.getValue(); + Double existingValue = cells.get(reducedAddress); + if (existingValue != null) + newValue += existingValue; + cells.put(reducedAddress, newValue); + } + } + + private TensorAddress removeDimension(String dimension, TensorAddress address) { + List<TensorAddress.Element> reducedAddress = new ArrayList<>(); + for (TensorAddress.Element element : address.elements()) + if ( ! element.dimension().equals(dimension)) + reducedAddress.add(element); + return TensorAddress.fromSorted(reducedAddress); + } + + /** Returns the result of taking this sum */ + public MapTensor result() { return new MapTensor(dimensions, cells); } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorFunction.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorFunction.java new file mode 100644 index 00000000000..db73626d6d0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorFunction.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; +import java.util.Set; +import java.util.function.UnaryOperator; + +/** + * Computes the tensor with some function to all the cells of the input tensor + * + * @author bratseth + */ +class TensorFunction { + + private final Set<String> dimensions; + private final ImmutableMap.Builder<TensorAddress, Double> cells = new ImmutableMap.Builder<>(); + + public TensorFunction(Tensor t, UnaryOperator<Double> f) { + dimensions = t.dimensions(); + for (Map.Entry<TensorAddress, Double> cell : t.cells().entrySet()) { + cells.put(cell.getKey(), f.apply(cell.getValue())); + } + } + + /** Returns the result of taking this sum */ + public MapTensor result() { + return new MapTensor(dimensions, cells.build()); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorMax.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorMax.java new file mode 100644 index 00000000000..d15e5092476 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorMax.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Takes the max of each cell of two tensors, see {@link Tensor#max} + * + * @author bratseth + */ +class TensorMax { + + private final Set<String> dimensions; + private final Map<TensorAddress, Double> cells = new HashMap<>(); + + public TensorMax(Tensor a, Tensor b) { + dimensions = TensorOperations.combineDimensions(a, b); + cells.putAll(a.cells()); + for (Map.Entry<TensorAddress, Double> bCell : b.cells().entrySet()) { + Double aValue = a.cells().get(bCell.getKey()); + if (aValue == null) + cells.put(bCell.getKey(), bCell.getValue()); + else + cells.put(bCell.getKey(), Math.max(aValue, bCell.getValue())); + } + } + + /** Returns the result of taking this sum */ + public Tensor result() { + return new MapTensor(dimensions, cells); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorMin.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorMin.java new file mode 100644 index 00000000000..e389dea3883 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorMin.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Takes the min of each cell of two tensors, see {@link Tensor#min} + * + * @author bratseth + */ +class TensorMin { + + private final Set<String> dimensions; + private final Map<TensorAddress, Double> cells = new HashMap<>(); + + public TensorMin(Tensor a, Tensor b) { + dimensions = TensorOperations.combineDimensions(a, b); + cells.putAll(a.cells()); + for (Map.Entry<TensorAddress, Double> bCell : b.cells().entrySet()) { + Double aValue = a.cells().get(bCell.getKey()); + if (aValue == null) + cells.put(bCell.getKey(), bCell.getValue()); + else + cells.put(bCell.getKey(), Math.min(aValue, bCell.getValue())); + } + } + + /** Returns the result of taking this sum */ + public Tensor result() { return new MapTensor(dimensions, cells); } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorOperations.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorOperations.java new file mode 100644 index 00000000000..aca306b914c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorOperations.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.collect.ImmutableSet; + +import java.util.Set; + +/** + * Functions on tensors + * + * @author bratseth + */ +class TensorOperations { + + /** + * A utility method which returns an ummutable set of the union of the dimensions + * of the two argument tensors. + * + * @return the combined dimensions as an unmodifiable set + */ + static Set<String> combineDimensions(Tensor a, Tensor b) { + ImmutableSet.Builder<String> setBuilder = new ImmutableSet.Builder<>(); + setBuilder.addAll(a.dimensions()); + setBuilder.addAll(b.dimensions()); + return setBuilder.build(); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorProduct.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorProduct.java new file mode 100644 index 00000000000..221bd985380 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorProduct.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.collect.ImmutableMap; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; + +/** + * Computes a <i>sparse tensor product</i>, see {@link Tensor#multiply} + * + * @author bratseth + */ +class TensorProduct { + + private final Set<String> dimensionsA, dimensionsB; + + private final Set<String> dimensions; + private final ImmutableMap.Builder<TensorAddress, Double> cells = new ImmutableMap.Builder<>(); + + public TensorProduct(Tensor a, Tensor b) { + dimensionsA = a.dimensions(); + dimensionsB = b.dimensions(); + + // Dimension product + dimensions = TensorOperations.combineDimensions(a, b); + + // Cell product (slow baseline implementation) + for (Map.Entry<TensorAddress, Double> aCell : a.cells().entrySet()) { + for (Map.Entry<TensorAddress, Double> bCell : b.cells().entrySet()) { + TensorAddress combinedAddress = combine(aCell.getKey(), bCell.getKey()); + if (combinedAddress == null) continue; // not combinable + cells.put(combinedAddress, aCell.getValue() * bCell.getValue()); + } + } + } + + private TensorAddress combine(TensorAddress a, TensorAddress b) { + List<TensorAddress.Element> combined = new ArrayList<>(); + combined.addAll(dense(a, dimensionsA)); + combined.addAll(dense(b, dimensionsB)); + Collections.sort(combined); + TensorAddress.Element previous = null; + for (ListIterator<TensorAddress.Element> i = combined.listIterator(); i.hasNext(); ) { + TensorAddress.Element current = i.next(); + if (previous != null && previous.dimension().equals(current.dimension())) { // an overlapping dimension + if (previous.label().equals(current.label())) + i.remove(); // a match: remove the duplicate + else + return null; // no match: a combination isn't viable + } + previous = current; + } + return TensorAddress.fromSorted(sparse(combined)); + } + + /** + * Returns a set of tensor elements which contains an entry for each dimension including "undefined" values + * (which are not present in the sparse elements list). + */ + private List<TensorAddress.Element> dense(TensorAddress sparse, Set<String> dimensions) { + if (sparse.elements().size() == dimensions.size()) return sparse.elements(); + + List<TensorAddress.Element> dense = new ArrayList<>(sparse.elements()); + for (String dimension : dimensions) { + if ( ! sparse.hasDimension(dimension)) + dense.add(new TensorAddress.Element(dimension, TensorAddress.Element.undefinedLabel)); + } + return dense; + } + + /** + * Removes any "undefined" entries from the given elements. + */ + private List<TensorAddress.Element> sparse(List<TensorAddress.Element> dense) { + List<TensorAddress.Element> sparse = new ArrayList<>(); + for (TensorAddress.Element element : dense) { + if ( ! element.label().equals(TensorAddress.Element.undefinedLabel)) + sparse.add(element); + } + return sparse; + } + + /** Returns the result of taking this product */ + public Tensor result() { + return new MapTensor(dimensions, cells.build()); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorSum.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorSum.java new file mode 100644 index 00000000000..85dfa289bd3 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorSum.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Takes the sum of two tensors, see {@link Tensor#add} + * + * @author bratseth + */ +class TensorSum { + + private final Set<String> dimensions; + private final Map<TensorAddress, Double> cells = new HashMap<>(); + + public TensorSum(Tensor a, Tensor b) { + dimensions = TensorOperations.combineDimensions(a, b); + cells.putAll(a.cells()); + for (Map.Entry<TensorAddress, Double> bCell : b.cells().entrySet()) { + cells.put(bCell.getKey(), a.cells().getOrDefault(bCell.getKey(), 0d) + bCell.getValue()); + } + } + + /** Returns the result of taking this sum */ + public Tensor result() { return new MapTensor(dimensions, cells); } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java new file mode 100644 index 00000000000..507a2f9f612 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -0,0 +1,195 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.annotations.Beta; +import com.google.common.collect.ImmutableList; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * A tensor type with its dimensions. This is immutable. + * <p> + * A dimension can be indexed (bound or unbound) or mapped. + * Currently, we only support tensor types where all dimensions have the same type. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +@Beta +public class TensorType { + + public static abstract class Dimension { + + public enum Type { indexedBound, indexedUnbound, mapped } + + private final String name; + + private Dimension(String name) { this.name = name; } + + public final String name() { return name; } + + /** Returns the size of this dimension if it is indexedUnbound, empty otherwise */ + public abstract Optional<Integer> size(); + + public abstract Type type(); + + @Override + public abstract String toString(); + + } + + public static class IndexedBoundDimension extends TensorType.Dimension { + + private final Optional<Integer> size; + + private IndexedBoundDimension(String name, int size) { + super(name); + if (size < 1) + throw new IllegalArgumentException("Size of bound dimension '" + name + "' must be at least 1"); + this.size = Optional.of(size); + } + + @Override + public Optional<Integer> size() { return size; } + + @Override + public Type type() { return Type.indexedBound; } + + @Override + public String toString() { return name() + "[" + size.get() + "]"; } + + } + + public static class IndexedUnboundDimension extends TensorType.Dimension { + + private IndexedUnboundDimension(String name) { + super(name); + } + + @Override + public Optional<Integer> size() { return Optional.empty(); } + + @Override + public Type type() { return Type.indexedUnbound; } + + @Override + public String toString() { return name() + "[]"; } + + } + + public static class MappedDimension extends TensorType.Dimension { + + private MappedDimension(String name) { + super(name); + } + + @Override + public Optional<Integer> size() { return Optional.empty(); } + + @Override + public Type type() { return Type.mapped; } + + @Override + public String toString() { return name() + "{}"; } + + } + + public static class Builder { + + private final Map<String, Dimension> dimensions = new LinkedHashMap<>(); + private Dimension prevDimension = null; + + private Builder add(Dimension dimension) { + if (!dimensions.isEmpty()) { + validateDimensionName(dimension); + validateDimensionType(dimension); + } + + dimensions.put(dimension.name(), dimension); + prevDimension = dimension; + return this; + } + + private void validateDimensionName(Dimension newDimension) { + Dimension prevDimension = dimensions.get(newDimension.name()); + if (prevDimension != null) { + throw new IllegalArgumentException("Expected all dimensions to have unique names, " + + "but '" + prevDimension + "' and '" + newDimension + "' have the same name"); + } + } + + private void validateDimensionType(Dimension newDimension) { + if (prevDimension.type() != newDimension.type()) { + throw new IllegalArgumentException("Expected all dimensions to have the same type, " + + "but '" + prevDimension + "' does not have the same type as '" + newDimension + "'"); + } + } + + public Builder indexedBound(String name, int size) { + return add(new IndexedBoundDimension(name, size)); + } + + public Builder indexedUnbound(String name) { + return add(new IndexedUnboundDimension(name)); + } + + public Builder mapped(String name) { + return add(new MappedDimension(name)); + } + + public TensorType build() { + return new TensorType(dimensions.values()); + } + } + + private final List<Dimension> dimensions; + + private TensorType(Collection<Dimension> dimensions) { + this.dimensions = ImmutableList.copyOf(dimensions); + } + + /** + * Returns a tensor type instance from a string on the format + * <code>tensor(dimension1, dimension2, ...)</code> + * where each dimension is either + * <ul> + * <li><code>dimension-name[]</code> - an unbound indexed dimension + * <li><code>dimension-name[int]</code> - an bound indexed dimension + * <li><code>dimension-name{}</code> - a mapped dimension + * </ul> + * Example: <code>tensor(x[10],y[20])</code> (a matrix) + */ + public static TensorType fromSpec(String specString) { + return TensorTypeParser.fromSpec(specString); + } + + /** Returns an immutable list of the dimensions of this */ + public List<Dimension> dimensions() { return dimensions; } + + @Override + public String toString() { + return "tensor(" + dimensions.stream().map(Dimension::toString).collect(Collectors.joining(",")) + ")"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TensorType that = (TensorType) o; + + if (!dimensions.equals(that.dimensions)) return false; + + return true; + } + + @Override + public int hashCode() { + return dimensions.hashCode(); + } +} + diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java new file mode 100644 index 00000000000..3d2e1663971 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.annotations.Beta; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Class for parsing a tensor type spec. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +@Beta +class TensorTypeParser { + + private final static String START_STRING = "tensor("; + private final static String END_STRING = ")"; + + private static final Pattern indexedPattern = Pattern.compile("(\\w+)\\[(\\d*)\\]"); + private static final Pattern mappedPattern = Pattern.compile("(\\w+)\\{\\}"); + + static TensorType fromSpec(String specString) { + if (!specString.startsWith(START_STRING) || !specString.endsWith(END_STRING)) { + throw new IllegalArgumentException("Tensor type spec must start with '" + START_STRING + "'" + + " and end with '" + END_STRING + "', but was '" + specString + "'"); + } + TensorType.Builder builder = new TensorType.Builder(); + String dimensionsSpec = specString.substring(START_STRING.length(), specString.length() - END_STRING.length()); + if (dimensionsSpec.isEmpty()) { + return builder.build(); + } + for (String element : dimensionsSpec.split(",")) { + String trimmedElement = element.trim(); + if (tryParseIndexedDimension(trimmedElement, builder)) { + } else if (tryParseMappedDimension(trimmedElement, builder)) { + } else { + throw new IllegalArgumentException("Failed parsing element '" + element + + "' in type spec '" + specString + "'"); + } + } + return builder.build(); + } + + private static boolean tryParseIndexedDimension(String element, TensorType.Builder builder) { + Matcher matcher = indexedPattern.matcher(element); + if (matcher.matches()) { + String dimensionName = matcher.group(1); + String dimensionSize = matcher.group(2); + if (dimensionSize.isEmpty()) { + builder.indexedUnbound(dimensionName); + } else { + builder.indexedBound(dimensionName, Integer.valueOf(dimensionSize)); + } + return true; + } + return false; + } + + private static boolean tryParseMappedDimension(String element, TensorType.Builder builder) { + Matcher matcher = mappedPattern.matcher(element); + if (matcher.matches()) { + String dimensionName = matcher.group(1); + builder.mapped(dimensionName); + return true; + } + return false; + } +} + diff --git a/vespajlib/src/main/java/com/yahoo/tensor/package-info.java b/vespajlib/src/main/java/com/yahoo/tensor/package-info.java new file mode 100644 index 00000000000..13ca7fa8a13 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/package-info.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Tensor data types + * + * @author bratseth + */ +@ExportPackage +@PublicApi +package com.yahoo.tensor; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/BinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/BinaryFormat.java new file mode 100644 index 00000000000..97d62d5169a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/BinaryFormat.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor.serialization; + +import com.google.common.annotations.Beta; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.Tensor; + +/** + * Representation of a specific binary format with functions for serializing a Tensor object into + * this format or de-serializing binary data into a Tensor object. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +@Beta +interface BinaryFormat { + + /** + * Serialize the given tensor into binary format. + */ + public void encode(GrowableByteBuffer buffer, Tensor tensor); + + /** + * Deserialize the given binary data into a Tensor object. + */ + public Tensor decode(GrowableByteBuffer buffer); +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/CompactBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/CompactBinaryFormat.java new file mode 100644 index 00000000000..0c1f04552f4 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/CompactBinaryFormat.java @@ -0,0 +1,113 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor.serialization; + +import com.google.common.annotations.Beta; +import com.google.common.collect.Sets; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.MapTensorBuilder; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorAddress; +import com.yahoo.text.Utf8; + +import java.util.*; + +/** + * Implementation of a compact binary format for a tensor on the form: + * + * Sorted dimensions = num_dimensions [dimension_str_len dimension_str_bytes]* + * Cells = num_cells [label_1_str_len label_1_str_bytes ... label_N_str_len label_N_str_bytes cell_value]* + * + * Note that the dimensions are sorted and the tensor address labels are given in the same sorted order. + * Unspecified labels are encoded as the empty string "". + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +@Beta +class CompactBinaryFormat implements BinaryFormat { + + @Override + public void encode(GrowableByteBuffer buffer, Tensor tensor) { + List<String> sortedDimensions = new ArrayList<>(tensor.dimensions()); + Collections.sort(sortedDimensions); + encodeDimensions(buffer, sortedDimensions); + encodeCells(buffer, tensor.cells(), sortedDimensions); + } + + private static void encodeDimensions(GrowableByteBuffer buffer, List<String> sortedDimensions) { + buffer.putInt1_4Bytes(sortedDimensions.size()); + for (String dimension : sortedDimensions) { + encodeString(buffer, dimension); + } + } + + private static void encodeCells(GrowableByteBuffer buffer, Map<TensorAddress, Double> cells, + List<String> sortedDimensions) { + buffer.putInt1_4Bytes(cells.size()); + for (Map.Entry<TensorAddress, Double> cellEntry : cells.entrySet()) { + encodeAddress(buffer, cellEntry.getKey(), sortedDimensions); + buffer.putDouble(cellEntry.getValue().doubleValue()); + } + } + + private static void encodeAddress(GrowableByteBuffer buffer, TensorAddress address, List<String> sortedDimensions) { + for (String dimension : sortedDimensions) { + Optional<TensorAddress.Element> element = + address.elements().stream().filter(elem -> elem.dimension().equals(dimension)).findFirst(); + String label = (element.isPresent() ? element.get().label() : ""); + encodeString(buffer, label); + } + } + + private static void encodeString(GrowableByteBuffer buffer, String value) { + byte[] stringBytes = Utf8.toBytes(value); + buffer.putInt1_4Bytes(stringBytes.length); + buffer.put(stringBytes); + } + + @Override + public Tensor decode(GrowableByteBuffer buffer) { + List<String> sortedDimensions = decodeDimensions(buffer); + MapTensorBuilder builder = new MapTensorBuilder(); + for (String dimension : sortedDimensions) { + builder.dimension(dimension); + } + decodeCells(buffer, builder, sortedDimensions); + return builder.build(); + } + + private static List<String> decodeDimensions(GrowableByteBuffer buffer) { + int numDimensions = buffer.getInt1_4Bytes(); + List<String> sortedDimensions = new ArrayList<>(); + for (int i = 0; i < numDimensions; ++i) { + sortedDimensions.add(decodeString(buffer)); + } + return sortedDimensions; + } + + private static void decodeCells(GrowableByteBuffer buffer, MapTensorBuilder builder, + List<String> sortedDimensions) { + int numCells = buffer.getInt1_4Bytes(); + for (int i = 0; i < numCells; ++i) { + MapTensorBuilder.CellBuilder cellBuilder = builder.cell(); + decodeAddress(buffer, cellBuilder, sortedDimensions); + cellBuilder.value(buffer.getDouble()); + } + } + + private static void decodeAddress(GrowableByteBuffer buffer, MapTensorBuilder.CellBuilder builder, + List<String> sortedDimensions) { + for (String dimension : sortedDimensions) { + String label = decodeString(buffer); + if (!label.isEmpty()) { + builder.label(dimension, label); + } + } + } + + private static String decodeString(GrowableByteBuffer buffer) { + int stringLength = buffer.getInt1_4Bytes(); + byte[] stringBytes = new byte[stringLength]; + buffer.get(stringBytes); + return Utf8.toString(stringBytes); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java new file mode 100644 index 00000000000..cdd26a11ac2 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor.serialization; + +import com.google.common.annotations.Beta; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.Tensor; + +/** + * Class used by clients for serializing a Tensor object into binary format or + * de-serializing binary data into a Tensor object. + * + * The actual binary format used is not a concern for the client and + * is hidden in this class and in the binary data. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +@Beta +public class TypedBinaryFormat { + + private static final int COMPACT_BINARY_FORMAT_TYPE = 1; + + public static byte[] encode(Tensor tensor) { + GrowableByteBuffer buffer = new GrowableByteBuffer(); + buffer.putInt1_4Bytes(COMPACT_BINARY_FORMAT_TYPE); + new CompactBinaryFormat().encode(buffer, tensor); + buffer.flip(); + byte[] result = new byte[buffer.remaining()]; + buffer.get(result); + return result; + } + + public static Tensor decode(byte[] data) { + GrowableByteBuffer buffer = GrowableByteBuffer.wrap(data); + int formatType = buffer.getInt1_4Bytes(); + switch (formatType) { + case COMPACT_BINARY_FORMAT_TYPE: + return new CompactBinaryFormat().decode(buffer); + default: + throw new IllegalArgumentException("Binary format type " + formatType + " is not a known format"); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/package-info.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/package-info.java new file mode 100644 index 00000000000..72027284bc1 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.tensor.serialization; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/text/AbstractUtf8Array.java b/vespajlib/src/main/java/com/yahoo/text/AbstractUtf8Array.java new file mode 100644 index 00000000000..1a11e30dd9d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/AbstractUtf8Array.java @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.nio.ByteBuffer; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public abstract class AbstractUtf8Array implements Comparable<AbstractUtf8Array> { + /** + * This will write the utf8 sequence to the given target. + */ + final public void writeTo(ByteBuffer target) { + target.put(getBytes(), getByteOffset(), getByteLength()); + } + + /** + * This will return the byte at the given position. + */ + public byte getByte(int index) { return getBytes()[getByteOffset() + index]; } + + /** + * + * @return Length in bytes of the utf8 sequence. + */ + public abstract int getByteLength(); + + /** + * Wraps the utf8 sequence in a ByteBuffer + * @return The wrapping buffer. + */ + public ByteBuffer wrap() { return ByteBuffer.wrap(getBytes(), getByteOffset(), getByteLength()); } + + /** + * + * @return The backing byte array. + */ + protected abstract byte [] getBytes(); + + public boolean isEmpty() { return getByteLength() == 0; } + + /** + * + * @return The offset in the backing array where the utf8 sequence starts. + */ + protected abstract int getByteOffset(); + @Override + public int hashCode() { + final int l = getByteLength(); + final int c = getByteOffset(); + final byte [] b = getBytes(); + int h = 0; + for (int i=0; i < l; i++) { + int v = b[c+i]; + h ^= v << ((i%4)*8); + } + return h; + } + @Override + public boolean equals(Object o) { + if (o instanceof AbstractUtf8Array) { + AbstractUtf8Array other = (AbstractUtf8Array)o; + return compareTo(other) == 0; + } else if (o instanceof String) { + return toString().equals(o); + } + return false; + } + + /** + * Will convert the utf8 sequence to a Java string + * @return The converted Java String + */ + @Override + public String toString() { + return Utf8.toString(getBytes(), getByteOffset(), getByteLength()); + } + + @Override + public int compareTo(AbstractUtf8Array rhs) { + final int l = getByteLength(); + final int rl = rhs.getByteLength(); + if (l < rl) { + return -1; + } else if (l > rl) { + return 1; + } else { + final byte [] b = getBytes(); + final byte [] rb = rhs.getBytes(); + final int c = getByteOffset(); + final int rc = rhs.getByteOffset(); + for (int i=0; i < l; i++) { + if (b[c+i] < rb[rc+i]) { + return -1; + } else if (b[c+i] > rb[rc+i]) { + return 1; + } + } + return 0; + } + } + + public Utf8Array ascii7BitLowerCase() { + byte [] upper = new byte[getByteLength()]; + + for (int i=0; i< upper.length; i++ ) { + byte b = getByte(i); + if ((b >= 0x41) && (b < (0x41+26))) { + b |= 0x20; // Lowercase + } + upper[i] = b; + } + return new Utf8Array(upper); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Ascii.java b/vespajlib/src/main/java/com/yahoo/text/Ascii.java new file mode 100644 index 00000000000..552a46fd36f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Ascii.java @@ -0,0 +1,226 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.util.Iterator; +import java.util.Set; +import java.util.TreeSet; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class Ascii { + + public final static char ESCAPE_CHAR = '\\'; + + public static String encode(String str, Charset charset, int... requiresEscape) { + return newEncoder(charset, requiresEscape).encode(str); + } + + public static String decode(String str, Charset charset) { + return newDecoder(charset).decode(str); + } + + public static Encoder newEncoder(Charset charset, int... requiresEscape) { + switch (requiresEscape.length) { + case 0: + return new Encoder(charset, new EmptyPredicate()); + case 1: + return new Encoder(charset, new SingletonPredicate(requiresEscape[0])); + default: + return new Encoder(charset, new ArrayPredicate(requiresEscape)); + } + } + + public static Decoder newDecoder(Charset charset) { + return new Decoder(charset); + } + + public static class Encoder { + + private final Charset charset; + private final EncodePredicate predicate; + + private Encoder(Charset charset, EncodePredicate predicate) { + this.charset = charset; + this.predicate = predicate; + } + + public String encode(String str) { + StringBuilder out = new StringBuilder(); + for (int c : new CodePointSequence(str)) { + if (c < 0x20 || c >= 0x7F || c == ESCAPE_CHAR || predicate.requiresEscape(c)) { + escape(c, out); + } else { + out.appendCodePoint(c); + } + } + return out.toString(); + } + + private void escape(int c, StringBuilder out) { + switch (c) { + case ESCAPE_CHAR: + out.append(ESCAPE_CHAR).append(ESCAPE_CHAR); + break; + case '\f': + out.append(ESCAPE_CHAR).append("f"); + break; + case '\n': + out.append(ESCAPE_CHAR).append("n"); + break; + case '\r': + out.append(ESCAPE_CHAR).append("r"); + break; + case '\t': + out.append(ESCAPE_CHAR).append("t"); + break; + default: + ByteBuffer buf = charset.encode(CharBuffer.wrap(Character.toChars(c))); + while (buf.hasRemaining()) { + out.append(ESCAPE_CHAR).append(String.format("x%02X", buf.get())); + } + break; + } + } + } + + public static class Decoder { + + private final Charset charset; + + private Decoder(Charset charset) { + this.charset = charset; + } + + public String decode(String str) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (Iterator<Integer> it = new CodePointIterator(str); it.hasNext(); ) { + int c = it.next(); + if (c == ESCAPE_CHAR) { + unescape(it, out); + } else { + ByteBuffer buf = charset.encode(CharBuffer.wrap(Character.toChars(c))); + while (buf.hasRemaining()) { + out.write(buf.get()); + } + } + } + return new String(out.toByteArray(), charset); + } + + private void unescape(Iterator<Integer> it, ByteArrayOutputStream out) { + int c = it.next(); + switch (c) { + case 'f': + out.write('\f'); + break; + case 'n': + out.write('\n'); + break; + case 'r': + out.write('\r'); + break; + case 't': + out.write('\t'); + break; + case 'x': + int x1 = it.next(); + int x2 = it.next(); + out.write((Character.digit(x1, 16) << 4) + + (Character.digit(x2, 16))); + break; + default: + out.write(c); + break; + } + } + } + + private static interface EncodePredicate { + + boolean requiresEscape(int codePoint); + } + + private static class EmptyPredicate implements EncodePredicate { + + @Override + public boolean requiresEscape(int codePoint) { + return false; + } + } + + private static class SingletonPredicate implements EncodePredicate { + + final int requiresEscape; + + private SingletonPredicate(int requiresEscape) { + this.requiresEscape = requiresEscape; + } + + @Override + public boolean requiresEscape(int codePoint) { + return codePoint == requiresEscape; + } + } + + private static class ArrayPredicate implements EncodePredicate { + + final Set<Integer> requiresEscape = new TreeSet<>(); + + private ArrayPredicate(int[] requiresEscape) { + for (int codePoint : requiresEscape) { + this.requiresEscape.add(codePoint); + } + } + + @Override + public boolean requiresEscape(int codePoint) { + return requiresEscape.contains(codePoint); + } + } + + private static class CodePointSequence implements Iterable<Integer> { + + final String str; + + CodePointSequence(String str) { + this.str = str; + } + + @Override + public Iterator<Integer> iterator() { + return new CodePointIterator(str); + } + } + + private static class CodePointIterator implements Iterator<Integer> { + + final String str; + int idx = 0; + + CodePointIterator(String str) { + this.str = str; + } + + @Override + public boolean hasNext() { + return idx < str.length(); + } + + @Override + public Integer next() { + int c = str.codePointAt(idx); + idx += Character.charCount(c); + return c; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/BooleanParser.java b/vespajlib/src/main/java/com/yahoo/text/BooleanParser.java new file mode 100644 index 00000000000..a17d821ff9d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/BooleanParser.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * Utility class parsing a string into a boolean. + * In contrast to Boolean.parseBoolean in the Java API this parser is strict. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class BooleanParser { + + /** + * Returns true if the input string is case insensitive equal to "true" and + * false if it is case insensitive equal to "false". + * In any other case an exception is thrown. + * + * @param s the string to parse + * @return true if s is "true", false if it is "false" + * @throws IllegalArgumentException if s is not null but neither "true" or "false" + * @throws NullPointerException if s is null + */ + public static boolean parseBoolean(String s) { + if (s==null) + throw new NullPointerException("Expected 'true' or 'false', got NULL"); + if (s.equalsIgnoreCase("false")) + return false; + if (s.equalsIgnoreCase("true")) + return true; + throw new IllegalArgumentException("Expected 'true' or 'false', got '" + s + "'"); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/CaseInsensitiveIdentifier.java b/vespajlib/src/main/java/com/yahoo/text/CaseInsensitiveIdentifier.java new file mode 100644 index 00000000000..258f5f74d14 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/CaseInsensitiveIdentifier.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 11.11.12 + * Time: 11:25 + * To change this template use File | Settings | File Templates. + */ +public class CaseInsensitiveIdentifier extends Identifier { + private final Identifier original; + + public CaseInsensitiveIdentifier(String s) { + this(new Utf8String(s)); + } + public CaseInsensitiveIdentifier(byte [] utf8) { + this(new Utf8Array(utf8)); + } + public CaseInsensitiveIdentifier(AbstractUtf8Array utf8) { + super(utf8.ascii7BitLowerCase()); + original = new Identifier(utf8); + } + public String toString() { return original.toString(); } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/DataTypeIdentifier.java b/vespajlib/src/main/java/com/yahoo/text/DataTypeIdentifier.java new file mode 100644 index 00000000000..364cb87d6f7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/DataTypeIdentifier.java @@ -0,0 +1,134 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 11.11.12 + * Time: 21:11 + * To change this template use File | Settings | File Templates. + */ +public class DataTypeIdentifier { + private static final byte [] ARRAY = {'a', 'r', 'r', 'a', 'y'}; + private static final byte [] ANNOTATIONREFERENCE = {'a','n','n','o','t','a','t','i','o','n','r','e','f','e','r','e','n','c','e'}; + private static final byte [] MAP = { 'm', 'a', 'p'}; + private static final byte [] WSET = {'w', 'e', 'i', 'g', 'h', 't', 'e', 'd', 's', 'e', 't'}; + private static final byte [] CREATEIFNONEXISTENT = {';','a', 'd', 'd'}; + private static final byte [] REMOVEIFZERO = {';','r', 'e', 'm', 'o', 'v', 'e'}; + private static final byte [] CREATANDREMOVE = {';','a', 'd', 'd',';','r', 'e', 'm', 'o', 'v', 'e'}; + private static final byte [] EMPTY = {}; + private Utf8String utf8; + public DataTypeIdentifier(String s) { + utf8 = new Utf8String(s); + verify(utf8.wrap().array()); + } + public DataTypeIdentifier(AbstractUtf8Array utf8) { + this.utf8 = new Utf8String(utf8); + verify(utf8.wrap().array()); + } + public DataTypeIdentifier(byte [] utf8) { + this(new Utf8Array(utf8)); + } + + private DataTypeIdentifier(final byte [] prefix, DataTypeIdentifier nested, final byte [] postfix) { + utf8 = new Utf8String(new Utf8Array(createPrefixDataType(prefix, nested, postfix))); + } + private DataTypeIdentifier(final byte [] prefix, DataTypeIdentifier key, DataTypeIdentifier value) { + utf8 = new Utf8String(new Utf8Array(createMapDataType(prefix, key, value))); + } + + public static DataTypeIdentifier createArrayDataTypeIdentifier(DataTypeIdentifier nested) { + return new DataTypeIdentifier(ARRAY, nested, EMPTY); + } + public static DataTypeIdentifier createAnnotationReferenceDataTypeIdentifier(DataTypeIdentifier nested) { + return new DataTypeIdentifier(ANNOTATIONREFERENCE, nested, EMPTY); + } + public static DataTypeIdentifier createMapDataTypeIdentifier(DataTypeIdentifier key, DataTypeIdentifier value) { + return new DataTypeIdentifier(MAP, key, value); + } + public static DataTypeIdentifier createWeightedSetTypeIdentifier(DataTypeIdentifier nested, boolean createIfNonExistent, boolean removeIfZero) { + return new DataTypeIdentifier(WSET, nested, createPostfix(createIfNonExistent, removeIfZero)); + } + @Override + public int hashCode() { + return utf8.hashCode(); + } + @Override + public boolean equals(Object obj) { + if (obj instanceof DataTypeIdentifier) { + return utf8.equals(((DataTypeIdentifier)obj).utf8); + } + return false; + } + @Override + public String toString() { + return utf8.toString(); + } + public final Utf8String getUtf8() { + return utf8; + } + private static byte [] createPostfix(boolean createIfNonExistent, boolean removeIfZero) { + if (createIfNonExistent && removeIfZero) { + return CREATANDREMOVE; + } else if (createIfNonExistent) { + return CREATEIFNONEXISTENT; + } else if (removeIfZero) { + return REMOVEIFZERO; + } + return EMPTY; + } + private static byte [] createPrefixDataType(final byte [] prefix, final DataTypeIdentifier nested, final byte [] postfix) { + byte [] whole = new byte[prefix.length + 2 + nested.utf8.getByteLength() + postfix.length]; + for (int i=0; i < prefix.length; i++) { + whole[i] = prefix[i]; + } + whole[prefix.length] = '<'; + for (int i = 0, m=nested.utf8.getByteLength(); i < m; i++ ) { + whole[prefix.length+1+i] = nested.utf8.getByte(i); + } + whole[prefix.length + 1 + nested.utf8.getByteLength()] = '>'; + for (int i = 0; i < postfix.length; i++) { + whole[prefix.length + 1 + nested.utf8.length() + 1 + i] = postfix[i]; + } + return whole; + } + private static byte [] createMapDataType(final byte [] prefix, final DataTypeIdentifier key, final DataTypeIdentifier value) { + byte [] whole = new byte[prefix.length + 3 + key.utf8.getByteLength() + value.utf8.getByteLength()]; + for (int i=0; i < prefix.length; i++) { + whole[i] = prefix[i]; + } + whole[prefix.length] = '<'; + for (int i = 0, m=key.utf8.getByteLength(); i < m; i++ ) { + whole[prefix.length+1+i] = key.utf8.getByte(i); + } + whole[prefix.length + 1 + key.utf8.getByteLength()] = ','; + for (int i = 0; i < value.utf8.getByteLength(); i++) { + whole[prefix.length + 1 + key.utf8.getByteLength() + 1 + i] = value.utf8.getByte(i); + } + whole[whole.length-1] = '>'; + return whole; + } + private static byte [] verify(final byte [] utf8) { + if (utf8.length > 0) { + verifyFirst(utf8[0], utf8); + for (int i=1; i < utf8.length; i++) { + verifyAny(utf8[i], utf8); + } + } + return utf8; + + } + private static boolean verifyFirst(byte c, byte [] identifier) { + if (!((c == '_') || ((c >= 'a') && (c <= 'z')))) { + throw new IllegalArgumentException("Illegal starting character '" + (char)c + "' of identifier '" + new Utf8String(new Utf8Array(identifier)).toString() +"'."); + } + return true; + } + private static boolean verifyAny(byte c, byte [] identifier) { + if (!((c == '_') || (c == '.') || ((c >= 'a') && (c <= 'z')) || ((c >= '0') && (c <= '9')))) { + throw new IllegalArgumentException("Illegal character '" + (char)c + "' of identifier '" + new Utf8String(new Utf8Array(identifier)).toString() +"'."); + } + return true; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/DoubleFormatter.java b/vespajlib/src/main/java/com/yahoo/text/DoubleFormatter.java new file mode 100644 index 00000000000..a11694f466c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/DoubleFormatter.java @@ -0,0 +1,532 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import com.google.common.annotations.Beta; + +/** + * Utility class to format a double into a String. + * <p> + * This is intended as a lower-cost replacement for the standard + * String.valueOf(double) since that method will cause lock + * contention if it's used too often. + * <p> + * Note that this implementation won't always produce the same results + * as java.lang.* formatting. + * <p> + * Also, this implementation is very poorly tested at the moment, so + * it should be used carefully, only in cases where you know the input + * will be well-defined and you don't need full precision. + * + * @author arnej27959 + */ + +@Beta +public final class DoubleFormatter { + + private static void tooSmall(StringBuilder target, long mantissa, int exponent) { + double carry = 0; + int prExp = 0; + while (exponent < 0) { + while (mantissa < (1L << 53)) { + carry *= 10.0; + mantissa *= 10; + ++prExp; + } + carry *= 0.5; + carry += 0.5*(mantissa & 1); + mantissa >>= 1; + ++exponent; + } + while (mantissa < (1L << 53)) { + carry *= 10.0; + mantissa *= 10; + ++prExp; + } + mantissa += (long)(carry+0.5); + + int[] befor = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + int b = 0; + for (int i = 0; mantissa > 0; i++) { + befor[i] = (int)(mantissa % 10); + mantissa /= 10; + ++b; + } + --b; + target.append((char)('0'+befor[b])); + target.append('.'); + if (b == 0) { + target.append('0'); + } else { + for (int i = b; i-- > 0; ) { + target.append((char)('0'+befor[i])); + } + } + prExp -= b; + target.append("E-"); + target.append(String.valueOf(prExp)); + } + + public static StringBuilder fmt(StringBuilder target, double v) { + append(target, v); + return target; + } + + public static String stringValue(double v) { + return fmt(new StringBuilder(), v).toString(); + } + + + //Hardcode some byte arrays to make them quickly available + public static final char[] INFINITY = {'I','n','f','i','n','i','t','y'}; + public static final char[] NaN = {'N','a','N'}; + public static final char[][] ZEROS = { + {}, + {'0'}, + {'0','0'}, + {'0','0','0'}, + {'0','0','0','0'}, + {'0','0','0','0','0'}, + {'0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0'}, + }; + + private static final char[] charForDigit = { + '0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f','g','h', + 'i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z' + }; + + //And required double related constants. + private static final long DoubleSignMask = 0x8000000000000000L; + private static final long DoubleExpMask = 0x7ff0000000000000L; + private static final long DoubleFractMask= ~(DoubleSignMask|DoubleExpMask); + private static final int DoubleExpShift = 52; + private static final int DoubleExpBias = 1023; + + private static final double[] d_tenthPowers = { + 1e-323D, 1e-322D, 1e-321D, 1e-320D, 1e-319D, 1e-318D, 1e-317D, 1e-316D, 1e-315D, 1e-314D, + 1e-313D, 1e-312D, 1e-311D, 1e-310D, 1e-309D, 1e-308D, 1e-307D, 1e-306D, 1e-305D, 1e-304D, + 1e-303D, 1e-302D, 1e-301D, 1e-300D, 1e-299D, 1e-298D, 1e-297D, 1e-296D, 1e-295D, 1e-294D, + 1e-293D, 1e-292D, 1e-291D, 1e-290D, 1e-289D, 1e-288D, 1e-287D, 1e-286D, 1e-285D, 1e-284D, + 1e-283D, 1e-282D, 1e-281D, 1e-280D, 1e-279D, 1e-278D, 1e-277D, 1e-276D, 1e-275D, 1e-274D, + 1e-273D, 1e-272D, 1e-271D, 1e-270D, 1e-269D, 1e-268D, 1e-267D, 1e-266D, 1e-265D, 1e-264D, + 1e-263D, 1e-262D, 1e-261D, 1e-260D, 1e-259D, 1e-258D, 1e-257D, 1e-256D, 1e-255D, 1e-254D, + 1e-253D, 1e-252D, 1e-251D, 1e-250D, 1e-249D, 1e-248D, 1e-247D, 1e-246D, 1e-245D, 1e-244D, + 1e-243D, 1e-242D, 1e-241D, 1e-240D, 1e-239D, 1e-238D, 1e-237D, 1e-236D, 1e-235D, 1e-234D, + 1e-233D, 1e-232D, 1e-231D, 1e-230D, 1e-229D, 1e-228D, 1e-227D, 1e-226D, 1e-225D, 1e-224D, + 1e-223D, 1e-222D, 1e-221D, 1e-220D, 1e-219D, 1e-218D, 1e-217D, 1e-216D, 1e-215D, 1e-214D, + 1e-213D, 1e-212D, 1e-211D, 1e-210D, 1e-209D, 1e-208D, 1e-207D, 1e-206D, 1e-205D, 1e-204D, + 1e-203D, 1e-202D, 1e-201D, 1e-200D, 1e-199D, 1e-198D, 1e-197D, 1e-196D, 1e-195D, 1e-194D, + 1e-193D, 1e-192D, 1e-191D, 1e-190D, 1e-189D, 1e-188D, 1e-187D, 1e-186D, 1e-185D, 1e-184D, + 1e-183D, 1e-182D, 1e-181D, 1e-180D, 1e-179D, 1e-178D, 1e-177D, 1e-176D, 1e-175D, 1e-174D, + 1e-173D, 1e-172D, 1e-171D, 1e-170D, 1e-169D, 1e-168D, 1e-167D, 1e-166D, 1e-165D, 1e-164D, + 1e-163D, 1e-162D, 1e-161D, 1e-160D, 1e-159D, 1e-158D, 1e-157D, 1e-156D, 1e-155D, 1e-154D, + 1e-153D, 1e-152D, 1e-151D, 1e-150D, 1e-149D, 1e-148D, 1e-147D, 1e-146D, 1e-145D, 1e-144D, + 1e-143D, 1e-142D, 1e-141D, 1e-140D, 1e-139D, 1e-138D, 1e-137D, 1e-136D, 1e-135D, 1e-134D, + 1e-133D, 1e-132D, 1e-131D, 1e-130D, 1e-129D, 1e-128D, 1e-127D, 1e-126D, 1e-125D, 1e-124D, + 1e-123D, 1e-122D, 1e-121D, 1e-120D, 1e-119D, 1e-118D, 1e-117D, 1e-116D, 1e-115D, 1e-114D, + 1e-113D, 1e-112D, 1e-111D, 1e-110D, 1e-109D, 1e-108D, 1e-107D, 1e-106D, 1e-105D, 1e-104D, + 1e-103D, 1e-102D, 1e-101D, 1e-100D, 1e-99D, 1e-98D, 1e-97D, 1e-96D, 1e-95D, 1e-94D, + 1e-93D, 1e-92D, 1e-91D, 1e-90D, 1e-89D, 1e-88D, 1e-87D, 1e-86D, 1e-85D, 1e-84D, + 1e-83D, 1e-82D, 1e-81D, 1e-80D, 1e-79D, 1e-78D, 1e-77D, 1e-76D, 1e-75D, 1e-74D, + 1e-73D, 1e-72D, 1e-71D, 1e-70D, 1e-69D, 1e-68D, 1e-67D, 1e-66D, 1e-65D, 1e-64D, + 1e-63D, 1e-62D, 1e-61D, 1e-60D, 1e-59D, 1e-58D, 1e-57D, 1e-56D, 1e-55D, 1e-54D, + 1e-53D, 1e-52D, 1e-51D, 1e-50D, 1e-49D, 1e-48D, 1e-47D, 1e-46D, 1e-45D, 1e-44D, + 1e-43D, 1e-42D, 1e-41D, 1e-40D, 1e-39D, 1e-38D, 1e-37D, 1e-36D, 1e-35D, 1e-34D, + 1e-33D, 1e-32D, 1e-31D, 1e-30D, 1e-29D, 1e-28D, 1e-27D, 1e-26D, 1e-25D, 1e-24D, + 1e-23D, 1e-22D, 1e-21D, 1e-20D, 1e-19D, 1e-18D, 1e-17D, 1e-16D, 1e-15D, 1e-14D, + 1e-13D, 1e-12D, 1e-11D, 1e-10D, 1e-9D, 1e-8D, 1e-7D, 1e-6D, 1e-5D, 1e-4D, + 1e-3D, 1e-2D, 1e-1D, 1e0D, 1e1D, 1e2D, 1e3D, 1e4D, + 1e5D, 1e6D, 1e7D, 1e8D, 1e9D, 1e10D, 1e11D, 1e12D, 1e13D, 1e14D, + 1e15D, 1e16D, 1e17D, 1e18D, 1e19D, 1e20D, 1e21D, 1e22D, 1e23D, 1e24D, + 1e25D, 1e26D, 1e27D, 1e28D, 1e29D, 1e30D, 1e31D, 1e32D, 1e33D, 1e34D, + 1e35D, 1e36D, 1e37D, 1e38D, 1e39D, 1e40D, 1e41D, 1e42D, 1e43D, 1e44D, + 1e45D, 1e46D, 1e47D, 1e48D, 1e49D, 1e50D, 1e51D, 1e52D, 1e53D, 1e54D, + 1e55D, 1e56D, 1e57D, 1e58D, 1e59D, 1e60D, 1e61D, 1e62D, 1e63D, 1e64D, + 1e65D, 1e66D, 1e67D, 1e68D, 1e69D, 1e70D, 1e71D, 1e72D, 1e73D, 1e74D, + 1e75D, 1e76D, 1e77D, 1e78D, 1e79D, 1e80D, 1e81D, 1e82D, 1e83D, 1e84D, + 1e85D, 1e86D, 1e87D, 1e88D, 1e89D, 1e90D, 1e91D, 1e92D, 1e93D, 1e94D, + 1e95D, 1e96D, 1e97D, 1e98D, 1e99D, 1e100D, 1e101D, 1e102D, 1e103D, 1e104D, + 1e105D, 1e106D, 1e107D, 1e108D, 1e109D, 1e110D, 1e111D, 1e112D, 1e113D, 1e114D, + 1e115D, 1e116D, 1e117D, 1e118D, 1e119D, 1e120D, 1e121D, 1e122D, 1e123D, 1e124D, + 1e125D, 1e126D, 1e127D, 1e128D, 1e129D, 1e130D, 1e131D, 1e132D, 1e133D, 1e134D, + 1e135D, 1e136D, 1e137D, 1e138D, 1e139D, 1e140D, 1e141D, 1e142D, 1e143D, 1e144D, + 1e145D, 1e146D, 1e147D, 1e148D, 1e149D, 1e150D, 1e151D, 1e152D, 1e153D, 1e154D, + 1e155D, 1e156D, 1e157D, 1e158D, 1e159D, 1e160D, 1e161D, 1e162D, 1e163D, 1e164D, + 1e165D, 1e166D, 1e167D, 1e168D, 1e169D, 1e170D, 1e171D, 1e172D, 1e173D, 1e174D, + 1e175D, 1e176D, 1e177D, 1e178D, 1e179D, 1e180D, 1e181D, 1e182D, 1e183D, 1e184D, + 1e185D, 1e186D, 1e187D, 1e188D, 1e189D, 1e190D, 1e191D, 1e192D, 1e193D, 1e194D, + 1e195D, 1e196D, 1e197D, 1e198D, 1e199D, 1e200D, 1e201D, 1e202D, 1e203D, 1e204D, + 1e205D, 1e206D, 1e207D, 1e208D, 1e209D, 1e210D, 1e211D, 1e212D, 1e213D, 1e214D, + 1e215D, 1e216D, 1e217D, 1e218D, 1e219D, 1e220D, 1e221D, 1e222D, 1e223D, 1e224D, + 1e225D, 1e226D, 1e227D, 1e228D, 1e229D, 1e230D, 1e231D, 1e232D, 1e233D, 1e234D, + 1e235D, 1e236D, 1e237D, 1e238D, 1e239D, 1e240D, 1e241D, 1e242D, 1e243D, 1e244D, + 1e245D, 1e246D, 1e247D, 1e248D, 1e249D, 1e250D, 1e251D, 1e252D, 1e253D, 1e254D, + 1e255D, 1e256D, 1e257D, 1e258D, 1e259D, 1e260D, 1e261D, 1e262D, 1e263D, 1e264D, + 1e265D, 1e266D, 1e267D, 1e268D, 1e269D, 1e270D, 1e271D, 1e272D, 1e273D, 1e274D, + 1e275D, 1e276D, 1e277D, 1e278D, 1e279D, 1e280D, 1e281D, 1e282D, 1e283D, 1e284D, + 1e285D, 1e286D, 1e287D, 1e288D, 1e289D, 1e290D, 1e291D, 1e292D, 1e293D, 1e294D, + 1e295D, 1e296D, 1e297D, 1e298D, 1e299D, 1e300D, 1e301D, 1e302D, 1e303D, 1e304D, + 1e305D, 1e306D, 1e307D, 1e308D + }; + + + /** + * Assumes i is positive. Returns the magnitude of i in base 10. + */ + private static long tenthPower(long i) + { + if (i < 10L) return 1; + else if (i < 100L) return 10L; + else if (i < 1000L) return 100L; + else if (i < 10000L) return 1000L; + else if (i < 100000L) return 10000L; + else if (i < 1000000L) return 100000L; + else if (i < 10000000L) return 1000000L; + else if (i < 100000000L) return 10000000L; + else if (i < 1000000000L) return 100000000L; + else if (i < 10000000000L) return 1000000000L; + else if (i < 100000000000L) return 10000000000L; + else if (i < 1000000000000L) return 100000000000L; + else if (i < 10000000000000L) return 1000000000000L; + else if (i < 100000000000000L) return 10000000000000L; + else if (i < 1000000000000000L) return 100000000000000L; + else if (i < 10000000000000000L) return 1000000000000000L; + else if (i < 100000000000000000L) return 10000000000000000L; + else if (i < 1000000000000000000L) return 100000000000000000L; + else return 1000000000000000000L; + } + + + private static int magnitude(double d) + { + //It works. What else can I say. + long doubleToLongBits = Double.doubleToLongBits(d); + int magnitude = + (int) ((((doubleToLongBits & DoubleExpMask) >> DoubleExpShift) - DoubleExpBias) * 0.301029995663981); + + if (magnitude < -323) + magnitude = -323; + else if (magnitude > 308) + magnitude = 308; + + if (d >= d_tenthPowers[magnitude+323]) + { + while(magnitude < 309 && d >= d_tenthPowers[magnitude+323]) + magnitude++; + magnitude--; + return magnitude; + } + else + { + while(magnitude > -324 && d < d_tenthPowers[magnitude+323]) + magnitude--; + return magnitude; + } + } + + static long[] l_tenthPowers = { + 1, + 10L, + 100L, + 1000L, + 10000L, + 100000L, + 1000000L, + 10000000L, + 100000000L, + 1000000000L, + 10000000000L, + 100000000000L, + 1000000000000L, + 10000000000000L, + 100000000000000L, + 1000000000000000L, + 10000000000000000L, + 100000000000000000L, + 1000000000000000000L, + }; + + public static long getNthDigit(long l, int n) + { + return (l/(tenthPower(l)/l_tenthPowers[n-1]))%10; + } + + + + + public static void append(StringBuilder s, double d) + { + if (d == Double.NEGATIVE_INFINITY) + s.append(NEGATIVE_INFINITY); + else if (d == Double.POSITIVE_INFINITY) + s.append(POSITIVE_INFINITY); + else if (d != d) + s.append(NaN); + else if (d == 0.0) + { + if ( (Double.doubleToLongBits(d) & DoubleSignMask) != 0) + s.append('-'); + s.append(DOUBLE_ZERO); + } + else + { + if (d < 0) + { + s.append('-'); + d = -d; + } + + if (d >= 0.001 && d < 0.01) + { + long i = (long) (d * 1E20); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + s.append(DOUBLE_ZERO2); + appendFractDigits(s, i,-1); + } + else if (d >= 0.01 && d < 0.1) + { + long i = (long) (d * 1E19); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + s.append(DOUBLE_ZERO); + appendFractDigits(s, i,-1); + } + else if (d >= 0.1 && d < 1) + { + long i = (long) (d * 1E18); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + s.append(DOUBLE_ZERO0); + appendFractDigits(s, i,-1); + } + else if (d >= 1 && d < 10) + { + long i = (long) (d * 1E17); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i,1); + } + else if (d >= 10 && d < 100) + { + long i = (long) (d * 1E16); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i,2); + } + else if (d >= 100 && d < 1000) + { + long i = (long) (d * 1E15); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i,3); + } + else if (d >= 1000 && d < 10000) + { + long i = (long) (d * 1E14); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i,4); + } + else if (d >= 10000 && d < 100000) + { + long i = (long) (d * 1E13); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i,5); + } + else if (d >= 100000 && d < 1000000) + { + long i = (long) (d * 1E12); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i,6); + } + else if (d >= 1000000 && d < 10000000) + { + long i = (long) (d * 1E11); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i,7); + } + else + { + int magnitude = magnitude(d); + long i; + if (magnitude < -280) { + long valueBits = Double.doubleToRawLongBits(d); + long mantissa = valueBits & 0x000fffffffffffffL; + int exponent = (int)((valueBits >> 52) & 0x7ff); + if (exponent == 0) { + tooSmall(s, mantissa, -1074); + } else { + mantissa |= 0x0010000000000000L; + exponent -= 1075; + tooSmall(s, mantissa, exponent); + } + } else { + i = (long) (d / d_tenthPowers[magnitude + 323 - 17]); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i, 1); + s.append('E'); + append(s, magnitude); + } + } + } + } + public static void append(StringBuilder s, int i) + { + if (i < 0) + { + if (i == Integer.MIN_VALUE) + { + //cannot make this positive due to integer overflow + s.append("-2147483648"); + } + s.append('-'); + i = -i; + } + int mag; + int c; + if (i < 10) + { + //one digit + s.append(charForDigit[i]); + } + else if (i < 100) + { + //two digits + s.append(charForDigit[i/10]); + s.append(charForDigit[i%10]); + } + else if (i < 1000) + { + //three digits + s.append(charForDigit[i/100]); + s.append(charForDigit[(c=i%100)/10]); + s.append(charForDigit[c%10]); + } + else if (i < 10000) + { + //four digits + s.append(charForDigit[i/1000]); + s.append(charForDigit[(c=i%1000)/100]); + s.append(charForDigit[(c%=100)/10]); + s.append(charForDigit[c%10]); + } + else if (i < 100000) + { + //five digits + s.append(charForDigit[i/10000]); + s.append(charForDigit[(c=i%10000)/1000]); + s.append(charForDigit[(c%=1000)/100]); + s.append(charForDigit[(c%=100)/10]); + s.append(charForDigit[c%10]); + } + else if (i < 1000000) + { + //six digits + s.append(charForDigit[i/100000]); + s.append(charForDigit[(c=i%100000)/10000]); + s.append(charForDigit[(c%=10000)/1000]); + s.append(charForDigit[(c%=1000)/100]); + s.append(charForDigit[(c%=100)/10]); + s.append(charForDigit[c%10]); + } + else if (i < 10000000) + { + //seven digits + s.append(charForDigit[i/1000000]); + s.append(charForDigit[(c=i%1000000)/100000]); + s.append(charForDigit[(c%=100000)/10000]); + s.append(charForDigit[(c%=10000)/1000]); + s.append(charForDigit[(c%=1000)/100]); + s.append(charForDigit[(c%=100)/10]); + s.append(charForDigit[c%10]); + } + else if (i < 100000000) + { + //eight digits + s.append(charForDigit[i/10000000]); + s.append(charForDigit[(c=i%10000000)/1000000]); + s.append(charForDigit[(c%=1000000)/100000]); + s.append(charForDigit[(c%=100000)/10000]); + s.append(charForDigit[(c%=10000)/1000]); + s.append(charForDigit[(c%=1000)/100]); + s.append(charForDigit[(c%=100)/10]); + s.append(charForDigit[c%10]); + } + else if (i < 1000000000) + { + //nine digits + s.append(charForDigit[i/100000000]); + s.append(charForDigit[(c=i%100000000)/10000000]); + s.append(charForDigit[(c%=10000000)/1000000]); + s.append(charForDigit[(c%=1000000)/100000]); + s.append(charForDigit[(c%=100000)/10000]); + s.append(charForDigit[(c%=10000)/1000]); + s.append(charForDigit[(c%=1000)/100]); + s.append(charForDigit[(c%=100)/10]); + s.append(charForDigit[c%10]); + } + else + { + //ten digits + s.append(charForDigit[i/1000000000]); + s.append(charForDigit[(c=i%1000000000)/100000000]); + s.append(charForDigit[(c%=100000000)/10000000]); + s.append(charForDigit[(c%=10000000)/1000000]); + s.append(charForDigit[(c%=1000000)/100000]); + s.append(charForDigit[(c%=100000)/10000]); + s.append(charForDigit[(c%=10000)/1000]); + s.append(charForDigit[(c%=1000)/100]); + s.append(charForDigit[(c%=100)/10]); + s.append(charForDigit[c%10]); + } + } + private static void appendFractDigits(StringBuilder s, long i, int decimalOffset) + { + long mag = tenthPower(i); + long c; + while ( i > 0 ) + { + c = i/mag; + s.append(charForDigit[(int) c]); + decimalOffset--; + if (decimalOffset == 0) + s.append('.'); + c *= mag; + if ( c <= i) + i -= c; + mag = mag/10; + } + if (i != 0) + s.append(charForDigit[(int) i]); + else if (decimalOffset > 0) + { + s.append(ZEROS[decimalOffset]); + decimalOffset = 1; + } + + decimalOffset--; + if (decimalOffset == 0) + s.append(DOT_ZERO); + else if (decimalOffset == -1) + s.append('0'); + } + + public static final char[] NEGATIVE_INFINITY = {'-','I','n','f','i','n','i','t','y'}; + public static final char[] POSITIVE_INFINITY = {'I','n','f','i','n','i','t','y'}; + public static final char[] DOUBLE_ZERO = {'0','.','0'}; + public static final char[] DOUBLE_ZERO2 = {'0','.','0','0'}; + public static final char[] DOUBLE_ZERO0 = {'0','.'}; + public static final char[] DOT_ZERO = {'.','0'}; + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/DoubleParser.java b/vespajlib/src/main/java/com/yahoo/text/DoubleParser.java new file mode 100644 index 00000000000..8dfe8f012f7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/DoubleParser.java @@ -0,0 +1,199 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * Utility class to parse a String into a double. + * <p> + * This is intended as a lower-cost replacement for the standard + * Double.parseDouble(String) since that method will cause lock + * contention if it's used too often. + * <p> + * Note that this implementation won't always produce the same results + * as java.lang.Double (low-order bits may differ), and it doesn't + * properly support denormalized numbers. + * <p> + * Also, this implementation is very poorly tested at the moment, so + * it should be used carefully, only in cases where you know the input + * will be well-defined and you don't need full precision. + * + * @author arnej27959 + */ +public final class DoubleParser { + + /** + * Utility method that parses a String and returns a double. + * + * @param data the String to parse + * @return double parsed value of the string + * @throws NumberFormatException if the string is not a well-formatted number + * @throws NullPointerException if the string is a null pointer + */ + public static double parse(String data) { + final int len = data.length(); + double result = 0; + boolean negative = false; + int beforePoint = 0; + int exponent = 0; + byte[] digits = new byte[25]; + int numDigits = 0; + + int i = 0; + while (i < len && Character.isWhitespace(data.charAt(i))) { + i++; + } + if (data.charAt(i) == '+') { + i++; + } else if (data.charAt(i) == '-') { + negative = true; + i++; + } + if (i + 3 <= len && data.substring(i, i+3).equals("NaN")) { + i += 3; + result = Double.NaN; + } else if (i + 8 <= len && data.substring(i, i+8).equals("Infinity")) { + i += 8; + if (negative) { + result = Double.NEGATIVE_INFINITY; + } else { + result = Double.POSITIVE_INFINITY; + } + } else { + while (i < len && Character.isDigit(data.charAt(i))) { + int dval = Character.digit(data.charAt(i), 10); + assert dval >= 0; + assert dval < 10; + if (numDigits < 25) { + digits[numDigits++] = (byte)dval; + } + ++beforePoint; + i++; + } + if (i < len && data.charAt(i) == '.') { + i++; + while (i < len && Character.isDigit(data.charAt(i))) { + int dval = Character.digit(data.charAt(i), 10); + assert dval >= 0; + assert dval < 10; + if (numDigits < 25) { + digits[numDigits++] = (byte)dval; + } + i++; + } + } + if (numDigits == 0) { + throw new NumberFormatException("No digits in number: '"+data+"'"); + } + if (i < len && (data.charAt(i) == 'e' || data.charAt(i) == 'E')) { + i++; + boolean expNeg = false; + int expDigits = 0; + if (data.charAt(i) == '+') { + i++; + } else if (data.charAt(i) == '-') { + expNeg = true; + i++; + } + while (i < len && Character.isDigit(data.charAt(i))) { + int dval = Character.digit(data.charAt(i), 10); + assert dval >= 0; + assert dval < 10; + exponent *= 10; + exponent += dval; + ++expDigits; + i++; + } + if (expDigits == 0) { + throw new NumberFormatException("Missing digits in exponent part: "+data); + } + if (expNeg) { + exponent = -exponent; + } + } + // System.out.println("parsed exp: "+exponent); + // System.out.println("before pt: "+beforePoint); + exponent += beforePoint; + exponent -= numDigits; + // System.out.println("adjusted exp: "+exponent); + for (int d = numDigits; d > 0; d--) { + double dv = digits[d-1]; + dv *= powTen(numDigits - d); + result += dv; + } + // System.out.println("digits sum: "+result); + while (exponent < -99) { + result *= powTen(-99); + exponent += 99; + } + while (exponent > 99) { + result *= powTen(99); + exponent -= 99; + } + // System.out.println("digits sum: "+result); + // System.out.println("exponent multiplier: "+powTen(exponent)); + result *= powTen(exponent); + + if (negative) { + result = -result; + } + } + while (i < len && Character.isWhitespace(data.charAt(i))) { + i++; + } + if (i < len) { + throw new NumberFormatException("Extra characters after number: "+data.substring(i)); + } + return result; + } + + private static double[] tens = { + 1.0e00, 1.0e01, 1.0e02, 1.0e03, 1.0e04, 1.0e05, 1.0e06, + 1.0e07, 1.0e08, 1.0e09, 1.0e10, 1.0e11, 1.0e12, 1.0e13, + 1.0e14, 1.0e15, 1.0e16, 1.0e17, 1.0e18, 1.0e19, 1.0e20, + 1.0e21, 1.0e22, 1.0e23, 1.0e24, 1.0e25, 1.0e26, 1.0e27, + 1.0e28, 1.0e29, 1.0e30, 1.0e31, 1.0e32, 1.0e33, 1.0e34, + 1.0e35, 1.0e36, 1.0e37, 1.0e38, 1.0e39, 1.0e40, 1.0e41, + 1.0e42, 1.0e43, 1.0e44, 1.0e45, 1.0e46, 1.0e47, 1.0e48, + 1.0e49, 1.0e50, 1.0e51, 1.0e52, 1.0e53, 1.0e54, 1.0e55, + 1.0e56, 1.0e57, 1.0e58, 1.0e59, 1.0e60, 1.0e61, 1.0e62, + 1.0e63, 1.0e64, 1.0e65, 1.0e66, 1.0e67, 1.0e68, 1.0e69, + 1.0e70, 1.0e71, 1.0e72, 1.0e73, 1.0e74, 1.0e75, 1.0e76, + 1.0e77, 1.0e78, 1.0e79, 1.0e80, 1.0e81, 1.0e82, 1.0e83, + 1.0e84, 1.0e85, 1.0e86, 1.0e87, 1.0e88, 1.0e89, 1.0e90, + 1.0e91, 1.0e92, 1.0e93, 1.0e94, 1.0e95, 1.0e96, 1.0e97, + 1.0e98, 1.0e99 + }; + + private static double[] tenths = { + 1.0e-00, 1.0e-01, 1.0e-02, 1.0e-03, 1.0e-04, 1.0e-05, + 1.0e-06, 1.0e-07, 1.0e-08, 1.0e-09, 1.0e-10, 1.0e-11, + 1.0e-12, 1.0e-13, 1.0e-14, 1.0e-15, 1.0e-16, 1.0e-17, + 1.0e-18, 1.0e-19, 1.0e-20, 1.0e-21, 1.0e-22, 1.0e-23, + 1.0e-24, 1.0e-25, 1.0e-26, 1.0e-27, 1.0e-28, 1.0e-29, + 1.0e-30, 1.0e-31, 1.0e-32, 1.0e-33, 1.0e-34, 1.0e-35, + 1.0e-36, 1.0e-37, 1.0e-38, 1.0e-39, 1.0e-40, 1.0e-41, + 1.0e-42, 1.0e-43, 1.0e-44, 1.0e-45, 1.0e-46, 1.0e-47, + 1.0e-48, 1.0e-49, 1.0e-50, 1.0e-51, 1.0e-52, 1.0e-53, + 1.0e-54, 1.0e-55, 1.0e-56, 1.0e-57, 1.0e-58, 1.0e-59, + 1.0e-60, 1.0e-61, 1.0e-62, 1.0e-63, 1.0e-64, 1.0e-65, + 1.0e-66, 1.0e-67, 1.0e-68, 1.0e-69, 1.0e-70, 1.0e-71, + 1.0e-72, 1.0e-73, 1.0e-74, 1.0e-75, 1.0e-76, 1.0e-77, + 1.0e-78, 1.0e-79, 1.0e-80, 1.0e-81, 1.0e-82, 1.0e-83, + 1.0e-84, 1.0e-85, 1.0e-86, 1.0e-87, 1.0e-88, 1.0e-89, + 1.0e-90, 1.0e-91, 1.0e-92, 1.0e-93, 1.0e-94, 1.0e-95, + 1.0e-96, 1.0e-97, 1.0e-98, 1.0e-99 + }; + + private static double powTen(int exponent) { + if (exponent > 0) { + assert exponent < 100; + return tens[exponent]; + } + if (exponent < 0) { + exponent = -exponent; + assert exponent < 100; + return tenths[exponent]; + } + return 1.0; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/ForwardWriter.java b/vespajlib/src/main/java/com/yahoo/text/ForwardWriter.java new file mode 100644 index 00000000000..a7876d620b9 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/ForwardWriter.java @@ -0,0 +1,157 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.IOException; + +/** + * Wraps another writer and also converting IOException to Exceptions. + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public class ForwardWriter extends GenericWriter { + + private final GenericWriter out; + + public ForwardWriter(GenericWriter writer) { + super(); + this.out = writer; + } + + @Override + public void write(char[] c, int offset, int bytes) { + try { + out.write(c, offset, bytes); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + @Override + public GenericWriter write(AbstractUtf8Array v) { + try { + out.write(v); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + @Override + public void write(String v) { + try { + out.write(v); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + @Override + public GenericWriter write(CharSequence c) { + try { + out.write(c); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + + @Override + public GenericWriter write(double d) { + try { + out.write(d); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public GenericWriter write(float f) { + try { + out.write(f); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public GenericWriter write(long v) { + try { + out.write(v); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + @Override + public void write(int v) { + try { + out.write(v); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + @Override + public GenericWriter write(short v) { + try { + out.write(v); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + @Override + public GenericWriter write(char c) { + try { + out.write(c); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + @Override + public GenericWriter write(byte b) { + try { + out.write(b); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + + @Override + public GenericWriter write(boolean v) { + try { + out.write(v); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public void flush() { + try { + out.flush(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() { + try { + out.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Gives access to the wrapped writer. + * @return wrapped writer. + */ + public GenericWriter getWriter() { return out; } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/GenericWriter.java b/vespajlib/src/main/java/com/yahoo/text/GenericWriter.java new file mode 100644 index 00000000000..0a07b617352 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/GenericWriter.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.IOException; +import java.io.Writer; + +/** + * This is a basic writer for presenting text. Its has the pattern as + * java.io.Writer, but it allows for more overrides for speed. + * This introduces additional interfaces in addition to the java.lang.Writer. + * The purpose is to allow for optimizations. + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ + +public abstract class GenericWriter extends Writer { +/* + public abstract void write(char [] c, int offset, int bytes); + public abstract void flush(); + public abstract void close(); + +*/ + public GenericWriter write(char c) throws java.io.IOException { + char t[] = new char[1]; + t[0] = c; + try { + write(t, 0, 1); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + public GenericWriter write(CharSequence s) throws java.io.IOException { + for (int i=0, m=s.length(); i < m; i++) { + write(s.charAt(i)); + } + return this; + } + + public GenericWriter write(long i) throws java.io.IOException { + write(String.valueOf(i)); + return this; + } + + public GenericWriter write(short i) throws java.io.IOException { + write(String.valueOf(i)); + return this; + } + public GenericWriter write(byte i) throws java.io.IOException { + write(String.valueOf(i)); + return this; + } + public GenericWriter write(double i) throws java.io.IOException { + write(String.valueOf(i)); + return this; + } + public GenericWriter write(float i) throws java.io.IOException { + write(String.valueOf(i)); + return this; + } + public GenericWriter write(boolean i) throws java.io.IOException { + write(String.valueOf(i)); + return this; + } + + public GenericWriter write(AbstractUtf8Array v) throws java.io.IOException { + write(v.toString()); + return this; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/HTML.java b/vespajlib/src/main/java/com/yahoo/text/HTML.java new file mode 100644 index 00000000000..e76a2f54d1f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/HTML.java @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + + +import java.util.Map; +import java.util.HashMap; + + +/** + * Static HTML escaping stuff + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class HTML { + static Object[][] entities = { + // {"#39", new Integer(39)}, // ' - apostrophe + {"quot", 34}, // " - double-quote + {"amp", 38}, // & - ampersand + {"lt", 60}, // < - less-than + {"gt", 62}, // > - greater-than + {"nbsp", 160}, // non-breaking space + {"copy", 169}, // \u00A9 - copyright + {"reg", 174}, // \u00AE - registered trademark + {"Agrave", 192}, // \u00C0 - uppercase A, grave accent + {"Aacute", 193}, // \u00C1 - uppercase A, acute accent + {"Acirc", 194}, // \u00C2 - uppercase A, circumflex accent + {"Atilde", 195}, // \u00C3 - uppercase A, tilde + {"Auml", 196}, // \u00C4 - uppercase A, umlaut + {"Aring", 197}, // \u00C5 - uppercase A, ring + {"AElig", 198}, // \u00C6 - uppercase AE + {"Ccedil", 199}, // \u00C7 - uppercase C, cedilla + {"Egrave", 200}, // \u00C8 - uppercase E, grave accent + {"Eacute", 201}, // \u00C9 - uppercase E, acute accent + {"Ecirc", 202}, // \u00CA - uppercase E, circumflex accent + {"Euml", 203}, // \u00CB - uppercase E, umlaut + {"Igrave", 204}, // \u00CC - uppercase I, grave accent + {"Iacute", 205}, // \u00CD - uppercase I, acute accent + {"Icirc", 206}, // \u00CE - uppercase I, circumflex accent + {"Iuml", 207}, // \u00CF - uppercase I, umlaut + {"ETH", 208}, // \u00D0 - uppercase Eth, Icelandic + {"Ntilde", 209}, // \u00D1 - uppercase N, tilde + {"Ograve", 210}, // \u00D2 - uppercase O, grave accent + {"Oacute", 211}, // \u00D3 - uppercase O, acute accent + {"Ocirc", 212}, // \u00D4 - uppercase O, circumflex accent + {"Otilde", 213}, // \u00D5 - uppercase O, tilde + {"Ouml", 214}, // \u00D6 - uppercase O, umlaut + {"Oslash", 216}, // \u00D8 - uppercase O, slash + {"Ugrave", 217}, // \u00D9 - uppercase U, grave accent + {"Uacute", 218}, // \u00DA - uppercase U, acute accent + {"Ucirc", 219}, // \u00DB - uppercase U, circumflex accent + {"Uuml", 220}, // \u00DC - uppercase U, umlaut + {"Yacute", 221}, // \u00DD - uppercase Y, acute accent + {"THORN", 222}, // \u00DE - uppercase THORN, Icelandic + {"szlig", 223}, // \u00DF - lowercase sharps, German + {"agrave", 224}, // \u00E0 - lowercase a, grave accent + {"aacute", 225}, // \u00E1 - lowercase a, acute accent + {"acirc", 226}, // \u00E2 - lowercase a, circumflex accent + {"atilde", 227}, // \u00E3 - lowercase a, tilde + {"auml", 228}, // \u00E4 - lowercase a, umlaut + {"aring", 229}, // \u00E5 - lowercase a, ring + {"aelig", 230}, // \u00E6 - lowercase ae + {"ccedil", 231}, // \u00E7 - lowercase c, cedilla + {"egrave", 232}, // \u00E8 - lowercase e, grave accent + {"eacute", 233}, // \u00E9 - lowercase e, acute accent + {"ecirc", 234}, // \u00EA - lowercase e, circumflex accent + {"euml", 235}, // \u00EB - lowercase e, umlaut + {"igrave", 236}, // \u00EC - lowercase i, grave accent + {"iacute", 237}, // \u00ED - lowercase i, acute accent + {"icirc", 238}, // \u00EE - lowercase i, circumflex accent + {"iuml", 239}, // \u00EF - lowercase i, umlaut + {"igrave", 236}, // \u00EC - lowercase i, grave accent + {"iacute", 237}, // \u00ED - lowercase i, acute accent + {"icirc", 238}, // \u00EE - lowercase i, circumflex accent + {"iuml", 239}, // \u00EF - lowercase i, umlaut + {"eth", 240}, // \u00F0 - lowercase eth, Icelandic + {"ntilde", 241}, // \u00F1 - lowercase n, tilde + {"ograve", 242}, // \u00F2 - lowercase o, grave accent + {"oacute", 243}, // \u00F3 - lowercase o, acute accent + {"ocirc", 244}, // \u00F4 - lowercase o, circumflex accent + {"otilde", 245}, // \u00F5 - lowercase o, tilde + {"ouml", 246}, // \u00F6 - lowercase o, umlaut + {"oslash", 248}, // \u00F8 - lowercase o, slash + {"ugrave", 249}, // \u00F9 - lowercase u, grave accent + {"uacute", 250}, // \u00FA - lowercase u, acute accent + {"ucirc", 251}, // \u00FB - lowercase u, circumflex accent + {"uuml", 252}, // \u00FC - lowercase u, umlaut + {"yacute", 253}, // \u00FD - lowercase y, acute accent + {"thorn", 254}, // \u00FE - lowercase thorn, Icelandic + {"yuml", 255}, // \u00FF - lowercase y, umlaut + {"euro", 8364}, // Euro symbol + }; + + static Map<String, Integer> e2i = new HashMap<>(); + static Map<Integer, String> i2e = new HashMap<>(); + + static { + for (Object[] entity : entities) { + e2i.put((String) entity[0], (Integer) entity[1]); + i2e.put((Integer) entity[1], (String) entity[0]); + } + } + + public static String htmlescape(String s1) { + if (s1 == null) return ""; + + int len = s1.length(); + // about 20% guess + StringBuilder buf = new StringBuilder((int) (len * 1.2)); + int i; + + for (i = 0; i < len; ++i) { + char ch = s1.charAt(i); + String entity = i2e.get((int) ch); + + if (entity == null) { + if (((int) ch) > 128) buf.append("&#").append((int) ch).append(";"); + else buf.append(ch); + } else { + buf.append("&").append(entity).append(";"); + } + } + return buf.toString(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Identifier.java b/vespajlib/src/main/java/com/yahoo/text/Identifier.java new file mode 100644 index 00000000000..2d74b61e1e0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Identifier.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 11.11.12 + * Time: 10:37 + * This class is used to represent a legal identifier of [a-zA-Z_][a-zA-Z_0-9]* + */ +public class Identifier extends Utf8Array { + public Identifier(String s) { + this(Utf8.toBytes(s)); + } + public Identifier(AbstractUtf8Array utf8) { + this(utf8.getBytes()); + } + public Identifier(byte [] utf8) { + super(verify(utf8)); + } + private static byte [] verify(final byte [] utf8) { + if (utf8.length > 0) { + verifyFirst(utf8[0], utf8); + for (int i=1; i < utf8.length; i++) { + verifyAny(utf8[i], utf8); + } + } + return utf8; + + } + private static boolean verifyFirst(byte c, byte [] identifier) { + if (!((c == '_') || ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z')))) { + throw new IllegalArgumentException("Illegal starting character '" + (char)c + "' of identifier '" + new Utf8String(new Utf8Array(identifier)).toString() +"'."); + } + return true; + } + private static boolean verifyAny(byte c, byte [] identifier) { + if (!((c == '_') || ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z')) || ((c >= '0') && (c <= '9')))) { + throw new IllegalArgumentException("Illegal character '" + (char)c + "' of identifier '" + new Utf8String(new Utf8Array(identifier)).toString() +"'."); + } + return true; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/JSON.java b/vespajlib/src/main/java/com/yahoo/text/JSON.java new file mode 100644 index 00000000000..33af017b81d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/JSON.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.util.Map; + +/** + * Static methods for working with the map textual format which is parsed by {@link MapParser} + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public final class JSON { + + /** No instances */ + private JSON() {} + + /** + * Outputs a map as a JSON 'object' string, provided that the map values + * are either + * <ul> + * <li>String + * <li>Number + * <li>Any object whose toString returns JSON + * </ul> + */ + public static String encode(Map<String, ?> map) { + StringBuilder b = new StringBuilder("{"); + for (Map.Entry<String,?> entry : map.entrySet()) { + b.append("\"").append(escape(entry.getKey())).append("\":"); + if (entry.getValue() instanceof String) + b.append("\"").append(escape(entry.getValue().toString())).append("\""); + else // Number, or some other object which returns JSON + b.append(entry.getValue()); + b.append(","); + } + if (b.length()>1) + b.setLength(b.length()-1); // remove last comma + b.append("}"); + return b.toString(); + } + + /** Returns the given string as a properly json escaped string */ + public static String escape(String s) { + StringBuilder b = null; // lazy create to optimize for "nothing to do" case + + for (int i=0; i < s.length(); i = s.offsetByCodePoints(i, 1)) { + final int codepoint = s.codePointAt(i); + if (codepoint == '"') { + if (b == null) + b = new StringBuilder(s.substring(0, i)); + b.append('\\'); + } + + if (b != null) + b.appendCodePoint(codepoint); + } + return b != null ? b.toString() : s; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/JSONWriter.java b/vespajlib/src/main/java/com/yahoo/text/JSONWriter.java new file mode 100644 index 00000000000..0df2b2d2d3a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/JSONWriter.java @@ -0,0 +1,202 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayDeque; +import java.util.Deque; + +/** + * A class which knows how to write JSON markup. All methods return this to + * enable chaining of method calls. + * Consider using the Jackson generator API instead, as that may be faster. + * + * @author bratseth + */ +public final class JSONWriter { + + /** A stack maintaining the "needs comma" state at the current level */ + private Deque<Boolean> needsComma=new ArrayDeque<>(); + + private static final char[] DIGITS = new char[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + + private final OutputStream stream; + + public JSONWriter(OutputStream stream) { + this.stream = stream; + } + + /** Called on the start of a field or array value */ + private void beginFieldOrArrayValue() throws IOException { + if (needsComma.getFirst()) { + write(","); + } + } + + /** Called on the end of a field or array value */ + private void endFieldOrArrayValue() { + setNeedsComma(); + } + + /** Begins an object field */ + public JSONWriter beginField(String fieldName) throws IOException { + beginFieldOrArrayValue(); + write("\"" + fieldName + "\":"); + return this; + } + + /** Ends an object field */ + public JSONWriter endField() throws IOException { + endFieldOrArrayValue(); + return this; + } + + /** Begins an array value */ + public JSONWriter beginArrayValue() throws IOException { + beginFieldOrArrayValue(); + return this; + } + + /** Ends an array value */ + public JSONWriter endArrayValue() throws IOException { + endFieldOrArrayValue(); + return this; + } + + /** Begin an object value */ + public JSONWriter beginObject() throws IOException { + write("{"); + needsComma.addFirst(Boolean.FALSE); + return this; + } + + /** End an object value */ + public JSONWriter endObject() throws IOException { + write("}"); + needsComma.removeFirst(); + return this; + } + + /** Begin an array value */ + public JSONWriter beginArray() throws IOException { + write("["); + needsComma.addFirst(Boolean.FALSE); + return this; + } + + /** End an array value */ + public JSONWriter endArray() throws IOException { + write("]"); + needsComma.removeFirst(); + return this; + } + + /** Writes a string value */ + public JSONWriter value(String value) throws IOException { + write("\"").write(escape(value)).write("\""); + return this; + } + + /** Writes a numeric value */ + public JSONWriter value(Number value) throws IOException { + write(value.toString()); + return this; + } + + /** Writes a boolean value */ + public JSONWriter value(boolean value) throws IOException { + write(Boolean.toString(value)); + return this; + } + + /** Writes a null value */ + public JSONWriter value() throws IOException { + write("null"); + return this; + } + + private void setNeedsComma() { + if (level() == 0) return; + needsComma.removeFirst(); + needsComma.addFirst(Boolean.TRUE); + } + + /** Returns the current nested level */ + private int level() { return needsComma.size(); } + + /** + * Writes a string directly as-is to the stream of this. + * + * @return this for convenience + */ + private JSONWriter write(String string) throws IOException { + if (string.length() == 0) return this; + stream.write(Utf8.toBytes(string)); + return this; + } + + /** + * Do JSON escaping of a string. + * + * @param in a string to escape + * @return a String suitable for use in JSON strings + */ + private String escape(final String in) { + final StringBuilder quoted = new StringBuilder((int) (in.length() * 1.2)); + return escape(in, quoted).toString(); + } + + /** + * Do JSON escaping of the incoming string to the "quoted" buffer. The + * buffer returned is the same as the one given in the "quoted" parameter. + * + * @param in a string to escape + * @param escaped the target buffer for escaped data + * @return the same buffer as given in the "quoted" parameter + */ + private StringBuilder escape(final String in, final StringBuilder escaped) { + for (final char c : in.toCharArray()) { + switch (c) { + case ('"'): + escaped.append("\\\""); + break; + case ('\\'): + escaped.append("\\\\"); + break; + case ('\b'): + escaped.append("\\b"); + break; + case ('\f'): + escaped.append("\\f"); + break; + case ('\n'): + escaped.append("\\n"); + break; + case ('\r'): + escaped.append("\\r"); + break; + case ('\t'): + escaped.append("\\t"); + break; + default: + if (c < 32) { + escaped.append("\\u").append(fourDigitHexString(c)); + } else { + escaped.append(c); + } + } + } + return escaped; + } + + private static char[] fourDigitHexString(final char c) { + final char[] hex = new char[4]; + int in = ((c) & 0xFFFF); + for (int i = 3; i >= 0; --i) { + hex[i] = DIGITS[in & 0xF]; + in >>>= 4; + } + return hex; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/JavaWriterWriter.java b/vespajlib/src/main/java/com/yahoo/text/JavaWriterWriter.java new file mode 100644 index 00000000000..b89092e9780 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/JavaWriterWriter.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.IOException; +import java.io.Writer; + +/** + * Wraps a simple java.lang.Writer. Of course you loose the possible optimizations. + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public final class JavaWriterWriter extends GenericWriter { + + final Writer out; + + public JavaWriterWriter(Writer writer) { + out = writer; + } + + @Override + public void write(char[] c, int offset, int bytes) { + try { + out.write(c, offset, bytes); + } catch (IOException e) { + throw new RuntimeException("Caught exception in Java writer.write.", e); + } + } + + @Override + public void flush() { + try { + out.flush(); + } catch (IOException e) { + throw new RuntimeException("Caught exception in Java writer.flush.", e); + } + } + + @Override + public void close() { + try { + out.close(); + } catch (IOException e) { + throw new RuntimeException("Caught exception in Java writer.close.", e); + } + } + public final Writer getWriter() { return out; } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/LanguageHacks.java b/vespajlib/src/main/java/com/yahoo/text/LanguageHacks.java new file mode 100644 index 00000000000..37fa01ccfa0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/LanguageHacks.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import static com.yahoo.text.Lowercase.toLowerCase; + +/** + * Language helper functions. + * + * @deprecated do not use + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +@Deprecated +public class LanguageHacks { + + /** + * Whether a language is in the CJK group. + */ + public static boolean isCJK(String language) { + if (language == null) return false; + + language = toLowerCase(language); + return "ja".equals(language) + || "ko".equals(language) + || language.startsWith("zh") + || language.startsWith("tw"); // TODO: tw is a bogus value? + } + + /** + * Whether there is desegmenting in this language. + */ + public static boolean yellDesegments(String language) { + if (language == null) return false; + + language = toLowerCase(language); + return "de".equals(language) || isCJK(language); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Lowercase.java b/vespajlib/src/main/java/com/yahoo/text/Lowercase.java new file mode 100644 index 00000000000..4babba29ab3 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Lowercase.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.util.Locale; + +/** + * The lower casing method to use in Vespa when doing string processing of data + * which is not to be handled as natural language data, e.g. field names or + * configuration paramaters. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class Lowercase { + + private static final char[] lowercase = new char[123]; + + static { + lowercase[0x41] = 'a'; + lowercase[0x42] = 'b'; + lowercase[0x43] = 'c'; + lowercase[0x44] = 'd'; + lowercase[0x45] = 'e'; + lowercase[0x46] = 'f'; + lowercase[0x47] = 'g'; + lowercase[0x48] = 'h'; + lowercase[0x49] = 'i'; + lowercase[0x4A] = 'j'; + lowercase[0x4B] = 'k'; + lowercase[0x4C] = 'l'; + lowercase[0x4D] = 'm'; + lowercase[0x4E] = 'n'; + lowercase[0x4F] = 'o'; + lowercase[0x50] = 'p'; + lowercase[0x51] = 'q'; + lowercase[0x52] = 'r'; + lowercase[0x53] = 's'; + lowercase[0x54] = 't'; + lowercase[0x55] = 'u'; + lowercase[0x56] = 'v'; + lowercase[0x57] = 'w'; + lowercase[0x58] = 'x'; + lowercase[0x59] = 'y'; + lowercase[0x5A] = 'z'; + + lowercase[0x61] = 'a'; + lowercase[0x62] = 'b'; + lowercase[0x63] = 'c'; + lowercase[0x64] = 'd'; + lowercase[0x65] = 'e'; + lowercase[0x66] = 'f'; + lowercase[0x67] = 'g'; + lowercase[0x68] = 'h'; + lowercase[0x69] = 'i'; + lowercase[0x6A] = 'j'; + lowercase[0x6B] = 'k'; + lowercase[0x6C] = 'l'; + lowercase[0x6D] = 'm'; + lowercase[0x6E] = 'n'; + lowercase[0x6F] = 'o'; + lowercase[0x70] = 'p'; + lowercase[0x71] = 'q'; + lowercase[0x72] = 'r'; + lowercase[0x73] = 's'; + lowercase[0x74] = 't'; + lowercase[0x75] = 'u'; + lowercase[0x76] = 'v'; + lowercase[0x77] = 'w'; + lowercase[0x78] = 'x'; + lowercase[0x79] = 'y'; + lowercase[0x7A] = 'z'; + } + + /** + * Return a lowercased version of the given string. Since this is language + * independent, this is more of a case normalization operation than + * lowercasing. Vespa code should <i>never</i> do lowercasing with implicit + * locale. + * + * @param in + * a string to lowercase + * @return a string containing only lowercase character + */ + public static String toLowerCase(String in) { + // def is picked from http://docs.oracle.com/javase/6/docs/api/java/lang/String.html#toLowerCase%28%29 + String lower = toLowerCasePrintableAsciiOnly(in); + return (lower == null) ? in.toLowerCase(Locale.ENGLISH) : lower; + } + public static String toUpperCase(String in) { + // def is picked from http://docs.oracle.com/javase/6/docs/api/java/lang/String.html#toLowerCase%28%29 + return in.toUpperCase(Locale.ENGLISH); + } + + private static String toLowerCasePrintableAsciiOnly(String in) { + boolean anyUpper = false; + for (int i = 0; i < in.length(); i++) { + char c = in.charAt(i); + if (c < 0x41) { //lower than A-Z + return null; + } + if (c > 0x5A && c < 0x61) { //between A-Z and a-z + return null; + } + if (c > 0x7A) { //higher than a-z + return null; + } + if (c != lowercase[c]) { + anyUpper = true; + } + } + if (!anyUpper) { + return in; + } + StringBuilder builder = new StringBuilder(in.length()); + for (int i = 0; i < in.length(); i++) { + builder.append((char) (in.charAt(i) | ((char) 0x20))); + } + return builder.toString(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/LowercaseIdentifier.java b/vespajlib/src/main/java/com/yahoo/text/LowercaseIdentifier.java new file mode 100644 index 00000000000..b0f5b023a38 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/LowercaseIdentifier.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 11.11.12 + * Time: 20:50 + * To change this template use File | Settings | File Templates. + */ +public class LowercaseIdentifier extends Identifier { + public LowercaseIdentifier(String s) { + this(Utf8.toBytes(s)); + } + public LowercaseIdentifier(AbstractUtf8Array utf8) { + this(utf8.getBytes()); + } + public LowercaseIdentifier(byte [] utf8) { + super(verify(utf8)); + } + private static byte [] verify(final byte [] utf8) { + for (int i=0; i < utf8.length; i++) { + verifyAny(utf8[i], utf8); + } + + return utf8; + + } + private static boolean verifyAny(byte c, byte [] identifier) { + if ((c >= 'A') && (c <= 'Z')) { + throw new IllegalArgumentException("Illegal uppercase character '" + (char)c + "' of identifier '" + new Utf8String(new Utf8Array(identifier)).toString() +"'."); + } + return true; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/MapParser.java b/vespajlib/src/main/java/com/yahoo/text/MapParser.java new file mode 100644 index 00000000000..e627ba9bb11 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/MapParser.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.util.HashMap; +import java.util.Map; + +/** + * <p>Superclasses of parsers of a map represented textually as + * <code>{key1:value1,"anystringkey":value2,'anystringkey2':value3 ...}</code>. + * This parser must be extended to override the way values are parsed and constructed.</p> + * + * <p>Example: To create a Double map parser:</p> + * <pre> + * public static final class DoubleMapParser extends MapParser<Double> { + * + * @Override + * protected Double parseValue(String value) { + * return Double.parseDouble(value); + * } + * + * } + * </pre> + * + * <p>Map parsers are NOT multithread safe, but are cheap to construct.</p> + * + * @author bratseth + * @since 5.1.15 + */ +public abstract class MapParser<VALUETYPE> extends SimpleMapParser { + + private Map<String, VALUETYPE> map; + + /** + * Convenience method doing return parse(s,new HashMap<String,VALUETYPE>()) + */ + public Map<String,VALUETYPE> parseToMap(String s) { + return parse(s,new HashMap<>()); + } + + /** + * Parses a map on the form <code>{key1:value1,key2:value2 ...}</code> + * + * @param string the textual representation of the map + * @param map the map to which the values will be added + * @return the input map instance for convenience + */ + public Map<String,VALUETYPE> parse(String string,Map<String,VALUETYPE> map) { + this.map = map; + parse(string); + return this.map; + } + + protected void handleKeyValue(String key, String value) { + map.put(key, parseValue(value)); + } + + protected abstract VALUETYPE parseValue(String value); + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/PositionedString.java b/vespajlib/src/main/java/com/yahoo/text/PositionedString.java new file mode 100644 index 00000000000..de2e349ef82 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/PositionedString.java @@ -0,0 +1,137 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * A string which has a current position. + * Useful for writing simple single-pass parsers. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.1.15 + */ +public class PositionedString { + + private String s; + private int p; + + /** + * Creates this from a given string. + */ + public PositionedString(String string) { + this.s=string; + } + + /** The complete string value of this */ + public String string() { return s; } + + /** The current position into this string */ + public int position() { return p; } + + /** Assigns the current position in the string */ + public void setPosition(int position) { p=position; } + + /** + * Consumes the character at this position. + * <br>Precondition: The character at this position is c. + * <br>Postcondition: The position is increased by 1 + * + * @param c the expected character at this + * @throws IllegalArgumentException if the character at this position is not c + */ + public void consume(char c) { + if (s.charAt(p++)!=c) + throw new IllegalArgumentException("Expected '" + c + "' " + at(p -1)); + } + + /** + * Consumes zero or more whitespace characters starting at the current position + */ + public void consumeSpaces() { + while (Character.isWhitespace(s.charAt(p))) + p++; + } + + /** + * Advances the position by 1 if the character at the current position is c. + * Does nothing otherwise. + * + * @return whether this consumed a c at the current position, or if it did nothing + */ + public boolean consumeOptional(char c) { + if (s.charAt(p)!=c) return false; + p++; + return true; + } + + /** + * Returns whether the character at the current position is c. + */ + public boolean peek(char c) { + return s.charAt(p)==c; + } + + /** + * Returns the position of the next occurrence of c, + * or -1 if there are no occurrences of c in the string after the current position. + */ + public int indexOf(char c) { + return s.indexOf(c,p); + } + + /** Adds n to the current position */ + public void skip(int n) { + p = p +n; + } + + /** + * Sets the position of this to the next occurrence of c after the current position. + * + * @param c the char to move the position to + * @return the substring between the current position and the new position at c + * @throws IllegalArgumentException if there was no occurrence of c after the current position + */ + public String consumeTo(char c) { + int nextC=indexOf(c); + if (nextC<0) + throw new IllegalArgumentException("Expected a string terminated by '" + c + "' " + at()); + String value=substring(nextC); + p=nextC; + return value; + } + + /** + * Returns the substring between the current position and <code>position</code> + * and advances the current position to <code>position</code> + */ + public String consumeToPosition(int position) { + String consumed=substring(position); + p=position; + return consumed; + } + + /** Returns a substring of this from the current position to the end argument */ + public String substring(int end) { + return string().substring(position(),end); + } + + /** Returns the substring of this string from the current position to the end */ + public String substring() { + return string().substring(position()); + } + + /** Returns a textual description of the current position, useful for appending to error messages. */ + public String at() { + return at(p); + } + + /** Returns a textual description of a given position, useful for appending to error messages. */ + public String at(int position) { + return "starting at position " + position + " but was '" + s.charAt(position) + "'"; + } + + /** Returns the string */ + @Override + public String toString() { + return s; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/SimpleMapParser.java b/vespajlib/src/main/java/com/yahoo/text/SimpleMapParser.java new file mode 100644 index 00000000000..a27563ebea1 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/SimpleMapParser.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.util.HashMap; +import java.util.Map; + +/** + * <p>Superclasses of parsers of a map represented textually as + * <code>{key1:value1,"anystringkey":value2,'anystringkey2':value3 ...}</code>. + * This parser must be extended to specify how to handle the key/value pairs.</p> + * + * <p>Example: To create a Double map parser:</p> + * <pre> + * public static final class DoubleMapParser extends MapParser<Double> { + * private Map<String, Double> map; + * + * ... + * + * @Override + * protected Double handleKeyValue(String key, String value) { + * map.put(key, Double.parseDouble(value)); + * } + * + * } + * </pre> + * + * <p>Map parsers are NOT multithread safe, but are cheap to construct.</p> + * + * @author bratseth + * @since 5.1.15 + */ +public abstract class SimpleMapParser { + + private PositionedString s; + + /** + * Parses a map on the form <code>{key1:value1,key2:value2 ...}</code> + * + * @param string the textual representation of the map + */ + public void parse(String string) { + try { + this.s=new PositionedString(string); + + s.consumeSpaces(); + s.consume('{'); + while ( ! s.peek('}')) { + s.consumeSpaces(); + String key=consumeKey(); + s.consume(':'); + s.consumeSpaces(); + consumeValue(key); + s.consumeOptional(','); + s.consumeSpaces(); + } + s.consume('}'); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("'" + s + "' is not a legal sparse vector string",e); + } + } + + private String consumeKey() { + if (s.consumeOptional('"')) { + String key=s.consumeTo('"'); + s.consume('"'); + return key; + } + else if (s.consumeOptional('\'')) { + String key=s.consumeTo('\''); + s.consume('\''); + return key; + } + else { + int keyEnd=findEndOfKey(); + if (keyEnd<0) + throw new IllegalArgumentException("Expected a key followed by ':' " + s.at()); + return s.consumeToPosition(keyEnd); + } + } + + protected int findEndOfKey() { + for (int peekI=s.position(); peekI<s.string().length(); peekI++) { + if (s.string().charAt(peekI)==':' || s.string().charAt(peekI)==',') + return peekI; + } + return -1; + } + + protected int findEndOfValue() { + for (int peekI=s.position(); peekI<s.string().length(); peekI++) { + if (s.string().charAt(peekI)==',' || s.string().charAt(peekI)=='}') + return peekI; + } + return -1; + } + + protected void consumeValue(String key) { + // find the next comma or bracket, whichever is next + int endOfValue=findEndOfValue(); + if (endOfValue<0) { + throw new IllegalArgumentException("Expected a value followed by ',' or '}' " + s.at()); + } + try { + handleKeyValue(key, s.substring(endOfValue)); + s.setPosition(endOfValue); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Expected a legal value from position " + s.position() + " to " + endOfValue + + " but was '" + s.substring(endOfValue) + "'", e); + } + } + + /** Returns the string being parsed along with its current position */ + public PositionedString string() { return s; } + + protected abstract void handleKeyValue(String key, String value); + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java new file mode 100644 index 00000000000..d65681e8f5b --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java @@ -0,0 +1,204 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.nio.charset.Charset; +import java.util.List; +import java.io.ByteArrayOutputStream; + +/** + * Escapes strings into and out of a format where they only contain printable characters. + * + * Need to duplicate escape / unescape of strings as we have in C++ for java version of system states. + * + * @author <a href="mailto:humbe@yahoo-inc.com">Haakon Humberset</a> + */ +public class StringUtilities { + private static Charset UTF8 = Charset.forName("utf8"); + + private static byte toHex(int val) { return (byte) (val < 10 ? '0' + val : 'a' + (val - 10)); } + + private static class ReplacementCharacters { + public byte needEscape[] = new byte[256]; + public byte replacement1[] = new byte[256]; + public byte replacement2[] = new byte[256]; + + public ReplacementCharacters() { + for (int i=0; i<256; ++i) { + if (i >= 32 && i <= 126) { + needEscape[i] = 0; + } else { + needEscape[i] = 3; + replacement1[i] = toHex((i >> 4) & 0xF); + replacement2[i] = toHex(i & 0xF); + } + } + makeSimpleEscape('"', '"'); + makeSimpleEscape('\\', '\\'); + makeSimpleEscape('\t', 't'); + makeSimpleEscape('\n', 'n'); + makeSimpleEscape('\r', 'r'); + makeSimpleEscape('\f', 'f'); + } + + private void makeSimpleEscape(char source, char dest) { + needEscape[source] = 1; + replacement1[source] = '\\'; + replacement2[source] = (byte) dest; + } + } + + private final static ReplacementCharacters replacementCharacters = new ReplacementCharacters(); + + public static String escape(String source) { return escape(source, '\0'); } + + /** + * Escapes strings into a format with only printable ASCII characters. + * + * @param source The string to escape + * @param delimiter Escape this character too, even if it is printable. + * @return The escaped string + */ + public static String escape(String source, char delimiter) { + byte bytes[] = source.getBytes(UTF8); + ByteArrayOutputStream result = new ByteArrayOutputStream(); + for (byte b : bytes) { + int val = b; + if (val < 0) val += 256; + if (b == delimiter) { + result.write('\\'); + result.write('x'); + result.write(toHex((val >> 4) & 0xF)); + result.write(toHex(val & 0xF)); + } else if (replacementCharacters.needEscape[val] == 0) { + result.write(b); + } else { + if (replacementCharacters.needEscape[val] == 3) { + result.write('\\'); + result.write('x'); + } + result.write(replacementCharacters.replacement1[val]); + result.write(replacementCharacters.replacement2[val]); + } + } + return new String(result.toByteArray(), UTF8); + } + + public static String unescape(String source) { + byte bytes[] = source.getBytes(UTF8); + ByteArrayOutputStream result = new ByteArrayOutputStream(); + for (int i=0; i<bytes.length; ++i) { + if (bytes[i] != '\\') { + result.write(bytes[i]); + continue; + } + if (i + 1 == bytes.length) throw new IllegalArgumentException("Found backslash at end of input"); + + if (bytes[i + 1] != (byte) 'x') { + switch (bytes[i + 1]) { + case '\\': result.write('\\'); break; + case '"': result.write('"'); break; + case 't': result.write('\t'); break; + case 'n': result.write('\n'); break; + case 'r': result.write('\r'); break; + case 'f': result.write('\f'); break; + default: + throw new IllegalArgumentException("Illegal escape sequence \\" + ((char) bytes[i+1]) + " found"); + } + ++i; + continue; + } + + if (i + 3 >= bytes.length) throw new IllegalArgumentException("Found \\x at end of input"); + + String hexdigits = "" + ((char) bytes[i + 2]) + ((char) bytes[i + 3]); + result.write((byte) Integer.parseInt(hexdigits, 16)); + i += 3; + } + return new String(result.toByteArray(), UTF8); + } + + /** + * Returns the given array flattened to string, with the given separator string + * @param array the array + * @param sepString or null + * @return imploded array + */ + public static String implode(String[] array, String sepString) { + if (array==null) return null; + StringBuilder ret = new StringBuilder(); + if (sepString==null) sepString=""; + for (int i = 0 ; i<array.length ; i++) { + ret.append(array[i]); + if (!(i==array.length-1)) ret.append(sepString); + } + return ret.toString(); + } + + /** + * Returns the given list flattened to one with newline between + * + * @return flattened string + */ + public static String implodeMultiline(List<String> lines) { + if (lines==null) return null; + return implode(lines.toArray(new String[0]), "\n"); + } + + /** + * This will truncate sequences in a string of the same character that exceed the maximum + * allowed length. + * + * @return The same string or a new one if truncation is done. + */ + public static String truncateSequencesIfNecessary(String text, int maxConsecutiveLength) { + char prev = 0; + int sequenceCount = 1; + for (int i = 0, m = text.length(); i < m ; i++) { + char curr = text.charAt(i); + if (prev == curr) { + sequenceCount++; + if (sequenceCount > maxConsecutiveLength) { + return truncateSequences(text, maxConsecutiveLength, i); + } + } else { + sequenceCount = 1; + prev = curr; + } + } + return text; + } + + private static String truncateSequences(String text, int maxConsecutiveLength, int firstTruncationPos) { + char [] truncated = text.toCharArray(); + char prev = truncated[firstTruncationPos]; + int sequenceCount = maxConsecutiveLength + 1; + int wp=firstTruncationPos; + for (int rp=wp+1; rp < truncated.length; rp++) { + char curr = truncated[rp]; + if (prev == curr) { + sequenceCount++; + if (sequenceCount <= maxConsecutiveLength) { + truncated[wp++] = curr; + } + } else { + truncated[wp++] = curr; + sequenceCount = 1; + prev = curr; + } + } + return String.copyValueOf(truncated, 0, wp); + } + + public static String stripSuffix(String string, String suffix) { + int index = string.lastIndexOf(suffix); + return index == -1 ? string : string.substring(0, index); + } + + /** + * Adds single quotes around object.toString + * Example: '12' + */ + public static String quote(Object object) { + return "'" + object.toString() + "'"; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8.java b/vespajlib/src/main/java/com/yahoo/text/Utf8.java new file mode 100644 index 00000000000..9126870117e --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8.java @@ -0,0 +1,595 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.ReadOnlyBufferException; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; + +/** + * utility class with functions for handling UTF-8 + * + * @author arnej27959 + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * + */ +public final class Utf8 { + + private static final byte [] TRUE = {(byte) 't', (byte) 'r', (byte) 'u', (byte) 'e'}; + private static final byte [] FALSE = {(byte) 'f', (byte) 'a', (byte) 'l', (byte) 's', (byte) 'e'}; + private static final byte[] LONG_MIN_VALUE_BYTES = String.valueOf(Long.MIN_VALUE).getBytes(StandardCharsets.UTF_8); + + /** Returns the Charset instance for UTF-8 */ + public static Charset getCharset() { + return StandardCharsets.UTF_8; + } + + /** To be used instead of String.String(byte[] bytes) */ + public static String toStringStd(byte[] data) { + return new String(data, StandardCharsets.UTF_8); + } + + /** + * Utility method as toString(byte[]). + * + * @param data + * bytes to decode + * @param offset + * index of first byte to decode + * @param length + * number of bytes to decode + * @return String decoded from UTF-8 + */ + public static String toString(byte[] data, int offset, int length) { + String s = toStringAscii(data, offset, length); + return s != null ? s : toString(ByteBuffer.wrap(data, offset, length)); + } + + /** + * Fetch a string from a ByteBuffer instance. ByteBuffer instances are + * stateful, so it is assumed to caller manipulates the instance's limit if + * the entire buffer is not a string. + * + * @param data + * The UTF-8 data source + * @return a decoded String + */ + public static String toString(ByteBuffer data) { + CharBuffer c = StandardCharsets.UTF_8.decode(data); + return c.toString(); + } + + /** + * Uses String.getBytes directly. + */ + public static byte[] toBytesStd(String str) { + return str.getBytes(StandardCharsets.UTF_8); + } + + /** + * Encode a long as its decimal representation, i.e. toAsciiBytes(15L) will + * return "15" encoded as UTF-8. In other words it is an optimized version + * of String.valueOf() followed by UTF-8 encoding. Avoid going through + * string in order to get a simple UTF-8 sequence. + * + * @param l + * value to represent as a decimal number encded as utf8 + * @return byte array + */ + public static byte[] toAsciiBytes(long l) { + // Handle Long.MIN_VALUE specifically, since it breaks all the assumptions + if (Long.MIN_VALUE == l) { + return LONG_MIN_VALUE_BYTES; + } + int count=1; + for (long v= l<0 ? -l : l; v >= 10; v=v/10, count++); + byte [] buf = new byte [count + ((l<0) ? 1 : 0)]; + int offset = 0; + if (l < 0) { + buf[offset++] = (byte) '-'; + l = -l; + } + for (count--; count >= 0; l=l/10, count--) { + buf[count+offset] = (byte)(0x30 + l%10); + } + return buf; + } + + public static byte [] toAsciiBytes(boolean v) { + return v ? TRUE : FALSE; + } + + /** + * Will try an optimistic approach to utf8 encoding. + * That is 4.6x faster that the brute encode for ascii, not accounting for reduced memory footprint and GC. + * @param str The string to encode. + * @return Utf8 encoded array + */ + public static byte[] toBytes(String str) { + byte [] utf8 = toBytesAscii(str); + return utf8 != null ? utf8 : str.getBytes(StandardCharsets.UTF_8); + } + /** + * Will try an optimistic approach to utf8 decoding. + * + * @param utf8 The string to encode. + * @return Utf8 encoded array + */ + public static String toString(byte [] utf8) { + String s = toStringAscii(utf8, 0, utf8.length); + return s != null ? s : new String(utf8, StandardCharsets.UTF_8); + } + + /** + * If String is purely ascii 7bit it will encode it as a byte array. + * @param str The string to encode + * @return Utf8 encoded array + */ + private static byte[] toBytesAscii(final CharSequence str) { + byte [] utf8 = new byte[str.length()]; + for (int i=0; i < utf8.length; i++) { + char c = str.charAt(i); + if ((c < 0) || (c >= 0x80)) { + return null; + } + utf8[i] = (byte)c; + } + return utf8; + } + + private static String toStringAscii(byte [] b, int offset, int length) { + if (length > 0) { + char [] s = new char[length]; + for (int i=0; i < length; i++) { + if (b[offset + i] >= 0) { + s[i] = (char)b[offset+i]; + } else { + return null; + } + } + return new String(s); + } else { + return ""; + } + } + + /** + * Utility method as toBytes(String). + * + * @param str + * String to encode + * @param offset + * index of first character to encode + * @param length + * number of characters to encode + * @return substring encoded as UTF-8 + */ + public static byte[] toBytes(String str, int offset, int length) { + CharBuffer c = CharBuffer.wrap(str, offset, offset + length); + ByteBuffer b = StandardCharsets.UTF_8.encode(c); + byte[] result = new byte[b.remaining()]; + b.get(result); + return result; + } + + /** + * Direct encoding of a String into an array. + * + * @param str + * string to encode + * @param srcOffset + * index of first character in string to encode + * @param srcLen + * number of characters in string to encode + * @param dst + * destination for encoded data + * @param dstOffset + * index of first position to write data + * @return the number of bytes written to the array. + */ + public static int toBytes(String str, int srcOffset, int srcLen, byte[] dst, int dstOffset) { + CharBuffer c = CharBuffer.wrap(str, srcOffset, srcOffset + srcLen); + ByteBuffer b = StandardCharsets.UTF_8.encode(c); + int encoded = b.remaining(); + b.get(dst, dstOffset, encoded); + return encoded; + } + + /** + * Encode a string directly into a ByteBuffer instance. + * + * <p> + * This method is somewhat more cumbersome than the rest of the helper + * methods in this library, as it is intended for use cases in the following + * style, if extraneous copying is highly undesirable: + * + * <pre> + * String[] a = {"abc", "def", "ghi\u00e8"}; + * int[] aLens = {3, 3, 5}; + * CharsetEncoder ce = Utf8.getNewEncoder(); + * ByteBuffer forWire = ByteBuffer.allocate(someNumber); + * + * for (int i = 0; i < a.length; i++) { + * forWire.putInt(aLens[i]); + * Utf8.toBytes(a[i], 0, a[i].length(), forWire, ce); + * } + * </pre> + * + * @see Utf8#getNewEncoder() + * + * @param src the string to encode + * @param srcOffset index of first character to encode + * @param srcLen number of characters to encode + * @param dst the destination ByteBuffer + * @param encoder the character encoder to use + */ + public static void toBytes(String src, int srcOffset, int srcLen, ByteBuffer dst, CharsetEncoder encoder) { + CharBuffer c = CharBuffer.wrap(src, srcOffset, srcOffset + srcLen); + encoder.encode(c, dst, true); + } + + /** + * Create a new UTF-8 encoder. + * + * @see Utf8#toBytes(String, int, int, ByteBuffer, CharsetEncoder) + */ + public static CharsetEncoder getNewEncoder() { + return StandardCharsets.UTF_8.newEncoder().onMalformedInput(CodingErrorAction.REPLACE) + .onUnmappableCharacter(CodingErrorAction.REPLACE); + } + + /** + * Count the number of bytes needed to represent a given sequence of 16-bit + * char values as a UTF-8 encoded array. This method is written to be cheap + * to invoke. + * + * Note: It is strongly assumed to character sequence is valid. + */ + public static int byteCount(CharSequence str) { return byteCount(str, 0, str.length()); } + + /** + * Count the number of bytes needed to represent a given sequence of 16-bit + * char values as a UTF-8 encoded array. This method is written to be cheap + * to invoke. + * + * Note: It is strongly assumed to character sequence is valid. + */ + public static int byteCount(CharSequence str, int offset, int length) { + int count = 0; + int barrier = offset + length; + int i = offset; + while (i < barrier) { + int codePoint = (int) str.charAt(i); + if (codePoint < 0x800) { + if (codePoint < 0x80) { + ++count; + } else { + count += 2; + } + ++i; + } else { + // bit masking to check (codePoint >= 0xd800 && codePoint < + // 0xe000) + if ((codePoint & 0xF800) == 0xD800) { + count += 4; + i += 2; + } else { + count += 3; + ++i; + } + } + } + return count; + } + + /** + * Count the number of Unicode code units ("UTF-16 characters") needed to + * represent a given array of UTF-8 characters. This method is written to be + * cheap to invoke. + * + * Note: It is strongly assumed the sequence is valid. + */ + public static int unitCount(byte[] utf8) { return unitCount(utf8, 0, utf8.length); } + + /** + * Count the number of Unicode code units ("UTF-16 characters") needed to + * represent a given array of UTF-8 characters. This method is written to be + * cheap to invoke. + * + * Note: It is strongly assumed the sequence is valid. + * + * @param utf8 + * raw data + * @param offset + * index of first byte of UTF-8 sequence to check + * @param length + * number of bytes in the UTF-8 sequence to check + */ + public static int unitCount(byte[] utf8, int offset, int length) { + int units = 0; + int barrier = offset + length; + int i = offset; + while (i < barrier) { + byte firstByte = utf8[i]; + if (firstByte >= -16) { + if (firstByte >= 0) { + ++units; + ++i; + } else { + units += 2; + i += 4; + } + } else { + if (firstByte >= -32) { + ++units; + i += 3; + } else { + ++units; + i += 2; + } + } + } + return units; + } + + /** + * Calculate the number of Unicode code units ("UTF-16 characters") needed + * to represent a given UTF-8 encoded code point. + * + * @param firstByte + * the first byte of a character encoded as UTF-8 + * @return the number of UTF-16 code units needed to represent the given + * code point + */ + public static int unitCount(byte firstByte) { + int units = 0; + if (firstByte >= -16) { + if (firstByte >= 0) { + units = 1; + } else { + units = 2; + } + } else { + units = 1; + } + return units; + } + + /** + * Inspects a byte assumed to be the first byte in a UTF8 to check how many + * bytes in total the sequence of bytes will use. + * + * @param firstByte + * the first byte of a UTF8 encoded character + * @return the number of bytes used to encode the character + */ + // To avoid code duplication, this function should be used by unitCount(), + // but then unitCount(byte[], int, int) would not be as tight. This class is in general + // meant to be safe to use in performance sensitive code. + public static int totalBytes(byte firstByte) { + if (firstByte >= -16) { + if (firstByte >= 0) { + return 1; + } else { + return 4; + } + } else { + if (firstByte >= -32) { + return 3; + } else { + return 2; + } + } + } + + /** + * Returns an integer array the length as the input string plus one. For + * every index in the array, the corresponding value gives the index into + * the UTF-8 byte sequence that can be created from the input. + * + * @param value + * a String to generate UTF-8 byte indexes from + * @return an array containing corresponding UTF-8 byte indexes + */ + public static int[] calculateBytePositions(CharSequence value) { + int[] positions = new int[value.length() + 1]; + + int bytePos = 0; + int barrier = value.length(); + int i = 0; + int codepointNo = 0; + positions[codepointNo++] = bytePos; + while (i < barrier) { + int codePoint = (int) value.charAt(i); + if (codePoint < 0x800) { + if (codePoint < 0x80) { + ++bytePos; + } else { + bytePos += 2; + } + ++i; + } else { + // bit masking to check (codePoint >= 0xd800 && codePoint < + // 0xe000) + if ((codePoint & 0xF800) == 0xD800) { + // double position write, as we have a surrogate pair + positions[codepointNo++] = bytePos; + bytePos += 4; + i += 2; + } else { + bytePos += 3; + ++i; + } + } + positions[codepointNo++] = bytePos; + } + return positions; + } + + /** + * Returns an array of the same length as the input array plus one. For + * every index in the array, the corresponding value gives the index into + * the Java string (UTF-16 sequence) that can be created from the input. + * + * @param utf8 + * a byte array containing a string encoded as UTF-8. Note: It is + * strongly assumed that this sequence is correct. + * @return an array containing corresponding UTF-16 character indexes. If input + * array is empty, returns an array containg a single zero. + */ + public static int[] calculateStringPositions(byte[] utf8) { + if (utf8.length == 0) { + return new int[] { 0 }; + } + int[] positions = new int[utf8.length + 1]; + int utf8BytePos = 0; + int charPos = 0; + int lastUtf8SequencePos = 0; + int utf8SequenceLen = 0; + while (utf8BytePos < utf8.length) { + utf8SequenceLen = totalBytes(utf8[utf8BytePos]); + lastUtf8SequencePos = utf8BytePos; + for (int utf8SequenceCnt = 0; utf8SequenceCnt < utf8SequenceLen; utf8SequenceCnt++) { + positions[utf8BytePos + utf8SequenceCnt] = charPos; + } + utf8BytePos += utf8SequenceLen; + charPos++; + } + //we need to check if the last UTF-8 sequence resulted in a surrogate pair: + int lastCharLen = unitCount(utf8, lastUtf8SequencePos, utf8SequenceLen); + positions[utf8.length] = charPos + lastCharLen - 1; + return positions; + } + + + /** + * Encode a valid Unicode codepoint as a sequence of UTF-8 bytes into a new allocated array. + * + * @param codepoint Unicode codepoint to encode + * @return number of bytes written + * @throws IndexOutOfBoundsException if there is insufficient room for the encoded data in the given array + */ + public static byte[] encode(int codepoint) { + byte[] destination = new byte[codePointAsUtf8Length(codepoint)]; + encode(codepoint, destination, 0); + return destination; + } + + /** + * Encode a valid Unicode codepoint as a sequence of UTF-8 bytes into an array. + * + * @param codepoint Unicode codepoint to encode + * @param destination array to write into + * @param offset index of first byte written + * @return index of the first byte after the last byte written (i.e. offset plus number of bytes written) + * @throws IndexOutOfBoundsException if there is insufficient room for the encoded data in the given array + */ + public static int encode(int codepoint, byte[] destination, int offset) { + int writeOffset = offset; + byte firstByte = firstByte(codepoint); + int leftToWrite = codePointAsUtf8Length(codepoint) - 1; + destination[writeOffset++] = firstByte; + while (leftToWrite-- > 0) { + destination[writeOffset++] = trailingOctet(codepoint, leftToWrite); + } + return writeOffset; + } + + /** + * Encode a valid Unicode codepoint as a sequence of UTF-8 bytes into a + * ByteBuffer. + * + * @param codepoint + * Unicode codepoint to encode + * @param destination + * buffer to write into + * @throws BufferOverflowException + * if the buffer's limit is met while writing (propagated from + * the ByteBuffer) + * @throws ReadOnlyBufferException + * if the buffer is read only (propagated from the ByteBuffer) + */ + public static void encode(int codepoint, ByteBuffer destination) { + byte firstByte = firstByte(codepoint); + int leftToWrite = codePointAsUtf8Length(codepoint) - 1; + destination.put(firstByte); + while (leftToWrite-- > 0) { + destination.put(trailingOctet(codepoint, leftToWrite)); + } + } + + /** + * Encode a valid Unicode codepoint as a sequence of UTF-8 bytes into an + * OutputStream. + * + * @param codepoint + * Unicode codepoint to encode + * @param destination + * buffer to write into + * @return number of bytes written + * @throws IOException + * propagated from stream + */ + public static int encode(int codepoint, OutputStream destination) throws IOException { + byte firstByte = firstByte(codepoint); + int toWrite = codePointAsUtf8Length(codepoint); + int leftToWrite = toWrite - 1; + destination.write(firstByte); + while (leftToWrite-- > 0) { + destination.write(trailingOctet(codepoint, leftToWrite)); + } + return toWrite; + } + + + private static byte trailingOctet(int codepoint, int leftToWrite) { + return (byte) (0x80 | ((codepoint >> (6 * leftToWrite)) & 0x3F)); + } + + private static byte firstByte(int codepoint) { + if (codepoint < 0x800) { + if (codepoint < 0x80) { + return (byte) codepoint; + } else { + return (byte) (0xC0 | codepoint >> 6); + } + } else { + if (codepoint < 0x10000) { + return (byte) (0xE0 | codepoint >> 12); + } else { + return (byte) (0xF0 | codepoint >> 18); + } + } + + } + + /** + * Return the number of octets needed to encode a valid Unicode codepoint as UTF-8. + * + * @param codepoint the Unicode codepoint to inspect + * @return the number of bytes needed for UTF-8 representation + */ + public static int codePointAsUtf8Length(int codepoint) { + if (codepoint < 0x800) { + if (codepoint < 0x80) { + return 1; + } else { + return 2; + } + } else { + if (codepoint < 0x10000) { + return 3; + } else { + return 4; + } + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8Array.java b/vespajlib/src/main/java/com/yahoo/text/Utf8Array.java new file mode 100644 index 00000000000..30b2e665392 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8Array.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + + +import java.nio.ByteBuffer; + +/** + * This is a primitive class that owns an array of utf8 encoded string. + * This is a class that has speed as its primary purpose. + * If you have a string, consider Utf8String + * If you have a large backing array consider Utf8PartialArray + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ + +public class Utf8Array extends AbstractUtf8Array { + + protected final byte[] utf8; + + /** + * This will simply wrap the given array assuming it is valid utf8. + * Note that the immutability of this primitive class depends on that the buffer + * is not modified after ownership has been transferred. + * @param utf8data The utf8 byte sequence. + */ + public Utf8Array(final byte[] utf8data) { + utf8 = utf8data; + } + + /** + * This will create a new array from the window given. No validation done. + * Note that this will copy data. You might also want to consider Utf8PartialArray + * @param utf8data The base array. + * @param offset The offset from where to copy from + * @param length The number of bytes that should be copied. + */ + public Utf8Array(byte[] utf8data, int offset, int length) { + this.utf8 = new byte[length]; + System.arraycopy(utf8data, offset, this.utf8, 0, length); + } + + /** + * This will fetch length bytes from the given buffer. + * @param buf The ByteBuffer to read from + * @param length number of bytes to read + */ + public Utf8Array(ByteBuffer buf, int length) { + this.utf8 = new byte[length]; + buf.get(this.utf8, 0, length); + } + + @Override + public byte[] getBytes() { + return utf8; + } + + @Override + public int getByteLength() { + return utf8.length; + } + + @Override + protected int getByteOffset() { + return 0; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8PartialArray.java b/vespajlib/src/main/java/com/yahoo/text/Utf8PartialArray.java new file mode 100644 index 00000000000..c6032e751b7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8PartialArray.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * This wraps a window in a backing byte array. Without doing any copying. + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public class Utf8PartialArray extends Utf8Array { + final int offset; + final int length; + + /** + * Takes ownership of the given byte array. And keeps note of where + * the interesting utf8 sequence start and its length. + * @param utf8data The backing byte array. + * @param offset The start of the utf8 sequence. + * @param bytes The length of the utf8 sequence. + */ + public Utf8PartialArray(byte[] utf8data, int offset, int bytes) { + super(utf8data); + this.offset = offset; + this.length = bytes; + } + @Override + public int getByteLength() { + return length; + } + + @Override + protected int getByteOffset() { + return offset; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8String.java b/vespajlib/src/main/java/com/yahoo/text/Utf8String.java new file mode 100644 index 00000000000..1f4dfc0d4f6 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8String.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + + +/** + * String with Utf8 backing. + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public final class Utf8String extends Utf8Array implements CharSequence +{ + private final String s; + + /** + * This will construct a utf8 backing of the given string. + * @param str The string that will be utf8 encoded + */ + public Utf8String(String str) { + super(Utf8.toBytes(str)); + s = str; + } + + /** + * This will create a string based on the utf8 sequence. + * @param utf8 The backing array + */ + public Utf8String(AbstractUtf8Array utf8) { + super(utf8.getBytes(), utf8.getByteOffset(), utf8.getByteLength()); + s = utf8.toString(); + } + + @Override + public char charAt(int index) { + return toString().charAt(index); + } + @Override + public int length() { + return toString().length(); + } + @Override + public CharSequence subSequence(int start, int end) { + return toString().subSequence(start, end); + } + + @Override + public boolean equals(Object o) { + if (o instanceof Utf8String) { + return s.equals(o.toString()); + } + return super.equals(o); + } + + @Override + public String toString() { + return s; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/XML.java b/vespajlib/src/main/java/com/yahoo/text/XML.java new file mode 100644 index 00000000000..c688d5f9722 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/XML.java @@ -0,0 +1,636 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.SAXParseException; + +/** + * Static XML utility methods + * + * @author Bjorn Borud + * @author Vegard Havdal + * @author bratseth + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class XML { + /** + * The point of this weird class and the jumble of abstract methods is + * linking the scan for characters that must be quoted into the quoting + * table, and making it actual work to make them go out of sync again. + */ + private static abstract class LegalCharacters { + // To quote http://www.w3.org/TR/REC-xml/ : + // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | + // [#x10000-#x10FFFF] + final boolean isLegal(final int codepoint, final boolean escapeLow, + final int stripCodePoint, final boolean isAttribute) { + if (codepoint == stripCodePoint) { + return removeCodePoint(); + } else if (codepoint < ' ') { + if (!escapeLow) { + return true; + } + switch (codepoint) { + case 0x09: + case 0x0a: + case 0x0d: + return true; + default: + return ctrlEscapeCodePoint(codepoint); + } + } else if (codepoint >= 0x20 && codepoint <= 0xd7ff) { + switch (codepoint) { + case '&': + return ampCodePoint(); + case '<': + return ltCodePoint(); + case '>': + return gtCodePoint(); + case '"': + return quotCodePoint(isAttribute); + default: + return true; + } + } else if ((codepoint >= 0xe000 && codepoint <= 0xfffd) + || (codepoint >= 0x10000 && codepoint <= 0x10ffff)) { + return true; + } else { + return filterCodePoint(codepoint); + + } + } + + private boolean quotCodePoint(final boolean isAttribute) { + if (isAttribute) { + quoteQuot(); + return false; + } else { + return true; + } + } + + private boolean filterCodePoint(final int codepoint) { + replace(codepoint); + return false; + } + + private boolean gtCodePoint() { + quoteGt(); + return false; + } + + private boolean ltCodePoint() { + quoteLt(); + return false; + } + + private boolean ampCodePoint() { + quoteAmp(); + return false; + } + + private boolean ctrlEscapeCodePoint(final int codepoint) { + ctrlEscape(codepoint); + return false; + } + + private boolean removeCodePoint() { + remove(); + return false; + } + + protected abstract void quoteQuot(); + + protected abstract void quoteGt(); + + protected abstract void quoteLt(); + + protected abstract void quoteAmp(); + + protected abstract void remove(); + + protected abstract void ctrlEscape(int codepoint); + + protected abstract void replace(int codepoint); + } + + private static final class Quote extends LegalCharacters { + + char[] lastQuoted; + private static final char[] EMPTY = new char[0]; + private static final char[] REPLACEMENT_CHARACTER = "\ufffd".toCharArray(); + private static final char[] AMP = "&".toCharArray(); + private static final char[] LT = "<".toCharArray(); + private static final char[] GT = ">".toCharArray(); + private static final char[] QUOT = """.toCharArray(); + + @Override + protected void remove() { + lastQuoted = EMPTY; + } + + @Override + protected void replace(final int codepoint) { + lastQuoted = REPLACEMENT_CHARACTER; + } + + @Override + protected void quoteQuot() { + lastQuoted = QUOT; + } + + @Override + protected void quoteGt() { + lastQuoted = GT; + } + + @Override + protected void quoteLt() { + lastQuoted = LT; + } + + @Override + protected void quoteAmp() { + lastQuoted = AMP; + } + + @Override + protected void ctrlEscape(final int codepoint) { + lastQuoted = REPLACEMENT_CHARACTER; + } + } + + private static final class Scan extends LegalCharacters { + + @Override + protected void quoteQuot() { + } + + @Override + protected void quoteGt() { + } + + @Override + protected void quoteLt() { + } + + @Override + protected void quoteAmp() { + } + + @Override + protected void remove() { + } + + @Override + protected void ctrlEscape(final int codepoint) { + } + + @Override + protected void replace(final int codepoint) { + } + } + + private static final Scan scanner = new Scan(); + + /** + * Replaces the characters that need to be escaped with their corresponding + * character entities. + * + * @param s1 + * String possibly containing characters that need to be escaped + * in XML + * + * @return Returns the input string with special characters that need to be + * escaped replaced by character entities. + */ + public static String xmlEscape(String s1) { + return xmlEscape(s1, true, true, null, -1); + } + + /** + * Replaces the characters that need to be escaped with their corresponding + * character entities. + * + * @param s1 + * String possibly containing characters that need to be escaped + * in XML + * @param isAttribute + * Is the input string to be used as an attribute? + * + * @return Returns the input string with special characters that need to be + * escaped replaced by character entities + */ + public static String xmlEscape(String s1, boolean isAttribute) { + return xmlEscape(s1, isAttribute, true, null, -1); + } + + /** + * Replaces the characters that need to be escaped with their corresponding + * character entities. + * + * @param s1 + * String possibly containing characters that need to be escaped + * in XML + * @param isAttribute + * Is the input string to be used as an attribute? + * + * + * @param stripCharacter + * any occurrence of this character is removed from the string + * + * @return Returns the input string with special characters that need to be + * escaped replaced by character entities + */ + public static String xmlEscape(String s1, boolean isAttribute, char stripCharacter) { + return xmlEscape(s1, isAttribute, true, null, (int) stripCharacter); + } + + /** + * Replaces the characters that need to be escaped with their corresponding + * character entities. + * + * @param s1 + * String possibly containing characters that need to be escaped + * in XML + * @param isAttribute + * Is the input string to be used as an attribute? + * + * @param escapeLowAscii + * Should ascii characters below 32 be escaped as well + * + * @return Returns the input string with special characters that need to be + * escaped replaced by character entities + */ + public static String xmlEscape(String s1, boolean isAttribute, boolean escapeLowAscii) { + return xmlEscape(s1, isAttribute, escapeLowAscii, null, -1); + } + + /** + * Replaces the characters that need to be escaped with their corresponding + * character entities. + * + * @param s1 + * String possibly containing characters that need to be escaped + * in XML + * @param isAttribute + * Is the input string to be used as an attribute? + * + * @param escapeLowAscii + * Should ascii characters below 32 be escaped as well + * + * @param stripCharacter + * any occurrence of this character is removed from the string + * + * @return Returns the input string with special characters that need to be + * escaped replaced by character entities + */ + public static String xmlEscape(String s1, boolean isAttribute, boolean escapeLowAscii, char stripCharacter) { + return xmlEscape(s1, isAttribute, escapeLowAscii, null, (int) stripCharacter); + } + + /** + * Replaces the following: + * <ul> + * <li>all ascii codes less than 32 except 9 (tab), 10 (nl) and 13 (cr) + * <li>ampersand (&) + * <li>less than (<) + * <li>larger than (>) + * <li>double quotes (") if isAttribute is <code>true</code> + * </ul> + * with character entities. + * + */ + public static String xmlEscape(String string, boolean isAttribute, StringBuilder buffer) { + return xmlEscape(string, isAttribute, true, buffer, -1); + } + + /** + * Replaces the following: + * <ul> + * <li>all ascii codes less than 32 except 9 (tab), 10 (nl) and 13 (cr) if + * escapeLowAscii is <code>true</code> + * <li>ampersand (&) + * <li>less than (<) + * <li>larger than (>) + * <li>double quotes (") if isAttribute is <code>true</code> + * </ul> + * with character entities. + * + */ + public static String xmlEscape(String string, boolean isAttribute, boolean escapeLowAscii, StringBuilder buffer) { + return xmlEscape(string, isAttribute, escapeLowAscii, buffer, -1); + } + + /** + * Replaces the following: + * <ul> + * <li>all ascii codes less than 32 except 9 (tab), 10 (nl) and 13 (cr) if + * escapeLowAscii is <code>true</code> + * <li>ampersand (&) + * <li>less than (<) + * <li>larger than (>) + * <li>double quotes (") if isAttribute is <code>true</code> + * </ul> + * with character entities. + * + * @param stripCodePoint + * any occurrence of this character is removed from the string + */ + public static String xmlEscape(String string, boolean isAttribute, boolean escapeLowAscii, + StringBuilder buffer, int stripCodePoint) { + // buffer and stripCodePoint changed order in the signature compared to + // the char based API to avoid wrong method being called + + // This is inner loop stuff, so we sacrifice a little for speed - + // no copying will occur until a character needing escaping is found + boolean legalCharacter = true; + Quote escaper; + int i = 0; + + for (i = 0; i < string.length() && legalCharacter; i = string.offsetByCodePoints(i, 1)) { + legalCharacter = scanner.isLegal(string.codePointAt(i), escapeLowAscii, stripCodePoint, isAttribute); + } + if (legalCharacter) { + return string; + } + + i = string.offsetByCodePoints(i, -1); // Back to the char needing escaping + escaper = new Quote(); + + if (buffer == null) { + buffer = new StringBuilder((int) (string.length() * 1.2)); + } + + // ugly appending zero length strings + if (i > 0) { + buffer.append(string.substring(0, i)); + } + + // i is at the first codepoint which needs replacing + // Don't guard against double-escaping, as: + // don't try to be clever (LCJ). + for (; i < string.length(); i = string.offsetByCodePoints(i, 1)) { + int codepoint = string.codePointAt(i); + if (escaper.isLegal(codepoint, escapeLowAscii, stripCodePoint, isAttribute)) { + buffer.appendCodePoint(codepoint); + } else { + buffer.append(escaper.lastQuoted); + } + } + return buffer.toString(); + } + + /** + * Returns the Document of an XML file reader + * + * @throws RuntimeException + * if the root Document cannot be returned + */ + public static Document getDocument(Reader reader) { + try { + return getDocumentBuilder().parse(new InputSource(reader)); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read '" + reader + "'", e); + } catch (SAXParseException e) { + throw new IllegalArgumentException("Could not parse '" + reader + "', error at line " + e.getLineNumber() + ", column " + e.getColumnNumber(), e); + } catch (SAXException e) { + throw new IllegalArgumentException("Could not parse '" + reader + "'", e); + } + } + + /** + * Returns the Document of the string XML payload + */ + public static Document getDocument(String string) { + return getDocument(new StringReader(string)); + } + + /** + * Creates a new XML DocumentBuilder + * + * @return a DocumentBuilder + * @throws RuntimeException + * if we fail to create one + */ + public static DocumentBuilder getDocumentBuilder() { + return getDocumentBuilder("com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl", null); + } + + /** + * Creates a new XML DocumentBuilder + * + * @param implementation + * which jaxp implementation should be used + * @param classLoader + * which class loader should be used when getting a new + * DocumentBuilder + * @throws RuntimeException + * if we fail to create one + * @return a DocumentBuilder + */ + public static DocumentBuilder getDocumentBuilder(String implementation, ClassLoader classLoader) { + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(implementation, classLoader); + factory.setNamespaceAware(true); + factory.setXIncludeAware(true); + return factory.newDocumentBuilder(); + } catch (ParserConfigurationException e) { + throw new RuntimeException("Could not create an XML builder"); + } + } + + /** + * Returns the child Element objects from a w3c dom spec + * + * @return List of elements. Empty list (never null) if none found or if the + * given element is null + */ + public static List<Element> getChildren(Element spec) { + List<Element> children = new ArrayList<>(); + if (spec == null) { + return children; + } + + NodeList childNodes = spec.getChildNodes(); + for (int i = 0; i < childNodes.getLength(); i++) { + Node child = childNodes.item(i); + if (child instanceof Element) { + children.add((Element) child); + } + } + return children; + } + + /** + * Returns the child Element objects with given name from a w3c dom spec + * + * @return List of elements. Empty list (never null) if none found or the + * given element is null + */ + public static List<Element> getChildren(Element spec, String name) { + List<Element> ret = new ArrayList<>(); + if (spec == null) { + return ret; + } + + NodeList children = spec.getChildNodes(); + if (children == null) { + return ret; + } + for (int i = 0; i < children.getLength(); i++) { + Node child = children.item(i); + if (child != null && child instanceof Element) { + if (child.getNodeName().equals(name)) { + ret.add((Element) child); + } + } + } + return ret; + } + + /** + * Gets the string contents of the given Element. Returns "", never null if + * the element is null, or has no content + */ + public static String getValue(Element e) { + if (e == null) { + return ""; + } + Node child = e.getFirstChild(); + if (child == null) { + return ""; + } + return child.getNodeValue(); + } + + /** Returns the first child with the given name, or null if none */ + public static Element getChild(Element e, String name) { + return (getChildren(e, name).size() >= 1) ? getChildren(e, name).get(0) : null; + } + + /** + * Returns the path to the given xml node, where each node name is separated + * by the given separator string. + * + * @param n + * The xml node to find path to + * @param sep + * The separator string + * @return The path to the xml node as a String + */ + public static String getNodePath(Node n, String sep) { + if (n == null) { + return ""; + } + StringBuffer ret = new StringBuffer(n.getNodeName()); + while ((n.getParentNode() != null) && !(n.getParentNode() instanceof Document)) { + n = n.getParentNode(); + ret.insert(0, sep).insert(0, n.getNodeName()); + } + return ret.toString(); + } + + + private static boolean inclusiveWithin(int x, int low, int high) { + return low <= x && x <= high; + } + + private static boolean nameStartSet(int codepoint) { + // NameStartChar ::= ":" | [A-Z] | "_" | [a-z] | [#xC0-#xD6] | + // [#xD8-#xF6] | [#xF8-#x2FF] | [#x370-#x37D] | [#x37F-#x1FFF] | + // [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] | [#x3001-#xD7FF] + // | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] | [#x10000-#xEFFFF] + + boolean valid; + if (codepoint < 0xC0) { + valid = inclusiveWithin(codepoint, 'a', 'z') + || inclusiveWithin(codepoint, 'A', 'Z') || codepoint == '_' + || codepoint == ':'; + } else { + valid = inclusiveWithin(codepoint, 0xC0, 0xD6) + || inclusiveWithin(codepoint, 0xD8, 0xF6) + || inclusiveWithin(codepoint, 0xF8, 0x2FF) + || inclusiveWithin(codepoint, 0x370, 0x37D) + || inclusiveWithin(codepoint, 0x37F, 0x1FFF) + || inclusiveWithin(codepoint, 0x200C, 0x200D) + || inclusiveWithin(codepoint, 0x2070, 0x218F) + || inclusiveWithin(codepoint, 0x2C00, 0x2FEF) + || inclusiveWithin(codepoint, 0x3001, 0xD7FF) + || inclusiveWithin(codepoint, 0xF900, 0xFDCF) + || inclusiveWithin(codepoint, 0xFDF0, 0xFFFD) + || inclusiveWithin(codepoint, 0x10000, 0xEFFFF); + } + return valid; + } + + private static boolean nameSetExceptStart(int codepoint) { + // "-" | "." | [0-9] | #xB7 | [#x0300-#x036F] | [#x203F-#x2040] + boolean valid; + if (codepoint < 0xB7) { + valid = inclusiveWithin(codepoint, '0', '9') || codepoint == '-' + || codepoint == '.'; + } else { + + valid = codepoint == '\u00B7' + || inclusiveWithin(codepoint, 0x300, 0x36F) + || inclusiveWithin(codepoint, 0x023F, 0x2040); + } + return valid; + } + + private static boolean nameChar(int codepoint, boolean first) { + // NameChar ::= NameStartChar | "-" | "." | [0-9] | #xB7 | [#x0300-#x036F] | [#x203F-#x2040] + boolean valid = nameStartSet(codepoint); + return first ? valid : valid || nameSetExceptStart(codepoint); + } + + + /** + * Check whether the name of a tag or attribute conforms to <a + * href="http://www.w3.org/TR/2006/REC-xml11-20060816/#sec-common-syn">XML + * 1.1 (Second Edition)</a>. This does not check against reserved names, it + * only checks the set of characters used. + * + * @param possibleName + * a possibly valid XML name + * @return true if the name may be used as an XML tag or attribute name + */ + public static boolean isName(CharSequence possibleName) { + final int barrier = possibleName.length(); + int i = 0; + boolean valid = true; + boolean first = true; + + if (barrier < 1) { + valid = false; + } + + while (valid && i < barrier) { + char c = possibleName.charAt(i++); + if (Character.isHighSurrogate(c)) { + valid = nameChar(Character.toCodePoint(c, possibleName.charAt(i++)), first); + } else { + valid = nameChar((int) c, first); + } + first = false; + } + return valid; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/XMLWriter.java b/vespajlib/src/main/java/com/yahoo/text/XMLWriter.java new file mode 100644 index 00000000000..ee5ff753c57 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/XMLWriter.java @@ -0,0 +1,410 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.Writer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * A stream wrapper which contains utility methods for writing xml. + * All methods return this for convenience. + * <p> + * The methods of this writer can be used in conjunction with writing tags in raw form directly to the writer + * if some care is taken to close start tags and insert line breaks explicitly. If all content is written + * using these methods, start tags are closed and newlines inserted automatically as appropriate. + * + * @author bratseth + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class XMLWriter extends ForwardWriter { + + /** Configuration */ + private final int maxIndentLevel, maxLineSeparatorLevel; + + /** The current list of parent tags */ + private final List<Utf8String> openTags = new ArrayList<>(); + private final List<Utf8String> unmodifiableOpenTags = Collections.unmodifiableList(openTags); + + /** Control state */ + private boolean inOpenStartTag, currentIsMultiline, isFirstInParent; + + /** Write markup directly to this with no encoding if it is non-null (an optimization) */ + private final boolean markupIsAscii; + static private final Utf8String SPACE = new Utf8String(" "); + static private final Utf8String INDENT = new Utf8String(" "); + static private final Utf8String ATTRIBUTE_START = new Utf8String("=\""); + static private final Utf8String ATTRIBUTE_END = new Utf8String("\""); + static private final Utf8String ENCODING_START = new Utf8String("<?xml version=\"1.0\" encoding=\""); + static private final Utf8String ENCODING_END = new Utf8String("\" ?>\n"); + static private final Utf8String LF = new Utf8String("\n"); + static private final Utf8String LT = new Utf8String("<"); + static private final Utf8String GT = new Utf8String(">"); + static private final Utf8String ELT = new Utf8String("</"); + static private final Utf8String EGT = new Utf8String("/>"); + /** + * Creates an XML wrapper of a writer having maxIndentLevel=10 and maxLineSeparatorLevel=1 + * + * @param writer the writer to which this writers (accessible from this by getWrapped) + */ + public XMLWriter(Writer writer) { + this(writer,10); + } + + /** + * Creates an XML wrapper of a writer having maxIndentLevel=10 and maxLineSeparatorLevel=1 + * + * @param writer the writer to which this writers (accessible from this by getWrapped) + * @param markupIsAscii set to false to make this encode markup (tags, attributes). By default encoding + * is skipped if the underlying stream uses utf encoding for performance (yes, this matters) + */ + public XMLWriter(Writer writer,boolean markupIsAscii) { + this(writer,10,markupIsAscii); + } + + /** + * Creates an XML wrapper of a writer having maxLineSeparatorLevel=1 + * + * @param writer the writer to which this writers (accessible from this by getWrapped) + * @param maxIndentLevel the max number of tag levels for which we'll continue to indent, or -1 to + * never indent. The top level tag is level 0, etc. + */ + public XMLWriter(Writer writer,int maxIndentLevel) { + this(writer,maxIndentLevel,1); + } + + /** + * Creates an XML wrapper of a writer having maxLineSeparatorLevel=1 + * + * @param writer the writer to which this writers (accessible from this by getWrapped) + * @param maxIndentLevel the max number of tag levels for which we'll continue to indent, or -1 to + * never indent. The top level tag is level 0, etc. + * @param markupIsAscii set to false to make this encode markup (tags, attributes). By default encoding + * is skipped if the underlying stream uses utf encoding for performance (yes, this matters) + */ + public XMLWriter(Writer writer,int maxIndentLevel,boolean markupIsAscii) { + this(writer,maxIndentLevel,1,markupIsAscii); + } + + /** + * Creates an XML wrapper of a writer + * + * @param writer the writer to which this writers (accessible from this by getWrapped) + * @param maxIndentLevel the max number of tag levels for which we'll continue to indent, or -1 to + * never indent. The top level tag is level 0, etc. + * @param maxLineSeparatorLevel the max number of tag levels for which we'll add a blank line separator, + * or -1 to never add line separators. + * The top level tag is level 0, etc. + */ + public XMLWriter(Writer writer,int maxIndentLevel,int maxLineSeparatorLevel) { + this(writer,maxIndentLevel,maxLineSeparatorLevel,true); + } + + /** + * Creates an XML wrapper of a writer + * + * @param writer the writer to which this writers (accessible from this by getWrapped) + * @param maxIndentLevel the max number of tag levels for which we'll continue to indent, or -1 to + * never indent. The top level tag is level 0, etc. + * @param maxLineSeparatorLevel the max number of tag levels for which we'll add a blank line separator, + * or -1 to never add line separators. + * The top level tag is level 0, etc. + * @param markupIsAscii set to false to make this encode markup (tags, attributes). By default encoding + * is skipped if the underlying stream uses utf encoding for performance (yes, this matters) + */ + public XMLWriter(Writer writer,int maxIndentLevel,int maxLineSeparatorLevel,boolean markupIsAscii) { + super(writer instanceof GenericWriter ? (GenericWriter)writer : new JavaWriterWriter(writer)); + this.maxIndentLevel=maxIndentLevel; + this.maxLineSeparatorLevel=maxLineSeparatorLevel; + this.markupIsAscii = markupIsAscii; + } + + /** Returns the input writer as-is if it is an XMLWriter instance. Returns new XMLWriter(writer) otherwise */ + @SuppressWarnings("resource") + public static XMLWriter from(Writer writer, int maxIndentLevel,int maxLineSeparatorLevel) { + return (writer instanceof XMLWriter) + ? (XMLWriter)writer + : new XMLWriter(writer, maxIndentLevel, maxLineSeparatorLevel); + } + + /** Returns the input writer as-is if it is an XMLWriter instance. Returns new XMLWriter(writer) otherwise */ + @SuppressWarnings("resource") + public static XMLWriter from(Writer writer) { + return (writer instanceof XMLWriter) + ? (XMLWriter)writer + : new XMLWriter(writer); + } + + public Writer getWrapped() { + return (getWriter() instanceof JavaWriterWriter) ? ((JavaWriterWriter)getWriter()).getWriter() : getWriter(); + } + + /** Writes the first line of an XML file */ + public void xmlHeader(String encoding) { + w(ENCODING_START).w(encoding).w(ENCODING_END); + } + + public XMLWriter openTag(String s) { + return openTag(new Utf8String(s)); + } + public XMLWriter openTag(Utf8String tag) { + closeStartTag(); + if (openTags.size()>0) { + w(LF); + if (isFirstInParent && openTags.size()<=maxLineSeparatorLevel) { + w(LF); + } + indent(); + } + w(LT).w(tag); + openTags.add(tag); + inOpenStartTag=true; + currentIsMultiline=false; + isFirstInParent=true; + return this; + } + + public XMLWriter closeTag() { + if (openTags.size()<=0) { + throw new RuntimeException("Called closeTag() when no tag was open"); + } + Utf8String lastOpenTag=openTags.remove(openTags.size()-1); + + if (inOpenStartTag) {// this tag has no content - use short form + w(EGT); + } + else { + if (currentIsMultiline) { + w(LF).indent(); + } + w(ELT).w(lastOpenTag).w(GT); + } + if (openTags.size()==0 || openTags.size()<=maxLineSeparatorLevel) { + w(LF); + } + inOpenStartTag=false; + currentIsMultiline=true; // When we go up from a subtag we are at a multiline tag (because it contains subtags) + isFirstInParent=false; // the next opened tag will not be first + return this; + } + + private XMLWriter indent() { + for (int i=0; i<openTags.size() && i<maxIndentLevel; i++) { + w(INDENT); + } + return this; + } + + /** + * Closes the start tag. Usually, it is not necessary to call this, as the other methods in this will do + * the right thing as needed. However, this can be called explicitly to allow content or subtags to be written + * by a regular write call which bypasses the logic in this. + * If a start tag is not currently open this has no effect. + */ + public XMLWriter closeStartTag() { + if (!inOpenStartTag) return this; + w(GT); + inOpenStartTag=false; + return this; + } + + /** + * Writes an attribute by XML.xmlEscape(value.toString(),false) + * + * @param name the name of the attribute. An exception is thrown if this is null + * @param value the value of the attribute. The empty string if the attribute is null or empty + */ + public XMLWriter forceAttribute(Utf8String name, Object value) { + String stringValue = value!=null ? value.toString() : ""; + allowAttribute(); + return w(SPACE).w(name).w(ATTRIBUTE_START).wTranscode(XML.xmlEscape(stringValue,true)).w(ATTRIBUTE_END); + } + + public XMLWriter forceAttribute(String name, Object value) { + return forceAttribute(new Utf8String(name), value); + } + + private void allowAttribute() { + if (!inOpenStartTag) { + throw new RuntimeException("Called writeAttribute() while not in an open start tag"); + } + } + /** + * Writes an attribute by its utf8 value + * + * @param name the name of the attribute. An exception is thrown if this is null + * @param value the value of the attribute. This method does nothing if the value is null or empty + */ + public XMLWriter attribute(Utf8String name, AbstractUtf8Array value) { + if (value.isEmpty()) return this; + allowAttribute(); + return w(SPACE).w(name).w(ATTRIBUTE_START).w(value).w(ATTRIBUTE_END); + } + + /** + * Writes an attribute by its utf8 value + * + * @param name the name of the attribute. An exception is thrown if this is null + * @param value the value of the attribute. This method does nothing if the value is null. + */ + public XMLWriter attribute(Utf8String name, Number value) { + if (value == null) return this; + allowAttribute(); + return w(SPACE).w(name).w(ATTRIBUTE_START).w(value).w(ATTRIBUTE_END); + } + + /** + * Writes an attribute by its utf8 value + * + * @param name the name of the attribute. An exception is thrown if this is null + * @param value the value of the attribute. + */ + public XMLWriter attribute(Utf8String name, long value) { + allowAttribute(); + return w(SPACE).w(name).w(ATTRIBUTE_START).w(value).w(ATTRIBUTE_END); + } + + /** + * Writes an attribute by its utf8 value + * + * @param name the name of the attribute. An exception is thrown if this is null + * @param value the value of the attribute. + */ + public XMLWriter attribute(Utf8String name, double value) { + allowAttribute(); + return w(SPACE).w(name).w(ATTRIBUTE_START).w(value).w(ATTRIBUTE_END); + } + + /** + * Writes an attribute by its utf8 value + * + * @param name the name of the attribute. An exception is thrown if this is null + * @param value the value of the attribute. This method does nothing if the value is null or empty + */ + public XMLWriter attribute(Utf8String name, boolean value) { + allowAttribute(); + return w(SPACE).w(name).w(ATTRIBUTE_START).w(value).w(ATTRIBUTE_END); + } + + /** + * Writes an attribute by XML.xmlEscape(value.toString(),false) + * + * @param name the name of the attribute. An exception is thrown if this is null + * @param value the value of the attribute. This method does nothing if the value is null or empty + */ + public XMLWriter attribute(Utf8String name, String value) { + if ((value == null) || value.isEmpty()) return this; + allowAttribute(); + return w(SPACE).w(name).w(ATTRIBUTE_START).wTranscode(XML.xmlEscape(value, true)).w(ATTRIBUTE_END); + } + + public XMLWriter attribute(String name, Object value) { + if (value==null) return this; + return attribute(new Utf8String(name), value.toString()); + } + + /** + * XML escapes and writes the content.toString(). If the content is null this does nothing but closing the start tag. + * + * @param content the content - output by XML.xmlEscape(content.toString()) + * @param multiline whether the content should be treated as multiline, + * such that the following end tag should appear on a new line + */ + public XMLWriter content(Object content,boolean multiline) { + closeStartTag(); + return (content==null) + ? this + : escapedContent(XML.xmlEscape(content.toString(),false),multiline); + } + + /** + * Writes the given string as-is. The content string <i>must</i> be XML escaped before calling this. + * If the content is null this does nothing but closing the start tag. + * + * @param content the content - output by XML.xmlEscape(content.toString()) + * @param multiline whether the content should be treated as multiline, + * such that the following end tag should appear on a new line + */ + public XMLWriter escapedContent(String content,boolean multiline) { + closeStartTag(); + if (content==null) return this; + if (multiline) currentIsMultiline=true; + return wTranscode(content); + } + + /** + * Writes the given US-ASCII only string as-is. + * If the content is <b>not</b> US-ASCII <i>only</i> this <i>may</i> cause + * incorrectly encoded content to be written. + * This is faster than using escapedContent as transcoding is skipped. + * <p> + * The content string <i>must</i> be XML escaped before calling this. + * If the content is null this does nothing but closing the start tag. + * + * @param content the content - output by XML.xmlEscape(content.toString()) + * @param multiline whether the content should be treated as multiline, + * such that the following end tag should appear on a new line + */ + public XMLWriter escapedAsciiContent(String content,boolean multiline) { + closeStartTag(); + if (content==null) return this; + if (multiline) currentIsMultiline=true; + return w(content); + } + + /** + * Writes the given string. If markup is us ascii (default), and the wrapped writer encodes in UTF, this will write + * the string <b>as is, with no transcoding</b> (for speed). Hence, this should never be used for just any content. + * + * @return this for consistency + */ + private final XMLWriter w(String s) { + return markupIsAscii ? w(new Utf8String(s)) : w(s); + } + + private final XMLWriter w(AbstractUtf8Array utf8) { + write(utf8); + return this; + } + private final XMLWriter w(long v) { + write(v); + return this; + } + private final XMLWriter w(boolean v) { + write(v); + return this; + } + private final XMLWriter w(double v) { + write(v); + return this; + } + private final XMLWriter w(Number v) { + write(v.toString()); + return this; + } + + /** Calls write(s) and returns this. Use this for general content which must be transcoded */ + private final XMLWriter wTranscode(String s) { + write(s); + return this; + } + + /** + * Returns a read only view of the currently open tags we are within, sorted by highest to + * lowest in the hierarchy + * Only used for testing. + */ + public List<Utf8String> openTags() { return unmodifiableOpenTags; } + + /** + * Returns true if the immediate parent (i.e the last element in openTags) + * is the tag with the given name + */ + public boolean isIn(Utf8String containingTag) { + return (openTags.size()!=0) && openTags.get(openTags.size()-1).equals(containingTag); + } + + public boolean isIn(String containingTag) { + return (openTags.size()!=0) && openTags.get(openTags.size()-1).equals(containingTag); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/package-info.java b/vespajlib/src/main/java/com/yahoo/text/package-info.java new file mode 100644 index 00000000000..f0322acbcc6 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.text; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/time/WallClockSource.java b/vespajlib/src/main/java/com/yahoo/time/WallClockSource.java new file mode 100644 index 00000000000..5fef1f94879 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/time/WallClockSource.java @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.time; + +import com.google.common.annotations.Beta; + +/** + * A source for high-resolution timestamps. + * + * @author arnej27959 + */ + +@Beta +public class WallClockSource { + + private volatile long offset; + + /** + * Obtain the current time in nanoseconds. + * The epoch is January 1, 1970 UTC just as for System.currentTimeMillis(), + * but with greater resolution. Note that the absolute accuracy may be + * no better than currentTimeMills(). + * @return nanoseconds since the epoch. + **/ + public final long currentTimeNanos() { + return System.nanoTime() + offset; + } + + /** + * Create a source with 1 millisecond accuracy at start. + **/ + WallClockSource() { + long actual = System.currentTimeMillis(); + actual *= 1000000; + this.offset = actual - System.nanoTime(); + initialAdjust(); + } + + /** adjust the clock source from currentTimeMillis() + * to ensure that it is no more than 1 milliseconds off. + * @return true if we want adjust called again soon + **/ + boolean adjust() { + long nanosB = System.nanoTime(); + long actual = System.currentTimeMillis(); + long nanosA = System.nanoTime(); + if (nanosA - nanosB > 100000) { + return true; // not a good time to adjust, try again soon + } + return adjustOffset(nanosB, actual, nanosA); + } + + private boolean adjustOffset(long before, long actual, long after) { + actual *= 1000000; // convert millis to nanos + if (actual > after + offset) { + // System.out.println("WallClockSource adjust UP "+(actual-after-offset)); + offset = actual - after; + return true; + } + if (actual + 999999 < before + offset) { + // System.out.println("WallClockSource adjust DOWN "+(before+offset-actual-999999)); + offset = actual + 999999 - before; + return true; + } + return false; + } + + private void initialAdjust() { + for (int i = 0; i < 100; i++) { + long nanosB = System.nanoTime(); + long actual = System.currentTimeMillis(); + long nanosA = System.nanoTime(); + adjustOffset(nanosB, actual, nanosA); + } + } + + + static private WallClockSource autoAdjustingInstance = new WallClockSource(); + + /** + * Get a WallClockSource which auto adjusts to wall clock time. + **/ + static public WallClockSource get() { + autoAdjustingInstance.startAdjuster(); + return autoAdjustingInstance; + } + + private Thread adjuster; + + private synchronized void startAdjuster() { + if (adjuster == null) { + adjuster = new AdjustThread(); + adjuster.setDaemon(true); + adjuster.start(); + } + } + + private class AdjustThread extends Thread { + public void run() { + int millis = 0; + int nanos = 313373; // random number + while (true) { + try { + sleep(millis, nanos); + if (++millis > 4321) { + millis = 1000; // do not sleep too long + } + } catch (InterruptedException e) { + return; + } + if (adjust()) { + // adjust more often in case clock jumped + millis = 0; + } + } + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/transaction/Mutex.java b/vespajlib/src/main/java/com/yahoo/transaction/Mutex.java new file mode 100644 index 00000000000..3c4168b77f3 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/transaction/Mutex.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.transaction; + +/** + * An auto closeable mutex + * + * @author bratseth + */ +public interface Mutex extends AutoCloseable { + + public void close(); + +} diff --git a/vespajlib/src/main/java/com/yahoo/transaction/NestedTransaction.java b/vespajlib/src/main/java/com/yahoo/transaction/NestedTransaction.java new file mode 100644 index 00000000000..4be0a32ffe8 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/transaction/NestedTransaction.java @@ -0,0 +1,200 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.transaction; + +import java.util.ArrayList; +import java.util.List; +import java.util.ListIterator; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * A transaction which may contain a list of transactions, typically to represent a distributed transaction + * over multiple systems. + * + * @author bratseth + */ +public final class NestedTransaction implements AutoCloseable { + + private static final Logger log = Logger.getLogger(NestedTransaction.class.getName()); + + /** Nested transactions with ordering constraints, in the order they are added */ + private final List<ConstrainedTransaction> transactions = new ArrayList<>(2); + + /** Transaction ordering pairs */ + //private final List<OrderingConstraint> transactionOrders = new ArrayList<>(2); + + /** A list of (non-transactional) operations to execute after this transaction has committed successfully */ + private final List<Runnable> onCommitted = new ArrayList<>(2); + + /** + * Adds a transaction to this. + * + * @param transaction the transaction to add + * @param before transaction classes which should commit after this, if present. It is beneficial + * to order transaction types from the least to most reliable. If conflicting ordering constraints are + * given this will not be detected at add time but the transaction will fail to commit + * @return this for convenience + */ + @SafeVarargs // don't warn on 'before' argument + @SuppressWarnings("varargs") // don't warn on passing 'before' to the nested class constructor + public final NestedTransaction add(Transaction transaction, Class<? extends Transaction> ... before) { + transactions.add(new ConstrainedTransaction(transaction, before)); + return this; + } + + /** Returns the transactions nested in this, as they will be committed. */ + public List<Transaction> transactions() { return organizeTransactions(transactions); } + + /** Perform a 2 phase commit */ + public void commit() { + List<Transaction> organizedTransactions = organizeTransactions(transactions); + + // First phase + for (Transaction transaction : organizedTransactions) + transaction.prepare(); + + // Second phase + for (ListIterator<Transaction> i = organizedTransactions.listIterator(); i.hasNext(); ) { + Transaction transaction = i.next(); + try { + transaction.commit(); + } + catch (Exception e) { + // Clean up committed part or log that we can't + i.previous(); + while (i.hasPrevious()) + i.previous().rollbackOrLog(); + throw new IllegalStateException("Transaction failed during commit", e); + } + } + + // After commit: Execute completion tasks + for (Runnable task : onCommitted) { + try { + task.run(); + } + catch (Exception e) { // Don't throw from here as that indicates transaction didn't complete + log.log(Level.WARNING, "A committed task in " + this + " caused an exception", e); + } + } + } + + public void onCommitted(Runnable runnable) { + onCommitted.add(runnable); + } + + /** Free up any temporary resources held by this */ + @Override + public void close() { + for (ConstrainedTransaction transaction : transactions) + transaction.transaction.close(); + } + + private List<Transaction> organizeTransactions(List<ConstrainedTransaction> transactions) { + return orderTransactions(combineTransactions(transactions), findOrderingConstraints(transactions)); + } + + /** Combines all transactions of the same type to one */ + private List<Transaction> combineTransactions(List<ConstrainedTransaction> transactions) { + List<Transaction> combinedTransactions = new ArrayList<>(transactions.size()); + for (List<Transaction> combinableTransactions : + transactions.stream().map(ConstrainedTransaction::transaction). + collect(Collectors.groupingBy(Transaction::getClass)).values()) { + Transaction combinedTransaction = combinableTransactions.get(0); + for (int i = 1; i < combinableTransactions.size(); i++) + combinedTransaction = combinedTransaction.add(combinableTransactions.get(i).operations()); + combinedTransactions.add(combinedTransaction); + } + return combinedTransactions; + } + + private List<OrderingConstraint> findOrderingConstraints(List<ConstrainedTransaction> transactions) { + List<OrderingConstraint> orderingConstraints = new ArrayList<>(1); + for (ConstrainedTransaction transaction : transactions) { + for (Class<? extends Transaction> afterThis : transaction.before()) + orderingConstraints.add(new OrderingConstraint(transaction.transaction().getClass(), afterThis)); + } + return orderingConstraints; + } + + /** Orders combined transactions consistent with the ordering constraints */ + private List<Transaction> orderTransactions(List<Transaction> transactions, List<OrderingConstraint> constraints) { + if (transactions.size() == 1) return transactions; + + List<Transaction> orderedTransactions = new ArrayList<>(); + for (Transaction transaction : transactions) + orderedTransactions.add(findSuitablePositionFor(transaction, orderedTransactions, constraints), transaction); + return orderedTransactions; + } + + private int findSuitablePositionFor(Transaction transaction, List<Transaction> orderedTransactions, + List<OrderingConstraint> constraints) { + for (int i = 0; i < orderedTransactions.size(); i++) { + Transaction candidateNextTransaction = orderedTransactions.get(i); + if ( ! mustBeAfter(candidateNextTransaction.getClass(), transaction.getClass(), constraints)) return i; + + // transaction must be after this: continue to next position + if (mustBeAfter(transaction.getClass(), candidateNextTransaction.getClass(), constraints)) // must be after && must be before + throw new IllegalStateException("Conflicting transaction ordering constraints between" + + transaction + " and " + candidateNextTransaction); + } + return orderedTransactions.size(); // add last as it must be after everything + } + + /** + * Returns whether transaction type B must be after type A according to the ordering constraints. + * This is the same as asking whether there is a path between node a and b in the bi-directional + * graph defined by the ordering constraints. + */ + private boolean mustBeAfter(Class<? extends Transaction> a, Class<? extends Transaction> b, + List<OrderingConstraint> constraints) { + for (OrderingConstraint fromA : findAllOrderingConstraintsFrom(a, constraints)) { + if (fromA.after().equals(b)) return true; + if (mustBeAfter(fromA.after(), b, constraints)) return true; + } + return false; + } + + private List<OrderingConstraint> findAllOrderingConstraintsFrom(Class<? extends Transaction> transactionType, + List<OrderingConstraint> constraints) { + return constraints.stream().filter(c -> c.before().equals(transactionType)).collect(Collectors.toList()); + } + + private static class ConstrainedTransaction { + + private final Transaction transaction; + + private final Class<? extends Transaction>[] before; + + public ConstrainedTransaction(Transaction transaction, Class<? extends Transaction>[] before) { + this.transaction = transaction; + this.before = before; + } + + public Transaction transaction() { return transaction; } + + /** Returns transaction types which should commit after this */ + public Class<? extends Transaction>[] before() { return before; } + + } + + private static class OrderingConstraint { + + private final Class<? extends Transaction> before, after; + + public OrderingConstraint(Class<? extends Transaction> before, Class<? extends Transaction> after) { + this.before = before; + this.after = after; + } + + public Class<? extends Transaction> before() { return before; } + + public Class<? extends Transaction> after() { return after; } + + @Override + public String toString() { return before + " -> " + after; } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/transaction/Transaction.java b/vespajlib/src/main/java/com/yahoo/transaction/Transaction.java new file mode 100644 index 00000000000..642438dda0a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/transaction/Transaction.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.transaction; + +import java.util.List; + +/** + * An interface for building a transaction and committing it. Implementations are required to atomically apply changes + * in the commit step or throw an exception if it fails. + * + * @author lulf + * @author bratseth + */ +public interface Transaction extends AutoCloseable { + + /** + * Adds an operation to this transaction. Return self for chaining. + * + * @param operation {@link Operation} to append + * @return self, for chaining + */ + Transaction add(Operation operation); + + /** + * Adds multiple operations to this transaction. Return self for chaining. + * + * @param operation {@link Operation} to append + * @return self, for chaining + */ + Transaction add(List<Operation> operation); + + /** + * Returns the operations of this. + * Ownership of the returned list is transferred to the caller. The ist may be ready only. + */ + List<Operation> operations(); + + /** + * Checks whether or not the transaction is able to commit in its current state and do any transient preparatory + * work to commit. + * + * @throws IllegalStateException if the transaction cannot be committed + */ + void prepare(); + + /** + * Commit this transaction. If this method returns, all operations in this transaction was committed + * successfully. Implementations of this must be exception safe or log a message of type severe if they partially + * alter state. + * + * @throws IllegalStateException if transaction failed. + */ + void commit(); + + /** + * This is called if the transaction should be rolled back after commit. If a rollback is not possible or + * supported. This must log a message of type severe with detailed information about the resulting state. + */ + void rollbackOrLog(); + + /** + * Closes and frees any resources allocated by this transaction. The transaction instance cannot be reused once + * closed. + */ + void close(); + + /** + * Operations that a transaction supports should implement this interface. + */ + public interface Operation { + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/transaction/package-info.java b/vespajlib/src/main/java/com/yahoo/transaction/package-info.java new file mode 100644 index 00000000000..72ac10d13d0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/transaction/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.transaction; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/vespajlib/src/main/java/com/yahoo/vespa/VersionTagger.java b/vespajlib/src/main/java/com/yahoo/vespa/VersionTagger.java new file mode 100644 index 00000000000..556bcc0e90d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/VersionTagger.java @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa; + +import java.io.*; +import java.util.HashMap; +import java.util.Map; + +/** + * This class generates a java class based on the vtag.map file generated by dist/getversion.pl + */ +public class VersionTagger { + public static final String V_TAG_PKG = "V_TAG_PKG"; + + VersionTagger() throws IOException { + } + + private static void printUsage(PrintStream out) { + out.println("Usage: java VersionTagger vtagmap pkgname outputdir"); + } + + public static void main(String[] args) { + if (args.length < 3) { + printUsage(System.err); + throw new RuntimeException("bad arguments to main(): vtag.map packageName outputDirectory [outputFormat (simple or vtag)]"); + } + try { + VersionTagger me = new VersionTagger(); + me.runProgram(args); + } catch (Exception e) { + System.err.println(e); + printUsage(System.err); + throw new RuntimeException(e); + } + } + + private Map<String, String> readVtagMap(String path) { + Map<String, String> map = new HashMap<>(); + try { + BufferedReader in = new BufferedReader(new FileReader(path)); + String line; + while ((line = in.readLine()) != null) { + String elements[] = line.split("\\s+", 2); + map.put(elements[0], elements[1]); + } + } catch (FileNotFoundException e) { + // Use default values + map.put("V_TAG", "NOTAG"); + map.put("V_TAG_DATE", "NOTAG"); + map.put("V_TAG_PKG", "6.9999.0"); + map.put("V_TAG_ARCH", "NOTAG"); + map.put("V_TAG_SYSTEM", "NOTAG"); + map.put("V_TAG_SYSTEM_REV", "NOTAG"); + map.put("V_TAG_BUILDER", "NOTAG"); + map.put("V_TAG_COMPONENT", "6.9999.0"); + } catch (IOException e) { + throw new RuntimeException(e); + } + return map; + } + private enum Format { + SIMPLE, + VTAG + } + + void runProgram(String[] args) throws IOException, InterruptedException { + + String vtagmapPath = args[0]; + String packageName = args[1]; + String dirName = args[2] + "/" + packageName.replaceAll("\\.", "/"); + Format format = args.length >= 4 ? Format.valueOf(args[3].toUpperCase()) : Format.SIMPLE; + File outDir = new File(dirName); + if (!outDir.isDirectory() && !outDir.mkdirs()) { + throw new IOException("could not create directory " + outDir); + } + + String className = format == Format.SIMPLE ? "VespaVersion" : "Vtag"; + String outFile = dirName + "/" + className +".java"; + FileOutputStream out = new FileOutputStream(outFile); + OutputStreamWriter writer = new OutputStreamWriter(out); + System.err.println("generating: " + outFile); + + Map<String, String> vtagMap = readVtagMap(vtagmapPath); + writer.write(String.format("package %s;\n\n", packageName)); + + if (format == Format.VTAG) { + writer.write("import com.yahoo.component.Version;\n"); + } + + writer.write(String.format("public class %s {\n", className)); + if (!vtagMap.containsKey(V_TAG_PKG)) { + throw new RuntimeException("V_TAG_PKG not present in map file"); + } + switch (format) { + case SIMPLE: + String version = vtagMap.get(V_TAG_PKG); + String elements[] = version.split("\\."); + writer.write(String.format(" public static final int major = %s;\n", elements[0])); + writer.write(String.format(" public static final int minor = %s;\n", elements[1])); + writer.write(String.format(" public static final int micro = %s;\n", elements[2])); + break; + case VTAG: + vtagMap.forEach((key, value) -> { + try { + writer.write(String.format(" public static final String %s = \"%s\";\n", key, value)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + writer.write(" public static final Version currentVersion = new Version(V_TAG_COMPONENT);\n"); + break; + } + writer.write("}\n"); + writer.close(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/BufferSerializer.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/BufferSerializer.java new file mode 100644 index 00000000000..cf5d2e28af3 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/BufferSerializer.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.text.Utf8; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * @author balder + */ +public class BufferSerializer implements Serializer, Deserializer { + protected GrowableByteBuffer buf; + + public BufferSerializer(GrowableByteBuffer buf) { this.buf = buf; } + public BufferSerializer(ByteBuffer buf) { this(new GrowableByteBuffer(buf)); } + public BufferSerializer(byte [] buf) { this(ByteBuffer.wrap(buf)); } + public BufferSerializer() { this(new GrowableByteBuffer()); } + public static BufferSerializer wrap(byte [] buf) { return new BufferSerializer(buf); } + public final GrowableByteBuffer getBuf() { return buf; } + protected final void setBuf(GrowableByteBuffer buf) { this.buf = buf; } + public Serializer putByte(FieldBase field, byte value) { buf.put(value); return this; } + public Serializer putShort(FieldBase field, short value) { buf.putShort(value); return this; } + public Serializer putInt(FieldBase field, int value) { buf.putInt(value); return this; } + public Serializer putLong(FieldBase field, long value) { buf.putLong(value); return this; } + public Serializer putFloat(FieldBase field, float value) { buf.putFloat(value); return this; } + public Serializer putDouble(FieldBase field, double value) { buf.putDouble(value); return this; } + public Serializer put(FieldBase field, byte[] value) { buf.put(value); return this; } + public Serializer put(FieldBase field, String value) { + byte [] utf8 = createUTF8CharArray(value); + putInt(null, utf8.length+1); + put(null, utf8); + putByte(null, (byte) 0); + return this; + } + public Serializer put(FieldBase field, ByteBuffer value) { buf.put(value); return this; } + public Serializer putInt1_4Bytes(FieldBase field, int value) { buf.putInt1_4Bytes(value); return this; } + public Serializer putInt2_4_8Bytes(FieldBase field, long value) { buf.putInt2_4_8Bytes(value); return this; } + public int position() { return buf.position(); } + public ByteOrder order() { return buf.order(); } + public void position(int pos) { buf.position(pos); } + public void order(ByteOrder v) { buf.order(v); } + public void flip() { buf.flip(); } + + public byte getByte(FieldBase field) { return buf.getByteBuffer().get(); } + public short getShort(FieldBase field) { return buf.getByteBuffer().getShort(); } + public int getInt(FieldBase field) { return buf.getByteBuffer().getInt(); } + public long getLong(FieldBase field) { return buf.getByteBuffer().getLong(); } + public float getFloat(FieldBase field) { return buf.getByteBuffer().getFloat(); } + public double getDouble(FieldBase field) { return buf.getByteBuffer().getDouble(); } + public byte [] getBytes(FieldBase field, int length) { + if (buf.remaining() < length) { + throw new IllegalArgumentException("Wanted " + length + " bytes, but I only had " + buf.remaining()); + } + byte [] bbuf =new byte [length]; + buf.getByteBuffer().get(bbuf); + return bbuf; + } + public String getString(FieldBase field) { + int length = getInt(null); + byte[] stringArray = new byte[length-1]; + buf.get(stringArray); + getByte(null); + return Utf8.toString(stringArray); + } + public int getInt1_4Bytes(FieldBase field) { return buf.getInt1_4Bytes(); } + public int getInt1_2_4Bytes(FieldBase field) { return buf.getInt1_2_4Bytes(); } + public long getInt2_4_8Bytes(FieldBase field) { return buf.getInt2_4_8Bytes(); } + public int remaining() { return buf.remaining(); } + + public static byte[] createUTF8CharArray(String input) { + if (input == null || input.length() < 1) { + return new byte[0]; + } + return Utf8.toBytes(input); + } + +} + diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Deserializer.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Deserializer.java new file mode 100644 index 00000000000..abd82f6b251 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Deserializer.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +/** + * @author balder + */ +public interface Deserializer { + byte getByte(FieldBase field); + short getShort(FieldBase field); + int getInt(FieldBase field); + long getLong(FieldBase field); + float getFloat(FieldBase field); + double getDouble(FieldBase field); + byte [] getBytes(FieldBase field, int length); + String getString(FieldBase field); +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/FieldBase.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/FieldBase.java new file mode 100644 index 00000000000..2a7f9cbff7a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/FieldBase.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class FieldBase { + private final String name; + + public FieldBase(String name) { + this.name = name; + } + + public final String getName() { + return name; + } + + @Override + public boolean equals(Object o) { + return this == o || o instanceof FieldBase && name.equalsIgnoreCase(((FieldBase) o).name); + } + + @Override + public int hashCode() { + return name.toLowerCase(java.util.Locale.US).hashCode(); + } + + public String toString() { + return "field " + name; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java new file mode 100644 index 00000000000..7bc9c2f8d6b --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java @@ -0,0 +1,368 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +import com.yahoo.collections.Pair; +import com.yahoo.text.Utf8; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; + +/** + * This is the base class to do cross-language serialization and deserialization of complete object structures without + * the need for a separate protocol. Each subclass needs to register itself using the {@link #registerClass(int, Class)} + * method, and override {@link #onGetClassId()} to return the same classId as the one registered. Creating an instance + * of an identifiable object is done through the {@link #create(Deserializer)} or {@link #createFromId(int)} factory + * methods. + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Identifiable extends Selectable implements Cloneable { + + private static Registry registry = null; + public static int classId = registerClass(1, Identifiable.class); + + /** + * Returns the class identifier of this class. This proxies the {@link #onGetClassId()} method that must be + * implemented by every subclass. + * + * @return The class identifier. + */ + public final int getClassId() { + return onGetClassId(); + } + + /** + * Returns the class identifier for which this class is registered. It is important that all subclasses match the + * return value of this with their call to {@link #registerClass(int, Class)}. + * + * @return The class identifier. + */ + protected int onGetClassId() { + return classId; + } + + /** + * Serializes the content of this class into the given byte buffer. This method serializes its own identifier into + * the buffer before invoking the {@link #serialize(Serializer)} method. + * + * @param buf The buffer to serialize to. + * @return The buffer argument, to allow chaining. + */ + public final Serializer serializeWithId(Serializer buf) { + buf.putInt(null, getClassId()); + return serialize(buf); + } + + /** + * Serializes the content (excluding the identifier) of this class into the given byte buffer. If you need the + * identifier serialized, use the {@link #serializeWithId(Serializer)} method instead. This method invokes the + * {@link #onSerialize(Serializer)} method. + * + * @param buf The buffer to serialize to. + * @return The buffer argument, to allow chaining. + */ + public final Serializer serialize(Serializer buf) { + onSerialize(buf); + return buf; + } + + /** + * Serializes the content of this class into the given + * buffer. This method must be implemented by all subclasses that + * have content. If the subclass has no other content than the + * semantics of its class type, this method does not need to be + * overloaded. + * + * @param buf The buffer to serialize to. + */ + protected void onSerialize(Serializer buf) { + // empty + } + + /** + * Deserializes the content of this class from the given byte buffer. This method deserialize a class identifier + * first, and asserts that this identifier matches the identifier of this class. This is usable if you have an + * instance of a class whose content you wish to retrieve from a buffer. + * + * @param buf The buffer to deserialize from. + * @return The buffer argument, to allow chaining. + * @throws IllegalArgumentException Thrown if the deserialized class identifier does not match this class. + */ + public final Deserializer deserializeWithId(Deserializer buf) { + int id = buf.getInt(null); + if (id != getClassId()) { + Class<?> spec = registry.get(id); + if (spec != null) { + throw new IllegalArgumentException( + "Can not deserialize class '" + getClass().getName() + "' (id " + getClassId() + ") from " + + "buffer containing class '" + spec.getName() + "' (id " + id + ")."); + } else { + throw new IllegalArgumentException( + "Can not deserialize class '" + getClass().getName() + "' (id " + getClassId() + ") from " + + "buffer containing unknown class id " + id + "."); + } + } + return deserialize(buf); + } + + /** + * Deserializes the content (excluding the identifier) of this class from the given byte buffer. If you need the + * identifier deserialized and verified, use the {@link #deserializeWithId(Deserializer)} method instead. This + * method invokes the {@link #onDeserialize(Deserializer)} method. + * + * @param buf The buffer to deserialize from. + * @return The buffer argument, to allow chaining. + */ + public final Deserializer deserialize(Deserializer buf) { + onDeserialize(buf); + return buf; + } + + /** + * Deserializes the content of this class from the given byte + * buffer. This method must be implemented by all subclasses that + * have content. If the subclass has no other content than the + * semantics of its class type, this method does not need to be + * overloaded. + * + * @param buf The buffer to deserialize from. + */ + protected void onDeserialize(Deserializer buf) { + // empty + } + + /** + * Declares that all subclasses of Identifiable supports clone() by _not_ throwing CloneNotSupported exceptions. + * + * @return A cloned instance of this. + * @throws AssertionError Thrown if a subclass does not implement clone(). + */ + @Override + public Identifiable clone() { + try { + return (Identifiable)super.clone(); + } catch (CloneNotSupportedException e) { + throw (AssertionError)new AssertionError("The cloneable structure has been broken.").initCause(e); + } + } + + @Override + public int hashCode() { + return getClassId(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Identifiable)) { + return false; + } + Identifiable rhs = (Identifiable)obj; + return (getClassId() == rhs.getClassId()); + } + + @Override + public String toString() { + ObjectDumper ret = new ObjectDumper(); + ret.visit("", this); + return ret.toString(); + } + + /** + * Registers the given class specification for the given identifier in the class registry. This method returns the + * supplied identifier, so that subclasses can declare a static classId member like so: + * + * <code>public static int classId = registerClass(<id>, <ClassName>.class);</code> + * + * @param id The class identifier to register with. + * @param spec The class to register. + * @return The identifier argument. + */ + protected static int registerClass(int id, Class<? extends Identifiable> spec) { + if (registry == null) { + registry = new Registry(); + } + registry.add(id, spec); + return id; + } + + /** + * Deserializes a single {@link Identifiable} object from the given byte buffer. The object itself may perform + * recursive deserialization of {@link Identifiable} objects, but there is no requirement that this method consumes + * the whole content of the buffer. + * + * @param buf The buffer to deserialize from. + * @return The instantiated object. + * @throws IllegalArgumentException Thrown if an unknown class is contained in the buffer. + */ + public static Identifiable create(Deserializer buf) { + int classId = buf.getInt(null); + Identifiable obj = createFromId(classId); + if (obj != null) { + obj.deserialize(buf); + } else { + throw new IllegalArgumentException("Failed creating class for classId " + classId); + } + return obj; + } + + /** + * Creates an instance of the class registered with the given identifier. If the indentifier is unknown, this method + * returns null. + * + * @param id The identifier of the class to instantiate. + * @return The instantiated object. + */ + public static Identifiable createFromId(int id) { + return registry.createFromId(id); + } + + /** + * This is a convenience method to allow serialization of an optional field. A single byte is added to the buffer + * indicating whether or not an object follows. If the object is not null, it is serialized following this flag. + * + * @param buf The buffer to serialize to. + * @param obj The object to serialize, may be null. + * @return The buffer, to allow chaining. + */ + protected static Serializer serializeOptional(Serializer buf, Identifiable obj) { + if (obj != null) { + buf.putByte(null, (byte)1); + obj.serializeWithId(buf); + } else { + buf.putByte(null, (byte)0); + } + return buf; + } + + /** + * This is a convenience method to allow deserialization of an optional field. See {@link + * #serializeOptional(Serializer, Identifiable)} for notes on this. + * + * @param buf The buffer to deserialize from. + * @return The instantiated object, or null. + */ + protected static Identifiable deserializeOptional(Deserializer buf) { + byte hasObject = buf.getByte(null); + if (hasObject == 1) { + return create(buf); + } + return null; + } + + /** + * Returns whether or not two objects are equal, taking into account that either can be null. + * + * @param lhs The left hand side of the comparison. + * @param rhs The right hand side of the comparison. + * @return True if both arguments are null or equal. + */ + protected static boolean equals(Object lhs, Object rhs) { + return !(lhs == null && rhs != null) && + !(lhs != null && rhs == null) && + ((lhs == null || lhs.equals(rhs))); + } + + /** + * This function needs to be implemented in such a way that it visits all its members. This is done by invoking the + * {@link com.yahoo.vespa.objects.ObjectVisitor#visit(String, Object)} on the visitor argument for all members. + * + * @param visitor The visitor that is to access the member data. + */ + public void visitMembers(ObjectVisitor visitor) { + visitor.visit("classId", getClassId()); + } + + /** + * This class implements the class registry used by {@link Identifiable} to allow for creation of classes from + * shared class identifiers. It's methods are proxied through {@link Identifiable#registerClass(int, Class)}, {@link + * Identifiable#createFromId(int)} and {@link Identifiable#create(Deserializer)}. + */ + private static class Registry { + + // The map from class id to class descriptor. + private HashMap<Integer, Pair<Class<? extends Identifiable>, Constructor<? extends Identifiable>>> typeMap = + new HashMap<>(); + + /** + * Adds an entry in the type map, pairing the given identifier with the given class specification. + * + * @param id The class identifier to register with. + * @param spec The class to register. + * @throws IllegalArgumentException Thrown if two classes attempt to register with the same identifier. + */ + private void add(int id, Class<? extends Identifiable> spec) { + Class<?> old = get(id); + if (old == null) { + Constructor<? extends Identifiable> constructor; + try { + constructor = spec.getConstructor(); + } catch (NoSuchMethodException e) { + constructor = null; + } + typeMap.put(id, new Pair<Class<? extends Identifiable>, Constructor<? extends Identifiable>>(spec, constructor)); + } else if (!spec.equals(old)) { + throw new IllegalArgumentException("Can not register class '" + spec.toString() + "' with id " + id + + ", because it already maps to class '" + old.toString() + "'."); + } + } + + /** + * Returns the class registered for the given identifier. + * + * @param id The identifer whose class to return. + * @return The class specification, may be null. + */ + private Class<? extends Identifiable> get(int id) { + Pair<Class<? extends Identifiable>, Constructor<? extends Identifiable>> pair = typeMap.get(id); + return (pair != null) ? pair.getFirst() : null; + } + + /** + * Creates an instance of the class mapped to by the given identifier. This method proxies {@link + * #createFromClass(Constructor)}. + * + * @param id The id of the class to create. + * @return The instantiated object. + */ + private Identifiable createFromId(int id) { + Pair<Class<? extends Identifiable>, Constructor<? extends Identifiable>> pair = typeMap.get(id); + return createFromClass((pair != null) ? pair.getSecond() : null); + } + + /** + * Creates an instance of a given class specification. All instantiation-type exceptions are consumed and + * wrapped inside a runtime exception so that calling methods can let this propagate without declaring them + * thrown. + * + * @param spec The class to instantiate. + * @return The instantiated object. + * @throws IllegalArgumentException Thrown if instantiation failed. + */ + private Identifiable createFromClass(Constructor<? extends Identifiable> spec) { + Identifiable obj = null; + if (spec != null) { + try { + obj = spec.newInstance(); + } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) { + throw new IllegalArgumentException("Failed to create object from class '" + + spec.getName() + "'.", e); + } + } + return obj; + } + } + + protected String getUtf8(Deserializer buf) { + int len = buf.getInt(null); + byte[] arr = buf.getBytes(null, len); + return Utf8.toString(arr); + } + + protected void putUtf8(Serializer buf, String val) { + byte[] raw = Utf8.toBytes(val); + buf.putInt(null, raw.length); + buf.put(null, raw); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Ids.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Ids.java new file mode 100644 index 00000000000..85647c58744 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Ids.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +/** + * This is a class containing the global ids that are given out. + * Must be in sync with version for c++ in staging_vespalib/src/vespalib/objects/ids.h + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public interface Ids { + public static int document = 0x1000; + public static int searchlib = 0x4000; + public static int vespa_configmodel = 0x7000; + public static int annotation = 0x10000; +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectDumper.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectDumper.java new file mode 100755 index 00000000000..42c9a09550d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectDumper.java @@ -0,0 +1,134 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +import com.yahoo.vespa.objects.ObjectVisitor; + +import java.lang.reflect.Array; +import java.util.List; + +/** + * This is a concrete object visitor that will build up a structured human-readable string representation of an object. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ObjectDumper extends ObjectVisitor { + + // The current string being written to. + private final StringBuilder str = new StringBuilder(); + + // The number of spaces to indent each level. + private final int indent; + + // The current indent level. + private int currIndent = 0; + + /** + * Create an object dumper with the default indent size. + */ + public ObjectDumper() { + this(4); + } + + /** + * Create an object dumper with the given indent size. + * + * @param indent indent size in number of spaces + */ + public ObjectDumper(int indent) { + this.indent = indent; + } + + /** + * Add a number of spaces equal to the current indent to the string we are building. + */ + private void addIndent() { + int n = currIndent; + for (int i = 0; i < n; ++i) { + str.append(' '); + } + } + + /** + * Add a complete line of output. Appropriate indentation will be added before the given string and a newline will + * be added after it. + * + * @param line the line we want to add + */ + private void addLine(String line) { + addIndent(); + str.append(line); + str.append('\n'); + } + + /** + * Open a subscope by increasing the current indent level + */ + private void openScope() { + currIndent += indent; + } + + /** + * Close a subscope by decreasing the current indent level + */ + private void closeScope() { + currIndent -= indent; + } + + /** + * Obtain the created object string representation. This object should be invoked after the complete object + * structure has been visited. + * + * @return object string representation + */ + @Override + public String toString() { + return str.toString(); + } + + // Inherit doc from ObjectVisitor. + @Override + public void openStruct(String name, String type) { + if (name == null || name.isEmpty()) { + addLine(type + " {"); + } else { + addLine(name + ": " + type + " {"); + } + openScope(); + } + + // Inherit doc from ObjectVisitor. + @Override + public void closeStruct() { + closeScope(); + addLine("}"); + } + + // Inherit doc from ObjectVisitor. + @Override + public void visit(String name, Object obj) { + if (obj == null) { + addLine(name + ": <NULL>"); + } else if (obj instanceof Identifiable) { + openStruct(name, obj.getClass().getSimpleName()); + ((Identifiable)obj).visitMembers(this); + closeStruct(); + } else if (obj instanceof String) { + addLine(name + ": '" + obj + "'"); + } else if (obj.getClass().isArray()) { + openStruct(name, obj.getClass().getComponentType().getSimpleName() + "[]"); + for (int i = 0, len = Array.getLength(obj); i < len; ++i) { + visit("[" + i + "]", Array.get(obj, i)); + } + closeStruct(); + } else if (obj instanceof List) { + openStruct(name, "List"); + List<?> lst = (List<?>) obj; + for (int i = 0; i < lst.size(); ++i) { + visit("[" + i + "]", lst.get(i)); + } + closeStruct(); + } else { + addLine(name + ": " + obj); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectOperation.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectOperation.java new file mode 100755 index 00000000000..7e652aa588c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectOperation.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +/** + * An operation that is able to operate on a generic object. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface ObjectOperation { + + /** + * Apply this operation to the given object. + * + * @param obj The object to operate on. + */ + public void execute(Object obj); +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectPredicate.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectPredicate.java new file mode 100755 index 00000000000..adc918ae696 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectPredicate.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +/** + * A predicate that is able to say either true or false when presented with a + * generic object. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface ObjectPredicate { + + /** + * Apply this predicate to the given object. + * + * @param obj The object to check. + * @return True or false. + */ + public boolean check(Object obj); +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectVisitor.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectVisitor.java new file mode 100755 index 00000000000..07c8e90f4b7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectVisitor.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +/** + * This is an abstract class used to visit structured objects. It contains a basic interface that is intended to be + * overridden by subclasses. As an extension to this class, the visit.hpp file contains various versions of the visit + * method that maps visitation of various types into invocations of the basic interface defined by this class. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class ObjectVisitor { + + /** + * Open a (sub-)structure + * + * @param name name of structure + * @param type type of structure + */ + public abstract void openStruct(String name, String type); + + /** + * Close a (sub-)structure + */ + public abstract void closeStruct(); + + /** + * Visits some object. + * + * @param name variable name + * @param obj object to visit + */ + public abstract void visit(String name, Object obj); +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Selectable.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Selectable.java new file mode 100644 index 00000000000..a49d09a212b --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Selectable.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * + * This class acts as an interface for traversing a tree, or a graph. + * Every non leaf Object implements {@link #selectMembers(ObjectPredicate, ObjectOperation)} implementing + * the actual traversal. You can then implement an {@link ObjectPredicate} to select which nodes you want to look at with + * your {@link ObjectOperation} + */ +public class Selectable { + + /** + * Apply the predicate to this object. If the predicate returns true, pass this object to the operation, otherwise + * invoke the {@link #selectMembers(ObjectPredicate, ObjectOperation)} method to locate sub-elements that might + * trigger the predicate. + * + * @param predicate component used to select (sub-)objects + * @param operation component performing some operation on the selected (sub-)objects + */ + public final void select(ObjectPredicate predicate, ObjectOperation operation) { + if (predicate.check(this)) { + operation.execute(this); + } else { + selectMembers(predicate, operation); + } + } + + /** + * Invoke {@link #select(ObjectPredicate, ObjectOperation)} on any member objects this object wants to expose + * through the selection mechanism. Overriding this method is optional, and which objects to expose is determined by + * the application logic of the object itself. + * + * @param predicate component used to select (sub-)objects + * @param operation component performing some operation on the selected (sub-)objects + */ + public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { + // empty + } + + public static void select(Selectable selectable, ObjectPredicate predicate, ObjectOperation operation) { + if (selectable != null) { + selectable.select(predicate, operation); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Serializer.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Serializer.java new file mode 100644 index 00000000000..a50252fc70c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Serializer.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +import java.nio.ByteBuffer; + +/** + * @author balder + */ +public interface Serializer { + Serializer putByte(FieldBase field, byte value); + Serializer putShort(FieldBase field, short value); + Serializer putInt(FieldBase field, int value); + Serializer putLong(FieldBase field, long value); + Serializer putFloat(FieldBase field, float value); + Serializer putDouble(FieldBase field, double value); + Serializer put(FieldBase field, byte[] value); + Serializer put(FieldBase field, ByteBuffer value); + Serializer put(FieldBase field, String value); +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/package-info.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/package-info.java new file mode 100644 index 00000000000..bb4b11d182a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.vespa.objects; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; |