diff options
Diffstat (limited to 'vespajlib')
334 files changed, 36839 insertions, 0 deletions
diff --git a/vespajlib/.gitignore b/vespajlib/.gitignore new file mode 100644 index 00000000000..06ae1debf2c --- /dev/null +++ b/vespajlib/.gitignore @@ -0,0 +1,11 @@ +.classpath +.project +.settings +archive +build +log +target +*.iml +*.ipr +*.iws +/pom.xml.build diff --git a/vespajlib/OWNERS b/vespajlib/OWNERS new file mode 100644 index 00000000000..67cd2820bb8 --- /dev/null +++ b/vespajlib/OWNERS @@ -0,0 +1 @@ +arnej27959 diff --git a/vespajlib/README b/vespajlib/README new file mode 100644 index 00000000000..caf560d229d --- /dev/null +++ b/vespajlib/README @@ -0,0 +1 @@ +Module for shared Java utility code in Vespa. diff --git a/vespajlib/developernotes/CharClassStats.java b/vespajlib/developernotes/CharClassStats.java new file mode 100644 index 00000000000..359b41766a2 --- /dev/null +++ b/vespajlib/developernotes/CharClassStats.java @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.util.*; + +public class CharClassStats { + + public static class TypeStat { + public final int typecode; + public final String name; + public final List<Integer> codepoints = new ArrayList<Integer>(); + + TypeStat(int typecode) { + this(typecode, "[???]"); + } + TypeStat(int typecode, String name) { + this.typecode = typecode; + this.name = name; + } + void addCodepoint(int codepoint) { + codepoints.add(codepoint); + } + } + + private static void init(Map<Integer, TypeStat> map) { + + TypeStat stat; + stat = new TypeStat(Character.COMBINING_SPACING_MARK, "COMBINING_SPACING_MARK"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.CONNECTOR_PUNCTUATION, "CONNECTOR_PUNCTUATION"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.CONTROL, "CONTROL"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.CURRENCY_SYMBOL, "CURRENCY_SYMBOL"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.DASH_PUNCTUATION, "DASH_PUNCTUATION"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.DECIMAL_DIGIT_NUMBER, "DECIMAL_DIGIT_NUMBER"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.ENCLOSING_MARK, "ENCLOSING_MARK"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.END_PUNCTUATION, "END_PUNCTUATION"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.FINAL_QUOTE_PUNCTUATION, "FINAL_QUOTE_PUNCTUATION"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.FORMAT, "FORMAT"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.INITIAL_QUOTE_PUNCTUATION, "INITIAL_QUOTE_PUNCTUATION"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.LETTER_NUMBER, "LETTER_NUMBER"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.LINE_SEPARATOR, "LINE_SEPARATOR"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.LOWERCASE_LETTER, "LOWERCASE_LETTER"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.MATH_SYMBOL, "MATH_SYMBOL"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.MODIFIER_LETTER, "MODIFIER_LETTER"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.MODIFIER_SYMBOL, "MODIFIER_SYMBOL"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.NON_SPACING_MARK, "NON_SPACING_MARK"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.OTHER_LETTER, "OTHER_LETTER"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.OTHER_NUMBER, "OTHER_NUMBER"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.OTHER_PUNCTUATION, "OTHER_PUNCTUATION"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.OTHER_SYMBOL, "OTHER_SYMBOL"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.PARAGRAPH_SEPARATOR, "PARAGRAPH_SEPARATOR"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.PRIVATE_USE, "PRIVATE_USE"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.SPACE_SEPARATOR, "SPACE_SEPARATOR"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.START_PUNCTUATION, "START_PUNCTUATION"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.SURROGATE, "SURROGATE"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.TITLECASE_LETTER, "TITLECASE_LETTER"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.UNASSIGNED, "UNASSIGNED"); + map.put(stat.typecode, stat); + stat = new TypeStat(Character.UPPERCASE_LETTER, "UPPERCASE_LETTER"); + map.put(stat.typecode, stat); + } + + public static void main(String[] args) { + Map<Integer, TypeStat> map = new HashMap<Integer, TypeStat>(); + + init(map); + + for (int codepoint = 0; codepoint <= 0x110000; codepoint++) { + int type = java.lang.Character.getType(codepoint); + + if (! map.containsKey(type)) { + map.put(type, new TypeStat(type)); + } + map.get(type).addCodepoint(codepoint); + } + + int[] codes = new int[map.size()]; + int numcodes = 0; + for (Integer type : map.keySet()) { + codes[numcodes++] = type; + } + Arrays.sort(codes); + for (int type : codes) { + TypeStat ts = map.get(type); + System.out.println("type "+type+" typecode="+ts.typecode+" name="+ts.name+" contains "+ts.codepoints.size()+" codepoints"); + } + } + +} diff --git a/vespajlib/developernotes/CopyOnWriteHashMapBenchmark.java b/vespajlib/developernotes/CopyOnWriteHashMapBenchmark.java new file mode 100644 index 00000000000..c1cb0cc3e6c --- /dev/null +++ b/vespajlib/developernotes/CopyOnWriteHashMapBenchmark.java @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public class RcuHashMapBenchmark { + static class Actor implements Runnable { + private final CopyOnWriteHashMap<Long, Long> m; + private long mSum = 0; + private long mMissRate = 0; + Actor(CopyOnWriteHashMap<Long, Long> m) { + this.m = m; + } + @Override + public void run() { + final int NUM_UPDATES=100; + final long NUM_LOOKUPS=10000000; + final List<Long> upd = new ArrayList<Long>(NUM_UPDATES); + upd.add(0l); + long missRate = 0; + long sum = 0; + for (long i=0; i < NUM_LOOKUPS; i++) { + long t = i%upd.size(); + Long v = m.get(upd.get((int)t)); + if (v == null) { + missRate++; + m.put(upd.get((int)t), i); + sum += i; + } else { + sum += v; + } + if (i%(NUM_LOOKUPS/NUM_UPDATES) == 0) { + upd.add((long)upd.size()); + } + } + synchronized (this) { + mSum = sum; + mMissRate = missRate; + } + } + long getSum() { synchronized (this) { return mSum; } } + long getMissRate() { synchronized (this) { return mMissRate;} } + } + RcuHashMapBenchmark(int numThreads) { + CopyOnWriteHashMap<Long, Long> m = new CopyOnWriteHashMap<Long, Long>(); + Thread[] threads = new Thread[numThreads]; + Actor [] actors = new Actor[threads.length]; + for (int i = 0; i < threads.length; ++i) { + Actor a = new Actor(m); + actors[i] = a; + threads[i] = new Thread(a); + } + runAll(threads); + long missRate=0; + long sum=0; + for (Actor a : actors) { + missRate += a.getMissRate(); + sum += a.getSum(); + System.out.println("Missrate: " + a.getMissRate() + " sum = " + a.getSum()); + } + System.out.println("Total Missrate: " + missRate + " sum = " + sum); + } + + private void runAll(Thread[] threads) { + for (Thread t : threads) { + t.start(); + } + for (Thread t : threads) { + try { + t.join(); + } catch (InterruptedException e) { + // nop + } + } + } + public static void main(String[] args) { + long start, end; + start = System.currentTimeMillis(); + new RcuHashMapBenchmark(1); + end = System.currentTimeMillis(); + System.out.println("Elapsed during warmup: " + (end - start) + " ms."); + for (int i=0; i < 16; i++) { + start = System.currentTimeMillis(); + new RcuHashMapBenchmark(i+1); + end = System.currentTimeMillis(); + System.out.println("Elapsed during " + (i+1) + " threads: " + (end - start) + " ms."); + } + + } +} diff --git a/vespajlib/developernotes/ThreadLocalDirectoryBenchmark.java b/vespajlib/developernotes/ThreadLocalDirectoryBenchmark.java new file mode 100644 index 00000000000..bc91d076e8d --- /dev/null +++ b/vespajlib/developernotes/ThreadLocalDirectoryBenchmark.java @@ -0,0 +1,230 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Benchmark to compare ThreadLocalDirectory with java.util.concurrent's atomic + * variables. Very low precision since it's an adapted unit test. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ThreadLocalDirectoryBenchmark { + private static final int ITERATIONS = 500000; + private final AtomicInteger atomicCounter = new AtomicInteger(0); + private volatile int volatileCounter = 0; + private int naiveCounter = 0; + + private static class SumUpdater implements ThreadLocalDirectory.Updater<Integer, Integer> { + + @Override + public Integer update(Integer current, Integer x) { + return Integer.valueOf(current.intValue() + x.intValue()); + } + + @Override + public Integer createGenerationInstance(Integer previous) { + return Integer.valueOf(0); + } + } + + private static class Counter implements Runnable { + ThreadLocalDirectory<Integer, Integer> r; + + Counter(ThreadLocalDirectory<Integer, Integer> r) { + this.r = r; + } + + @Override + public void run() { + LocalInstance<Integer, Integer> s = r.getLocalInstance(); + for (int i = 0; i < ITERATIONS; ++i) { + r.update(Integer.valueOf(i), s); + } + } + } + + private static class MutableSumUpdater implements ThreadLocalDirectory.Updater<IntWrapper, IntWrapper> { + + @Override + public IntWrapper update(IntWrapper current, IntWrapper x) { + current.counter += x.counter; + return current; + } + + @Override + public IntWrapper createGenerationInstance(IntWrapper previous) { + return new IntWrapper(); + } + } + + private static class IntWrapper { + public int counter = 0; + } + + private static class WrapperCounter implements Runnable { + ThreadLocalDirectory<IntWrapper, IntWrapper> r; + + WrapperCounter(ThreadLocalDirectory<IntWrapper, IntWrapper> r) { + this.r = r; + } + + @Override + public void run() { + LocalInstance<IntWrapper, IntWrapper> s = r.getLocalInstance(); + IntWrapper w = new IntWrapper(); + for (int i = 0; i < ITERATIONS; ++i) { + w.counter = i; + r.update(w, s); + } + } + } + + private class AtomicCounter implements Runnable { + @Override + public void run() { + for (int i = 0; i < ITERATIONS; ++i) { + atomicCounter.addAndGet(i); + } + } + } + + /** + * This just bangs on a shared volatile to give an idea of the basic cost of + * sharing a single variable with a memory barrier. + */ + private class VolatileSillyness implements Runnable { + + @Override + public void run() { + for (int i = 0; i < ITERATIONS; ++i) { + volatileCounter += i; + } + } + } + + /** + * This just bangs on a shared to give some sort of lower bound for time + * elapsed. + */ + private class SillySillyness implements Runnable { + + @Override + public void run() { + for (int i = 0; i < ITERATIONS; ++i) { + naiveCounter += i; + } + } + } + + private void sumFromMultipleThreads() { + SumUpdater updater = new SumUpdater(); + ThreadLocalDirectory<Integer, Integer> s = new ThreadLocalDirectory<Integer, Integer>(updater); + Thread[] threads = new Thread[500]; + for (int i = 0; i < 500; ++i) { + Counter c = new Counter(s); + threads[i] = new Thread(c); + } + runAll(threads); + List<Integer> measurements = s.fetch(); + long sum = 0; + for (Integer i : measurements) { + sum += i.intValue(); + } + System.out.println("Sum from all threads: " + sum); + } + + private void sumMutableFromMultipleThreads() { + MutableSumUpdater updater = new MutableSumUpdater(); + ThreadLocalDirectory<IntWrapper, IntWrapper> s = new ThreadLocalDirectory<IntWrapper, IntWrapper>(updater); + Thread[] threads = new Thread[500]; + for (int i = 0; i < 500; ++i) { + WrapperCounter c = new WrapperCounter(s); + threads[i] = new Thread(c); + } + runAll(threads); + List<IntWrapper> measurements = s.fetch(); + long sum = 0; + for (IntWrapper i : measurements) { + sum += i.counter; + } + System.out.println("Sum from all threads: " + sum); + } + + private void sumAtomicFromMultipleThreads() { + Thread[] threads = new Thread[500]; + for (int i = 0; i < 500; ++i) { + AtomicCounter c = new AtomicCounter(); + threads[i] = new Thread(c); + } + runAll(threads); + System.out.println("Sum from all threads: " + atomicCounter.get()); + } + + private void overwriteVolatileFromMultipleThreads() { + Thread[] threads = new Thread[500]; + for (int i = 0; i < 500; ++i) { + VolatileSillyness c = new VolatileSillyness(); + threads[i] = new Thread(c); + } + runAll(threads); + System.out.println("Checksum from all threads: " + volatileCounter); + } + + private void overwriteIntegerFromMultipleThreads() { + Thread[] threads = new Thread[500]; + for (int i = 0; i < 500; ++i) { + SillySillyness c = new SillySillyness(); + threads[i] = new Thread(c); + } + runAll(threads); + System.out.println("Checksum from all threads: " + volatileCounter); + } + + private void runAll(Thread[] threads) { + for (Thread t : threads) { + t.start(); + } + for (Thread t : threads) { + try { + t.join(); + } catch (InterruptedException e) { + // nop + } + } + } + + public static void main(String[] args) { + ThreadLocalDirectoryBenchmark benchmark = new ThreadLocalDirectoryBenchmark(); + long end; + System.out.println("ThreadLocalDirectory<Integer, Integer>"); + long start = System.currentTimeMillis(); + benchmark.sumFromMultipleThreads(); + end = System.currentTimeMillis(); + System.out.println("Elapsed using threadlocals: " + (end - start) + " ms."); + System.out.println("AtomicInteger"); + start = System.currentTimeMillis(); + benchmark.sumAtomicFromMultipleThreads(); + end = System.currentTimeMillis(); + System.out.println("Elapsed using atomic integer: " + (end - start) + " ms."); + System.out.println("volatile int += volatile int"); + start = System.currentTimeMillis(); + benchmark.overwriteVolatileFromMultipleThreads(); + end = System.currentTimeMillis(); + System.out.println("Elapsed using single shared volatile: " + (end - start) + " ms."); + System.out.println("int += int"); + start = System.currentTimeMillis(); + benchmark.overwriteIntegerFromMultipleThreads(); + end = System.currentTimeMillis(); + System.out.println("Checksum: " + benchmark.naiveCounter); + System.out.println("Elapsed using shared int: " + (end - start) + " ms."); + System.out.println("ThreadLocalDirectory<IntWrapper, IntWrapper>"); + start = System.currentTimeMillis(); + benchmark.sumMutableFromMultipleThreads(); + end = System.currentTimeMillis(); + System.out.println("Elapsed using threadlocal with mutable int wrapper: " + (end - start) + " ms."); + } + +} diff --git a/vespajlib/developernotes/Utf8MicroBencmark.java b/vespajlib/developernotes/Utf8MicroBencmark.java new file mode 100644 index 00000000000..ff1ae4ce3a2 --- /dev/null +++ b/vespajlib/developernotes/Utf8MicroBencmark.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ + +public class Utf8MicroBencmark { + public void benchmark(int sizeInK) { + String [] l = new String[1000]; + for (int i=0; i < l.length; i++) { + l[i] = "typical ascii string" + i; + } + System.out.println("Warming up..."); + utf8encode(l, 10000); // warm-up + utf8encodeFast(l, 10000); + + long startTime, endTime, sum; + System.out.println("Starting benchmark ..."); + startTime=System.currentTimeMillis(); + sum = utf8encode(l, sizeInK); + endTime=System.currentTimeMillis(); + System.out.println("Utf8 encoding " + sizeInK + "k strings took " + (endTime-startTime) + "ms generating " + sum + "bytes"); + startTime=System.currentTimeMillis(); + sum = utf8encodeFast(l, sizeInK); + endTime=System.currentTimeMillis(); + System.out.println("Utf8 fast encoding " + sizeInK + "k strings took " + (endTime-startTime) + "ms generating " + sum + "bytes"); + } + + private long utf8encode(String [] l, int sizeInK) { + long sum = 0; + for (int i=0; i<1000*sizeInK; i++) { + sum += Utf8.toBytesStd(l[i%l.length]).length; + } + return sum; + } + private long utf8encodeFast(String [] l, int sizeInK) { + long sum = 0; + for (int i=0; i<1000*sizeInK; i++) { + sum += Utf8.toBytes(l[i%l.length]).length; + } + return sum; + } + + public static void main(String[] args) { + new Utf8MicroBencmark().benchmark(10000); + } + +} diff --git a/vespajlib/developernotes/XMLMicroBenchmark.java b/vespajlib/developernotes/XMLMicroBenchmark.java new file mode 100644 index 00000000000..6a9d02e1c45 --- /dev/null +++ b/vespajlib/developernotes/XMLMicroBenchmark.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * It is what it says + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class XMLMicroBenchmark { + + public void benchmark(int sizeInK) { + System.out.println("Warming up..."); + escapeStrings(1000); // warm-up + + System.out.println("Starting benchmark..."); + long startTime=System.currentTimeMillis(); + escapeStrings(sizeInK); + long endTime=System.currentTimeMillis(); + System.out.println("Done.\nEscaping " + sizeInK + "k strings took " + (endTime-startTime) + "ms"); + } + + private void escapeStrings(int sizeInK) { + for (int i=0; i<1000*sizeInK; i++) { + XML.xmlEscape("foobar" + i,true,true,'\u001f'); + } + } + + public static void main(String[] args) { + new XMLMicroBenchmark().benchmark(10000); + } + +} diff --git a/vespajlib/developernotes/XMLWriterMicroBenchmark.java b/vespajlib/developernotes/XMLWriterMicroBenchmark.java new file mode 100644 index 00000000000..67570d54ea6 --- /dev/null +++ b/vespajlib/developernotes/XMLWriterMicroBenchmark.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import com.yahoo.io.ByteWriter; + +import java.io.ByteArrayOutputStream; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; + +/** + * It is what it says + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class XMLWriterMicroBenchmark { + + private final ByteArrayOutputStream output; + private final XMLWriter xmlWriter; + + public XMLWriterMicroBenchmark(boolean optimize) { + // setup + output=new ByteArrayOutputStream(); + Charset cs = Charset.forName("utf-8"); + CharsetEncoder encoder = cs.newEncoder(); + xmlWriter=new XMLWriter(new ByteWriter(output, encoder), optimize); + } + + public void benchmark(int sizeInK,boolean verifyOutput) { + System.out.println("Warming up..."); + writeStrings(1000); // warm-up + + System.out.println("Starting benchmark..."); + long startTime=System.currentTimeMillis(); + writeStrings(sizeInK); + long endTime=System.currentTimeMillis(); + System.out.println("Done.\nWriting " + sizeInK + "k strings took " + (endTime-startTime) + "ms"); + + if (verifyOutput) { + System.out.println("First 1k of output:"); + String result=null; + try { result=output.toString("utf-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } + System.out.println(result.substring(0,Math.min(500,result.length()))); + } + } + + private void writeStrings(int sizeInK) { + for (int i=0; i<1000*sizeInK; i++) { + xmlWriter.openTag("dummytag").content(i,false).closeTag(); + } + } + + public static void main(String[] args) { + System.out.println("Unoptimized: -------------------------"); + new XMLWriterMicroBenchmark(false).benchmark(10000,false); + System.out.println("Optimized: ------------------------"); + new XMLWriterMicroBenchmark(true).benchmark(10000,false); + } + + +} diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml new file mode 100644 index 00000000000..38028810afb --- /dev/null +++ b/vespajlib/pom.xml @@ -0,0 +1,114 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>vespajlib</artifactId> + <packaging>container-plugin</packaging> + <version>6-SNAPSHOT</version> + <description> + Library for use in Java components of Vespa. Shared code which did + not fit anywhere else. + </description> + <repositories> + <repository> + <id>apache-org</id> + <name>apache.org Repository for Maven</name> + <url>https://repository.apache.org/content/groups/public</url> + <releases> + <updatePolicy>never</updatePolicy> + </releases> + <snapshots> + <enabled>false</enabled> + </snapshots> + </repository> + </repositories> + + <dependencies> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>net.jpountz.lz4</groupId> + <artifactId>lz4</artifactId> + </dependency> + <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-library</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>yolean</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-deploy-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <configuration> + <updateReleaseInfo>true</updateReleaseInfo> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/vespajlib/src/main/java/com/yahoo/api/annotations/.gitignore b/vespajlib/src/main/java/com/yahoo/api/annotations/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/api/annotations/.gitignore diff --git a/vespajlib/src/main/java/com/yahoo/api/annotations/PackageMarker.java b/vespajlib/src/main/java/com/yahoo/api/annotations/PackageMarker.java new file mode 100644 index 00000000000..cbd09fbf69f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/api/annotations/PackageMarker.java @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.api.annotations; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.SOURCE) +public @interface PackageMarker { } diff --git a/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryPrefix.java b/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryPrefix.java new file mode 100644 index 00000000000..e207e584115 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryPrefix.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.binaryprefix; + +/** + * Represents binary prefixes. + * @author tonytv + */ +public enum BinaryPrefix { + //represents the binary prefix 2^(k*10) + unit(0), + kilo(1, 'K'), + mega(2, 'M'), + giga(3, 'G'), + tera(4, 'T'), + peta(5, 'P'), + exa(6, 'E'), + zetta(7, 'Z'), + yotta(8, 'Y'); + + private final int k; + public final char symbol; + + private BinaryPrefix(int k, char symbol) { + this.k = k; + this.symbol = symbol; + } + + private BinaryPrefix(int k) { + this(k, (char)0); + } + + /* In most cases, BinaryScaledAmount should be prefered instead of this */ + public double convertFrom(double value, BinaryPrefix binaryPrefix) { + return value * Math.pow(2, + 10 * (binaryPrefix.k - k)); + } + + public static BinaryPrefix fromSymbol(char c) { + for (BinaryPrefix binaryPrefix : values()) { + if (binaryPrefix.symbol == c) + return binaryPrefix; + } + throw new RuntimeException("No such binary prefix: " + c); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryScaledAmount.java b/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryScaledAmount.java new file mode 100644 index 00000000000..303674bb504 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryScaledAmount.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.binaryprefix; + +/** + * An amount scaled by a binary prefix. + * + * <p> + * Examples: 2 kilo, 2 mega, ... + * </p> + * + * @author tonytv + */ +public final class BinaryScaledAmount { + public final double amount; + public final BinaryPrefix binaryPrefix; + + public BinaryScaledAmount(double amount, BinaryPrefix binaryPrefix) { + this.amount = amount; + this.binaryPrefix = binaryPrefix; + } + + public BinaryScaledAmount() { + this(0, BinaryPrefix.unit); + } + + public long as(BinaryPrefix newBinaryPrefix) { + return Math.round(newBinaryPrefix.convertFrom(amount, binaryPrefix)); + } + + public boolean equals(BinaryScaledAmount candidate) { + return BinaryPrefix.unit.convertFrom(amount, binaryPrefix) == + BinaryPrefix.unit.convertFrom(candidate.amount, candidate.binaryPrefix); + } + + public BinaryScaledAmount multiply(double d) { + return new BinaryScaledAmount(d*amount, binaryPrefix); + } + + public BinaryScaledAmount divide(double d) { + return multiply(1/d); + } + + @Override + public boolean equals(Object candidate) { + if (!(candidate instanceof BinaryScaledAmount)) { + return false; + } else { + return equals((BinaryScaledAmount)candidate); + } + } + + @Override + public int hashCode() { + return (int)BinaryPrefix.unit.convertFrom(amount, binaryPrefix); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/binaryprefix/package-info.java b/vespajlib/src/main/java/com/yahoo/binaryprefix/package-info.java new file mode 100644 index 00000000000..e1dada71ade --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/binaryprefix/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.binaryprefix; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/cache/Cache.java b/vespajlib/src/main/java/com/yahoo/cache/Cache.java new file mode 100644 index 00000000000..bfc3f3010aa --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/cache/Cache.java @@ -0,0 +1,276 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.cache; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * <p>A generic cache which keeps the total memory consumed by its content + * below a configured maximum.</p> + * + * <p>Thread safe.</p> + * + * @author vegardh + */ +public class Cache<K, V> { + private Map<CacheKey<K>,CacheValue<K, V>> content=new LinkedHashMap<>(12500, 1.0f, true); + private SizeCalculator calc = new SizeCalculator(); + private long maxSizeBytes; + + private long currentSizeBytes=0; + + /** The time an element is allowed to live, negative for indefinite lifespan */ + private long timeToLiveMillis=-1; + + /** The max allowed size of an entry, negative for no limit */ + private long maxEntrySizeBytes=10000; + + /** + * Creates a new cache + * + * @param maxSizeBytes the max size in bytes this cache is permitted to consume, + * including Result objects and Query keys + * @param timeToLiveMillis a negative value means unlimited time + * @param maxEntrySizeBytes never cache objects bigger than this, negative for no such limit + */ + public Cache(long maxSizeBytes,long timeToLiveMillis, long maxEntrySizeBytes) { + this.maxSizeBytes=maxSizeBytes; + this.timeToLiveMillis=timeToLiveMillis; + this.maxEntrySizeBytes=maxEntrySizeBytes; + } + + private synchronized CacheValue<K, V> synchGet(CacheKey<K> k) { + return content.get(k); + } + + private synchronized boolean synchPut(K key,V value, long keySizeBytes, long valueSizeBytes) { + // log.info("Put "+key.toString()+ " key size:"+keySizeBytes+" val size:"+valueSizeBytes); + if ((valueSizeBytes+keySizeBytes)>maxSizeBytes) { + return false; + } + makeRoomForBytes(valueSizeBytes+keySizeBytes); + CacheKey<K> cacheKey = new CacheKey<>(keySizeBytes, key); + CacheValue<K, V> cacheValue; + if (timeToLiveMillis<0) { + cacheValue=new CacheValue<>(valueSizeBytes,value, cacheKey); + } else { + cacheValue=new AgingCacheValue<>(valueSizeBytes,value, cacheKey); + } + currentSizeBytes+=(valueSizeBytes+keySizeBytes); + content.put(cacheKey, cacheValue); + return true; + } + + /** + * Attempts to add a value to the cache + * + * @param key the key of the value + * @param value the value to add + * @return true if the value was added, false if it could not be added + */ + public boolean put(K key,V value) { + long keySizeBytes=calc.sizeOf(key); + long valueSizeBytes=calc.sizeOf(value); + if (tooBigToCache(keySizeBytes+valueSizeBytes)) { + return false; + } + return synchPut(key, value, keySizeBytes, valueSizeBytes); + } + + /** + * Don't cache elems that are too big, even if there's space + * @return true if the argument is too big to cache. + */ + private boolean tooBigToCache(long totalSize) { + if (maxEntrySizeBytes<0) { + return false; + } + if (totalSize > maxEntrySizeBytes) { + return true; + } + return false; + } + + private void makeRoomForBytes(long bytes) { + if ((maxSizeBytes-currentSizeBytes) > bytes) { + return; + } + if (content.isEmpty()) { + return; + } + for (Iterator<Map.Entry<CacheKey<K>, CacheValue<K, V>>> i = content.entrySet().iterator() ; i.hasNext() ; ) { + Map.Entry<CacheKey<K>, CacheValue<K, V>> entry = i.next(); + CacheKey<K> key = entry.getKey(); + CacheValue<K, V> value = entry.getValue(); + // Can't call this.remove(), breaks iterator. + i.remove(); // Access order: first ones are LRU. + currentSizeBytes-=key.sizeBytes(); + currentSizeBytes-=value.sizeBytes(); + if ((maxSizeBytes-currentSizeBytes) > bytes) { + break; + } + } + } + + public boolean containsKey(K k) { + return content.containsKey(new CacheKey<>(-1, k)); + } + + /** Returns a value, if it is present in the cache */ + public V get(K key) { + // Currently it works to make a new CacheKey object without size + // because we have changed hashCode() there. + CacheKey<K> cacheKey = new CacheKey<>(-1, key); + CacheValue<K, V> value=synchGet(cacheKey); + if (value==null) { + return null; + } + if (timeToLiveMillis<0) { + return value.value(); + } + + if (value.expired(timeToLiveMillis)) { + // There was a value, which has now expired + remove(key); + return null; + } else { + return value.value(); + } + } + + /** + * Removes a cache value if present + * + * @return true if the value was removed, false if it was not present + */ + public synchronized boolean remove(K key) { + CacheValue<K, V> value=content.remove(key); + if (value==null) { + return false; + } + currentSizeBytes-=value.sizeBytes(); + currentSizeBytes-=value.getKey().sizeBytes(); + return true; + } + + public long getTimeToLiveMillis() { + return timeToLiveMillis; + } + + public int size() { + return content.size(); + } + + private static class CacheKey<K> { + private long sizeBytes; + private K key; + public CacheKey(long sizeBytes,K key) { + this.sizeBytes=sizeBytes; + this.key=key; + } + + public long sizeBytes() { + return sizeBytes; + } + + public K getKey() { + return key; + } + + public int hashCode() { + return key.hashCode(); + } + + @SuppressWarnings("rawtypes") + public boolean equals(Object k) { + if (key==null) { + return false; + } + if (k==null) { + return false; + } + if (k instanceof CacheKey) { + return key.equals(((CacheKey)k).getKey()); + } + return false; + } + } + + private static class CacheValue<K, V> { + private long sizeBytes; + private V value; + private CacheKey<K> key; + public CacheValue(long sizeBytes, V value, CacheKey<K> key) { + this.sizeBytes=sizeBytes; + this.value=value; + this.key = key; + } + + public boolean expired(long ttl) { + return false; + } + + public V value() { + return value; + } + + public long sizeBytes() { + return sizeBytes; + } + + public CacheKey<K> getKey() { + return key; + } + } + + private static class AgingCacheValue<K, V> extends CacheValue<K, V> { + private long birthTimeMillis; + + public AgingCacheValue(long sizeBytes,V value, CacheKey<K> key) { + super(sizeBytes,value, key); + this.birthTimeMillis=System.currentTimeMillis(); + } + + public long ageMillis() { + return System.currentTimeMillis()-birthTimeMillis; + } + + public boolean expired(long ttl) { + return (ageMillis() >= ttl); + } + } + + /** + * Empties the cache + */ + public synchronized void clear() { + content.clear(); + currentSizeBytes=0; + } + + /** + * Collection of keys. + */ + public Collection<K> getKeys() { + Collection<K> ret = new ArrayList<>(); + for (Iterator<CacheKey<K>> i = content.keySet().iterator(); i.hasNext();) { + ret.add(i.next().getKey()); + } + return ret; + } + + /** + * Collection of values. + */ + public Collection<V> getValues() { + Collection<V> ret = new ArrayList<>(); + for (Iterator<CacheValue<K, V>> i = content.values().iterator(); i.hasNext();) { + ret.add(i.next().value()); + } + return ret; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/cache/SizeCalculator.java b/vespajlib/src/main/java/com/yahoo/cache/SizeCalculator.java new file mode 100644 index 00000000000..677a3fb07e6 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/cache/SizeCalculator.java @@ -0,0 +1,175 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.cache; + +import java.lang.reflect.Array; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; + +/** + * Size calculator for objects. + * Thread safe. + * @author vegardh + * @see <a href="http://www.javaspecialists.co.za/archive/Issue078.html">MemoryCounter by Dr H M Kabutz</a> + */ +public class SizeCalculator { + + private static class ObjectSet { + private final Map<Object, Object> map = new IdentityHashMap<>(); + + public boolean had(Object obj) { + if (map.containsKey(obj)) { + return true; + } + map.put(obj, null); + return false; + } + } + + private int getPointerSize() { + return 4; + } + + private int getClassSize() { + return 8; + } + + private int getArraySize() { + return 16; + } + + @SuppressWarnings("serial") + private final IdentityHashMap<Class<?>, Integer> primitiveSizes = new IdentityHashMap<Class<?>, Integer>() { + { + put(boolean.class, 1); + put(byte.class, 1); + put(char.class, 2); + put(short.class, 2); + put(int.class, 4); + put(float.class, 4); + put(double.class, 8); + put(long.class, 8); + } + }; + + // Only called on un-visited objects and only with array. + private long sizeOfArray(Object a, ObjectSet visitedObjects) { + long sum = getArraySize(); + int length = Array.getLength(a); + if (length == 0) { + return sum; + } + Class<?> elementClass = a.getClass().getComponentType(); + if (elementClass.isPrimitive()) { + sum += length * (primitiveSizes.get(elementClass)); + return sum; + } else { + for (int i = 0; i < length; i++) { + Object val = Array.get(a, i); + sum += getPointerSize(); + sum += sizeOfObject(val, visitedObjects); + } + return sum; + } + } + + private long getSumOfFields(Class<?> clas, Object obj, + ObjectSet visitedObjects) { + long sum = 0; + Field[] fields = clas.getDeclaredFields(); + for (Field field : fields) { + if (!Modifier.isStatic(field.getModifiers())) { + if (field.getType().isPrimitive()) { + sum += primitiveSizes.get(field.getType()); + } else { + sum += getPointerSize(); + field.setAccessible(true); + try { + sum += sizeOfObject(field.get(obj), visitedObjects); + } catch (IllegalArgumentException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + } + } + return sum; + } + + // Skip literal strings + private boolean isIntern(Object obj) { + if (obj instanceof String) { + if (obj == ((String) obj).intern()) { + return true; + } + } + return false; + } + + // Only called on non-visited non-arrays. + private long sizeOfNonArray(Class<?> clas, Object obj, + ObjectSet visitedObjects) { + if (isIntern(obj)) { + return 0; + } + long sum = getClassSize(); + while (clas != null) { + sum += getSumOfFields(clas, obj, visitedObjects); + clas = clas.getSuperclass(); + } + return sum; + } + + private long sizeOfObject(Object obj, ObjectSet visitedObjects) { + if (obj == null) { + return 0; + } + if (visitedObjects.had(obj)) { + return 0; + } + Class<?> clas = obj.getClass(); + if (clas.isArray()) { + return sizeOfArray(obj, visitedObjects); + } + return sizeOfNonArray(clas, obj, visitedObjects); + } + + /** + * Returns the heap size of an object/array + * + * @return Number of bytes for object, approximately + */ + public long sizeOf(Object value) { + ObjectSet visitedObjects = new ObjectSet(); + return sizeOfObject(value, visitedObjects); + } + + /** + * Returns the heap size of two objects/arrays, common objects counted only + * once + * + * @return Number of bytes for objects, approximately + */ + public long sizeOf(Object value1, Object value2) { + ObjectSet visitedObjects = new ObjectSet(); + return sizeOfObject(value1, visitedObjects) + + sizeOfObject(value2, visitedObjects); + } + + /** + * The approximate size in bytes for a list of objects, viewed as a closure, + * ie. common objects are counted only once. + * + * @return total number of bytes + */ + public long sizeOf(List<?> objects) { + ObjectSet visitedObjects = new ObjectSet(); + long sum = 0; + for (Object o : objects) { + sum += sizeOfObject(o, visitedObjects); + } + return sum; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/ArraySet.java b/vespajlib/src/main/java/com/yahoo/collections/ArraySet.java new file mode 100644 index 00000000000..8df46e113a2 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/ArraySet.java @@ -0,0 +1,251 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Set; + +/** + * A Set implementation with low allocation cost. It should only be used for + * small number of objects, as it is implemented as scanning an ArrayList for + * equality matches. In other words: Performance will only be acceptable for + * <i>small</i> sets. + * + * <p> + * The rationale for this class is the high cost of the object identifier used + * in IdentityHashMap, where the key set is often used as an identity set. + * </p> + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author balder + * @since 5.1.4 + * + * @param <E> + * the type contained in the Set + */ +public final class ArraySet<E> implements Set<E> { + private class ArrayIterator<T> implements Iterator<E> { + private int i = -1; + private boolean removed = false; + + @Override + public boolean hasNext() { + return i + 1 < size; + } + + @SuppressWarnings("unchecked") + @Override + public E next() { + if (!hasNext()) { + throw new NoSuchElementException("No more elements available"); + } + removed = false; + return (E) entries[++i]; + } + + @Override + public void remove() { + if (removed) { + throw new IllegalStateException( + "Trying to remove same element twice."); + } + if (i == -1) { + throw new IllegalStateException( + "Trying to remove before entering iterator."); + } + delete(i--); + removed = true; + } + + } + + private Object[] entries; + private int size = 0; + + /** + * Create a set with an initial capacity of initSize. The internal array + * will grow automatically with a linear growth rate if more elements than + * initSize are added. + * + * @param initSize + * initial size of internal element array + */ + public ArraySet(final int initSize) { + entries = new Object[initSize]; + } + + /** + * Expose the index in the internal array of a given object. -1 is returned + * if the object is not present in the internal array. + * + * @param e + * an object to check whether exists in this set + * @return the index of the argument e in the internal array, or -1 if the + * object is not present + */ + public int indexOf(final Object e) { + for (int i = 0; i < size; ++i) { + if (e.equals(entries[i])) { + return i; + } + } + return -1; + } + + private void clean() { + int offset = 0; + for (int i = 0; i < size; ++i) { + if (entries[i] == null) { + ++offset; + } else { + entries[i - offset] = entries[i]; + } + } + size -= offset; + } + + private void grow() { + entries = Arrays.copyOf(entries, entries.length * 2 + 1); + } + + private void append(final Object arg) { + if (size == entries.length) { + grow(); + } + entries[size++] = arg; + } + + @Override + public boolean add(final E arg) { + final int i = indexOf(arg); + if (i >= 0) { + return false; + } + append(arg); + return true; + } + + @Override + public boolean addAll(final Collection<? extends E> arg) { + boolean changed = false; + for (final E entry : arg) { + changed |= add(entry); + } + return changed; + } + + @Override + public void clear() { + size = 0; + } + + @Override + public boolean contains(final Object arg) { + return indexOf(arg) >= 0; + } + + /** + * This is an extremely expensive implementation of + * {@link Set#containsAll(Collection)}. It is implemented as O(n**2). + */ + @Override + public boolean containsAll(final Collection<?> arg) { + for (final Object entry : arg) { + if (indexOf(entry) < 0) { + return false; + } + } + return true; + } + + @Override + public boolean isEmpty() { + return size == 0; + } + + @Override + public Iterator<E> iterator() { + return new ArrayIterator<E>(); + } + + private void delete(int i) { + if (i < 0 || i >= size) { + return; + } + --size; + while (i < size) { + entries[i] = entries[i + 1]; + ++i; + } + entries[i] = null; + } + + @Override + public boolean remove(final Object arg) { + final int i = indexOf(arg); + if (i < 0) { + return false; + } + delete(i); + return true; + } + + /** + * This is an extremely expensive implementation of + * {@link Set#removeAll(Collection)}. It is implemented as O(n**2). + */ + @Override + public boolean removeAll(final Collection<?> arg) { + boolean changed = false; + for (final Object entry : arg) { + final int i = indexOf(entry); + if (i >= 0) { + entries[i] = null; + changed = true; + } + } + if (changed) { + clean(); + } + return changed; + } + + /** + * This is an extremely expensive implementation of + * {@link Set#retainAll(Collection)}. It is implemented as O(n**2). + */ + @Override + public boolean retainAll(final Collection<?> arg) { + boolean changed = false; + for (int i = 0; i < size; ++i) { + final Object entry = entries[i]; + if ( !arg.contains(entry)) { + entries[i] = null; + changed = true; + } + } + if (changed) { + clean(); + } + return changed; + } + + @Override + public int size() { + return size; + } + + @Override + public Object[] toArray() { + return Arrays.copyOf(entries, size); + } + + @SuppressWarnings("unchecked") + @Override + public <T> T[] toArray(final T[] arg) { + return Arrays.copyOf(entries, size, (Class<T[]>) arg.getClass()); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/BobHash.java b/vespajlib/src/main/java/com/yahoo/collections/BobHash.java new file mode 100644 index 00000000000..b942c4e78f0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/BobHash.java @@ -0,0 +1,200 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import com.yahoo.text.Utf8; + +/** + * <p>A Java port of Michael Susag's BobHash in FastLib. This version is + * specifically done to be bit compatible with the one in FastLib, as it + * is used in decoding packets from FastServer.</p> + * + * <p>Hash function based on + * <a href="http://burtleburtle.net/bob/hash/index.html"> + * http://burtleburtle.net/bob/hash/index.html</a> + * by Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. You may use this + * code any way you wish, private, educational, or commercial. It's free.</p> + * + * @author Michael Susag + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * + * + */ + +public class BobHash { + + /** + * mix -- mix 3 32-bit values reversibly. + * For every delta with one or two bits set, and the deltas of all three + * high bits or all three low bits, whether the original value of a,b,c + * is almost all zero or is uniformly distributed, + * If mix() is run forward or backward, at least 32 bits in a,b,c + * have at least 1/4 probability of changing. + * If mix() is run forward, every bit of c will change between 1/3 and + * 2/3 of the time. (Well, 22/100 and 78/100 for some 2-bit deltas.) + * mix() was built out of 36 single-cycle latency instructions in a + * structure that could supported 2x parallelism, like so: + * + * <pre> + * a -= b; + * a -= c; x = (c>>13); + * b -= c; a ^= x; + * b -= a; x = (a<<8); + * c -= a; b ^= x; + * c -= b; x = (b>>13); + * ... + * </pre> + * + * <p> + * Unfortunately, superscalar Pentiums and Sparcs can't take advantage + * of that parallelism. They've also turned some of those single-cycle + * latency instructions into multi-cycle latency instructions. Still, + * this is the fastest good hash I could find. There were about 2^^68 + * to choose from. I only looked at a billion or so. + */ + private static int[] mix(int a, int b, int c) { + a -= b; a -= c; a ^= (c >>> 13); + b -= c; b -= a; b ^= (a << 8); + c -= a; c -= b; c ^= (b >>> 13); + a -= b; a -= c; a ^= (c >>> 12); + b -= c; b -= a; b ^= (a << 16); + c -= a; c -= b; c ^= (b >>> 5); + a -= b; a -= c; a ^= (c >>> 3); + b -= c; b -= a; b ^= (a << 10); + c -= a; c -= b; c ^= (b >>> 15); + + return new int[]{ a, b, c }; + } + + /** + * Transform a byte to an int viewed as an unsigned byte. + */ + private static int unsign(byte x) { + int y; + + y = 0xFF & x; + return y; + } + + /** + * Hashes a string, by calling hash(byte[] key,int initval) with + * the utf-8 bytes of the string as key and 0 as initval. + * Note: This is copying the string content, change implementation to + * use efficiently on large strings. + * + * <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ + public static int hash(String key) { + return hash(Utf8.toBytes(key), 0); + } + + /** + * The hash function + * + * <p> + * hash() -- hash a variable-length key into a 32-bit value<br> + * k : the key (the unaligned variable-length array of bytes)<br> + * len : the length of the key, counting by bytes<br> + * initval : can be any 4-byte value + * + * <p> + * Returns a 32-bit value. Every bit of the key affects every bit of + * the return value. Every 1-bit and 2-bit delta achieves avalanche. + * About 6*len+35 instructions. + * + * <p> + * The best hash table sizes are powers of 2. There is no need to do + * mod a prime (mod is sooo slow!). If you need less than 32 bits, + * use a bitmask. For example, if you need only 10 bits, do + * h = (h & hashmask(10)); + * In which case, the hash table should have hashsize(10) elements. + * + * If you are hashing n strings (ub1 **)k, do it like this: + * for (i=0, h=0; i<n; ++i) h = hash( k[i], len[i], h); + * + * <p> + * By Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. You may use this + * code any way you wish, private, educational, or commercial. It's free. + * + * <p> + * See http://burtleburtle.net/bob/hash/evahash.html + * Use for hash table lookup, or anything where one collision in 2^^32 is + * acceptable. Do NOT use for cryptographic purposes. + * + * @param k the key + * @param initval the previous hash, or an arbitrary value + * @return A 32 bit hash value + */ + @SuppressWarnings("fallthrough") + public static int hash(byte[] k, int initval) { + int a, b, c, len; + int offset = 0; + int[] abcBuffer; + + /* Set up the internal state */ + len = k.length; + a = b = 0x9e3779b9; /* the golden ratio; an arbitrary value */ + c = initval; /* the previous hash value */ + + // handle most of the key + while (len >= 12) { + a += (unsign(k[offset + 0]) + (unsign(k[offset + 1]) << 8) + + (unsign(k[offset + 2]) << 16) + + (unsign(k[offset + 3]) << 24)); + b += (unsign(k[offset + 4]) + (unsign(k[offset + 5]) << 8) + + (unsign(k[offset + 6]) << 16) + + (unsign(k[offset + 7]) << 24)); + c += (unsign(k[offset + 8]) + (unsign(k[offset + 9]) << 8) + + (unsign(k[offset + 10]) << 16) + + (unsign(k[offset + 11]) << 24)); + abcBuffer = mix(a, b, c); + a = abcBuffer[0]; + b = abcBuffer[1]; + c = abcBuffer[2]; + offset += 12; + len -= 12; + } + + // handle the last 11 bytes + c += k.length; + switch (len) { + // all the case statements fall through + case 11: + c += (unsign(k[offset + 10]) << 24); + + case 10: + c += (unsign(k[offset + 9]) << 16); + + case 9: + c += (unsign(k[offset + 8]) << 8); + + /* the first byte of c is reserved for the length */ + case 8: + b += (unsign(k[offset + 7]) << 24); + + case 7: + b += (unsign(k[offset + 6]) << 16); + + case 6: + b += (unsign(k[offset + 5]) << 8); + + case 5: + b += unsign(k[offset + 4]); + + case 4: + a += (unsign(k[offset + 3]) << 24); + + case 3: + a += (unsign(k[offset + 2]) << 16); + + case 2: + a += (unsign(k[offset + 1]) << 8); + + case 1: + a += unsign(k[offset + 0]); + + /* case 0: nothing left to add */ + } + abcBuffer = mix(a, b, c); + return abcBuffer[2]; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/ByteArrayComparator.java b/vespajlib/src/main/java/com/yahoo/collections/ByteArrayComparator.java new file mode 100644 index 00000000000..c0f630a92e5 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/ByteArrayComparator.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +/** + * Utility class which is useful when implementing <code>Comparable</code> and one needs to + * compare byte arrays as instance variables. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class ByteArrayComparator { + /** + * Compare the arguments. Shorter arrays are always considered + * smaller than longer arrays. For arrays of equal lengths, the elements + * are compared one-by-one. Whenever two corresponding elements in the + * two arrays are non-equal, the method returns. If all elements at + * corresponding positions in the two arrays are equal, the arrays + * are considered equal. + * + * @param first a byte array to be compared + * @param second a byte array to be compared + * @return 0 if the arguments are equal, -1 if the first argument is smaller, 1 if the second argument is smaller + * @throws NullPointerException if any of the arguments are null + */ + public static int compare(byte[] first, byte[] second) { + if (first.length < second.length) { + return -1; + } + if (first.length > second.length) { + return 1; + } + + //lengths are equal, compare contents + for (int i = 0; i < first.length; i++) { + if (first[i] < second[i]) { + return -1; + } else if (first[i] > second[i]) { + return 1; + } + //values at index i are equal, continue... + } + + //we haven't returned yet; contents must be equal: + return 0; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/CollectionComparator.java b/vespajlib/src/main/java/com/yahoo/collections/CollectionComparator.java new file mode 100644 index 00000000000..8999f0cac9c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/CollectionComparator.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.Collection; +import java.util.Iterator; + +/** + * Utility class which is useful when implementing <code>Comparable</code> and one needs to + * compare Collections of Comparables as instance variables. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class CollectionComparator { + /** + * Compare the arguments. Shorter Collections are always considered + * smaller than longer Collections. For Collections of equal lengths, the elements + * are compared one-by-one. Whenever two corresponding elements in the + * two Collections are non-equal, the method returns. If all elements at + * corresponding positions in the two Collections are equal, the Collections + * are considered equal. + * + * @param first a Collection of Comparables to be compared + * @param second a Collection of Comparables to be compared + * @return 0 if the arguments are equal, -1 if the first argument is smaller, 1 if the second argument is smaller + * @throws NullPointerException if any of the arguments are null + */ + @SuppressWarnings({"rawtypes", "unchecked"}) + public static int compare(Collection<? extends Comparable> first, Collection<? extends Comparable> second) { + if (first.size() < second.size()) { + return -1; + } + if (first.size() > second.size()) { + return 1; + } + + //sizes are equal, compare contents + Iterator<? extends Comparable> firstIt = first.iterator(); + Iterator<? extends Comparable> secondIt = second.iterator(); + + while (firstIt.hasNext()) { + // FIXME: unchecked casting + Comparable itemFirst = firstIt.next(); + Comparable itemSecond = secondIt.next(); + int comp = itemFirst.compareTo(itemSecond); + if (comp != 0) { + return comp; + } + //values are equal, continue... + } + + //we haven't returned yet; contents must be equal: + return 0; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/CollectionUtil.java b/vespajlib/src/main/java/com/yahoo/collections/CollectionUtil.java new file mode 100644 index 00000000000..ddcc6e97dff --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/CollectionUtil.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + * Utilities for java collections + * + * @author tonytv + * @author gjoranv + * @since 5.1.8 + */ +public class CollectionUtil { + + /** + * Returns a String containing the string representation of all elements from + * the given collection, separated by the separator string. + * + * @param collection The collection + * @param sep The separator string + * @return A string: elem(0) + sep + ... + elem(N) + */ + public static String mkString(Collection<?> collection, String sep) { + return mkString(collection, "", sep, ""); + } + + /** + * Returns a String containing the string representation of all elements from + * the given collection, using a start string, separator strings, and an end string. + * + * @param collection The collection + * @param start The start string + * @param sep The separator string + * @param end The end string + * @param <T> The element type + * @return A string: start + elem(0) + sep + ... + elem(N) + end + */ + public static <T> String mkString(Collection<T> collection, String start, String sep, String end) { + return collection.stream() + .map(T::toString) + .collect(Collectors.joining(sep, start, end)); + } + + /** + * Returns true if the contents of the two given collections are equal, ignoring order. + */ + public static boolean equalContentsIgnoreOrder(Collection<?> c1, Collection<?> c2) { + return c1.size() == c2.size() && + c1.containsAll(c2); + } + + /** + * Returns the symmetric difference between two collections, i.e. the set of elements + * that occur in exactly one of the collections. + */ + public static <T> Set<T> symmetricDifference(Collection<? extends T> c1, Collection<? extends T> c2) { + Set<T> diff1 = new HashSet<>(c1); + diff1.removeAll(c2); + + Set<T> diff2 = new HashSet<>(c2); + diff2.removeAll(c1); + + diff1.addAll(diff2); + return diff1; + } + + /** + * Returns the subset of elements from the given collection that can be cast to the reference + * type, defined by the given Class object. + */ + public static <T> Collection<T> filter(Collection<?> collection, Class<T> lowerBound) { + List<T> result = new ArrayList<>(); + for (Object element : collection) { + if (lowerBound.isInstance(element)) { + result.add(lowerBound.cast(element)); + } + } + return result; + } + + /** + * Returns the first element in a collection according to iteration order. + * Returns null if the collection is empty. + */ + public static <T> T first(Collection<T> collection) { + return collection.isEmpty()? null: collection.iterator().next(); + } + + public static <T> Optional<T> firstMatching(T[] array, Predicate<? super T> predicate) { + for (T t: array) { + if (predicate.test(t)) + return Optional.of(t); + } + return Optional.empty(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/ConcurrentResourcePool.java b/vespajlib/src/main/java/com/yahoo/collections/ConcurrentResourcePool.java new file mode 100644 index 00000000000..98cc443fd71 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/ConcurrentResourcePool.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 13.11.12 + * Time: 20:57 + * To change this template use File | Settings | File Templates. + */ +public class ConcurrentResourcePool<T> implements Iterable<T> { + + private final Queue<T> pool = new ConcurrentLinkedQueue<>(); + private final ResourceFactory<T> factory; + + public ConcurrentResourcePool(ResourceFactory<T> factory) { + this.factory = factory; + } + + public final T alloc() { + final T e = pool.poll(); + return e != null ? e : factory.create(); + } + + public final void free(T e) { + pool.offer(e); + } + + @Override + public Iterator<T> iterator() { + return pool.iterator(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/CopyOnWriteHashMap.java b/vespajlib/src/main/java/com/yahoo/collections/CopyOnWriteHashMap.java new file mode 100644 index 00000000000..43f38c67e4d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/CopyOnWriteHashMap.java @@ -0,0 +1,154 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import com.google.common.annotations.Beta; + +import java.util.AbstractMap; +import java.util.AbstractSet; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +/** + * A hashmap wrapper which defers cloning of the enclosed map until it is written. + * Use this to make clones cheap in maps which are often not further modified. + * <p> + * As with regular maps, this can only be used safely if the content of the map is immutable. + * If not, the {@link #copyMap} method can be overridden to perform a deep clone. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +@Beta +public class CopyOnWriteHashMap<K,V> extends AbstractMap<K,V> implements Cloneable { + + private Map<K,V> map; + + /** True when this class is allowed to write to the map */ + private boolean writable = true; + + /** Lazily initialized view */ + private transient Set<Map.Entry<K,V>> entrySet = null; + + public CopyOnWriteHashMap() { + this.map = new HashMap<>(); + } + + public CopyOnWriteHashMap(int capacity) { + this.map = new HashMap<>(capacity); + } + + public CopyOnWriteHashMap(Map<K,V> map) { + this.map = new HashMap<>(map); + } + + private void makeReadOnly() { + writable = false; + } + + private void makeWritable() { + if (writable) return; + map = copyMap(map); + writable = true; + entrySet = null; + } + + /** + * Make a copy of the given map with the requisite deepness. + * This default implementation does return new HashMap<>(original); + */ + protected Map<K,V> copyMap(Map<K,V> original) { + return new HashMap<>(original); + } + + @SuppressWarnings("unchecked") + public CopyOnWriteHashMap<K,V> clone() { + try { + CopyOnWriteHashMap<K,V> clone = (CopyOnWriteHashMap<K,V>)super.clone(); + this.makeReadOnly(); + clone.makeReadOnly(); + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + + @Override + public Set<Entry<K, V>> entrySet() { + if (entrySet == null) + entrySet = new EntrySet(); + return entrySet; + } + + @Override + public V put(K key, V value) { + makeWritable(); + return map.put(key, value); + } + + /** Override to avoid using iterator.remove */ + @Override + public V remove(Object key) { + makeWritable(); + return map.remove(key); + } + + private final class EntrySet extends AbstractSet<Map.Entry<K,V>> { + + public Iterator<Map.Entry<K,V>> iterator() { + return new EntryIterator(); + } + + @SuppressWarnings("unchecked") + public boolean contains(Object o) { + if ( ! (o instanceof Map.Entry)) return false; + Map.Entry<K,V> entry = (Map.Entry<K,V>) o; + Object candidate = map.get(entry.getKey()); + if (candidate == null) return entry.getValue()==null; + return candidate.equals(entry.getValue()); + } + + public boolean remove(Object o) { + makeWritable(); + return map.remove(o) !=null; + } + + public int size() { + return map.size(); + } + + public void clear() { map.clear(); } + + } + + /** + * An entry iterator which does not allow removals if the map wasn't already modifiable + * There is no sane way to implement that given that the wrapped map changes mid iteration. + */ + private class EntryIterator implements Iterator<Map.Entry<K,V>> { + + /** Wrapped iterator */ + private Iterator<Map.Entry<K,V>> mapIterator; + + public EntryIterator() { + mapIterator = map.entrySet().iterator(); + } + + public final boolean hasNext() { + return mapIterator.hasNext(); + } + + public Entry<K,V> next() { + return mapIterator.next(); + } + + public void remove() { + if ( ! writable) + throw new UnsupportedOperationException("Cannot perform the copy-on-write operation during iteration"); + mapIterator.remove(); + } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/FreezableArrayList.java b/vespajlib/src/main/java/com/yahoo/collections/FreezableArrayList.java new file mode 100644 index 00000000000..e145a08be09 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/FreezableArrayList.java @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.Collection; +import java.util.List; + +/** + * An array list which can be frozen to disallow further edits. + * After freezing, edit operations will throw UnsupportedOperationException. + * Freezable lists may optionally allow new items to be added to the end of the list also after freeze. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.20 + */ +public class FreezableArrayList<ITEM> extends ListenableArrayList<ITEM> { + + private static final long serialVersionUID = 5900452593651895638L; + + private final boolean permitAddAfterFreeze; + private boolean frozen = false; + + /** Creates a freezable array list which does not permit adds after freeze */ + public FreezableArrayList() { + this(false); + } + + /** Creates a freezable array list which does not permit adds after freeze */ + public FreezableArrayList(int initialCapacity) { + this(false, initialCapacity); + } + + public FreezableArrayList(boolean permitAddAfterFreeze) { + this.permitAddAfterFreeze = permitAddAfterFreeze; + } + + public FreezableArrayList(boolean permitAddAfterFreeze, int initialCapacity) { + super(initialCapacity); + this.permitAddAfterFreeze = permitAddAfterFreeze; + } + + /** Irreversibly freezes the content of this */ + public void freeze() { + this.frozen = true; + } + + @Override + public boolean add(ITEM e) { + if ( ! permitAddAfterFreeze) throwIfFrozen(); + return super.add(e); + } + + @Override + public void add(int index, ITEM e) { + throwIfFrozen(); + super.add(index, e); + } + + @Override + public boolean addAll(Collection<? extends ITEM> a) { + if ( ! permitAddAfterFreeze) throwIfFrozen(); + return super.addAll(a); + } + + @Override + public boolean addAll(int index, Collection<? extends ITEM> a) { + throwIfFrozen(); + return super.addAll(index, a); + } + + @Override + public ITEM set(int index, ITEM e) { + throwIfFrozen(); + return super.set(index, e); + } + + @Override + public ITEM remove(int index) { + throwIfFrozen(); + return super.remove(index); + } + + @Override + public boolean remove(Object o) { + throwIfFrozen(); + return super.remove(o); + } + + @Override + public void clear() { + throwIfFrozen(); + super.clear(); + } + + @Override + protected void removeRange(int fromIndex, int toIndex) { + throwIfFrozen(); + super.removeRange(fromIndex, toIndex); + } + + @Override + public boolean removeAll(Collection<?> c) { + throwIfFrozen(); + return super.removeAll(c); + } + + @Override + public boolean retainAll(Collection<?> c) { + throwIfFrozen(); + return super.retainAll(c); + } + + private void throwIfFrozen() { + if ( frozen ) + throw new UnsupportedOperationException(this + " is frozen"); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/Hashlet.java b/vespajlib/src/main/java/com/yahoo/collections/Hashlet.java new file mode 100644 index 00000000000..86e82bb3241 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/Hashlet.java @@ -0,0 +1,226 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + + +/** + * Lightweight hash map from key to value with limited + * functionality. This class lets you build a map from key to + * value. The value for a key may be overwritten and the put and get + * methods have the same semantics as for normal Java Maps, but there + * is no remove operation. Also, there is no iterator support, but + * keys and values can be accessed directly by index. The access order + * of keys and values are defined by the insert order of the keys. The + * goal of this class is to reduce the amount of object that are + * allocated by packing everything into two internal arrays. The keys + * and values are packed in an Object array and the hash table and + * entries are packed in an int array. The internal arrays are not + * created until space is needed. The default initial capacity is 16 + * entries. If you know you need much more space than this, you can + * explicitly reserve more space before starting to insert values. The + * maximum load factor is 0.7 and drops slightly with increasing + * capacity. + * + * @author <a href="mailto:havardpe@yahoo-inc.com">Havard Pettersen</a> + **/ +public final class Hashlet<K, V> { + + private static final int[] emptyHash = new int[1]; + private int capacity = 0; + private int hashSize() { return (capacity + (capacity / 2) - 1); } + private int used = 0; + private Object[] store; + private int[] hash = emptyHash; + + /** + * Create an empty Hashlet. + **/ + public Hashlet() {} + + /** + * Create a Hashlet that is a shallow copy of another Hashlet. + * + * @param hashlet the Hashlet to copy. + **/ + public Hashlet(Hashlet<K, V> hashlet) { + if (hashlet.used > 0) { + capacity = hashlet.capacity; + used = hashlet.used; + store = new Object[hashlet.store.length]; + hash = new int[hashlet.hash.length]; + System.arraycopy(hashlet.store, 0, store, 0, store.length); + System.arraycopy(hashlet.hash, 0, hash, 0, hash.length); + } + } + + /** + * Reserve space for more key value pairs. This method is used by + * the put method to perform rehashing when needed. It can be + * invoked directly by the application to reduce the number of + * rehashes needed to insert a large number of entries. + * + * @param n the number of additional entries to reserve space for + **/ + public void reserve(int n) { + if (used + n > capacity) { + final int c = capacity; + if (capacity == 0) { + capacity = 16; + } + while (used + n > capacity) { + capacity *= 2; + } + final Object[] s = store; + store = new Object[capacity * 2]; + hash = new int[hashSize() + (capacity * 2)]; + if (c > 0) { + System.arraycopy(s, 0, store, 0, used); + System.arraycopy(s, c, store, capacity, used); + for (int i = 0; i < used; i++) { + int prev = Math.abs(s[i].hashCode() % hashSize()); + int entry = hash[prev]; + while (entry != 0) { + prev = entry + 1; + entry = hash[prev]; + } + final int insertIdx = (hashSize() + (i * 2)); + hash[prev] = insertIdx; + hash[insertIdx] = i; + } + } + } + } + + /** + * The current size. This is the number of key value pairs + * currently stored in this object. + * + * @return current size + **/ + public int size() { + return used; + } + + /** + * Obtain a key. Keys are accessed in the order they were first + * inserted. + * + * @return the requested key + * @param i the index of the key, must be in the range [0, size() - 1] + **/ + @SuppressWarnings("unchecked") + public K key(int i) { + return (K) store[i]; + } + + /** + * Obtain a value. Values are accessed in the order in which + * theirs keys were first inserted. + * + * @return the requested value + * @param i the index of the value, must be in the range [0, size() - 1] + **/ + @SuppressWarnings("unchecked") + public V value(int i) { + return (V) store[capacity + i]; + } + + /** + * This will replace the value at the index give. + * + * @param i the index of the value, must be in the range [0, size() - 1] + * @param value The new value you want to set for this index. + * @return previous value + */ + public V setValue(int i, V value) { + V prev = value(i); + store[capacity + i] = value; + return prev; + } + + /** + * Associate a value with a specific key. + * + * @return the old value for the key, if it was already present + * @param key the key + * @param value the value + **/ + public V put(K key, V value) { + reserve(1); + int prev = Math.abs(key.hashCode() % hashSize()); + int entry = hash[prev]; + while (entry != 0) { + final int idx = hash[entry]; + if (store[idx].equals(key)) { // found entry + @SuppressWarnings("unchecked") + final V ret = (V) store[capacity + idx]; + store[capacity + idx] = value; + return ret; + } + prev = entry + 1; + entry = hash[prev]; + } + final int insertIdx = (hashSize() + (used * 2)); + hash[prev] = insertIdx; + hash[insertIdx] = used; + store[used] = key; + store[capacity + (used++)] = value; + return null; + } + + /** + * Obtain the value for a specific key. + * + * @return the value for a key, or null if not found + * @param key the key + **/ + public V get(Object key) { + int index = getIndexOfKey(key); + return (index != -1) ? value(index) : null; + } + + /** + * Finds the index where the key,value pair is stored. + * @param key to look for + * @return the index where the key is found or -1 if it is not found + */ + public int getIndexOfKey(Object key) { + int entry = hash[Math.abs(key.hashCode() % hashSize())]; + while (entry != 0) { + final int idx = hash[entry]; + if (store[idx].equals(key)) { // found entry + return idx; + } + entry = hash[entry + 1]; + } + return -1; + } + + @Override + public int hashCode() { + int h = 0; + for (int i = 0; i < used; i++) { + h += key(i).hashCode(); + V v = value(i); + if (v != null) { + h += v.hashCode(); + } + } + return h; + } + + @Override + public boolean equals(Object o) { + if (! (o instanceof Hashlet) ) return false; + Hashlet<?, ?> rhs = (Hashlet<?, ?>) o; + if (used != rhs.used) return false; + for (int i = 0; i < used; i++) { + int bi = rhs.getIndexOfKey(key(i)); + if (bi == -1) return false; + Object a = value(i); + Object b = rhs.value(bi); + boolean equal = (a == null) ? b == null : a.equals(b); + if ( !equal ) return false; + } + return true; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/IntArrayComparator.java b/vespajlib/src/main/java/com/yahoo/collections/IntArrayComparator.java new file mode 100644 index 00000000000..21c6f514cbf --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/IntArrayComparator.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +/** + * Utility class which is useful when implementing <code>Comparable</code> and one needs to + * compare int arrays as instance variables. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class IntArrayComparator { + /** + * Compare the arguments. Shorter arrays are always considered + * smaller than longer arrays. For arrays of equal lengths, the elements + * are compared one-by-one. Whenever two corresponding elements in the + * two arrays are non-equal, the method returns. If all elements at + * corresponding positions in the two arrays are equal, the arrays + * are considered equal. + * + * @param first an int array to be compared + * @param second an int array to be compared + * @return 0 if the arguments are equal, -1 if the first argument is smaller, 1 if the second argument is smaller + * @throws NullPointerException if any of the arguments are null + */ + public static int compare(int[] first, int[] second) { + if (first.length < second.length) { + return -1; + } + if (first.length > second.length) { + return 1; + } + + //lengths are equal, compare contents + for (int i = 0; i < first.length; i++) { + if (first[i] < second[i]) { + return -1; + } else if (first[i] > second[i]) { + return 1; + } + //values at index i are equal, continue... + } + + //we haven't returned yet; contents must be equal: + return 0; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/LazyMap.java b/vespajlib/src/main/java/com/yahoo/collections/LazyMap.java new file mode 100644 index 00000000000..1e1a75402eb --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/LazyMap.java @@ -0,0 +1,271 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.AbstractMap; +import java.util.AbstractSet; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Set; + +/** + * @author <a href="mailto:simon@hult-thoresen.com">Simon Thoresen Hult</a> + */ +public abstract class LazyMap<K, V> implements Map<K, V> { + + private Map<K, V> delegate = newEmpty(); + + @Override + public final int size() { + return delegate.size(); + } + + @Override + public final boolean isEmpty() { + return delegate.isEmpty(); + } + + @Override + public final boolean containsKey(Object key) { + return delegate.containsKey(key); + } + + @Override + public final boolean containsValue(Object value) { + return delegate.containsValue(value); + } + + @Override + public final V get(Object key) { + return delegate.get(key); + } + + @Override + public final V put(K key, V value) { + return delegate.put(key, value); + } + + @Override + public final V remove(Object key) { + return delegate.remove(key); + } + + @Override + public final void putAll(Map<? extends K, ? extends V> m) { + delegate.putAll(m); + } + + @Override + public final void clear() { + delegate.clear(); + } + + @Override + public final Set<K> keySet() { + return delegate.keySet(); + } + + @Override + public final Collection<V> values() { + return delegate.values(); + } + + @Override + public final Set<Entry<K, V>> entrySet() { + return delegate.entrySet(); + } + + @Override + public final int hashCode() { + return delegate.hashCode(); + } + + @Override + public final boolean equals(Object obj) { + return obj == this || (obj instanceof Map && delegate.equals(obj)); + } + + private Map<K, V> newEmpty() { + return new EmptyMap(); + } + + private Map<K, V> newSingleton(K key, V value) { + return new SingletonMap(key, value); + } + + protected abstract Map<K, V> newDelegate(); + + final Map<K, V> getDelegate() { + return delegate; + } + + class EmptyMap extends AbstractMap<K, V> { + + @Override + public V put(K key, V value) { + delegate = newSingleton(key, value); + return null; + } + + @Override + public void putAll(Map<? extends K, ? extends V> m) { + switch (m.size()) { + case 0: + break; + case 1: + Entry<? extends K, ? extends V> entry = m.entrySet().iterator().next(); + put(entry.getKey(), entry.getValue()); + break; + default: + delegate = newDelegate(); + delegate.putAll(m); + break; + } + } + + @Override + public Set<Entry<K, V>> entrySet() { + return Collections.emptySet(); + } + } + + class SingletonMap extends AbstractMap<K, V> { + + final K key; + V value; + + SingletonMap(K key, V value) { + this.key = key; + this.value = value; + } + + @Override + public V put(K key, V value) { + if (containsKey(key)) { + V oldValue = this.value; + this.value = value; + return oldValue; + } else { + delegate = newDelegate(); + delegate.put(this.key, this.value); + return delegate.put(key, value); + } + } + + @Override + public void putAll(Map<? extends K, ? extends V> m) { + switch (m.size()) { + case 0: + break; + case 1: + Entry<? extends K, ? extends V> entry = m.entrySet().iterator().next(); + put(entry.getKey(), entry.getValue()); + break; + default: + delegate = newDelegate(); + delegate.put(this.key, this.value); + delegate.putAll(m); + break; + } + } + + @Override + public Set<Entry<K, V>> entrySet() { + return new AbstractSet<Entry<K, V>>() { + + @Override + public Iterator<Entry<K, V>> iterator() { + return new Iterator<Entry<K, V>>() { + + boolean hasNext = true; + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public Entry<K, V> next() { + if (hasNext) { + hasNext = false; + return new Entry<K, V>() { + + @Override + public K getKey() { + return key; + } + + @Override + public V getValue() { + return value; + } + + @Override + public V setValue(V value) { + V oldValue = SingletonMap.this.value; + SingletonMap.this.value = value; + return oldValue; + } + + @Override + public int hashCode() { + return Objects.hashCode(key) + Objects.hashCode(value) * 31; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof Entry)) { + return false; + } + @SuppressWarnings("unchecked") + Entry<K, V> rhs = (Entry<K, V>)obj; + if (!Objects.equals(key, rhs.getKey())) { + return false; + } + if (!Objects.equals(value, rhs.getValue())) { + return false; + } + return true; + } + }; + } else { + throw new NoSuchElementException(); + } + } + + @Override + public void remove() { + if (hasNext) { + throw new IllegalStateException(); + } else { + delegate = newEmpty(); + } + } + }; + } + + @Override + public int size() { + return 1; + } + }; + } + } + + public static <K, V> LazyMap<K, V> newHashMap() { + return new LazyMap<K, V>() { + + @Override + protected Map<K, V> newDelegate() { + return new HashMap<>(); + } + }; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/LazySet.java b/vespajlib/src/main/java/com/yahoo/collections/LazySet.java new file mode 100644 index 00000000000..356b194c51f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/LazySet.java @@ -0,0 +1,225 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.AbstractSet; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Set; + +/** + * @author <a href="mailto:simon@hult-thoresen.com">Simon Thoresen Hult</a> + */ +public abstract class LazySet<E> implements Set<E> { + + private Set<E> delegate = newEmpty(); + + @Override + public final int size() { + return delegate.size(); + } + + @Override + public final boolean isEmpty() { + return delegate.isEmpty(); + } + + @Override + public final boolean contains(Object o) { + return delegate.contains(o); + } + + @Override + public final Iterator<E> iterator() { + return delegate.iterator(); + } + + @Override + public final Object[] toArray() { + return delegate.toArray(); + } + + @Override + public final <T> T[] toArray(T[] a) { + // noinspection SuspiciousToArrayCall + return delegate.toArray(a); + } + + @Override + public final boolean add(E e) { + return delegate.add(e); + } + + @Override + public final boolean remove(Object o) { + return delegate.remove(o); + } + + @Override + public final boolean containsAll(Collection<?> c) { + return delegate.containsAll(c); + } + + @Override + public final boolean addAll(Collection<? extends E> c) { + return delegate.addAll(c); + } + + @Override + public final boolean retainAll(Collection<?> c) { + return delegate.retainAll(c); + } + + @Override + public final boolean removeAll(Collection<?> c) { + return delegate.removeAll(c); + } + + @Override + public final void clear() { + delegate.clear(); + } + + @Override + public final int hashCode() { + return delegate.hashCode(); + } + + @Override + public final boolean equals(Object obj) { + return obj == this || (obj instanceof Set && delegate.equals(obj)); + } + + private Set<E> newEmpty() { + return new EmptySet(); + } + + private Set<E> newSingleton(E e) { + return new SingletonSet(e); + } + + protected abstract Set<E> newDelegate(); + + final Set<E> getDelegate() { + return delegate; + } + + class EmptySet extends AbstractSet<E> { + + @Override + public Iterator<E> iterator() { + return Collections.emptyIterator(); + } + + @Override + public int size() { + return 0; + } + + @Override + public boolean add(E e) { + delegate = newSingleton(e); + return true; + } + + @Override + public boolean addAll(Collection<? extends E> c) { + switch (c.size()) { + case 0: + return false; + case 1: + add(c.iterator().next()); + return true; + default: + delegate = newDelegate(); + delegate.addAll(c); + return true; + } + } + } + + class SingletonSet extends AbstractSet<E> { + + final E element; + + SingletonSet(E e) { + this.element = e; + } + + @Override + public Iterator<E> iterator() { + return new Iterator<E>() { + + boolean hasNext = true; + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public E next() { + if (hasNext) { + hasNext = false; + return element; + } else { + throw new NoSuchElementException(); + } + } + + @Override + public void remove() { + if (hasNext) { + throw new IllegalStateException(); + } else { + delegate = newEmpty(); + } + } + }; + } + + @Override + public int size() { + return 1; + } + + @Override + public boolean add(E e) { + if (contains(e)) { + return false; + } else { + delegate = newDelegate(); + delegate.add(element); + delegate.add(e); + return true; + } + } + + @Override + public boolean addAll(Collection<? extends E> c) { + switch (c.size()) { + case 0: + return false; + case 1: + return add(c.iterator().next()); + default: + delegate = newDelegate(); + delegate.add(element); + delegate.addAll(c); + return true; + } + } + } + + public static <E> LazySet<E> newHashSet() { + return new LazySet<E>() { + + @Override + protected Set<E> newDelegate() { + return new HashSet<>(); + } + }; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/ListMap.java b/vespajlib/src/main/java/com/yahoo/collections/ListMap.java new file mode 100644 index 00000000000..ab2c97fda17 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/ListMap.java @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.commons.lang.builder.ToStringBuilder; + +import java.util.*; + +/** + * A map holding multiple items at each key (using ArrayList and HashMap). + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ListMap<K, V> { + + private boolean frozen = false; + + private Map<K, List<V>> map; + + public ListMap() { + this(HashMap.class); + } + + @SuppressWarnings("unchecked") + public ListMap(@SuppressWarnings("rawtypes") Class<? extends Map> implementation) { + try { + this.map = implementation.newInstance(); + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } + + /** Puts an element into this. Multiple elements at the same position are added to the list at this key */ + public void put(K key, V value) { + List<V> list = map.get(key); + if (list == null) { + list = new ArrayList<>(); + map.put(key, list); + } + list.add(value); + } + + public void removeAll(K key) { + map.remove(key); + } + + public boolean removeValue(K key, V value) { + List<V> list = map.get(key); + if (list != null) + return list.remove(value); + else + return false; + } + + /** + * Removes the value at the given index. + * + * @return the removed value + * @throws IndexOutOfBoundsException if there is no value at the given index for this key + */ + public V removeValue(K key, int index) { + List<V> list = map.get(key); + if (list != null) + return list.remove(index); + else + throw new IndexOutOfBoundsException("The list at '" + key + "' is empty"); + } + + /** + * Returns the List containing the elements with this key, or an empty list + * if there are no elements. The list returned is unmodifiable. + */ + public List<V> get(K key) { + List<V> list = map.get(key); + if (list == null) + return ImmutableList.of();; + return ImmutableList.copyOf(list); + } + + /** The same as get */ + public List<V> getList(K key) { + return get(key); + } + + /** Returns the entries of this. Entries will be unmodifiable if this is frozen. */ + public Set<Map.Entry<K,List<V>>> entrySet() { return map.entrySet(); } + + /** Returns the keys of this */ + public Set<K> keySet() { return map.keySet(); } + + /** Returns the list values of this */ + public Collection<List<V>> values() { return map.values(); } + + /** + * Irreversibly prevent changes to the content of this. + * If this is already frozen, this method does nothing. + */ + public void freeze() { + if (frozen) return; + + for (Map.Entry<K,List<V>> entry : map.entrySet()) + entry.setValue(ImmutableList.copyOf(entry.getValue())); + this.map = ImmutableMap.copyOf(this.map); + } + + /** Returns whether this allows changes */ + public boolean isFrozen() { return frozen; } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this); + } + + /** Returns the number of keys in this map */ + public int size() { return map.size(); } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/ListenableArrayList.java b/vespajlib/src/main/java/com/yahoo/collections/ListenableArrayList.java new file mode 100644 index 00000000000..1b77e97d159 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/ListenableArrayList.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * An array list which notifies listeners after one or more items are added + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.1.17 + */ +@SuppressWarnings("serial") +public class ListenableArrayList<ITEM> extends ArrayList<ITEM> { + + private List<Runnable> listeners = null; + + public ListenableArrayList() {} + + public ListenableArrayList(int initialCapacity) { + super(initialCapacity); + } + + @Override + public boolean add(ITEM e) { + boolean result = super.add(e); + notifyListeners(); + return result; + } + + @Override + public void add(int index, ITEM e) { + super.add(index, e); + notifyListeners(); + } + + @Override + public boolean addAll(Collection<? extends ITEM> a) { + boolean result = super.addAll(a); + notifyListeners(); + return result; + } + + @Override + public boolean addAll(int index, Collection<? extends ITEM> a) { + boolean result = super.addAll(index, a); + notifyListeners(); + return result; + } + + @Override + public ITEM set(int index, ITEM e) { + ITEM result = super.set(index, e); + notifyListeners(); + return result; + } + + /** + * Adds a listener which is invoked whenever elements are added to this. + * This may not be invoked once for each added element. + */ + public void addListener(Runnable listener) { + if (listeners == null) + listeners = new ArrayList<>(); + listeners.add(listener); + } + + private void notifyListeners() { + if (listeners == null) return; + for (Runnable listener : listeners) + listener.run(); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/MD5.java b/vespajlib/src/main/java/com/yahoo/collections/MD5.java new file mode 100644 index 00000000000..b80a823eff0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/MD5.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import com.yahoo.text.Utf8; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * Convenience class for hashing a String with MD5, and either returning + * an int with the 4 LSBytes, or the whole 12-byte MD5 hash. + * <p> + * Note that instantiating this class can be expensive, so re-using instances + * is a good idea. + * <p> + * This class is not thread safe. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class MD5 { + public static final ThreadLocal<MessageDigest> md5 = new MD5Factory(); + + private static class MD5Factory extends ThreadLocal<MessageDigest> { + + @Override + protected MessageDigest initialValue() { + return createMD5(); + } + } + private static MessageDigest createMD5() { + try { + return MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + final private MessageDigest digester; + public MD5() { + try { + digester = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new AssertionError("MD5 algorithm not found."); + } + } + + public int hash(String s) { + byte[] md5 = digester.digest(Utf8.toBytes(s)); + int hash = 0; + assert (md5.length == 16); + + //produce an int by using only the 32 lsb: + int byte1 = (((int) md5[12]) << 24) & 0xFF000000; + int byte2 = (((int) md5[13]) << 16) & 0x00FF0000; + int byte3 = (((int) md5[14]) << 8) & 0x0000FF00; + int byte4 = (((int) md5[15])) & 0x000000FF; + + hash |= byte1; + hash |= byte2; + hash |= byte3; + hash |= byte4; + return hash; + } + + public byte[] hashFull(String s) { + return digester.digest(Utf8.toBytes(s)); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/MethodCache.java b/vespajlib/src/main/java/com/yahoo/collections/MethodCache.java new file mode 100644 index 00000000000..5dd9f68e5cc --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/MethodCache.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import com.yahoo.concurrent.CopyOnWriteHashMap; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 6/12/13 + * Time: 9:03 AM + * To change this template use File | Settings | File Templates. + */ +public final class MethodCache { + private final String methodName; + private final CopyOnWriteHashMap<String, Method> cache = new CopyOnWriteHashMap<>(); + + public MethodCache(String methodName) { + this.methodName = methodName; + } + + public final Method get(Object object) { + Method m = cache.get(object.getClass().getName()); + if (m == null) { + m = lookupMethod(object); + if (m != null) { + cache.put(object.getClass().getName(), m); + } + } + return m; + } + private Method lookupMethod(Object object) { + try { + return object.getClass().getMethod(methodName); + } catch (NoSuchMethodException e) { + return null; + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/Pair.java b/vespajlib/src/main/java/com/yahoo/collections/Pair.java new file mode 100644 index 00000000000..8969e1b1021 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/Pair.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +/** + * An immutable pair of objects. This implements equals and hashCode by delegating to the + * pair objects. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class Pair<F, S> { + + /** The first member for the pair. May be null. */ + private final F first; + /** The second member for the pair. May be null. */ + private final S second; + + /** Creates a pair. Each member may be set to null. */ + public Pair(final F first, final S second) { + this.first = first; + this.second = second; + } + + /** Returns the first member. This may be null. */ + public F getFirst() { return first; } + + /** Returns the second member. This may be null. */ + public S getSecond() { return second; } + + @Override + public int hashCode() { + return ( first != null ? first.hashCode() : 0 ) + + ( second != null ? 17*second.hashCode() : 0) ; + } + + @Override + public boolean equals(final Object o) { + if (o == this) return true; + if (!(o instanceof Pair)) return false; + + @SuppressWarnings("rawtypes") + final Pair other = (Pair) o; + return equals(this.first, other.first) + && equals(this.second, other.second); + } + + private static boolean equals(final Object a, final Object b) { + if (a == null) return b == null; + return a.equals(b); + } + + @Override + public String toString() { + return "(" + first + "," + second + ")"; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/PredicateSplit.java b/vespajlib/src/main/java/com/yahoo/collections/PredicateSplit.java new file mode 100644 index 00000000000..68dd39dc283 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/PredicateSplit.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.List; +import java.util.ArrayList; +import java.util.function.Predicate; + +/** + * Class holding the result of a partition-by-predicate operation. + **/ +public class PredicateSplit<E> { + public final List<E> falseValues; /// list of values where the predicate returned false + public final List<E> trueValues; /// list of values where the predicate returned true + + private PredicateSplit() { + falseValues = new ArrayList<E>(); + trueValues = new ArrayList<E>(); + } + + /** + * Perform a partition-by-predicate operation. + * Each value in the input is tested by the predicate and + * added to either the falseValues list or the trueValues list. + * @param collection The input collection. + * @param predicate A test for selecting the target list. + * @return Two lists bundled in an object. + **/ + public static <V> PredicateSplit<V> partition(Iterable<V> collection, Predicate<? super V> predicate) + { + PredicateSplit<V> r = new PredicateSplit<V>(); + for (V value : collection) { + if (predicate.test(value)) { + r.trueValues.add(value); + } else { + r.falseValues.add(value); + } + } + return r; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/ResourceFactory.java b/vespajlib/src/main/java/com/yahoo/collections/ResourceFactory.java new file mode 100644 index 00000000000..44d99f78cfe --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/ResourceFactory.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public abstract class ResourceFactory<T> { + + public abstract T create(); +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/ResourcePool.java b/vespajlib/src/main/java/com/yahoo/collections/ResourcePool.java new file mode 100644 index 00000000000..dcf73425f6d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/ResourcePool.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; + +/** + * <p>This implements a simple stack based resource pool. If you are out of resources new are allocated from the + * factory.</p> + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public final class ResourcePool<T> implements Iterable<T> { + + private final Deque<T> pool = new ArrayDeque<>(); + private final ResourceFactory<T> factory; + + public ResourcePool(ResourceFactory<T> factory) { + this.factory = factory; + } + + public final T alloc() { + return pool.isEmpty() ? factory.create() : pool.pop(); + } + + public final void free(T e) { + pool.push(e); + } + + @Override + public Iterator<T> iterator() { + return pool.iterator(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java b/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java new file mode 100644 index 00000000000..177a4e6720b --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java @@ -0,0 +1,260 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Set; + +/** + * A Set implementation which only considers object identity. It should only be + * used for small number of objects, as it is implemented as scanning an + * ArrayList for identity matches. In other words: Performance will only be + * acceptable for <i>small</i> sets. + * + * <p> + * The rationale for this class is the high cost of the object identifier used + * in IdentityHashMap, where the key set is often used as an identity set. + * </p> + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @since 5.1.4 + * @see java.util.IdentityHashMap + * + * @param <E> + * the type contained in the Set + */ +public final class TinyIdentitySet<E> implements Set<E> { + private class ArrayIterator<T> implements Iterator<E> { + private int i = -1; + private boolean removed = false; + + @Override + public boolean hasNext() { + return i + 1 < size; + } + + @SuppressWarnings("unchecked") + @Override + public E next() { + if (!hasNext()) { + throw new NoSuchElementException("No more elements available"); + } + removed = false; + return (E) entries[++i]; + } + + @Override + public void remove() { + if (removed) { + throw new IllegalStateException( + "Trying to remove same element twice."); + } + if (i == -1) { + throw new IllegalStateException( + "Trying to remove before entering iterator."); + } + delete(i--); + removed = true; + } + + } + + private Object[] entries; + private int size = 0; + + /** + * Create a set with an initial capacity of initSize. The internal array + * will grow automatically with a linear growth rate if more elements than + * initSize are added. + * + * @param initSize + * initial size of internal element array + */ + public TinyIdentitySet(final int initSize) { + entries = new Object[initSize]; + } + + /** + * Expose the index in the internal array of a given object. -1 is returned + * if the object is not present in the internal array. + * + * @param e + * an object to check whether exists in this set + * @return the index of the argument e in the internal array, or -1 if the + * object is not present + */ + public int indexOf(final Object e) { + for (int i = 0; i < size; ++i) { + if (e == entries[i]) { + return i; + } + } + return -1; + } + + private void clean() { + int offset = 0; + for (int i = 0; i < size; ++i) { + if (entries[i] == null) { + ++offset; + } else { + entries[i - offset] = entries[i]; + } + } + size -= offset; + } + + private void grow() { + // linear growth, as we should always be working on small sets + entries = Arrays.copyOf(entries, entries.length + 10); + } + + private void append(final Object arg) { + if (size == entries.length) { + grow(); + } + entries[size++] = arg; + } + + @Override + public boolean add(final E arg) { + final int i = indexOf(arg); + if (i >= 0) { + return false; + } + append(arg); + return true; + } + + @Override + public boolean addAll(final Collection<? extends E> arg) { + boolean changed = false; + for (final E entry : arg) { + changed |= add(entry); + } + return changed; + } + + @Override + public void clear() { + size = 0; + } + + @Override + public boolean contains(final Object arg) { + return indexOf(arg) >= 0; + } + + /** + * This is an extremely expensive implementation of + * {@link Set#containsAll(Collection)}. It is implemented as O(n**2). + */ + @Override + public boolean containsAll(final Collection<?> arg) { + for (final Object entry : arg) { + if (indexOf(entry) < 0) { + return false; + } + } + return true; + } + + @Override + public boolean isEmpty() { + return size == 0; + } + + @Override + public Iterator<E> iterator() { + return new ArrayIterator<E>(); + } + + private void delete(int i) { + if (i < 0 || i >= size) { + return; + } + --size; + while (i < size) { + entries[i] = entries[i + 1]; + ++i; + } + entries[i] = null; + } + + @Override + public boolean remove(final Object arg) { + final int i = indexOf(arg); + if (i < 0) { + return false; + } + delete(i); + return true; + } + + /** + * This is an extremely expensive implementation of + * {@link Set#removeAll(Collection)}. It is implemented as O(n**2). + */ + @Override + public boolean removeAll(final Collection<?> arg) { + boolean changed = false; + for (final Object entry : arg) { + final int i = indexOf(entry); + if (i >= 0) { + entries[i] = null; + changed = true; + } + } + if (changed) { + clean(); + } + return changed; + } + + /** + * This is an extremely expensive implementation of + * {@link Set#retainAll(Collection)}. It is implemented as O(n**2). + */ + @Override + public boolean retainAll(final Collection<?> arg) { + boolean changed = false; + for (int i = 0; i < size; ++i) { + final Object entry = entries[i]; + boolean exists = false; + // cannot use Collection.contains(), as we want identity + for (final Object v : arg) { + if (v == entry) { + exists = true; + break; + } + } + if (!exists) { + entries[i] = null; + changed = true; + } + } + if (changed) { + clean(); + } + return changed; + } + + @Override + public int size() { + return size; + } + + @Override + public Object[] toArray() { + return Arrays.copyOf(entries, size); + } + + @SuppressWarnings("unchecked") + @Override + public <T> T[] toArray(final T[] arg) { + return Arrays.copyOf(entries, size, (Class<T[]>) arg.getClass()); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/Tuple2.java b/vespajlib/src/main/java/com/yahoo/collections/Tuple2.java new file mode 100644 index 00000000000..4a817381f9c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/Tuple2.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +/** + * A representation of a pair of values, typically of different types. + * + * <p> + * This class is to avoid littering a class with thin wrapper objects for + * passing around e.g. the state of an operation and the result value. Using + * this class may be correct, but it is a symptom that you may want to redesign + * your code. (Should you pass mutable objects to the method instead? Create a + * new class and do the work inside that class instead? Etc.) + * </p> + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class Tuple2<T1, T2> { + + public final T1 first; + public final T2 second; + + public Tuple2(final T1 first, final T2 second) { + this.first = first; + this.second = second; + } + + /** + * hashCode() will always throw UnsupportedOperationException. The reason is + * this class is not meant for being put in Container implementation or + * similar use where Java generics will lead to a type unsafe maintenance + * nightmare. + * + * @throws UnsupportedOperationException + * will always throw this when invoked + */ + @Override + public int hashCode() { + throw new UnsupportedOperationException( + "com.yahoo.collections.Tuple2<T1, T2> does not support equals(Object) by design. Refer to JavaDoc for details."); + } + + /** + * equals(Object) will always throw UnsupportedOperationException. The + * intention is always using the objects contained in the tuple directly. + * + * @param obj + * ignored + * @throws UnsupportedOperationException + * will always throw this when invoked + */ + @Override + public boolean equals(final Object obj) { + throw new UnsupportedOperationException( + "com.yahoo.collections.Tuple2<T1, T2> does not support equals(Object) by design. Refer to JavaDoc for details."); + } + + /** + * Human readable string representation which invokes the contained + * instances' toString() implementation. + */ + @Override + public String toString() { + return "Tuple2(" + first + ", " + second + ")"; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/collections/package-info.java b/vespajlib/src/main/java/com/yahoo/collections/package-info.java new file mode 100644 index 00000000000..7f09f0dd3d7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/collections/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.collections; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java b/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java new file mode 100644 index 00000000000..a689884db0a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +/** + * Compression type enum. + * + * @author bratseth + */ +public enum CompressionType { + + // Do not change the type->ordinal association. The gap is due to historic types no longer supported. + NONE((byte) 0), + INCOMPRESSIBLE((byte) 5), + LZ4((byte) 6); + + private byte code; + + CompressionType(byte code) { + this.code = code; + } + + public byte getCode() { + return code; + } + + /** + * Returns whether this type represent actually compressed data + */ + public boolean isCompressed() { + return this != NONE && this != INCOMPRESSIBLE; + } + + public static CompressionType valueOf(byte value) { + switch (value) { + case ((byte) 0): + return NONE; + case ((byte) 5): + return INCOMPRESSIBLE; + case ((byte) 6): + return LZ4; + default: + throw new IllegalArgumentException("Unknown compression type ordinal " + value); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java new file mode 100644 index 00000000000..664ceaea7dc --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java @@ -0,0 +1,154 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import java.util.Arrays; +import java.util.Optional; + +/** + * Compressor which can compress and decompress in various formats. + * This class is thread safe. Creating a reusable instance is faster than creating instances as needed. + * + * @author bratseth + */ +public class Compressor { + + private final CompressionType type; + private final int level; + private final double compressionThresholdFactor; + private final int compressMinSizeBytes; + + private final LZ4Factory factory = LZ4Factory.fastestInstance(); + + /** Creates a compressor with default settings. */ + public Compressor() { + this(CompressionType.LZ4); + } + + /** Creates a compressor with a default compression type. */ + public Compressor(CompressionType type) { + this(type, 9, 0.95, 0); + } + + /** + * Creates a compressor. + * + * @param type the type of compression to use to compress data + * @param level a number between 0 and 9 where a higher value means more compression + * @param compressionThresholdFactor the compression factor we need to achieve to return the compressed data + * instead of raw data + * @param compressMinSizeBytes the minimal input data size to perform compression + */ + public Compressor(CompressionType type, int level, double compressionThresholdFactor, int compressMinSizeBytes) { + this.type = type; + this.level = level; + this.compressionThresholdFactor = compressionThresholdFactor; + this.compressMinSizeBytes = compressMinSizeBytes; + } + + /** Returns the default compression type used by this */ + public CompressionType type() { return type; } + + /** Returns the compression level this will use - a number between 0 and 9 where higher means more compression */ + public int level() { return level; } + + /** Returns the compression factor we need to achieve to return compressed rather than raw data */ + public double compressionThresholdFactor() { return compressionThresholdFactor; } + + /** Returns the minimal data size required to perform compression */ + public int compressMinSizeBytes() { return compressMinSizeBytes; } + + /** + * Compresses some data + * + * @param requestedCompression the desired compression type, which will be used if the data is deemed suitable. + * Not all the existing types are actually supported. + * @param data the data to compress. This array is only read by this method. + * @param uncompressedSize uncompressedSize the size in bytes of the data array. If this is not present, it is + * assumed that the size is the same as the data array size, i.e that it is completely + * filled with uncompressed data. + * @return the compression result + * @throws IllegalArgumentException if the compression type is not supported + */ + public Compression compress(CompressionType requestedCompression, byte[] data, Optional<Integer> uncompressedSize) { + switch (requestedCompression) { + case NONE: + data = uncompressedSize.isPresent() ? Arrays.copyOf(data, uncompressedSize.get()) : data; + return new Compression(CompressionType.NONE, data); + case LZ4: + int dataSize = uncompressedSize.isPresent() ? uncompressedSize.get() : data.length; + if (dataSize < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, data); + LZ4Compressor compressor = level < 7 ? factory.fastCompressor() : factory.highCompressor(); + byte[] compressedData = compressor.compress(data); + if (compressedData.length + 8 >= dataSize * compressionThresholdFactor) + return new Compression(CompressionType.INCOMPRESSIBLE, data); + return new Compression(CompressionType.LZ4, compressedData); + default: + throw new IllegalArgumentException(requestedCompression + " is not supported"); + } + } + /** Compresses some data using the compression type of this compressor */ + public Compression compress(CompressionType requestedCompression, byte[] data) { return compress(type, data, Optional.empty()); } + /** Compresses some data using the compression type of this compressor */ + public Compression compress(byte[] data, int uncompressedSize) { return compress(type, data, Optional.of(uncompressedSize)); } + /** Compresses some data using the compression type of this compressor */ + public Compression compress(byte[] data) { return compress(type, data, Optional.empty()); } + + /** + * Decompresses some data + * + * @param compression the compression type used + * @param compressedData the compressed data. This array is only read by this method. + * @param compressedDataOffset the offset in the compressed data at which to start decompression + * @param expectedUncompressedSize the uncompressed size in bytes of this data + * @param expectedCompressedSize the expected compressed size of the data in bytes, optionally for validation with LZ4. + * @return the uncompressed data, of the given size + * @throws IllegalArgumentException if the compression type is not supported + * @throws IllegalStateException if the expected compressed size is non-empty and specifies a different size than the actual size + */ + public byte[] decompress(CompressionType compression, byte[] compressedData, int compressedDataOffset, + int expectedUncompressedSize, Optional<Integer> expectedCompressedSize) { + switch (compression) { + case NONE: case INCOMPRESSIBLE: // return a copy of the requested slize of the input buffer + int endPosition = expectedCompressedSize.isPresent() ? compressedDataOffset + expectedCompressedSize.get() : compressedData.length; + return Arrays.copyOfRange(compressedData, compressedDataOffset, endPosition); + case LZ4: + byte[] uncompressedLZ4Data = new byte[expectedUncompressedSize]; + int compressedSize = factory.fastDecompressor().decompress(compressedData, compressedDataOffset, + uncompressedLZ4Data, 0, expectedUncompressedSize); + if (expectedCompressedSize.isPresent() && compressedSize != expectedCompressedSize.get()) + throw new IllegalStateException("Compressed size mismatch. Expected " + compressedSize + ". Got " + expectedCompressedSize.get()); + return uncompressedLZ4Data; + default: + throw new IllegalArgumentException(compression + " is not supported"); + } + } + /** Decompress some data */ + public byte[] decompress(byte[] compressedData, CompressionType compression, int uncompressedSize) { + return decompress(compression, compressedData, 0, uncompressedSize, Optional.empty()); + } + + public static class Compression { + + private final CompressionType compressionType; + + private final byte[] data; + + public Compression(CompressionType compressionType, byte[] data) { + this.compressionType = compressionType; + this.data = data; + } + + /** + * Returns the compression type used to compress this data. + * This will be either the requested compression or INCOMPRESSIBLE. + */ + public CompressionType type() { return compressionType; } + + /** Returns the uncompressed data in a buffer which gets owned by the caller */ + public byte[] data() { return data; } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/compress/IntegerCompressor.java b/vespajlib/src/main/java/com/yahoo/compress/IntegerCompressor.java new file mode 100644 index 00000000000..cbf12bd3d94 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/compress/IntegerCompressor.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +import java.nio.ByteBuffer; + +/** + * TODO: balder + */ +public class IntegerCompressor { + + public static void putCompressedNumber(int n, ByteBuffer buf) { + int negative = n < 0 ? 0x80 : 0x0; + if (negative != 0) { + n = -n; + } + if (n < (0x1 << 5)) { + byte b = (byte)(n | negative); + buf.put(b); + } else if (n < (0x1 << 13)) { + n = n | 0x4000 | (negative << 8); + buf.putShort((short)n); + } else if ( n < (0x1 << 29)) { + n = n | 0x60000000 | (negative << 24); + buf.putInt(n); + } else { + throw new IllegalArgumentException("Number '" + ((negative != 0) ? -n : n) + "' too big, must extend encoding"); + } + } + + public static void putCompressedPositiveNumber(int n, ByteBuffer buf) { + if (n < 0) { + throw new IllegalArgumentException("Number '" + n + "' must be positive"); + } + if (n < (0x1 << 6)) { + buf.put((byte)n); + } else if (n < (0x1 << 14)) { + n = n | 0x8000; + buf.putShort((short)n); + } else if ( n < (0x1 << 30)) { + n = n | 0xc0000000; + buf.putInt(n); + } else { + throw new IllegalArgumentException("Number '" + n + "' too big, must extend encoding"); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/compress/package-info.java b/vespajlib/src/main/java/com/yahoo/compress/package-info.java new file mode 100644 index 00000000000..52529a290e4 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/compress/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.compress; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/CopyOnWriteHashMap.java b/vespajlib/src/main/java/com/yahoo/concurrent/CopyOnWriteHashMap.java new file mode 100644 index 00000000000..e15a3734094 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/CopyOnWriteHashMap.java @@ -0,0 +1,94 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * <p>This is a thread hash map for small collections that are stable once built. Until it is stable there will be a + * race among all threads missing something in the map. They will then clone the map add the missing stuff and then put + * it back as active again. Here are no locks, but the cost is that inserts will happen a lot more than necessary. The + * map reference is volatile, but on most multicpu machines that has no cost unless modified.</p> + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public class CopyOnWriteHashMap<K, V> implements Map<K, V> { + + private volatile HashMap<K, V> map = new HashMap<>(); + + @Override + public int size() { + return map.size(); + } + + @Override + public boolean isEmpty() { + return map.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return map.containsKey(key); + } + + @Override + public boolean containsValue(Object value) { + return map.containsValue(value); + } + + @Override + public V get(Object key) { + return map.get(key); + } + + @Override + public V put(K key, V value) { + HashMap<K, V> next = new HashMap<>(map); + V old = next.put(key, value); + map = next; + return old; + } + + @Override + @SuppressWarnings("SuspiciousMethodCalls") + public V remove(Object key) { + HashMap<K, V> prev = map; + if (!prev.containsKey(key)) { + return null; + } + HashMap<K, V> next = new HashMap<>(prev); + V old = next.remove(key); + map = next; + return old; + } + + @Override + public void putAll(Map<? extends K, ? extends V> m) { + HashMap<K, V> next = new HashMap<>(map); + next.putAll(m); + map = next; + } + + @Override + public void clear() { + map = new HashMap<>(); + } + + @Override + public Set<K> keySet() { + return map.keySet(); + } + + @Override + public Collection<V> values() { + return map.values(); + } + + @Override + public Set<Entry<K, V>> entrySet() { + return map.entrySet(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java b/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java new file mode 100644 index 00000000000..38c5bafc0d6 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; + +/** + * A simple thread factory that decorates <code>Executors.defaultThreadFactory()</code> + * and sets all created threads to be daemon threads. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DaemonThreadFactory implements ThreadFactory { + private ThreadFactory defaultThreadFactory = Executors.defaultThreadFactory(); + private String prefix = null; + + /** + * Creates a deamon thread factory that creates threads with the default names + * provided by <code>Executors.defaultThreadFactory()</code>. + */ + public DaemonThreadFactory() { + } + + /** + * Creates a deamon thread factory that creates threads with the default names + * provided by <code>Executors.defaultThreadFactory()</code> prepended by the + * specified prefix. + * + * @param prefix the thread name prefix to use + */ + public DaemonThreadFactory(String prefix) { + this.prefix = prefix; + } + + public String getPrefix() { + return prefix; + } + + @Override + public Thread newThread(Runnable runnable) { + Thread t = defaultThreadFactory.newThread(runnable); + t.setDaemon(true); + if (prefix != null) { + t.setName(prefix + t.getName()); + } + return t; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/EventBarrier.java b/vespajlib/src/main/java/com/yahoo/concurrent/EventBarrier.java new file mode 100644 index 00000000000..389fe8a85ea --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/EventBarrier.java @@ -0,0 +1,140 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent;
+
+
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Reference implementation of the 'Incremental Minimal Event Barrier'
+ * algorithm. An event in this context is defined to be something that
+ * happens during a time interval. An event barrier is a time interval
+ * for which events may start before or end after, but not both. The
+ * problem solved by the algorithm is to determine the minimal event
+ * barrier starting at a given time. In other words; wait for the
+ * currently active events to complete. The most natural use of this
+ * algorithm would be to make a thread wait for events happening in
+ * other threads to complete.
+ *
+ * @author <a href="mailto:havardpe@yahoo-inc.com">Haavard Pettersen</a>
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class EventBarrier {
+
+ private final List<Entry> queue = new LinkedList<>();
+ private int barrierToken = 0;
+ private int eventCount = 0;
+
+ /**
+ * At creation there are no active events and no pending barriers.
+ */
+ public EventBarrier() {
+ // empty
+ }
+
+ /**
+ * Obtain the current number of active events. This method is
+ * intended for testing and debugging.
+ *
+ * @return Number of active events.
+ */
+ int getNumEvents() {
+ int cnt = eventCount;
+ for (Entry entry : queue) {
+ cnt += entry.eventCount;
+ }
+ return cnt;
+ }
+
+ /**
+ * Obtain the current number of pending barriers. This method is
+ * intended for testing and debugging.
+ *
+ * @return Number of pending barriers.
+ */
+ int getNumBarriers() {
+ return queue.size();
+ }
+
+ /**
+ * Signal the start of an event. The value returned from this
+ * method must later be passed to the completeEvent method when
+ * signaling the completion of the event.
+ *
+ * @return Opaque token identifying the started event.
+ */
+ public int startEvent() {
+ ++eventCount;
+ return barrierToken;
+ }
+
+ /**
+ * Signal the completion of an event. The value passed to this
+ * method must be the same as the return value previously obtained
+ * from the startEvent method. This method will signal the
+ * completion of all pending barriers that were completed by the
+ * completion of this event.
+ *
+ * @param token Opaque token identifying the completed event.
+ */
+ public void completeEvent(int token) {
+ if (token == this.barrierToken) {
+ --eventCount;
+ return;
+ }
+ --queue.get(queue.size() - (this.barrierToken - token)).eventCount;
+ while (!queue.isEmpty() && queue.get(0).eventCount == 0) {
+ queue.remove(0).handler.completeBarrier();
+ }
+ }
+
+ /**
+ * Initiate the detection of the minimal event barrier starting
+ * now. If this method returns false it means that no events were
+ * currently active and the minimal event barrier was infinitely
+ * small. If this method returns false the handler will not be
+ * notified of the completion of the barrier. If this method
+ * returns true it means that the started barrier is pending and
+ * that the handler passed to this method will be notified of its
+ * completion at a later time.
+ *
+ * @param handler Handler notified of the completion of the barrier.
+ * @return True if a barrier was started, false if no events were active.
+ */
+ public boolean startBarrier(BarrierWaiter handler) {
+ if (eventCount == 0 && queue.isEmpty()) {
+ return false;
+ }
+ queue.add(new Entry(eventCount, handler));
+ ++barrierToken;
+ eventCount = 0;
+ return true;
+ }
+
+ /**
+ * Declares the interface required to wait for the detection of a
+ * minimal event barrier. An object that implements this is passed
+ * to the {@link EventBarrier#startBarrier(BarrierWaiter)}.
+ */
+ public interface BarrierWaiter {
+
+ /**
+ * Callback invoked by the thread that detected the minimal
+ * event barrier. Once this is called, all events taking place
+ * at or before the corresponding call to {@link
+ * EventBarrier#startBarrier(BarrierWaiter)} have ended.
+ */
+ public void completeBarrier();
+ }
+
+ private static class Entry {
+
+ int eventCount;
+ final BarrierWaiter handler;
+
+ Entry(int eventCount, BarrierWaiter handler) {
+ this.eventCount = eventCount;
+ this.handler = handler;
+ }
+ }
+}
diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/LocalInstance.java b/vespajlib/src/main/java/com/yahoo/concurrent/LocalInstance.java new file mode 100644 index 00000000000..c2d19831810 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/LocalInstance.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import com.yahoo.concurrent.ThreadLocalDirectory.ObservableUpdater; +import com.yahoo.concurrent.ThreadLocalDirectory.Updater; + +/** + * Only for use along with ThreadLocalDirectory. A thread local data container + * instance. The class is visible to avoid indirection through the internal + * {@link ThreadLocal} in ThreadLocalDirectory if possible, but has no user + * available methods. + * + * @param AGGREGATOR + * the structure to insert produced data into + * @param SAMPLE + * type of produced data to insert from each participating thread + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class LocalInstance<AGGREGATOR, SAMPLE> { + /** + * The current generation of data produced from a single thread, where + * generation is the period between two subsequent calls to + * ThreadLocalDirectory.fetch(). + */ + private AGGREGATOR current; + + // see comment on setRegistered(boolean) for locking explanation + private boolean isRegistered = false; + private final Object lock = new Object(); + + LocalInstance(Updater<AGGREGATOR, SAMPLE> updater) { + current = updater.createGenerationInstance(null); + } + + boolean update(SAMPLE x, Updater<AGGREGATOR, SAMPLE> updater) { + synchronized (lock) { + current = updater.update(current, x); + return isRegistered; + } + } + + AGGREGATOR getAndReset(Updater<AGGREGATOR, SAMPLE> updater) { + AGGREGATOR previous; + synchronized (lock) { + previous = current; + current = updater.createGenerationInstance(previous); + setRegistered(false); + } + return previous; + } + + AGGREGATOR copyCurrent(ObservableUpdater<AGGREGATOR, SAMPLE> updater) { + AGGREGATOR view; + synchronized (lock) { + view = updater.copy(current); + } + return view; + } + + // This is either set by the putting thread or the fetching thread. If + // it is set by the putting thread, then there is no memory barrier, + // because it is only _read_ in the putting thread. If it is set by the + // fetching thread, then the memory barrier is this.lock. This + // roundabout way is to avoid creating many-to-many memory barrier and + // locking relationships. + void setRegistered(boolean isRegistered) { + this.isRegistered = isRegistered; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/Receiver.java b/vespajlib/src/main/java/com/yahoo/concurrent/Receiver.java new file mode 100644 index 00000000000..339d8002c4f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/Receiver.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import com.yahoo.collections.Tuple2; + +/** + * A class for sending single messages between threads with timeout. Typical use + * would be + * + * <pre> + * Receiver<SomeMessage> receiver = new Receiver<SomeMessage>(); + * SomeRunnable runnable = new SomeRunnable(receiver); + * Thread worker = new Thread(runnable); + * worker.start(); + * Pair<Receiver.MessageState, SomeMessage> answer = receiver.get(500L); + * </pre> + * + * ... and in the worker thread simply + * + * <pre> + * receiver.put(new SomeMessage(...)) + * </pre> + * + * <p> + * Any number of threads may wait for the same message. Sending null references + * is supported. The object is intended for delivering only single message, + * there is no support for recycling it. + * </p> + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class Receiver<T> { + /** + * MessageState is the reason for returning from get(). If a message is + * received before timeout, the state will be VALID. If no message is + * received before timeout, state is TIMEOUT. + */ + public enum MessageState { + VALID, TIMEOUT; + }; + private final Object lock = new Object(); + private T message = null; + private boolean received = false; + + /** + * Make a message available for consumers. + * + * @param message the message to send + * @throws IllegalStateException if a message has already been received here + */ + public void put(T message) { + synchronized (lock) { + if (received) { + throw new IllegalStateException("Multiple puts on a single Receiver instance is not legal."); + } + this.message = message; + received = true; + lock.notifyAll(); + } + } + + /** + * Wait for up to "timeout" milliseconds for an incoming message. This hides + * spurious wakeup, but InterruptedException will be propagated. + * + * @param timeout + * maximum time to wait for message in milliseconds + * @return a Pair instance containing the reason for returning and the + * message possible received + * @throws InterruptedException if the waiting thread is interrupted + */ + public Tuple2<MessageState, T> get(long timeout) throws InterruptedException { + long barrier = System.currentTimeMillis() + timeout; + synchronized (lock) { + while (!received) { + long t = System.currentTimeMillis(); + if (t >= barrier) { + return new Tuple2<>(MessageState.TIMEOUT, null); + } + lock.wait(barrier - t); + } + return new Tuple2<>(MessageState.VALID, message); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java b/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java new file mode 100644 index 00000000000..5aa4990a86a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This is an implementation of {@link Timer} that is backed by an actual system timer.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public enum SystemTimer implements Timer {
+
+ INSTANCE;
+
+ private volatile long millis;
+
+ private SystemTimer() {
+ millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
+ Thread thread = new Thread() {
+
+ @Override
+ public void run() {
+ while (true) {
+ millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
+ try {
+ Thread.sleep(1);
+ } catch (InterruptedException e) {
+ break;
+ }
+ }
+ }
+ };
+ thread.setDaemon(true);
+ thread.start();
+ }
+
+ @Override
+ public long milliTime() {
+ return millis;
+ }
+}
diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadFactoryFactory.java b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadFactoryFactory.java new file mode 100644 index 00000000000..5be6da8c66d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadFactoryFactory.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 24.04.13 + * Time: 19:00 + * To change this template use File | Settings | File Templates. + */ +public class ThreadFactoryFactory { + static public synchronized ThreadFactory getThreadFactory(String name) { + PooledFactory p = factory.get(name); + if (p == null) { + p = new PooledFactory(name); + factory.put(name, p); + } + return p.getFactory(false); + } + static public synchronized ThreadFactory getDaemonThreadFactory(String name) { + PooledFactory p = factory.get(name); + if (p == null) { + p = new PooledFactory(name); + factory.put(name, p); + } + return p.getFactory(true); + } + private static class PooledFactory { + private static class Factory implements ThreadFactory { + final ThreadGroup group; + final AtomicInteger threadNumber = new AtomicInteger(1); + final String namePrefix; + final boolean isDaemon; + + Factory(final String name, boolean isDaemon) { + this.isDaemon = isDaemon; + final SecurityManager s = System.getSecurityManager(); + group = (s != null) + ? s.getThreadGroup() + : Thread.currentThread().getThreadGroup(); + namePrefix = name; + } + + @Override + public Thread newThread(final Runnable r) { + final Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0); + if (t.isDaemon() != isDaemon) { + t.setDaemon(isDaemon); + } + if (t.getPriority() != Thread.NORM_PRIORITY) { + t.setPriority(Thread.NORM_PRIORITY); + } + return t; + } + } + PooledFactory(String name) { + this.name = name; + } + ThreadFactory getFactory(boolean isDaemon) { + return new Factory(name + "-" + poolId.getAndIncrement() + "-thread-", isDaemon); + + } + private final String name; + private final AtomicInteger poolId = new AtomicInteger(1); + } + static private Map<String, PooledFactory> factory = new HashMap<>(); +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java new file mode 100644 index 00000000000..ef2273bdb25 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadLocalDirectory.java @@ -0,0 +1,346 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import java.util.ArrayList; +import java.util.List; + +/** + * A class for multiple producers and potentially multiple consumers (usually + * only one). + * + * <p> + * The consuming threads always unregisters the data producers when doing + * fetch(). This is the reason for having to do update through the directory. + * The reason for this is otherwise, we would either get reference leaks from + * registered objects belonging to dead threads if we did not unregister + * instances, otherwise the sampling thread would have to unregister the + * instance, and then we would create a memory relationship between all + * producing threads, which is exactly what this class aims to avoid. + * </p> + * + * <p> + * A complete example from a test: + * </p> + * + * <pre> + * private static class SumUpdater implements ThreadLocalDirectory.Updater<Integer, Integer> { + * + * {@literal @}Override + * public Integer update(Integer current, Integer x) { + * return Integer.valueOf(current.intValue() + x.intValue()); + * } + * + * {@literal @}Override + * public Integer createGenerationInstance(Integer previous) { + * return Integer.valueOf(0); + * } + * } + * + * ... then the producers does (where r is in instance of + * ThreadLocalDirectory)... + * + * {@literal @}Override + * public void run() { + * LocalInstance<Integer, Integer> s = r.getLocalInstance(); + * for (int i = 0; i < 500; ++i) { + * r.update(Integer.valueOf(i), s); + * } + * } + * + * ... and the consumer... + * + * List<Integer> measurements = s.fetch() + * </pre> + * + * <p> + * Invoking r.fetch() will produce a list of integers from all the participating + * threads at any time. + * </p> + * + * <p> + * Refer to e.g. com.yahoo.search.statistics.PeakQpsSearcher for a production + * example. + * </p> + * + * @param AGGREGATOR + * the type input data is aggregated into + * @param SAMPLE + * the type of input data + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class ThreadLocalDirectory<AGGREGATOR, SAMPLE> { + /** + * Factory interface to create the data container for each generation of + * samples, and putting data into it. + * + * <p> + * The method for actual insertion of a single sample into the current data + * generation exists separate from LocalInstance.AGGREGATOR to make it + * possible to use e.g. Integer and List as AGGREGATOR types. + * </p> + * + * <p> + * The allocation and sampling is placed in the same class, since you always + * need to implement both. + * </p> + * + * @param AGGREGATOR + * The type of the data container to produce + * @param SAMPLE + * The type of the incoming data to store in the container. + */ + public interface Updater<AGGREGATOR, SAMPLE> { + /** + * Create data container to receive produced data. This is invoked once + * on every instance every time ThreadLocalDirectory.fetch() is invoked. + * This might be an empty list, creating a new counter set to zero, or + * even copying the current state of LocalInstance.current. + * LocalInstance.current will be set to the value received from this + * factory after invokation this method. + * + * <p> + * The first time this method is invoked for a thread, previous will be + * null. + * </p> + * + * <p> + * If using mutable objects, an implementation should always create a + * new instance in this method, as the previous data generation will be + * transmitted to the consuming thread. This obviously does not matter + * if using immutable (value) objects. + * </p> + * + * <p> + * Examples: + * </p> + * + * <p> + * Using a mutable aggregator (a list of integers): + * </p> + * + * <pre> + * if (previous == null) { + * return new ArrayList<Integer>(); + * } else { + * return new ArrayList<Integer>(previous.size()); + * } + * </pre> + * + * <p> + * Using an immutable aggregator (an integer): + * </p> + * + * <pre> + * return Integer.valueOf(0); + * </pre> + * + * @return a fresh structure to receive data + */ + public AGGREGATOR createGenerationInstance(AGGREGATOR previous); + + /** + * Insert a data element of type S into the current generation of data + * carrier T. This could be e.g. adding to a list, putting into a local + * histogram or increasing a counter. + * + * <p> + * The method may or may not return a fresh instance of the current + * value for each invokation, if using a mutable aggregator the typical + * case will be returning the same instance for the new and old value of + * current, while if using an immutable aggregator, one is forced to + * return new instances. + * </p> + * + * <p> + * Examples: + * </p> + * + * <p> + * Using a mutable aggregator (a list of instances of type SAMPLE): + * </p> + * + * <pre> + * current.add(x); + * return current; + * </pre> + * + * <p> + * Using an immutable aggregator (Integer) while also using Integer as + * type for SAMPLE: + * </p> + * + * <pre> + * return Integer.valueOf(current.intValue() + x.intValue()); + * </pre> + * + * @param current + * the current generation's data container + * @param x + * the data to insert + * @return the new current value, may be the same as previous + */ + public AGGREGATOR update(AGGREGATOR current, SAMPLE x); + } + + /** + * Implement this interface to be able to view the contents of a + * ThreadLocalDirectory without resetting the local instances in each + * thread. + * + * @param <AGGREGATOR> + * as for {@link Updater} + * @param <SAMPLE> + * as for {@link Updater} + * @see ThreadLocalDirectory#view() + */ + public interface ObservableUpdater<AGGREGATOR, SAMPLE> extends + Updater<AGGREGATOR, SAMPLE> { + /** + * Create an application specific copy of the AGGREGATOR for a thread. + * + * @param current + * the AGGREGATOR instance to copy + * @return a copy of the incoming parameter + */ + public AGGREGATOR copy(AGGREGATOR current); + } + + private final ThreadLocal<LocalInstance<AGGREGATOR, SAMPLE>> local = new ThreadLocal<>(); + private final Object directoryLock = new Object(); + private List<LocalInstance<AGGREGATOR, SAMPLE>> directory = new ArrayList<>(); + private final Updater<AGGREGATOR, SAMPLE> updater; + private final ObservableUpdater<AGGREGATOR, SAMPLE> observableUpdater; + + public ThreadLocalDirectory(Updater<AGGREGATOR, SAMPLE> updater) { + this.updater = updater; + if (updater instanceof ObservableUpdater) { + observableUpdater = (ObservableUpdater<AGGREGATOR, SAMPLE>) updater; + } else { + observableUpdater = null; + } + } + + private void put(LocalInstance<AGGREGATOR, SAMPLE> q) { + // Has to set registered before adding to the list. Otherwise, the + // instance might be removed from the list, set as unregistered, and + // then the local thread might happily remove that information. The Java + // memory model is a guarantuee for the minimum amount of visibility, + // not a definition of the actual amount. + q.setRegistered(true); + synchronized (directoryLock) { + directory.add(q); + } + } + + /** + * Fetch the current set of sampled data, and reset state of all thread + * local instances. The producer threads will not alter data in the list + * returned from this method. + * + * @return a list of data from all producer threads + */ + public List<AGGREGATOR> fetch() { + List<AGGREGATOR> contained; + List<LocalInstance<AGGREGATOR, SAMPLE>> previous; + int previousIntervalSize; + + synchronized (directoryLock) { + previousIntervalSize = directory.size(); + previous = directory; + directory = new ArrayList<>( + previousIntervalSize); + } + contained = new ArrayList<>(previousIntervalSize); + // Yes, this is an inconsistence about when the registered state is + // reset and when the thread local is removed from the list. + // LocalInstance.isRegistered tells whether the data is available to + // some consumer, not whether the LocalInstance is a member of the + // directory. + for (LocalInstance<AGGREGATOR, SAMPLE> x : previous) { + contained.add(x.getAndReset(updater)); + } + return contained; + } + + /** + * Get a view of the current data. This requires this ThreadLocalDirectory + * to have been instantiated with an updater implementing ObservableUpdater. + * + * @return a list of a copy of the current data in all producer threads + * @throws IllegalStateException + * if the updater does not implement {@link ObservableUpdater} + */ + public List<AGGREGATOR> view() { + if (observableUpdater == null) { + throw new IllegalStateException("Does not use observable updaters."); + } + List<LocalInstance<AGGREGATOR, SAMPLE>> current; + List<AGGREGATOR> view; + synchronized (directoryLock) { + current = new ArrayList<>( + directory); + } + view = new ArrayList<>(current.size()); + for (LocalInstance<AGGREGATOR, SAMPLE> x : current) { + view.add(x.copyCurrent(observableUpdater)); + } + return view; + } + + private LocalInstance<AGGREGATOR, SAMPLE> getOrCreateLocal() { + LocalInstance<AGGREGATOR, SAMPLE> current = local.get(); + if (current == null) { + current = new LocalInstance<>(updater); + local.set(current); + } + return current; + } + + /** + * Expose the thread local for the running thread, for use in conjunction + * with update(SAMPLE, LocalInstance<AGGREGATOR, SAMPLE>). + * + * @return the current thread's local instance + */ + public LocalInstance<AGGREGATOR, SAMPLE> getLocalInstance() { + return getOrCreateLocal(); + } + + /** + * Input data from a producer thread. + * + * @param x + * the data to insert + */ + public void update(SAMPLE x) { + update(x, getOrCreateLocal()); + } + + /** + * Update a value with a given thread local instance. + * + * <p> + * If a producer thread is to insert a series of data, it is desirable to + * limit the number of memory transactions to the theoretical minimum. Since + * reading a thread local is the memory equivalence of reading a volatile, + * it is then useful to avoid re-reading the running threads' input + * instance. For this scenario, fetch the running thread's instance with + * getLocalInstance(), and then insert the produced data with the multiple + * calls necessary to update(SAMPLE, LocalInstance<AGGREGATOR, SAMPLE>). + * </p> + * + * @param x + * the data to insert + * @param localInstance + * the local data insertion instance + */ + public void update(SAMPLE x, LocalInstance<AGGREGATOR, SAMPLE> localInstance) { + boolean isRegistered; + isRegistered = localInstance.update(x, updater); + if (!isRegistered) { + put(localInstance); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadRobustList.java b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadRobustList.java new file mode 100644 index 00000000000..8a79db6a6eb --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadRobustList.java @@ -0,0 +1,151 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * A list which tolerates concurrent adds from one other thread while it is + * read. More precisely: <i>This list is guaranteed to provide a self-consistent + * read view regardless of the internal order in which the primitive mutating + * operations on it are observed from the reading thread.</i> + * <p> + * This is useful for traced information as there may be timed out threads + * working on the structure after it is returned upwards for consumption. + * + * @since 4.2 + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ThreadRobustList<T> implements Iterable<T> { + + private Object[] items; + + /** Index of the next item */ + private int next = 0; + + public ThreadRobustList() { + this(10); + } + + public ThreadRobustList(final int initialCapacity) { + items = new Object[initialCapacity]; + } + + public void add(final T item) { + Object[] workItems = items; + if (next >= items.length) { + final int newLength = 20 + items.length * 2; + workItems = Arrays.copyOf(workItems, newLength); + workItems[next++] = item; + items = workItems; + } else { + workItems[next++] = item; + } + } + + /** + * Returns an iterator over the elements of this. This iterator does not + * support remove. + */ + @Override + public Iterator<T> iterator() { + return new ThreadRobustIterator(items); + } + + /** + * Returns an iterator over the elements of this, starting at the last + * element and working backwards. This iterator does not support remove. + */ + public Iterator<T> reverseIterator() { + return new ThreadRobustReverseIterator(items); + } + + public boolean isEmpty() { + return next == 0; + } + + private class ThreadRobustIterator implements Iterator<T> { + + private final Object[] items; + + private int nextIndex = 0; + + public ThreadRobustIterator(final Object[] items) { + this.items = items; + } + + public @Override + void remove() { + throw new UnsupportedOperationException( + "remove() is not supported on thread robust list iterators"); + } + + @SuppressWarnings("unchecked") + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException("No more elements"); + } + + return (T) items[nextIndex++]; + } + + @Override + public boolean hasNext() { + if (nextIndex >= items.length) { + return false; + } + if (items[nextIndex] == null) { + return false; + } + return true; + } + + } + + private class ThreadRobustReverseIterator implements Iterator<T> { + + private final Object[] items; + + private int nextIndex; + + public ThreadRobustReverseIterator(final Object[] items) { + this.items = items; + nextIndex = findLastAssignedIndex(items); + } + + private int findLastAssignedIndex(final Object[] items) { + for (int i = items.length - 1; i >= 0; i--) { + if (items[i] != null) { + return i; + } + } + return -1; + } + + public @Override + void remove() { + throw new UnsupportedOperationException( + "remove() is not supported on thread robust list iterators"); + } + + @SuppressWarnings("unchecked") + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException("No more elements"); + } + + return (T) items[nextIndex--]; + } + + @Override + public boolean hasNext() { + return nextIndex >= 0; + } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java b/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java new file mode 100644 index 00000000000..aefbfafb7b1 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent;
+
+/**
+ * This interface wraps access to some timer that can be used to measure elapsed time, in milliseconds. This
+ * abstraction allows for unit testing the behavior of time-based constructs.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public interface Timer {
+
+ /**
+ * Returns the current value of some arbitrary timer, in milliseconds. This method can only be used to measure
+ * elapsed time and is not related to any other notion of system or wall-clock time.
+ *
+ * @return The current value of the timer, in milliseconds.
+ */
+ public long milliTime();
+}
diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/package-info.java b/vespajlib/src/main/java/com/yahoo/concurrent/package-info.java new file mode 100644 index 00000000000..dd0d639166d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.concurrent; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/data/access/ArrayTraverser.java b/vespajlib/src/main/java/com/yahoo/data/access/ArrayTraverser.java new file mode 100644 index 00000000000..99d79dd8eb5 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/ArrayTraverser.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access; + +/** + * Callback interface for traversing arrays. + * Implement this and call Inspector.traverse() + * and you will get one callback for each array entry. + **/ +public interface ArrayTraverser { + /** + * Callback function to implement. + * @param idx array index for the current array entry. + * @param inspector accessor for the current array entry's value. + **/ + public void entry(int idx, Inspector inspector); +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/Inspectable.java b/vespajlib/src/main/java/com/yahoo/data/access/Inspectable.java new file mode 100644 index 00000000000..bf3344a0fe9 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/Inspectable.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access; + +/** + * Minimal API to implement for objects containing or exposing + * structured, generic, schemaless data. Use this when it's + * impractical to implement the Inspector interface directly. + **/ +public interface Inspectable { + /** get an Inspector exposing this object's structured data. */ + public Inspector inspect(); +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/Inspector.java b/vespajlib/src/main/java/com/yahoo/data/access/Inspector.java new file mode 100644 index 00000000000..0b0061792bd --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/Inspector.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access; + + +import java.util.Map; + +/** + * This is a generic API for accessing structured, generic, schemaless data. + * An inspector is a handle to a value that has one of 8 specific types: + * EMPTY, the 5 scalar types BOOL, LONG, DOUBLE, STRING, or DATA, the + * simple list-like ARRAY container and the struct-like OBJECT container. + * Instrospection methods are available, but you can also use accessors + * with a default value if you expect a certain type and just want your + * default value if some field doesn't exist or was of the wrong type. + **/ +public interface Inspector extends Inspectable { + + /** + * Check if the inspector is valid. + * If you try to access a field or array entry that does not exist, + * you will get an invalid Inspector returned. + */ + public boolean valid(); + + /** Get the type of an inspector */ + public Type type(); + + /** Get the number of entries in an ARRAY (always returns 0 for non-arrays) */ + public int entryCount(); + + /** Get the number of fields in an OBJECT (always returns 0 for non-objects) */ + public int fieldCount(); + + /** Access the inspector's value if it's a BOOLEAN; otherwise throws exception */ + public boolean asBool(); + + /** Access the inspector's value if it's a LONG (or DOUBLE); otherwise throws exception */ + public long asLong(); + + /** Access the inspector's value if it's a DOUBLE (or LONG); otherwise throws exception */ + public double asDouble(); + + /** Access the inspector's value if it's a STRING; otherwise throws exception */ + public String asString(); + + /** + * Access the inspector's value (in utf-8 representation) if it's + * a STRING; otherwise throws exception + **/ + public byte[] asUtf8(); + + /** Access the inspector's value if it's DATA; otherwise throws exception */ + public byte[] asData(); + + /** Get the inspector's value (or the supplied default), never throws */ + public boolean asBool(boolean defaultValue); + + /** Get the inspector's value (or the supplied default), never throws */ + public long asLong(long defaultValue); + + /** Get the inspector's value (or the supplied default), never throws */ + public double asDouble(double defaultValue); + + /** Get the inspector's value (or the supplied default), never throws */ + public String asString(String defaultValue); + + /** Get the inspector's value (or the supplied default), never throws */ + public byte[] asUtf8(byte[] defaultValue); + + /** Get the inspector's value (or the supplied default), never throws */ + public byte[] asData(byte[] defaultValue); + + /** + * Traverse an array value, performing callbacks for each entry. + * + * If the current Inspector is connected to an array value, + * perform callbacks to the given traverser for each entry + * contained in the array. Otherwise a no-op. + * @param at traverser callback object. + **/ + @SuppressWarnings("overloads") + public void traverse(ArrayTraverser at); + + /** + * Traverse an object value, performing callbacks for each field. + * + * If the current Inspector is connected to an object value, + * perform callbacks to the given traverser for each field + * contained in the object. Otherwise a no-op. + * @param ot traverser callback object. + **/ + @SuppressWarnings("overloads") + public void traverse(ObjectTraverser ot); + + /** + * Access an array entry. + * + * If the current Inspector doesn't connect to an array value, + * or the given array index is out of bounds, the returned + * Inspector will be invalid. + * @param idx array index. + * @return a new Inspector for the entry value. + **/ + public Inspector entry(int idx); + + /** + * Access an field in an object. + * + * If the current Inspector doesn't connect to an object value, or + * the object value does not contain a field with the given symbol + * name, the returned Inspector will be invalid. + * @param name symbol name. + * @return a new Inspector for the field value. + **/ + public Inspector field(String name); + + /** + * Convert an array to an iterable list. Other types will just + * return an empty list. + **/ + public Iterable<Inspector> entries(); + + /** + * Convert an object to an iterable list of (name, value) pairs. + * Other types will just return an empty list. + **/ + public Iterable<Map.Entry<String,Inspector>> fields(); +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/ObjectTraverser.java b/vespajlib/src/main/java/com/yahoo/data/access/ObjectTraverser.java new file mode 100644 index 00000000000..90ff360d63d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/ObjectTraverser.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access; + +/** + * Callback interface for traversing objects. + * Implement this and call Inspector.traverse() + * and you will get one callback for each field in an object. + **/ +public interface ObjectTraverser { + /** + * Callback function to implement. + * @param name the name of the current field. + * @param inspector accessor for the current field's value. + **/ + public void field(String name, Inspector inspector); +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/Type.java b/vespajlib/src/main/java/com/yahoo/data/access/Type.java new file mode 100644 index 00000000000..e09b0ba9adc --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/Type.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access; + +/** + * Enumeration of all possible types accessed by the Inspector API. + * Note that: + * - the EMPTY type is used as a placeholder where data is missing. + * - all integers are put into LONGs; the encoding takes care of + * packing small integers compactly so this is also efficient. + * - likeweise DOUBLE is the only floating-point type, but "simple" + * numbers (like 0.0 or 1.0) are packed compactly anyway. + * - DATA can be used anything for wrapping anything else serialized + * as an array of bytes. + * - maps should be represented as an ARRAY of OBJECTs where each + * object has the fields "key" and "value". + **/ +public enum Type { + EMPTY, BOOL, LONG, DOUBLE, STRING, DATA, ARRAY, OBJECT; +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/package-info.java b/vespajlib/src/main/java/com/yahoo/data/access/package-info.java new file mode 100644 index 00000000000..904686c5d78 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.data.access; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/data/access/simple/JsonRender.java b/vespajlib/src/main/java/com/yahoo/data/access/simple/JsonRender.java new file mode 100644 index 00000000000..6acc43c2198 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/simple/JsonRender.java @@ -0,0 +1,166 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access.simple; + +import com.yahoo.text.DoubleFormatter; +import com.yahoo.data.access.*; + +/** + * Encodes json from an inspectable object. + * + * @author arnej27959 + */ +public final class JsonRender +{ + public static StringBuilder render(Inspectable value, + StringBuilder target, + boolean compact) + { + StringEncoder enc = new StringEncoder(target, compact); + enc.encode(value.inspect()); + return target; + } + + public static final class StringEncoder implements ArrayTraverser, ObjectTraverser + { + private final StringBuilder out; + private boolean head = true; + private boolean compact; + private int level = 0; + + public StringEncoder(StringBuilder out, boolean compact) { + this.out = out; + this.compact = compact; + } + + public void encode(Inspector top) { + encodeValue(top); + if (!compact) { + out.append('\n'); + } + } + + private void encodeEMPTY() { + out.append("null"); + } + + private void encodeBOOL(boolean value) { + out.append(value ? "true" : "false"); + } + + private void encodeLONG(long value) { + out.append(String.valueOf(value)); + } + + private void encodeDOUBLE(double value) { + if (Double.isNaN(value) || Double.isInfinite(value)) { + out.append("null"); + } else { + out.append(DoubleFormatter.stringValue(value)); + } + } + + static final char[] hex = "0123456789ABCDEF".toCharArray(); + + private void encodeSTRING(String value) { + out.append('"'); + for (char c : value.toCharArray()) { + switch (c) { + case '"': out.append('\\').append('"'); break; + case '\\': out.append('\\').append('\\'); break; + case '\b': out.append('\\').append('b'); break; + case '\f': out.append('\\').append('f'); break; + case '\n': out.append('\\').append('n'); break; + case '\r': out.append('\\').append('r'); break; + case '\t': out.append('\\').append('t'); break; + default: + if (c > 0x1f && c < 127) { + out.append(c); + } else { // requires escaping according to RFC 4627 + out.append('\\').append('u'); + out.append(hex[(c >> 12) & 0xf]); + out.append(hex[(c >> 8) & 0xf]); + out.append(hex[(c >> 4) & 0xf]); + out.append(hex[c & 0xf]); + } + } + } + out.append('"'); + } + + private void encodeDATA(byte[] value) { + out.append('"'); + out.append("0x"); + for (int pos = 0; pos < value.length; pos++) { + out.append(hex[(value[pos] >> 4) & 0xf]); + out.append(hex[value[pos] & 0xf]); + } + out.append('"'); + } + + private void encodeARRAY(Inspector inspector) { + openScope("["); + ArrayTraverser at = this; + inspector.traverse(at); + closeScope("]"); + } + + private void encodeOBJECT(Inspector inspector) { + openScope("{"); + ObjectTraverser ot = this; + inspector.traverse(ot); + closeScope("}"); + } + + private void openScope(String opener) { + out.append(opener); + level++; + head = true; + } + + private void closeScope(String closer) { + level--; + separate(false); + out.append(closer); + } + + private void encodeValue(Inspector inspector) { + switch(inspector.type()) { + case EMPTY: encodeEMPTY(); return; + case BOOL: encodeBOOL(inspector.asBool()); return; + case LONG: encodeLONG(inspector.asLong()); return; + case DOUBLE: encodeDOUBLE(inspector.asDouble()); return; + case STRING: encodeSTRING(inspector.asString()); return; + case DATA: encodeDATA(inspector.asData()); return; + case ARRAY: encodeARRAY(inspector); return; + case OBJECT: encodeOBJECT(inspector); return; + } + assert false : "Should not be reached"; + } + + private void separate(boolean useComma) { + if (!head && useComma) { + out.append(','); + } else { + head = false; + } + if (!compact) { + out.append("\n"); + for (int lvl = 0; lvl < level; lvl++) { out.append(" "); } + } + } + + public void entry(int idx, Inspector inspector) { + separate(true); + encodeValue(inspector); + } + + public void field(String name, Inspector inspector) { + separate(true); + encodeSTRING(name); + out.append(':'); + if (!compact) + out.append(' '); + encodeValue(inspector); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/simple/Value.java b/vespajlib/src/main/java/com/yahoo/data/access/simple/Value.java new file mode 100644 index 00000000000..baf5ce3cc54 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/simple/Value.java @@ -0,0 +1,215 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access.simple; + + +import com.yahoo.data.access.*; +import java.util.Collections; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.ArrayList; +import java.nio.charset.StandardCharsets; + + +public class Value implements Inspector { + private static Value empty = new EmptyValue(); + private static Value invalid = new Value(); + private static byte[] empty_array = new byte[0]; + public static Inspector empty() { return empty; } + public static Inspector invalid() { return invalid; } + public Inspector inspect() { return this; } + public boolean valid() { return false; } + public Type type() { return Type.EMPTY; } + public int entryCount() { return 0; } + public int fieldCount() { return 0; } + public boolean asBool() { throw new IllegalStateException("invalid data access!"); } + public long asLong() { throw new IllegalStateException("invalid data access!"); } + public double asDouble() { throw new IllegalStateException("invalid data access!"); } + public java.lang.String asString() { throw new IllegalStateException("invalid data access!"); } + public byte[] asUtf8() { throw new IllegalStateException("invalid data access!"); } + public byte[] asData() { throw new IllegalStateException("invalid data access!"); } + public boolean asBool(boolean defaultValue) { return defaultValue; } + public long asLong(long defaultValue) { return defaultValue; } + public double asDouble(double defaultValue) { return defaultValue; } + public java.lang.String asString(java.lang.String defaultValue) { return defaultValue; } + public byte[] asUtf8(byte[] defaultValue) { return defaultValue; } + public byte[] asData(byte[] defaultValue) { return defaultValue; } + public void traverse(ArrayTraverser at) {} + public void traverse(ObjectTraverser ot) {} + public Inspector entry(int idx) { return invalid; } + public Inspector field(java.lang.String name) { return invalid; } + public Iterable<Inspector> entries() { return Collections.emptyList(); } + public Iterable<Map.Entry<java.lang.String,Inspector>> fields() { return Collections.emptyList(); } + public StringBuilder writeJson(StringBuilder target) { + return JsonRender.render(this, target, true); + } + public String toJson() { return writeJson(new StringBuilder()).toString(); } + public String toString() { return toJson(); } + static public class EmptyValue extends Value { + public boolean valid() { return true; } + public boolean asBool() { return false; } + public long asLong() { return 0L; } + public double asDouble() { return 0.0; } + public java.lang.String asString() { return ""; } + public byte[] asUtf8() { return empty_array; } + public byte[] asData() { return empty_array; } + } + static public class BoolValue extends Value { + private boolean value; + public BoolValue(boolean v) { value = v; } + public boolean valid() { return true; } + public Type type() { return Type.BOOL; } + public boolean asBool() { return value; } + public boolean asBool(boolean x) { return value; } + } + static public class LongValue extends Value { + private long value; + public LongValue(long v) { value = v; } + public boolean valid() { return true; } + public Type type() { return Type.LONG; } + public long asLong() { return value; } + public double asDouble() { return (double)value; } + public long asLong(long x) { return value; } + public double asDouble(double x) { return (double)value; } + } + static public class DoubleValue extends Value { + private double value; + public DoubleValue(double v) { value = v; } + public boolean valid() { return true; } + public Type type() { return Type.DOUBLE; } + public double asDouble() { return value; } + public long asLong() { return (long)value; } + public double asDouble(double x) { return value; } + public long asLong(long x) { return (long)value; } + } + static public class StringValue extends Value { + private java.lang.String string_value = null; + private byte[] utf8_value = null; + private void handle_null() { + if (string_value == null && utf8_value == null) { + string_value = ""; + utf8_value = empty_array; + } + } + public StringValue(java.lang.String v) { + string_value = v; + handle_null(); + } + public StringValue(byte[] v) { + utf8_value = v; + handle_null(); + } + public boolean valid() { return true; } + public Type type() { return Type.STRING; } + public java.lang.String asString() { + if (string_value == null) { + string_value = new java.lang.String(utf8_value, StandardCharsets.UTF_8); + } + return string_value; + } + public java.lang.String asString(java.lang.String x) { return asString(); } + public byte[] asUtf8() { + if (utf8_value == null) { + utf8_value = string_value.getBytes(StandardCharsets.UTF_8); + } + return utf8_value; + } + public byte[] asUtf8(byte[] x) { return asUtf8(); } + } + static public class DataValue extends Value { + private byte[] value; + public DataValue(byte[] v) { + value = v; + if (v == null) { + value = empty_array; + } + } + public boolean valid() { return true; } + public Type type() { return Type.DATA; } + public byte[] asData() { return value; } + public byte[] asData(byte[] x) { return value; } + } + static public class ArrayValue extends Value { + private List<Inspector> values = new ArrayList<>(); + public boolean valid() { return true; } + public Type type() { return Type.ARRAY; } + public int entryCount() { return values.size(); } + public Inspector entry(int idx) { + if (idx < 0 || idx >= values.size()) { + return invalid; + } + return values.get(idx); + } + public void traverse(ArrayTraverser at) { + int idx = 0; + for (Inspector i: values) { + at.entry(idx++, i); + } + } + public Iterable<Inspector> entries() { + return Collections.unmodifiableList(values); + } + public ArrayValue add(Inspector v) { + if (v == null || !v.valid()) { + throw new IllegalArgumentException("tried to add an invalid value to an array"); + } + values.add(v); + return this; + } + public ArrayValue add(java.lang.String value) { + return add(new Value.StringValue(value)); + } + public ArrayValue add(long value) { + return add(new Value.LongValue(value)); + } + public ArrayValue add(int value) { + return add(new Value.LongValue(value)); + } + public ArrayValue add(double value) { + return add(new Value.DoubleValue(value)); + } + } + static public class ObjectValue extends Value { + private Map<java.lang.String,Inspector> values = new LinkedHashMap<>(); + public boolean valid() { return true; } + public Type type() { return Type.OBJECT; } + public int fieldCount() { return values.size(); } + public Inspector field(java.lang.String name) { + Inspector v = values.get(name); + if (v == null) { + return invalid; + } + return v; + } + public void traverse(ObjectTraverser ot) { + for (Map.Entry<java.lang.String,Inspector> i: values.entrySet()) { + ot.field(i.getKey(), i.getValue()); + } + } + public Iterable<Map.Entry<java.lang.String,Inspector>> fields() { + return Collections.<java.lang.String,Inspector>unmodifiableMap(values).entrySet(); + } + public ObjectValue put(java.lang.String name, Inspector v) { + if (name == null) { + throw new IllegalArgumentException("field name was <null>"); + } + if (v == null || !v.valid()) { + throw new IllegalArgumentException("tried to put an invalid value into an object"); + } + values.put(name, v); + return this; + } + public ObjectValue put(java.lang.String name, java.lang.String value) { + return put(name, new Value.StringValue(value)); + } + public ObjectValue put(java.lang.String name, long value) { + return put(name, new Value.LongValue(value)); + } + public ObjectValue put(java.lang.String name, int value) { + return put(name, new Value.LongValue(value)); + } + public ObjectValue put(java.lang.String name, double value) { + return put(name, new Value.DoubleValue(value)); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/simple/package-info.java b/vespajlib/src/main/java/com/yahoo/data/access/simple/package-info.java new file mode 100644 index 00000000000..730dc508b21 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/simple/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.data.access.simple; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/data/access/slime/SlimeAdapter.java b/vespajlib/src/main/java/com/yahoo/data/access/slime/SlimeAdapter.java new file mode 100644 index 00000000000..adfadfe8bb8 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/slime/SlimeAdapter.java @@ -0,0 +1,156 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access.slime; + + +import java.util.Map; +import java.util.AbstractMap; +import java.util.List; +import java.util.ArrayList; + + +public final class SlimeAdapter implements com.yahoo.data.access.Inspector { + private com.yahoo.slime.Inspector inspector; + public SlimeAdapter(com.yahoo.slime.Inspector inspector) { this.inspector = inspector; } + @Override public boolean equals(Object rhs) { + if (!(rhs instanceof SlimeAdapter)) { + return false; + } + return inspector.equals(((SlimeAdapter)rhs).inspector); + } + @Override public int hashCode() { return inspector.hashCode(); } + @Override public String toString() { return inspector.toString(); } + public com.yahoo.data.access.Inspector inspect() { return this; } + public boolean valid() { return inspector.valid(); } + public com.yahoo.data.access.Type type() { + switch(inspector.type()) { + case NIX: return com.yahoo.data.access.Type.EMPTY; + case BOOL: return com.yahoo.data.access.Type.BOOL; + case LONG: return com.yahoo.data.access.Type.LONG; + case DOUBLE: return com.yahoo.data.access.Type.DOUBLE; + case STRING: return com.yahoo.data.access.Type.STRING; + case DATA: return com.yahoo.data.access.Type.DATA; + case ARRAY: return com.yahoo.data.access.Type.ARRAY; + case OBJECT: return com.yahoo.data.access.Type.OBJECT; + } + return com.yahoo.data.access.Type.EMPTY; + } + private boolean verify(com.yahoo.slime.Type ok_type_a) { + com.yahoo.slime.Type my_type = inspector.type(); + return (valid() && (my_type == ok_type_a)); + } + private boolean verify(com.yahoo.slime.Type ok_type_a, + com.yahoo.slime.Type ok_type_b) + { + com.yahoo.slime.Type my_type = inspector.type(); + return (valid() && (my_type == ok_type_a || my_type == ok_type_b)); + } + private boolean verify(com.yahoo.slime.Type ok_type_a, + com.yahoo.slime.Type ok_type_b, + com.yahoo.slime.Type ok_type_c) + { + com.yahoo.slime.Type my_type = inspector.type(); + return (valid() && (my_type == ok_type_a || my_type == ok_type_b || my_type == ok_type_c)); + } + public int entryCount() { return inspector.entries(); } + public int fieldCount() { return inspector.fields(); } + public boolean asBool() { + if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.BOOL)) { + throw new IllegalStateException("invalid data extraction!"); + } + return inspector.asBool(); + } + public long asLong() { + if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.LONG, com.yahoo.slime.Type.DOUBLE)) { + throw new IllegalStateException("invalid data extraction!"); + } + return inspector.asLong(); + } + public double asDouble() { + if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.DOUBLE, com.yahoo.slime.Type.LONG)) { + throw new IllegalStateException("invalid data extraction!"); + } + return inspector.asDouble(); + } + public String asString() { + if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.STRING)) { + throw new IllegalStateException("invalid data extraction!"); + } + return inspector.asString(); + } + public byte[] asUtf8() { + if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.STRING)) { + throw new IllegalStateException("invalid data extraction!"); + } + return inspector.asUtf8(); + } + public byte[] asData() { + if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.DATA)) { + throw new IllegalStateException("invalid data extraction!"); + } + return inspector.asData(); + } + public boolean asBool(boolean defaultValue) { + if (!verify(com.yahoo.slime.Type.BOOL)) { + return defaultValue; + } + return inspector.asBool(); + } + public long asLong(long defaultValue) { + if (!verify(com.yahoo.slime.Type.LONG, com.yahoo.slime.Type.DOUBLE)) { + return defaultValue; + } + return inspector.asLong(); + } + public double asDouble(double defaultValue) { + if (!verify(com.yahoo.slime.Type.DOUBLE, com.yahoo.slime.Type.LONG)) { + return defaultValue; + } + return inspector.asDouble(); + } + public String asString(String defaultValue) { + if (!verify(com.yahoo.slime.Type.STRING)) { + return defaultValue; + } + return inspector.asString(); + } + public byte[] asUtf8(byte[] defaultValue) { + if (!verify(com.yahoo.slime.Type.STRING)) { + return defaultValue; + } + return inspector.asUtf8(); + } + public byte[] asData(byte[] defaultValue) { + if (!verify(com.yahoo.slime.Type.DATA)) { + return defaultValue; + } + return inspector.asData(); + } + public void traverse(final com.yahoo.data.access.ArrayTraverser at) { + inspector.traverse(new com.yahoo.slime.ArrayTraverser() { + public void entry(int idx, com.yahoo.slime.Inspector inspector) { at.entry(idx, new SlimeAdapter(inspector)); } + }); + } + public void traverse(final com.yahoo.data.access.ObjectTraverser ot) { + inspector.traverse(new com.yahoo.slime.ObjectTraverser() { + public void field(String name, com.yahoo.slime.Inspector inspector) { ot.field(name, new SlimeAdapter(inspector)); } + }); + } + public com.yahoo.data.access.Inspector entry(int idx) { return new SlimeAdapter(inspector.entry(idx)); } + public com.yahoo.data.access.Inspector field(String name) { return new SlimeAdapter(inspector.field(name)); } + public Iterable<com.yahoo.data.access.Inspector> entries() { + final List<com.yahoo.data.access.Inspector> list = new ArrayList<>(); + inspector.traverse(new com.yahoo.slime.ArrayTraverser() { + public void entry(int idx, com.yahoo.slime.Inspector inspector) { list.add(new SlimeAdapter(inspector)); } + }); + return list; + } + public Iterable<Map.Entry<String,com.yahoo.data.access.Inspector>> fields() { + final List<Map.Entry<String,com.yahoo.data.access.Inspector>> list = new ArrayList<>(); + inspector.traverse(new com.yahoo.slime.ObjectTraverser() { + public void field(String name, com.yahoo.slime.Inspector inspector) { + list.add(new AbstractMap.SimpleImmutableEntry<String,com.yahoo.data.access.Inspector>(name, new SlimeAdapter(inspector))); + } + }); + return list; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/data/access/slime/package-info.java b/vespajlib/src/main/java/com/yahoo/data/access/slime/package-info.java new file mode 100644 index 00000000000..bf6ae26baee --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/slime/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.data.access.slime; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/data/inspect/slime/.gitignore b/vespajlib/src/main/java/com/yahoo/data/inspect/slime/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/inspect/slime/.gitignore diff --git a/vespajlib/src/main/java/com/yahoo/errorhandling/Results.java b/vespajlib/src/main/java/com/yahoo/errorhandling/Results.java new file mode 100644 index 00000000000..310f679c883 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/errorhandling/Results.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.errorhandling; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * @author tonytv + */ +public class Results<DATA, ERROR> { + private final List<DATA> data; + private final List<ERROR> errors; + + public Results(List<DATA> data, List<ERROR> errors) { + this.data = ImmutableList.copyOf(data); + this.errors = ImmutableList.copyOf(errors); + } + + public boolean hasErrors() { + return !errors.isEmpty(); + } + + public List<DATA> data() { + return data; + } + + public List<ERROR> errors() { + return errors; + } + + public static class Builder<DATA, ERROR> { + private final List<DATA> data = new ArrayList<>(); + private final List<ERROR> errors = new ArrayList<>(); + + public void addData(DATA d) { + data.add(d); + } + + public void addAllData(Collection<? extends DATA> d) { + data.addAll(d); + } + + public void addError(ERROR e) { + errors.add(e); + } + + public void addAllErrors(Collection<? extends ERROR> e) { + errors.addAll(e); + } + + public Results<DATA, ERROR> build() { + return new Results<>(data, errors); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/errorhandling/package-info.java b/vespajlib/src/main/java/com/yahoo/errorhandling/package-info.java new file mode 100644 index 00000000000..4d07d20053d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/errorhandling/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.errorhandling; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/geo/BoundingBoxParser.java b/vespajlib/src/main/java/com/yahoo/geo/BoundingBoxParser.java new file mode 100644 index 00000000000..001386cd4b0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/geo/BoundingBoxParser.java @@ -0,0 +1,147 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.geo; + +import com.yahoo.text.DoubleParser; + + +/** + * Class for parsing a bounding box in text format: + * "n=37.44899,s=37.3323,e=-121.98241,w=-122.06566" + * + * <pre> + * Input from: + * http://gws.maps.yahoo.com/findlocation?q=sunnyvale,ca&amp;flags=X + * which gives this format: + * <boundingbox> + * <north>37.44899</north><south>37.3323</south><east>-121.98241</east><west>-122.06566</west> + * </boundingbox> + * it's also easy to use the geoplanet bounding box + * <boundingBox> + * <southWest> + * <latitude>40.183868</latitude> + * <longitude>-74.819519</longitude> + * </southWest> + * <northEast> + * <latitude>40.248291</latitude> + * <longitude>-74.728798</longitude> + * </northEast> + * </boundingBox> + * can be input as: + * s=40.183868,w=-74.819519,n=40.248291,e=-74.728798 + * </pre> + * + * @author Arne J + */ +public class BoundingBoxParser { + + // return variables + public double n = 0.0; + public double s = 0.0; + public double e = 0.0; + public double w = 0.0; + + /** + * parse the given string as a bounding box and return a parser object with parsed coordinates in member variables + * @throws IllegalArgumentException if the input is malformed in any way + **/ + public BoundingBoxParser(String bb) { + this.parseString = bb; + this.len = bb.length(); + parse(); + } + + private final String parseString; + private final int len; + private int pos = 0; + + private char getNextChar() throws IllegalArgumentException { + if (pos == len) { + pos++; + return 0; + } else if (pos > len) { + throw new IllegalArgumentException("position after end of string"); + } else { + return parseString.charAt(pos++); + } + } + + private boolean isCompassDirection(char ch) { + return (ch == 'N' || ch == 'S' || ch == 'E' || ch == 'W' || + ch == 'n' || ch == 's' || ch == 'e' || ch == 'w'); + } + + private int lastNumStartPos = 0; + + private char nsew = 0; + private boolean doneN = false; + private boolean doneS = false; + private boolean doneE = false; + private boolean doneW = false; + + private void parse() { + do { + char ch = getNextChar(); + if (isCompassDirection(ch) && nsew == 0) { + if (ch == 'n' || ch =='N') { + nsew = 'n'; + } else if (ch == 's' || ch == 'S') { + nsew = 's'; + } else if (ch == 'e' || ch == 'E') { + nsew = 'e'; + } else if (ch == 'w' || ch == 'W') { + nsew = 'w'; + } + lastNumStartPos = 0; + } + if (ch == '=' || ch == ':') { + if (nsew != 0) { + lastNumStartPos = pos; + } + } + if (ch == ',' || ch == 0 || ch == ' ') { + if (nsew != 0 && lastNumStartPos > 0) { + String sub = parseString.substring(lastNumStartPos, pos-1); + try { + double v = DoubleParser.parse(sub); + if (nsew == 'n') { + if (doneN) { + throw new IllegalArgumentException("multiple limits for 'n' boundary"); + } + n = v; + doneN = true; + } else if (nsew == 's') { + if (doneS) { + throw new IllegalArgumentException("multiple limits for 's' boundary"); + } + s = v; + doneS = true; + } else if (nsew == 'e') { + if (doneE) { + throw new IllegalArgumentException("multiple limits for 'e' boundary"); + } + e = v; + doneE = true; + } else if (nsew == 'w') { + if (doneW) { + throw new IllegalArgumentException("multiple limits for 'w' boundary"); + } + w = v; + doneW = true; + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Could not parse "+nsew+" limit '"+sub+"' as a number"); + } + nsew = 0; + } + } + } while (pos <= len); + + if (doneN && doneS && doneE && doneW) { + return; + } else { + throw new IllegalArgumentException("Missing bounding box limits, n="+doneN+" s="+doneS+" e="+doneE+" w="+doneW); + } + } + +} + diff --git a/vespajlib/src/main/java/com/yahoo/geo/DegreesParser.java b/vespajlib/src/main/java/com/yahoo/geo/DegreesParser.java new file mode 100644 index 00000000000..40398a2b1a0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/geo/DegreesParser.java @@ -0,0 +1,284 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.geo; + +/** + * utility for parsing geographical coordinates + * + * @author arnej27959 + **/ +public class DegreesParser { + /** + * the parsed latitude (degrees north if positive) + **/ + public double latitude = 0; + /** + * the parsed longitude (degrees east if positive) + **/ + public double longitude = 0; + + private boolean isDigit(char ch) { + return (ch >= '0' && ch <= '9'); + } + private boolean isCompassDirection(char ch) { + return (ch == 'N' || ch == 'S' || ch == 'E' || ch == 'W'); + } + + private String parseString = null; + private int len = 0; + private int pos = 0; + + private char getNextChar() throws IllegalArgumentException { + if (pos == len) { + pos++; + return 0; + } else if (pos > len) { + throw new IllegalArgumentException("position after end of string"); + } else { + return parseString.charAt(pos++); + } + } + + /** + * Parse the given string. + * + * The string must contain both a latitude and a longitude, + * separated by a semicolon, in any order. A latitude must + * contain "N" or "S" and a number signifying degrees north or + * south. A longitude must contain "E" or "W" and a number + * signifying degrees east or west. No signs or spaces are + * allowed. + * <br> + * Fractional degrees are recommended as the main input format, + * but degrees plus fractional minutes may be used for testing. + * You can use the degree sign (U+00B0 as seen in unicode at + * http://www.unicode.org/charts/PDF/U0080.pdf) to separate + * degrees from minutes, put the direction (NSEW) between as a + * separator, or use a small letter 'o' as a replacement for the + * degrees sign. + * <br> + * Some valid input formats: <br> + * "N37.416383;W122.024683" → Sunnyvale <br> + * "37N24.983;122W01.481" → same <br> + * "N37\u00B024.983;W122\u00B001.481" → same <br> + * "N63.418417;E10.433033" → Trondheim <br> + * "N63o25.105;E10o25.982" → same <br> + * "E10o25.982;N63o25.105" → same <br> + * "N63.418417;E10.433033" → same <br> + * "63N25.105;10E25.982" → same <br> + * @param latandlong Latitude and longitude separated by semicolon. + * + **/ + public DegreesParser(String latandlong) throws IllegalArgumentException { + this.parseString = latandlong; + this.len = parseString.length(); + + char ch = getNextChar(); + + boolean latSet = false; + boolean longSet = false; + + double degrees = 0.0; + double minutes = 0.0; + double seconds = 0.0; + boolean degSet = false; + boolean minSet = false; + boolean secSet = false; + boolean dirSet = false; + boolean foundDot = false; + boolean foundDigits = false; + + boolean findingLatitude = false; + boolean findingLongitude = false; + + double sign = 0.0; + + int lastpos = -1; + + do { + boolean valid = false; + if (pos == lastpos) { + throw new RuntimeException("internal logic error at '"+parseString+"' pos:"+pos); + } else { + lastpos = pos; + } + + // first, see if we can find some number + double accum = 0.0; + + if (isDigit(ch) || ch == '.') { + valid = true; + if (foundDigits) { + throw new IllegalArgumentException("found digits after not consuming previous digits"); + } + double divider = 1.0; + foundDot = false; + while (isDigit(ch)) { + foundDigits = true; + accum *= 10; + accum += (ch - '0'); + ch = getNextChar(); + } + if (ch == '.') { + foundDot = true; + ch = getNextChar(); + while (isDigit(ch)) { + foundDigits = true; + accum *= 10; + accum += (ch - '0'); + divider *= 10; + ch = getNextChar(); + } + } + if (!foundDigits) { + throw new IllegalArgumentException("just a . is not a valid number"); + } + accum /= divider; + } + + // next, did we find a separator after the number? + // degree sign is a separator after degrees, before minutes + if (ch == '\u00B0' || ch == 'o') { + valid = true; + if (degSet) { + throw new IllegalArgumentException("degrees sign only valid just after degrees"); + } + if (!foundDigits) { + throw new IllegalArgumentException("must have number before degrees sign"); + } + if (foundDot) { + throw new IllegalArgumentException("cannot have fractional degrees before degrees sign"); + } + ch = getNextChar(); + } + // apostrophe is a separator after minutes, before seconds + if (ch == '\'') { + if (minSet || !degSet || !foundDigits) { + throw new IllegalArgumentException("minutes sign only valid just after minutes"); + } + if (foundDot) { + throw new IllegalArgumentException("cannot have fractional minutes before minutes sign"); + } + ch = getNextChar(); + } + + // if we found some number, assign it into the next unset variable + if (foundDigits) { + valid = true; + if (degSet) { + if (minSet) { + if (secSet) { + throw new IllegalArgumentException("extra number after full field"); + } else { + seconds = accum; + secSet = true; + } + } else { + minutes = accum; + minSet = true; + if (foundDot) { + secSet = true; + } + } + } else { + degrees = accum; + degSet = true; + if (foundDot) { + minSet = true; + secSet = true; + } + } + foundDot = false; + foundDigits = false; + } + + // there needs to be a direction (NSEW) somewhere, too + if (isCompassDirection(ch)) { + valid = true; + if (dirSet) { + throw new IllegalArgumentException("already set direction once, cannot add direction: "+ch); + } + dirSet = true; + if (ch == 'S' || ch == 'W') { + sign = -1; + } else { + sign = 1; + } + if (ch == 'E' || ch == 'W') { + findingLongitude = true; + } else { + findingLatitude = true; + } + ch = getNextChar(); + } + + // lastly, did we find the end-of-string or a separator between lat and long? + if (ch == 0 || ch == ';' || ch == ' ') { + valid = true; + + if (!dirSet) { + throw new IllegalArgumentException("end of field without any compass direction seen"); + } + if (!degSet) { + throw new IllegalArgumentException("end of field without any number seen"); + } + degrees += minutes / 60.0; + degrees += seconds / 3600.0; + degrees *= sign; + + if (findingLatitude) { + if (latSet) { + throw new IllegalArgumentException("found latitude (N or S) twice"); + } + if (degrees < -90.0 || degrees > 90.0) { + throw new IllegalArgumentException("out of range [-90,+90]: "+degrees); + } + latitude = degrees; + latSet = true; + } else if (findingLongitude) { + if (longSet) { + throw new IllegalArgumentException("found longitude (E or W) twice"); + } + if (degrees < -180.0 || degrees > 180.0) { + throw new IllegalArgumentException("out of range [-180,+180]: "+degrees); + } + longitude = degrees; + longSet = true; + } else { + throw new IllegalArgumentException("no direction found"); + } + // reset + degrees = 0.0; + minutes = 0.0; + seconds = 0.0; + degSet = false; + minSet = false; + secSet = false; + dirSet = false; + foundDot = false; + foundDigits = false; + findingLatitude = false; + findingLongitude = false; + sign = 0.0; + + if (ch == 0) { + break; + } else { + ch = getNextChar(); + } + } + + if (!valid) { + throw new IllegalArgumentException("invalid character: "+ch); + } + + } while (ch != 0); + + if (!latSet) { + throw new IllegalArgumentException("missing latitude"); + } + if (!longSet) { + throw new IllegalArgumentException("missing longitude"); + } + // everything parsed OK + } +} diff --git a/vespajlib/src/main/java/com/yahoo/geo/ZCurve.java b/vespajlib/src/main/java/com/yahoo/geo/ZCurve.java new file mode 100644 index 00000000000..3e24316363e --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/geo/ZCurve.java @@ -0,0 +1,201 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.geo; + +/** + * Contains utility methods for a Z-curve (Morton-order) encoder and + * decoder. + * + * @author gjoranv + */ +public class ZCurve { + /** + * Encode two 32 bit integers by bit-interleaving them into one 64 bit + * integer value. The x-direction owns the least significant bit (bit + * 0). Both x and y can have negative values. + * + * <p> + * This is a time-efficient implementation. In the first step, the input + * value is split in two blocks, one containing the most significant bits, and + * the other containing the least significant bits. The most significant block + * is then shifted left for as many bits it contains. For each following step + * every block from the previous step is split in the same manner, with a + * least and most significant block, and the most significant blocks are shifted + * left for as many bits they contain (half the number from the previous step). + * This continues until each block has only one bit. + * + * <p> + * This algorithm works by placing the LSB of all blocks in the correct position + * after the bit-shifting is done in each step. This algorithm is quite similar + * to computing the Hamming Weight (or population count) of a bit + * string, see http://en.wikipedia.org/wiki/Hamming_weight. + * + * <p> + * Efficiency considerations: The encoding operations in this method + * should require 42 cpu operations, of which many can be executed + * in parallell. Practical experiments show that one call takes ~15 ns + * on a 64-bit Intel Xeon processor @2.33GHz, or 35 cycles. This gives + * an efficiency gain of just ~17% due to the CPUs ability to process + * parallell instructions, compared to ~50% for the slow method. + * But still it is 5 times faster. + * + * @param x x value + * @param y y value + * @return The bit-interleaved long containing x and y. + */ + public static long encode(int x, int y) { + long xl = (long)x; + long yl = (long)y; + + long rx = ((xl & 0x00000000ffff0000L) << 16) | (xl & 0x000000000000ffffL); + long ry = ((yl & 0x00000000ffff0000L) << 16) | (yl & 0x000000000000ffffL); + + rx = ((rx & 0xff00ff00ff00ff00L) << 8) | (rx & 0x00ff00ff00ff00ffL); + ry = ((ry & 0xff00ff00ff00ff00L) << 8) | (ry & 0x00ff00ff00ff00ffL); + + rx = ((rx & 0xf0f0f0f0f0f0f0f0L) << 4) | (rx & 0x0f0f0f0f0f0f0f0fL); + ry = ((ry & 0xf0f0f0f0f0f0f0f0L) << 4) | (ry & 0x0f0f0f0f0f0f0f0fL); + + rx = ((rx & 0xccccccccccccccccL) << 2) | (rx & 0x3333333333333333L); + ry = ((ry & 0xccccccccccccccccL) << 2) | (ry & 0x3333333333333333L); + + rx = ((rx & 0xaaaaaaaaaaaaaaaaL) << 1) | (rx & 0x5555555555555555L); + ry = ((ry & 0xaaaaaaaaaaaaaaaaL) << 1) | (ry & 0x5555555555555555L); + + return (rx | (ry << 1)); + } + + + /** + * Decode a z-value into the original two integers. Returns an + * array of two Integers, x and y in indices 0 and 1 respectively. + * + * @param z The bit-interleaved long containing x and y. + * @return Array of two Integers, x and y. + */ + public static int[] decode(long z) { + int[] xy = new int[2]; + + long xl = z & 0x5555555555555555L; + long yl = z & 0xaaaaaaaaaaaaaaaaL; + + xl = ((xl & 0xccccccccccccccccL) >> 1) | (xl & 0x3333333333333333L); + yl = ((yl & 0xccccccccccccccccL) >> 1) | (yl & 0x3333333333333333L); + + xl = ((xl & 0xf0f0f0f0f0f0f0f0L) >> 2) | (xl & 0x0f0f0f0f0f0f0f0fL); + yl = ((yl & 0xf0f0f0f0f0f0f0f0L) >> 2) | (yl & 0x0f0f0f0f0f0f0f0fL); + + xl = ((xl & 0xff00ff00ff00ff00L) >> 4) | (xl & 0x00ff00ff00ff00ffL); + yl = ((yl & 0xff00ff00ff00ff00L) >> 4) | (yl & 0x00ff00ff00ff00ffL); + + xl = ((xl & 0xffff0000ffff0000L) >> 8) | (xl & 0x0000ffff0000ffffL); + yl = ((yl & 0xffff0000ffff0000L) >> 8) | (yl & 0x0000ffff0000ffffL); + + xl = ((xl & 0xffffffff00000000L) >> 16) | (xl & 0x00000000ffffffffL); + yl = ((yl & 0xffffffff00000000L) >> 16) | (yl & 0x00000000ffffffffL); + + xy[0] = (int)xl; + xy[1] = (int)(yl >> 1); + return xy; + } + + + + /** + * Encode two integers by bit-interleaving them into one Long + * value. The x-direction owns the least significant bit (bit + * 0). Both x and y can have negative values. + * <br> + * Efficiency considerations: If Java compiles and runs this code + * as efficiently as would be the case with a good c-compiler, it + * should require 5 cpu operations per bit with optimal usage of + * the CPUs registers on a 64 bit processor(2 bit-shifts, 1 OR, 1 + * AND, and 1 conditional jump for the for-loop). This would + * correspond to 320+ cycles with no parallell execution. + * Practical experiments show that one call takes ~75 ns on a + * 64-bit Intel Xeon processor @2.33GHz, or 175 cycles. This gives + * an efficiency gain of ~50% due to the CPUs ability to perform + * several instructions in one clock-cycle. Here, it is probably the + * bit-shifts that can be done independently of the AND an OR + * operations, which must be done in sequence. + * + * @param x x value + * @param y y value + * @return The bit-interleaved long containing x and y. + */ + public static long encode_slow(int x, int y) { + long z = 0L; + long xl = (long)x; + long yl = (long)y; + + long mask = 1L; + for (int i=0; i<32; i++) { + long bit = (xl << i) & mask; + z |= bit; + //System.out.println("xs "+ i + ": " + toFullBinaryString(xl << i)); + //System.out.println("m "+ i + ": " + toFullBinaryString(mask)); + //System.out.println("bit "+ i + ": " + toFullBinaryString(bit)); + //System.out.println("z "+ i + ": " + toFullBinaryString(z)); + mask = mask << 2; + } + + mask = 2L; + for (int i=1; i<=32; i++) { + long bit = (yl << i) & mask; + z |= bit; + mask = mask << 2; + } + return z; + } + + /** + * Decode a z-value into the original two integers. Returns an + * array of two Integers, x and y in indices 0 and 1 respectively. + * + * @param z The bit-interleaved long containing x and y. + * @return Array of two Integers, x and y. + */ + public static int[] decode_slow(long z) { + int[] xy = new int[2]; + long xl = 0L; + long yl = 0L; + + long mask = 1L; + for (int i=0; i<32; i++) { + long bit = (z >> i) & mask; + xl |= bit; + //System.out.println("bits : m lm lm lm lm lm lm lm l"); + //System.out.println("zs "+ i + ": " + toFullBinaryString(z >> i)); + //System.out.println("m "+ i + ": " + toFullBinaryString(mask)); + //System.out.println("bit "+ i + ": " + toFullBinaryString(bit)); + //System.out.println("xl "+ i + ": " + toFullBinaryString(xl)); + mask = mask << 1; + } + + mask = 1L; + for (int i=1; i<=32; i++) { + long bit = (z >> i) & mask; + yl |= bit; + mask = mask << 1; + } + xy[0] = (int)xl; + xy[1] = (int)yl; + return xy; + } + + /** + * Debugging utility that returns a long value as binary string + * including the leading zeroes. + */ + public static String toFullBinaryString(long l) { + StringBuilder s = new StringBuilder(64); + for (int i=0; i<Long.numberOfLeadingZeros(l); i++) { + s.append('0'); + } + if (l == 0) { + s.deleteCharAt(0); + } + s.append(Long.toBinaryString(l)); + return s.toString(); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/geo/package-info.java b/vespajlib/src/main/java/com/yahoo/geo/package-info.java new file mode 100644 index 00000000000..2e515809012 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/geo/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.geo; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/io/AbstractByteWriter.java b/vespajlib/src/main/java/com/yahoo/io/AbstractByteWriter.java new file mode 100644 index 00000000000..65016ff5384 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/AbstractByteWriter.java @@ -0,0 +1,129 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import com.yahoo.text.GenericWriter; +import com.yahoo.text.AbstractUtf8Array; +import com.yahoo.text.Utf8; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; + +/** + * Base class for writers needing to accept binary data. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public abstract class AbstractByteWriter extends GenericWriter implements + WritableByteTransmitter { + + protected final CharsetEncoder encoder; + protected final BufferChain buffer; + protected final CharBuffer charBuffer = CharBuffer.allocate(2); + + protected AbstractByteWriter(final CharsetEncoder encoder) { + this.encoder = encoder; + buffer = new BufferChain(this); + } + + /** Returns the charset this encodes its output in */ + public Charset getEncoding() { + return encoder.charset(); + } + + @Override + public GenericWriter write(AbstractUtf8Array v) throws java.io.IOException { + buffer.append(v); + return this; + } + + @Override + public GenericWriter write(long v) throws java.io.IOException { + buffer.append(Utf8.toAsciiBytes(v)); + return this; + } + + /** + * Do note, if writing the first character of a surrogate pair, the next + * character written must be the second part of the pair. If this is not the + * case, the surrogate will be omitted from output. + */ + @Override + public void write(int v) throws java.io.IOException { + char c = (char) v; + if (Character.isSurrogate(c)) { + charBuffer.append(c); + if (!charBuffer.hasRemaining()) { + charBuffer.flip(); + buffer.append(charBuffer, encoder); + charBuffer.clear(); + } + } else { + charBuffer.clear(); // to nuke misplaced singleton surrogates + charBuffer.append((char) v); + charBuffer.flip(); + buffer.append(charBuffer, encoder); + charBuffer.clear(); + } + } + + @Override + public GenericWriter write(double v) throws java.io.IOException { + buffer.append(Utf8.toBytes(String.valueOf(v))); + return this; + } + @Override + public GenericWriter write(float v) throws java.io.IOException { + buffer.append(Utf8.toBytes(String.valueOf(v))); + return this; + } + + @Override + public GenericWriter write(short v) throws java.io.IOException { + buffer.append(Utf8.toAsciiBytes(v)); + return this; + } + @Override + public GenericWriter write(boolean v) throws java.io.IOException { + buffer.append(Utf8.toAsciiBytes(v)); + return this; + } + + @Override + public void write(final char[] cbuf, final int offset, final int len) + throws java.io.IOException { + final CharBuffer in = CharBuffer.wrap(cbuf, offset, len); + buffer.append(in, encoder); + } + + public void append(final ByteBuffer alreadyEncoded) + throws java.io.IOException { + buffer.append(alreadyEncoded); + } + + public void append(final byte alreadyEncoded) throws java.io.IOException { + buffer.append(alreadyEncoded); + } + + public void append(final byte[] alreadyEncoded) throws java.io.IOException { + buffer.append(alreadyEncoded); + } + + public void append(final byte[] alreadyEncoded, final int offset, + final int length) throws java.io.IOException { + buffer.append(alreadyEncoded, offset, length); + } + + /** + * Return the number of bytes this writer will produce for the underlying + * layer. That is, it sums the length of the raw bytes received and the + * number of bytes in the written strings after encoding. + * + * @return the number of bytes appended to this writer + */ + public long appended() { + return buffer.appended(); + } +}
\ No newline at end of file diff --git a/vespajlib/src/main/java/com/yahoo/io/Acceptor.java b/vespajlib/src/main/java/com/yahoo/io/Acceptor.java new file mode 100644 index 00000000000..62f19c186f6 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/Acceptor.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.io.IOException; +import java.nio.channels.SocketChannel; +import java.nio.channels.ServerSocketChannel; +import java.util.logging.Logger; +import java.util.logging.Level; +import java.net.InetSocketAddress; + + +/** + * Class for accepting new connections in separate thread. + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * + */ +public class Acceptor extends Thread { + private static Logger log = Logger.getLogger(Acceptor.class.getName()); + + private int port; + ServerSocketChannel socket; + private Listener listener; + private boolean initialized = false; + private ConnectionFactory factory; + private FatalErrorHandler fatalErrorHandler; + + public Acceptor(Listener listener, ConnectionFactory factory, int port) { + super("Acceptor-" + listener.getName() + "-" + port); + this.listener = listener; + this.factory = factory; + this.port = port; + } + + public Acceptor listen() throws IOException { + socket = ServerSocketChannel.open(); + socket.configureBlocking(true); + socket.socket().setReuseAddress(true); + socket.socket().bind(new InetSocketAddress(port)); + initialized = true; + return this; + } + + /** + * Register a handler for fatal errors. + * + * @param f The FatalErrorHandler instance to be registered + */ + public synchronized void setFatalErrorHandler(FatalErrorHandler f) { + fatalErrorHandler = f; + } + + public void run() { + try { + log.fine("Acceptor thread started"); + if (!initialized) { + log.severe("Acceptor was not initialized. aborting"); + return; + } + + while (!isInterrupted()) { + SocketChannel c = null; // hush jikes + + try { + c = socket.accept(); + c.configureBlocking(false); + listener.addNewConnection(factory.newConnection(c, listener)); + } catch (java.nio.channels.IllegalBlockingModeException e) { + log.log(Level.SEVERE, "Unable to set nonblocking", e); + try { + if (c != null) { + c.close(); + } + } catch (IOException ee) {} + } catch (IOException e) { + log.log(Level.WARNING, + "Error accepting connection on port=" + port, e); + try { + if (c != null) { + c.close(); + } + } catch (IOException ee) {} + } + } + } catch (Throwable t) { + if (fatalErrorHandler != null) { + fatalErrorHandler.handle(t, null); + } + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/Blob.java b/vespajlib/src/main/java/com/yahoo/io/Blob.java new file mode 100644 index 00000000000..808371e7b58 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/Blob.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.ByteBuffer; + +/** + * A Blob contains opaque data in the form of a byte array. + **/ +public class Blob { + + /** + * Shared empty array. + **/ + private static byte[] empty = new byte[0]; + + /** + * Internal data, will never be 'null'. + **/ + private byte[] data; + + /** + * Create a Blob containing an empty byte array. + **/ + public Blob() { + data = empty; + } + + /** + * Create a Blob containg a copy of a subset of the given byte + * array. + **/ + public Blob(byte[] src, int offset, int length) { + data = new byte[length]; + System.arraycopy(src, offset, data, 0, length); + } + + /** + * Create a Blob containing a copy of the given byte array. + **/ + public Blob(byte[] src) { + this(src, 0, src.length); + } + + /** + * Create a Blob containing a copy of the data held by the given + * blob. + **/ + public Blob(Blob src) { + this(src.data); + } + + /** + * Create a Blob containing a number of bytes read from a byte + * buffer. + **/ + public Blob(ByteBuffer src, int length) { + data = new byte[length]; + src.get(data); + } + + /** + * Create a Blob containing all bytes that could be read from a + * byte buffer. + **/ + public Blob(ByteBuffer src) { + this(src, src.remaining()); + } + + /** + * Obtain the internal data held by this object. + * + * @return internal data + **/ + public byte[] get() { + return data; + } + + /** + * Write the data held by this object to the given byte buffer. + * + * @param dst where to write the contained data + **/ + public void write(ByteBuffer dst) { + dst.put(data); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/BufferChain.java b/vespajlib/src/main/java/com/yahoo/io/BufferChain.java new file mode 100644 index 00000000000..fc9fadc64ca --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/BufferChain.java @@ -0,0 +1,147 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import com.yahoo.text.AbstractUtf8Array; + +import java.io.IOException; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.util.ArrayList; +import java.util.List; + +/** + * Data store for AbstractByteWriter. Tested in unit tests for ByteWriter. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class BufferChain { + // refer to the revision history of ByteWriter for more information about + // the reasons behind the sizing of BUFFERSIZE, WATERMARK and MAXBUFFERS + static final int BUFFERSIZE = 4096; + static final int WATERMARK = 1024; + static final int MAXBUFFERS = 50; + static { + //noinspection ConstantConditions + assert BUFFERSIZE > WATERMARK; + } + private final List<ByteBuffer> buffers = new ArrayList<>(); + private final WritableByteTransmitter endpoint; + private ByteBuffer current = ByteBuffer.allocate(BUFFERSIZE); + private long appended = 0L; + + public BufferChain(final WritableByteTransmitter endpoint) { + this.endpoint = endpoint; + } + + public void append(final byte b) throws IOException { + makeRoom(1); + current.put(b); + } + private final boolean shouldCopy(int length) { + return (length < WATERMARK); + } + private final void makeRoom(int length) throws IOException { + if (current.remaining() < length) { + scratch(); + } + } + public void append(AbstractUtf8Array v) throws IOException { + final int length = v.getByteLength(); + if (shouldCopy(length)) { + makeRoom(length); + v.writeTo(current); + } else { + append(v.wrap()); + } + } + public void append(final byte[] alreadyEncoded) throws java.io.IOException { + if (alreadyEncoded.length > 0) { + append(alreadyEncoded, 0, alreadyEncoded.length); + } + } + + public void append(final byte[] alreadyEncoded, final int offset, final int length) throws java.io.IOException { + if (shouldCopy(length)) { + makeRoom(length); + current.put(alreadyEncoded, offset, length); + } else { + append(ByteBuffer.wrap(alreadyEncoded, offset, length)); + } + } + + public void append(final ByteBuffer alreadyEncoded) throws java.io.IOException { + if (alreadyEncoded.remaining() == 0) { + return; + } + final int length = alreadyEncoded.limit() - alreadyEncoded.position(); + if (shouldCopy(length)) { + makeRoom(length); + current.put(alreadyEncoded); + } else { + scratch(); + add(alreadyEncoded); + } + } + private final void add(final ByteBuffer buf) { + buffers.add(buf); + appended += buf.limit(); + } + + public void append(final CharBuffer toEncode, final CharsetEncoder encoder) + throws java.io.IOException { + CoderResult overflow; + do { + overflow = encoder.encode(toEncode, current, true); + if (overflow.isOverflow()) { + scratch(); + } else if (overflow.isError()) { + try { + toEncode.get(); + } catch (final BufferUnderflowException e) { + // Give up if we can't discard some presumptively malformed + // or unmappable data + break; + } + } + } while (!overflow.isUnderflow()); + } + + private void scratch() throws java.io.IOException { + if (!possibleFlush() && current.position() != 0) { + current.flip(); + add(current); + current = ByteBuffer.allocate(BUFFERSIZE); + } + } + + private boolean possibleFlush() throws java.io.IOException { + if (buffers.size() > MAXBUFFERS) { + flush(); + return true; + } + return false; + } + + public void flush() throws IOException { + for (final ByteBuffer b : buffers) { + endpoint.send(b); + } + buffers.clear(); + if (current.position() > 0) { + current.flip(); + appended += current.limit(); + endpoint.send(current); + current = ByteBuffer.allocate(BUFFERSIZE); + } + } + + /** + * @return number of bytes written to this buffer + */ + public long appended() { + return appended + current.position(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java b/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java new file mode 100644 index 00000000000..8345f97f291 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import com.yahoo.text.Utf8; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.CharsetEncoder; + +/** + * A buffered writer which accepts byte arrays in addition to character arrays. + * + * @author <a href="mailt:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ByteWriter extends AbstractByteWriter { + private final OutputStream stream; + + public ByteWriter(final OutputStream stream, final CharsetEncoder encoder) { + super(encoder); + this.stream = stream; + } + public ByteWriter(final OutputStream stream) { + super(Utf8.getNewEncoder()); + this.stream = stream; + } + + @Override + public void send(final ByteBuffer b) throws IOException { + // we know from how BufferChain works we have a backing array + stream.write(b.array(), b.position() + b.arrayOffset(), b.limit() - b.position()); + } + + @Override + public void close() throws java.io.IOException { + buffer.flush(); + // Unit tests in prelude depends on the stream _not_ being flushed, it + // is necessary for Jetty to write content length headers, it seems. + // stream.flush(); + stream.close(); + } + + @Override + public void flush() throws IOException { + buffer.flush(); + // Unit tests in prelude depends on the stream _not_ being flushed, it + // is necessary for Jetty to write content length headers, it seems. + // stream.flush(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/Connection.java b/vespajlib/src/main/java/com/yahoo/io/Connection.java new file mode 100644 index 00000000000..18b91ff3b42 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/Connection.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.channels.SocketChannel; +import java.io.IOException; + + +/** + * Connection interface is the abstraction for an operating + * asynchronous NIO connection. One is created for each + * "accept" on the channel. + * + * @author <a href="mailto:travisb@yahoo-inc.com">Bob Travis</a> + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * + */ +public interface Connection { + + /** + * called when the channel can accept a write, and is + * enabled for writing + */ + public void write() throws IOException; + + /** + * Called when the channel can accept a read, and is + * enabled for reading + */ + public void read() throws IOException; + + /** + * Called when the channel should be closed. + */ + public void close() throws IOException; + + /** + * Called when a socket has completed connecting to its + * destination. (Asynchronous connect) + */ + public void connect() throws IOException; + + /** + * called to get the correct initial SelectionKey operation + * flags for the next Select cycle, for this channel + */ + public int selectOps(); + + /** + * Called to get the SocketChannel for this Connection. + * + * @return Returns the SocketChannel representing this connection + */ + public SocketChannel socketChannel(); +} + diff --git a/vespajlib/src/main/java/com/yahoo/io/ConnectionFactory.java b/vespajlib/src/main/java/com/yahoo/io/ConnectionFactory.java new file mode 100644 index 00000000000..a0bdedfba7d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/ConnectionFactory.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00F8rn Borud</a> + */ + +import java.nio.channels.SocketChannel; + + +/** + * A factory interface used for associating SocketChannel and Listener + * information with the application's Connection object. + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author <a href="mailto:travisb@yahoo-inc.com">Bob Travis</a> + */ +public interface ConnectionFactory { + public Connection newConnection(SocketChannel channel, Listener listener); +} diff --git a/vespajlib/src/main/java/com/yahoo/io/FatalErrorHandler.java b/vespajlib/src/main/java/com/yahoo/io/FatalErrorHandler.java new file mode 100644 index 00000000000..11b7d1cbc66 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/FatalErrorHandler.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* -*- c-basic-offset: 4 -*- + * + * $Id$ + * + */ +package com.yahoo.io; + + +import java.util.logging.Logger; +import java.util.logging.Level; + + +/** + * What to do if a fatal condition happens in an IO component. + * + * <P> + * TODO: We need to re-think this design a bit. First off, we + * probably need to make the interface an abstract class + * or a pure interface type. Second we provide a few + * default implementations which are named after what policy + * they implement -- like SystemExitOnError etc. Also, + * runnables that have fatal error handling capability should + * probably implement a standard interface for get/set etc. + * Also, we should encourage application authors to provide + * their own, application specific error handlers rather than + * relying on the default. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ + +public class FatalErrorHandler { + protected static final Logger log = Logger.getLogger(FatalErrorHandler.class.getName()); + + /** + * Do something reasonable when a an Error occurs. + * + * Override this to change behavior. Default behavior is to log + * the error, then exit. + * + * @param t The Throwable causing the handler to be activated. + * @param context The object calling the handler. + */ + public void handle(Throwable t, Object context) { + try { + log.log(Level.SEVERE, "Exiting due to error", t); + } finally { + Runtime.getRuntime().halt(1); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java b/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java new file mode 100644 index 00000000000..85b249432d4 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java @@ -0,0 +1,186 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.channels.WritableByteChannel; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Stack; +import java.util.LinkedList; +import java.util.Iterator; +import java.nio.ByteBuffer; + + +/** + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class GrowableBufferOutputStream extends OutputStream { +// private static final int MINIMUM_BUFFERSIZE = (64 * 1024); + private ByteBuffer lastBuffer; + private ByteBuffer directBuffer; + private LinkedList<ByteBuffer> bufferList = new LinkedList<>(); + private Stack<ByteBuffer> recycledBuffers = new Stack<>(); + + private int bufferSize; + private int maxBuffers; + + public GrowableBufferOutputStream(int bufferSize, int maxBuffers) { + this.bufferSize = bufferSize; + this.maxBuffers = maxBuffers; + lastBuffer = ByteBuffer.allocate(bufferSize); + directBuffer = ByteBuffer.allocateDirect(bufferSize); + } + + @Override + public void write(byte[] cbuf, int off, int len) throws IOException { + if (lastBuffer.remaining() >= len) { + lastBuffer.put(cbuf, off, len); + return; + } + + int residue = len; + + while (residue > 0) { + int newOffset = len - residue; + int toWrite = Math.min(lastBuffer.remaining(), residue); + + lastBuffer.put(cbuf, newOffset, toWrite); + residue -= toWrite; + if (residue != 0) { + extend(); + } + } + } + + @Override + public void write(byte[] b) throws IOException { + write(b,0,b.length); + } + + @Override + public String toString() { + return "GrowableBufferOutputStream, writable size " + writableSize() + + " bytes, " + numWritableBuffers() + " buffers, last buffer" + + " position " + lastBuffer.position() + ", last buffer limit " + + lastBuffer.limit(); + } + + public void write(int b) { + if (lastBuffer.remaining() == 0) { + extend(); + } + lastBuffer.put((byte) b); + } + + @Override + public void flush() { + // if the last buffer is untouched we do not need to do anything; if + // it has been touched we call extend(), which enqueues the buffer + // and allocates or recycles a buffer for us + if (lastBuffer.position() > 0) { + extend(); + } + } + + @Override + public void close() { + flush(); + } + + public int channelWrite(WritableByteChannel channel) throws IOException { + ByteBuffer buffer; + int totalWritten = 0; + + while (!bufferList.isEmpty()) { + buffer = bufferList.getFirst(); + int written = 0; + + synchronized (directBuffer) { + directBuffer.clear(); + directBuffer.put(buffer); + directBuffer.flip(); + written = channel.write(directBuffer); + int left = directBuffer.remaining(); + + if (left > 0) { + int oldpos = buffer.position(); + + buffer.position(oldpos - left); + } + totalWritten += written; + } + + // if we've completed writing this buffer we can dispose of it + if (buffer.remaining() == 0) { + bufferList.removeFirst(); + recycleBuffer(buffer); + } + + // if we didn't write any bytes we terminate + if (written == 0) { + break; + } + } + + return totalWritten; + } + + public int numWritableBuffers() { + return bufferList.size(); + } + + public void clear() { + flush(); + bufferList.clear(); + } + + public void clearCache() { + recycledBuffers.clear(); + } + + public void clearAll() { + clear(); + clearCache(); + } + + public int writableSize() { + Iterator<ByteBuffer> it = bufferList.iterator(); + int size = 0; + + while (it.hasNext()) { + size += (it.next()).remaining(); + } + + return size; + } + + public ByteBuffer[] getWritableBuffers() { + flush(); + ByteBuffer[] result = new ByteBuffer[numWritableBuffers()]; + return bufferList.toArray(result); + } + + private void extend() { + enqueueBuffer(lastBuffer); + + if (recycledBuffers.empty()) { + lastBuffer = ByteBuffer.allocate(bufferSize); + } else { + lastBuffer = recycledBuffers.pop(); + lastBuffer.clear(); + } + } + + private void enqueueBuffer(ByteBuffer buffer) { + buffer.flip(); + bufferList.addLast(buffer); + } + + private void recycleBuffer(ByteBuffer buffer) { + if (recycledBuffers.size() >= maxBuffers) { + return; + } + recycledBuffers.push(buffer); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/io/GrowableByteBuffer.java b/vespajlib/src/main/java/com/yahoo/io/GrowableByteBuffer.java new file mode 100644 index 00000000000..c33882052b4 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/GrowableByteBuffer.java @@ -0,0 +1,746 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.*; + +/** + * GrowableByteBuffer encapsulates a ByteBuffer and grows it as needed. + * The implementation is safe and simple (and certainly a bit inefficient) + * - when growing the buffer a new buffer + * is allocated, the old contents are copied into the new buffer, + * and the new buffer's position is set to the position of the old + * buffer. + * It is possible to set a growth factor. The default is 2.0, meaning that + * the buffer will double its size when growing. + * + * Note that NO methods are re-implemented (except growing the buffer, + * of course), all are delegated to the encapsulated ByteBuffer. + * This also includes toString(), hashCode(), equals() and compareTo(). + * + * No methods except getByteBuffer() expose the encapsulated + * ByteBuffer, which is intentional. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class GrowableByteBuffer implements Comparable<GrowableByteBuffer> { + public static final int DEFAULT_BASE_SIZE = 64*1024; + public static final float DEFAULT_GROW_FACTOR = 2.0f; + private ByteBuffer buffer; + private float growFactor; + private int mark = -1; + + //NOTE: It might have been better to subclass HeapByteBuffer, + //but that class is package-private. Subclassing ByteBuffer would involve + //implementing a lot of abstract methods, which would mean reinventing + //some (too many) wheels. + + //CONSTRUCTORS: + + public GrowableByteBuffer() { + this(DEFAULT_BASE_SIZE, DEFAULT_GROW_FACTOR); + } + + public GrowableByteBuffer(int baseSize, float growFactor) { + setGrowFactor(growFactor); + //NOTE: We MUST NEVER have a base size of 0, since checkAndGrow() will go into an infinite loop then + if (baseSize < 16) baseSize = 16; + buffer = ByteBuffer.allocate(baseSize); + } + + public GrowableByteBuffer(int baseSize) { + this(baseSize, DEFAULT_GROW_FACTOR); + } + + public GrowableByteBuffer(ByteBuffer buffer) { + this(buffer, DEFAULT_GROW_FACTOR); + } + + public GrowableByteBuffer(ByteBuffer buffer, float growFactor) { + this.buffer = buffer; + setGrowFactor(growFactor); + } + + + //ACCESSORS: + + public float getGrowFactor() { + return growFactor; + } + + public void setGrowFactor(float growFactor) { + if (growFactor <= 1.00f) { + throw new IllegalArgumentException("Growth factor must be greater than 1.00f, otherwise buffer will never grow!"); + } + this.growFactor = growFactor; + } + + public ByteBuffer getByteBuffer() { + return buffer; + } + + //PRIVATE GROWTH METHODS + + //TODO: Implement more efficient buffer growth + //Allocating a new buffer and copying the old buffer into the new one + //is a simple and uncomplicated strategy. + //For performance, it would be much better to have a linked list of + //ByteBuffers and keep track of global position etc., much like + //GrowableBufferOutputStream does it. + + protected void grow(int newSize) { + //create new buffer: + ByteBuffer newByteBuf; + if (buffer.isDirect()) { + newByteBuf = ByteBuffer.allocateDirect(newSize); + } else { + newByteBuf = ByteBuffer.allocate(newSize); + } + //set same byte order: + newByteBuf.order(buffer.order()); + + //copy old contents and set correct position: + int oldPos = buffer.position(); + newByteBuf.position(0); + buffer.position(0); + newByteBuf.put(buffer); + newByteBuf.position(oldPos); + + //set same mark: + if (mark >= 0) { + newByteBuf.position(mark); + newByteBuf.mark(); + newByteBuf.position(oldPos); + } + + //NOTE: No need to preserve "read-only" property, + //since a read-only buffer cannot grow and will never + //reach this point anyway + + //NOTE: No need to preserve "limit" property, it would be + //pointless to grow then... + + //set new buffer to be our buffer: + buffer = newByteBuf; + } + + private void accomodate(int putSize) { + int bufPos = buffer.position(); + int bufSize = buffer.capacity(); + int bufRem = bufSize - bufPos; + + if (bufRem >= putSize) return; + + while (bufRem < putSize) { + bufSize = (int) ((((float) bufSize) * growFactor) + 100.0); + bufRem = bufSize - bufPos; + } + + grow(bufSize); + } + + //VESPA-ENCODED INTEGERS: + + /** + * Writes a 62-bit positive integer to the buffer, using 2, 4, or 8 bytes. + * + * @param number the integer to write + */ + public void putInt2_4_8Bytes(long number) { + if (number < 0L) { + throw new IllegalArgumentException("Cannot encode negative number."); + } else if (number > 0x3FFFFFFFFFFFFFFFL) { + throw new IllegalArgumentException("Cannot encode number larger than 2^62."); + } + + if (number < 0x8000L) { + //length 2 bytes + putShort((short) number); + } else if (number < 0x40000000L) { + //length 4 bytes + putInt(((int) number) | 0x80000000); + } else { + //length 8 bytes + putLong(number | 0xC000000000000000L); + } + } + + /** + * Writes a 32 bit positive integer (or 31 bit unsigned) to the buffer, + * using 4 bytes. + * + * @param number the integer to write + */ + public void putInt2_4_8BytesAs4(long number) { + if (number < 0L) { + throw new IllegalArgumentException("Cannot encode negative number."); + } else if (number > 0x7FFFFFFFL) { + throw new IllegalArgumentException("Cannot encode number larger than 2^31-1."); + } + putInt(((int) number) | 0x80000000); + } + + /** + * Reads a 62-bit positive integer from the buffer, which was written using 2, 4, or 8 bytes. + * + * @return the integer read + */ + public long getInt2_4_8Bytes() { + byte flagByte = get(); + position(position() - 1); + + if ((flagByte & 0x80) != 0) { + if ((flagByte & 0x40) != 0) { + //length 8 bytes + return getLong() & 0x3FFFFFFFFFFFFFFFL; + } else { + //length 4 bytes + return getInt() & 0x3FFFFFFF; + } + } else { + //length 2 bytes + return getShort(); + } + } + + /** + * Computes the size used for storing the given integer using 2, 4 or 8 bytes. + * + * @param number the integer to check length of + * @return the number of bytes used to store it; 2, 4 or 8 + */ + public static int getSerializedSize2_4_8Bytes(long number) { + if (number < 0L) { + throw new IllegalArgumentException("Cannot encode negative number."); + } else if (number > 0x3FFFFFFFFFFFFFFFL) { + throw new IllegalArgumentException("Cannot encode number larger than 2^62."); + } + + if (number < 0x8000L) { + //length 2 bytes + return 2; + } else if (number < 0x40000000L) { + //length 4 bytes + return 4; + } else { + //length 8 bytes + return 8; + } + } + + /** + * Writes a 30-bit positive integer to the buffer, using 1, 2, or 4 bytes. + * + * @param number the integer to write + */ + public void putInt1_2_4Bytes(int number) { + if (number < 0) { + throw new IllegalArgumentException("Cannot encode negative number"); + } else if (number > 0x3FFFFFFF) { + throw new IllegalArgumentException("Cannot encode number larger than 2^30."); + } + + if (number < 0x80) { + //length 1 byte + put((byte) number); + } else if (number < 0x4000) { + //length 2 bytes + putShort((short) (((short)number) | ((short) 0x8000))); + } else { + //length 4 bytes + putInt(number | 0xC0000000); + } + } + + /** + * Writes a 30-bit positive integer to the buffer, using 4 bytes. + * + * @param number the integer to write + */ + public void putInt1_2_4BytesAs4(int number) { + if (number < 0) { + throw new IllegalArgumentException("Cannot encode negative number"); + } else if (number > 0x3FFFFFFF) { + throw new IllegalArgumentException("Cannot encode number larger than 2^30."); + } + putInt(number | 0xC0000000); + } + + /** + * Reads a 30-bit positive integer from the buffer, which was written using 1, 2, or 4 bytes. + * + * @return the integer read + */ + public int getInt1_2_4Bytes() { + byte flagByte = get(); + position(position() - 1); + + if ((flagByte & 0x80) != 0) { + if ((flagByte & 0x40) != 0) { + //length 4 bytes + return getInt() & 0x3FFFFFFF; + } else { + //length 2 bytes + return getShort() & 0x3FFF; + } + } else { + //length 1 byte + return get(); + } + } + + /** + * Computes the size used for storing the given integer using 1, 2 or 4 bytes. + * + * @param number the integer to check length of + * @return the number of bytes used to store it; 1, 2 or 4 + */ + public static int getSerializedSize1_2_4Bytes(int number) { + if (number < 0) { + throw new IllegalArgumentException("Cannot encode negative number"); + } else if (number > 0x3FFFFFFF) { + throw new IllegalArgumentException("Cannot encode number larger than 2^30."); + } + + if (number < 0x80) { + //length 1 byte + return 1; + } else if (number < 0x4000) { + //length 2 bytes + return 2; + } else { + //length 4 bytes + return 4; + } + } + + /** + * Writes a 31-bit positive integer to the buffer, using 1 or 4 bytes. + * + * @param number the integer to write + */ + public void putInt1_4Bytes(int number) { + if (number < 0) { + throw new IllegalArgumentException("Cannot encode negative number"); + } + //no need to check upper boundary, since INT_MAX == 2^31 + + if (number < 0x80) { + //length 1 byte + put((byte) number); + } else { + //length 4 bytes + putInt(number | 0x80000000); + } + } + + /** + * Writes a 31-bit positive integer to the buffer, using 4 bytes. + * + * @param number the integer to write + */ + public void putInt1_4BytesAs4(int number) { + if (number < 0) { + throw new IllegalArgumentException("Cannot encode negative number"); + } + //no need to check upper boundary, since INT_MAX == 2^31 + putInt(number | 0x80000000); + } + + /** + * Reads a 31-bit positive integer from the buffer, which was written using 1 or 4 bytes. + * + * @return the integer read + */ + public int getInt1_4Bytes() { + byte flagByte = get(); + position(position() - 1); + + if ((flagByte & 0x80) != 0) { + //length 4 bytes + return getInt() & 0x7FFFFFFF; + } else { + //length 1 byte + return get(); + } + } + + /** + * Computes the size used for storing the given integer using 1 or 4 bytes. + * + * @param number the integer to check length of + * @return the number of bytes used to store it; 1 or 4 + */ + public static int getSerializedSize1_4Bytes(int number) { + if (number < 0) { + throw new IllegalArgumentException("Cannot encode negative number"); + } + //no need to check upper boundary, since INT_MAX == 2^31 + + if (number < 0x80) { + //length 1 byte + return 1; + } else { + //length 4 bytes + return 4; + } + } + + //METHODS OF ENCAPSULATED BYTEBUFFER: + public static GrowableByteBuffer allocate(int capacity) { + return new GrowableByteBuffer(ByteBuffer.allocate(capacity)); + } + public static GrowableByteBuffer allocate(int capacity, float growFactor) { + return new GrowableByteBuffer(ByteBuffer.allocate(capacity), growFactor); + } + public static GrowableByteBuffer allocateDirect(int capacity) { + return new GrowableByteBuffer(ByteBuffer.allocateDirect(capacity)); + } + public static GrowableByteBuffer allocateDirect(int capacity, float growFactor) { + return new GrowableByteBuffer(ByteBuffer.allocateDirect(capacity), growFactor); + } + public final byte[] array() { + return buffer.array(); + } + public final int arrayOffset() { + return buffer.arrayOffset(); + } + public CharBuffer asCharBuffer() { + return buffer.asCharBuffer(); + } + public DoubleBuffer asDoubleBuffer() { + return buffer.asDoubleBuffer(); + } + public FloatBuffer asFloatBuffer() { + return buffer.asFloatBuffer(); + } + public IntBuffer asIntBuffer() { + return buffer.asIntBuffer(); + } + public LongBuffer asLongBuffer() { + return buffer.asLongBuffer(); + } + public GrowableByteBuffer asReadOnlyBuffer() { + return new GrowableByteBuffer(buffer.asReadOnlyBuffer(), growFactor); + } + public ShortBuffer asShortBuffer() { + return buffer.asShortBuffer(); + } + public GrowableByteBuffer compact() { + buffer.compact(); + return this; + } + public int compareTo(GrowableByteBuffer that) { + return buffer.compareTo(that.buffer); + } + public GrowableByteBuffer duplicate() { + return new GrowableByteBuffer(buffer.duplicate(), growFactor); + } + public boolean equals(Object obj) { + if (!(obj instanceof GrowableByteBuffer)) { + return false; + } + GrowableByteBuffer rhs = (GrowableByteBuffer)obj; + if (!buffer.equals(rhs.buffer)) { + return false; + } + return true; + } + public byte get() { + return buffer.get(); + } + public GrowableByteBuffer get(byte[] dst) { + buffer.get(dst); + return this; + } + public GrowableByteBuffer get(byte[] dst, int offset, int length) { + buffer.get(dst, offset, length); + return this; + } + public byte get(int index) { + return buffer.get(index); + } + public char getChar() { + return buffer.getChar(); + } + public char getChar(int index) { + return buffer.getChar(index); + } + public double getDouble() { + return buffer.getDouble(); + } + public double getDouble(int index) { + return buffer.getDouble(index); + } + public float getFloat() { + return buffer.getFloat(); + } + public float getFloat(int index) { + return buffer.getFloat(index); + } + public int getInt() { + return buffer.getInt(); + } + public int getInt(int index) { + return buffer.getInt(index); + } + public long getLong() { + return buffer.getLong(); + } + public long getLong(int index) { + return buffer.getLong(index); + } + public short getShort() { + return buffer.getShort(); + } + public short getShort(int index) { + return buffer.getShort(index); + } + public boolean hasArray() { + return buffer.hasArray(); + } + public int hashCode() { + return buffer.hashCode(); + } + public boolean isDirect() { + return buffer.isDirect(); + } + public ByteOrder order() { + return buffer.order(); + } + public GrowableByteBuffer order(ByteOrder bo) { + buffer.order(bo); + return this; + } + + public GrowableByteBuffer put(byte b) { + try { + buffer.put(b); + } catch (BufferOverflowException e) { + accomodate(1); + buffer.put(b); + } + return this; + } + public GrowableByteBuffer put(byte[] src) { + + accomodate(src.length); + buffer.put(src); + return this; + } + public GrowableByteBuffer put(byte[] src, int offset, int length) { + + accomodate(length); + buffer.put(src, offset, length); + return this; + } + public GrowableByteBuffer put(ByteBuffer src) { + accomodate(src.remaining()); + buffer.put(src); + return this; + } + public GrowableByteBuffer put(GrowableByteBuffer src) { + + accomodate(src.remaining()); + buffer.put(src.buffer); + return this; + } + // XXX: the put{Type}(index, value) methods do not handle index > position + public GrowableByteBuffer put(int index, byte b) { + try { + buffer.put(index, b); + } catch (IndexOutOfBoundsException e) { + accomodate(1); + buffer.put(index, b); + } + return this; + } + public GrowableByteBuffer putChar(char value) { + try { + buffer.putChar(value); + } catch (BufferOverflowException e) { + accomodate(2); + buffer.putChar(value); + } + return this; + } + public GrowableByteBuffer putChar(int index, char value) { + try { + buffer.putChar(index, value); + } catch (IndexOutOfBoundsException e) { + accomodate(2); + buffer.putChar(index, value); + } + return this; + } + public GrowableByteBuffer putDouble(double value) { + try { + buffer.putDouble(value); + } catch (BufferOverflowException e) { + accomodate(8); + buffer.putDouble(value); + } + return this; + } + public GrowableByteBuffer putDouble(int index, double value) { + try { + buffer.putDouble(index, value); + } catch (IndexOutOfBoundsException e) { + accomodate(8); + buffer.putDouble(index, value); + } + return this; + } + public GrowableByteBuffer putFloat(float value) { + try { + buffer.putFloat(value); + } catch (BufferOverflowException e) { + accomodate(4); + buffer.putFloat(value); + } + return this; + } + public GrowableByteBuffer putFloat(int index, float value) { + try { + buffer.putFloat(index, value); + } catch (IndexOutOfBoundsException e) { + accomodate(4); + buffer.putFloat(index, value); + } + return this; + } + public GrowableByteBuffer putInt(int value) { + try { + buffer.putInt(value); + } catch (BufferOverflowException e) { + accomodate(4); + buffer.putInt(value); + } + return this; + } + public GrowableByteBuffer putInt(int index, int value) { + try { + buffer.putInt(index, value); + } catch (IndexOutOfBoundsException e) { + accomodate(4); + buffer.putInt(index, value); + } + return this; + } + public GrowableByteBuffer putLong(int index, long value) { + try { + buffer.putLong(index, value); + } catch (IndexOutOfBoundsException e) { + accomodate(8); + buffer.putLong(index, value); + } + return this; + } + public GrowableByteBuffer putLong(long value) { + try { + buffer.putLong(value); + } catch (BufferOverflowException e) { + accomodate(8); + buffer.putLong(value); + } + return this; + } + public GrowableByteBuffer putShort(int index, short value) { + try { + buffer.putShort(index, value); + } catch (IndexOutOfBoundsException e) { + accomodate(2); + buffer.putShort(index, value); + } + return this; + } + public GrowableByteBuffer putShort(short value) { + try { + buffer.putShort(value); + } catch (BufferOverflowException e) { + accomodate(2); + buffer.putShort(value); + } + return this; + } + + /** + * Behaves as ByteBuffer slicing, but the internal buffer will no longer be + * shared if one of the buffers is forced to grow. + * + * @return a new buffer with shared contents + * @see ByteBuffer#slice() + */ + public GrowableByteBuffer slice() { + ByteBuffer b = buffer.slice(); + return new GrowableByteBuffer(b, growFactor); + } + + public String toString() { + return "GrowableByteBuffer" + + "[pos="+ position() + + " lim=" + limit() + + " cap=" + capacity() + + " grow=" + growFactor + + "]"; + } + public static GrowableByteBuffer wrap(byte[] array) { + return new GrowableByteBuffer(ByteBuffer.wrap(array)); + } + public static GrowableByteBuffer wrap(byte[] array, float growFactor) { + return new GrowableByteBuffer(ByteBuffer.wrap(array), growFactor); + } + public static GrowableByteBuffer wrap(byte[] array, int offset, int length) { + return new GrowableByteBuffer(ByteBuffer.wrap(array, offset, length)); + } + public static GrowableByteBuffer wrap(byte[] array, int offset, int length, float growFactor) { + return new GrowableByteBuffer(ByteBuffer.wrap(array, offset, length), growFactor); + } + + //METHODS FROM ENCAPSULATED BUFFER: + + public final int capacity() { + return buffer.capacity(); + } + public final void clear() { + buffer.clear(); + mark = -1; + } + public final void flip() { + buffer.flip(); + mark = -1; + } + public final boolean hasRemaining() { + return buffer.hasRemaining(); + } + public final boolean isReadOnly() { + return buffer.isReadOnly(); + } + public final int limit() { + return buffer.limit(); + } + public final void limit(int newLimit) { + buffer.limit(newLimit); + if (mark > newLimit) mark = -1; + } + public final void mark() { + buffer.mark(); + mark = position(); + } + public final int position() { + return buffer.position(); + } + public final void position(int newPosition) { + buffer.position(newPosition); + if (mark > newPosition) mark = -1; + } + public final int remaining() { + return buffer.remaining(); + } + public final void reset() { + buffer.reset(); + } + public final void rewind() { + buffer.rewind(); + mark = -1; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/HexDump.java b/vespajlib/src/main/java/com/yahoo/io/HexDump.java new file mode 100644 index 00000000000..65a6f8d2b5a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/HexDump.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +/** + * @author bratseth + */ +public class HexDump { + + private static final String HEX_CHARS = "0123456789ABCDEF"; + + public static String toHexString(byte[] buf) { + if (buf == null) { + return null; + } + StringBuilder sb = new StringBuilder(); + for (byte b : buf) { + int x = b; + if (x < 0) { + x += 256; + } + sb.append(HEX_CHARS.charAt(x / 16)); + sb.append(HEX_CHARS.charAt(x % 16)); + } + return sb.toString(); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java new file mode 100644 index 00000000000..61687f92659 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java @@ -0,0 +1,441 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.util.List; +import java.nio.charset.Charset; +import java.nio.ByteBuffer; + + +/** + * <p>Some static io convenience methods.</p> + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public abstract class IOUtils { + static private final Charset utf8Charset = Charset.forName("utf-8"); + + /** Closes a writer, or does nothing if the writer is null */ + public static void closeWriter(Writer writer) { + if (writer == null) return; + try { writer.close(); } catch (IOException e) {} + } + + /** Closes a reader, or does nothing if the reader is null */ + public static void closeReader(Reader reader) { + if (reader == null) return; + try { reader.close(); } catch (IOException e) {} + } + + /** Closes an input stream, or does nothing if the stream is null */ + public static void closeInputStream(InputStream stream) { + if (stream == null) return; + try { stream.close(); } catch (IOException e) {} + } + + /** Closes an output stream, or does nothing if the stream is null */ + public static void closeOutputStream(OutputStream stream) { + if (stream == null) return; + try { stream.close(); } catch (IOException e) {} + } + + /** + * Creates a buffered reader + * + * @param filename the name or path of the file + * @param encoding the encoding of the file, for instance "UTF-8" + */ + public static BufferedReader createReader(File filename, String encoding) throws IOException { + return new BufferedReader(new InputStreamReader(new FileInputStream(filename), encoding)); + } + + /** + * Creates a buffered reader + * + * @param filename the name or path of the file + * @param encoding the encoding of the file, for instance "UTF-8" + */ + public static BufferedReader createReader(String filename, String encoding) throws IOException { + return new BufferedReader(new InputStreamReader(new FileInputStream(filename), encoding)); + } + + /** Creates a buffered reader in the default encoding */ + public static BufferedReader createReader(String filename) throws IOException { + return new BufferedReader(new FileReader(filename)); + } + + /** + * Creates a buffered writer, + * and the directories to contain it if they do not exist + * + * @param filename the name or path of the file + * @param encoding the encoding to use, for instance "UTF-8" + * @param append whether to append to the files if it exists + */ + public static BufferedWriter createWriter(String filename, String encoding, boolean append) throws IOException { + createDirectory(filename); + return new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filename, append), encoding)); + } + + /** + * Creates a buffered writer, + * and the directories to contain it if they do not exist + * + * @param file the file to write to + * @param encoding the encoding to use, for instance "UTF-8" + * @param append whether to append to the files if it exists + */ + public static BufferedWriter createWriter(File file, String encoding, boolean append) throws IOException { + createDirectory(file.getAbsolutePath()); + return new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file, append),encoding)); + } + + /** + * Creates a buffered writer in the default encoding + * + * @param filename the name or path of the file + * @param append whether to append to the files if it exists + */ + public static BufferedWriter createWriter(String filename, boolean append) throws IOException { + createDirectory(filename); + return new BufferedWriter(new FileWriter(filename, append)); + } + + /** + * Creates a buffered writer in the default encoding + * + * @param file the file to write to + * @param append whether to append to the files if it exists + */ + public static BufferedWriter createWriter(File file, boolean append) throws IOException { + createDirectory(file.getAbsolutePath()); + return new BufferedWriter(new FileWriter(file, append)); + } + + /** Creates the directory path of this file if it does not exist */ + public static void createDirectory(String filename) { + File directory = new File(filename).getParentFile(); + + if (directory != null) + directory.mkdirs(); + } + + /** + * Copies the n first lines of a file to another file. + * If the out file exists it will be overwritten + * + * @throws IOException if copying fails + */ + public static void copy(String inFile, String outFile, int lineCount) throws IOException { + BufferedReader reader = null; + BufferedWriter writer = null; + + try { + reader = createReader(inFile); + writer = createWriter(outFile, false); + int c; + + int newLines = 0; + while (-1 != (c=reader.read()) && newLines<lineCount) { + writer.write(c); + if (c=='\n') + newLines++; + } + } finally { + closeReader(reader); + closeWriter(writer); + } + } + + /** + * Copies a file to another file. + * If the out file exists it will be overwritten. + * NOTE: Not an optimal implementation currently. + * + * @throws IOException if copying fails + */ + public static void copy(String inFile, String outFile) throws IOException { + BufferedReader reader=null; + BufferedWriter writer=null; + + try { + reader = createReader(inFile); + writer = createWriter(outFile, false); + int c; + while (-1 != (c = reader.read()) ) + writer.write(c); + } finally { + closeReader(reader); + closeWriter(writer); + } + } + + /** + * Copies a file to another file. + * If the out file exists it will be overwritten. + * NOTE: Not an optimal implementation currently. + */ + public static void copy(File inFile, File outFile) throws IOException { + copy(inFile.toString(),outFile.toString()); + } + + /** + * Copies all files and subdirectories in a directory to another. + * Any existing files are overwritten. + * + * @param sourceLocation the source directory + * @param targetLocation the target directory + * @param maxRecurseLevel if this is 1, only files immediately in sourceLocation are copied, + * if it is 2, then files contained in immediate subdirectories are copied, etc. + * If it is 0, sourceLocation will only be copied if it is a file, not a directory. + * If it is negative, recursion is infinite. + * @throws IOException if copying any file fails. This will typically result in some files being copied and + * others not, i.e this method is not exception safe + */ + public static void copyDirectory(File sourceLocation , File targetLocation, int maxRecurseLevel) throws IOException { + copyDirectory(sourceLocation, targetLocation, maxRecurseLevel, new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return true; + } + }); + } + + /** + * Copies all files and subdirectories in a directory to another. + * Any existing files are overwritten. + * + * @param sourceLocation the source directory + * @param targetLocation the target directory + * @param maxRecurseLevel if this is 1, only files immediately in sourceLocation are copied, + * if it is 2, then files contained in immediate subdirectories are copied, etc. + * If it is 0, sourceLocation will only be copied if it is a file, not a directory. + * If it is negative, recursion is infinite. + * @param filter Only copy files passing through filter. + * @throws IOException if copying any file fails. This will typically result in some files being copied and + * others not, i.e this method is not exception safe + */ + public static void copyDirectory(File sourceLocation , File targetLocation, int maxRecurseLevel, FilenameFilter filter) throws IOException { + if ( ! sourceLocation.isDirectory()) { // copy file + InputStream in=null; + OutputStream out=null; + try { + in = new FileInputStream(sourceLocation); + out = new FileOutputStream(targetLocation); + // Copy the bits from instream to outstream + byte[] buf = new byte[1024]; + int len; + while ((len = in.read(buf)) > 0) { + out.write(buf, 0, len); + } + } + finally { + closeInputStream(in); + closeOutputStream(out); + } + } + else if (maxRecurseLevel!=0) { // copy directory if allowed + if (!targetLocation.exists()) + targetLocation.mkdirs(); + + String[] children = sourceLocation.list(filter); + for (int i=0; i<children.length; i++) + copyDirectory(new File(sourceLocation, children[i]), + new File(targetLocation, children[i]), + maxRecurseLevel-1); + } + } + + /** + * Copies all files and subdirectories (infinitely recursively) in a directory to another. + * Any existing files are overwritten. + * + * @param sourceLocation the source directory + * @param targetLocation the target directory + * @throws IOException if copying any file fails. This will typically result in some files being copied and + * others not, i.e this method is not exception safe + */ + public static void copyDirectory(File sourceLocation , File targetLocation) throws IOException { + copyDirectory(sourceLocation, targetLocation, -1); + } + + /** + * Copies the whole source directory (infinitely recursively) into the target directory. + * @throws IOException if copying any file fails. This will typically result in some files being copied and + * others not, i.e this method is not exception safe + */ + public static void copyDirectoryInto(File sourceLocation, File targetLocation) throws IOException { + File destination = new File(targetLocation, sourceLocation.getAbsoluteFile().getName()); + copyDirectory(sourceLocation, destination); + } + + /** + * Returns the number of line in a file. + * If the files does not exists, 0 is returned + */ + public static int countLines(String file) { + BufferedReader reader = null; + int lineCount = 0; + + try { + reader = createReader(file,"utf8"); + while (reader.readLine() != null) + lineCount++; + return lineCount; + } catch (IOException e) { + return lineCount; + } finally { + closeReader(reader); + } + + } + + /** + * Returns a list containing the lines in the given file as strings + * + * @return a list of Strings for the lines of the file, in order + * @throws IOException if the file could not be read + */ + public static List<String> getLines(String fileName) throws IOException { + BufferedReader reader = null; + + try { + List<String> lines = new java.util.ArrayList<>(); + + reader = createReader(fileName,"utf8"); + String line; + + while (null != (line = reader.readLine())) + lines.add(line); + return lines; + } finally { + closeReader(reader); + } + } + + /** + * Recursive deletion of directories + */ + public static boolean recursiveDeleteDir(File dir) { + if (dir.isDirectory()) { + String[] children = dir.list(); + + for (String child : children) { + boolean success = recursiveDeleteDir(new File(dir, child)); + + if (!success) return false; + } + } + + // The directory is now empty so delete it + return dir.delete(); + } + + /** + * Encodes string as UTF-8 into ByteBuffer + */ + public static ByteBuffer utf8ByteBuffer(String s) { + return utf8Charset.encode(s); + } + + /** + * Reads the contents of a UTF-8 text file into a String. + * + * @param file the file to read, or null + * @return the file content as a string, or null if the input file was null + */ + public static String readFile(File file) throws IOException { + try { + if (file == null) return null; + return new String(Files.readAllBytes(file.toPath()), "utf-8"); + } + catch (NoSuchFileException e) { + throw new NoSuchFileException("Could not find file '" + file.getAbsolutePath() + "'"); + } + } + + /** + * Reads all the content of the given array, in chunks of at max chunkSize + */ + public static byte[] readBytes(InputStream stream, int chunkSize) throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + + int nRead; + byte[] data = new byte[chunkSize]; + while ((nRead = stream.read(data, 0, data.length)) != -1) + buffer.write(data, 0, nRead); + buffer.flush(); + return buffer.toByteArray(); + } + + /** + * Reads the content of a file into a byte array + */ + public static byte[] readFileBytes(File file) throws IOException { + long lengthL = file.length(); + if (lengthL>Integer.MAX_VALUE) + throw new IllegalArgumentException("File too big for byte array: "+file.getCanonicalPath()); + + InputStream in = null; + try { + in = new FileInputStream(file); + int length = (int)lengthL; + byte[] array = new byte[length]; + int offset = 0; + int count=0; + while (offset < length && (count = in.read(array, offset, (length - offset)))>=0) + offset += count; + return array; + } + finally { + if (in != null) + in.close(); + } + } + + /** + * Reads all data from a reader into a string. Uses a buffer to speed up reading. + */ + public static String readAll(Reader reader) throws IOException { + StringBuilder ret=new StringBuilder(); + BufferedReader buffered = new BufferedReader(reader); + int c; + while ((c=buffered.read())!=-1) + ret.appendCodePoint(c); + buffered.close(); + return ret.toString(); + } + + /** Convenience method for closing a list of readers. Does nothing if the given reader list is null. */ + public static void closeAll(List<Reader> readers) { + if (readers==null) return; + for (Reader reader : readers) + closeReader(reader); + } + + /** + * Writes the given string to the file + */ + public static void writeFile(File file, String text, boolean append) throws IOException { + BufferedWriter out = null; + try { + out = createWriter(file, append); + out.write(text); + } + finally { + closeWriter(out); + } + } + + /** + * Writes the given string to the file + */ + public static void writeFile(String file, String text, boolean append) throws IOException { + writeFile(new File(file), text, append); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/io/Listener.java b/vespajlib/src/main/java/com/yahoo/io/Listener.java new file mode 100644 index 00000000000..134cf828c60 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/Listener.java @@ -0,0 +1,564 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + + +import java.io.IOException; +import java.nio.channels.ClosedChannelException; +import java.net.InetSocketAddress; + +import java.util.Iterator; +import java.util.Map; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.ArrayList; + +import java.util.logging.Logger; +import java.util.logging.Level; + +import java.nio.channels.SocketChannel; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.Selector; +import java.nio.channels.SelectionKey; + + +/** + * A basic Reactor implementation using NIO. + * + * @author <a href="mailto:travisb@yahoo-inc.com">Bob Travis</a> + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * + */ +public class Listener extends Thread { + private static Logger log = Logger.getLogger(Listener.class.getName()); + private Selector selector; + Map<Integer, Acceptor> acceptors = new HashMap<>(); + Map<ServerSocketChannel, ConnectionFactory> factories = new IdentityHashMap<>(); + + private FatalErrorHandler fatalErrorHandler; + + private List<SelectLoopHook> selectLoopPreHooks; + private List<SelectLoopHook> selectLoopPostHooks; + + final private LinkedList<Connection> newConnections = new LinkedList<>(); + + // queue of SelectionKeys that need to be updated + final private LinkedList<UpdateInterest> modifyInterestOpsQueue = new LinkedList<>(); + + public Listener(String name) { + super("Listener-" + name); + + try { + selector = Selector.open(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + log.fine(name + " listener created " + this); + } + + /** + * Register a handler for fatal errors. + * + * @param f The FatalErrorHandler instance to be registered + */ + public synchronized void setFatalErrorHandler(FatalErrorHandler f) { + fatalErrorHandler = f; + } + + /** + * Add pre-select loop hook. Not threadsafe so please do this + * during initial setup before you start the listener. + */ + public void addSelectLoopPreHook(SelectLoopHook hook) { + if (selectLoopPreHooks == null) { + selectLoopPreHooks = new ArrayList<>(5); + } + selectLoopPreHooks.add(hook); + } + + /** + * Add pre-select loop hook. Not threadsafe so please do this + * during initial setup before you start the listener. + */ + public void addSelectLoopPostHook(SelectLoopHook hook) { + if (selectLoopPostHooks == null) { + selectLoopPostHooks = new ArrayList<>(5); + } + selectLoopPostHooks.add(hook); + } + + /** + * Run all the select loop pre hooks + */ + private void runSelectLoopPreHooks() { + if (selectLoopPreHooks == null) { + return; + } + + for (SelectLoopHook hook : selectLoopPreHooks) { + hook.selectLoopHook(true); + } + } + + /** + * Run all the select loop post hooks + */ + private void runSelectLoopPostHooks() { + if (selectLoopPostHooks == null) { + return; + } + + for (SelectLoopHook hook : selectLoopPostHooks) { + hook.selectLoopHook(false); + } + } + + /** + * Add a listening port and create an Acceptor thread which accepts + * new connections on this port. + * + * @param factory The connection factory for new connections + * on this port + * @param port The port we are going to listen to. + */ + public synchronized void listen(ConnectionFactory factory, int port) + throws IOException { + // make sure we have only one acceptor per listen port + if (acceptors.containsKey(port)) { + log.warning("Already listening to port=" + port); + return; + } + + Acceptor a = new Acceptor(this, factory, port); + + // inherit the fatal error handling of listener + if (fatalErrorHandler != null) { + a.setFatalErrorHandler(fatalErrorHandler); + } + + a.listen().start(); + acceptors.put(port, a); + } + + /** + * Add a listening port without creating a separate acceptor + * thread. + * + * @param factory The connection factory for new connections + * on this port + * @param port The port we are going to listen to. + */ + public synchronized void listenNoAcceptor(ConnectionFactory factory, int port) + throws IOException { + ServerSocketChannel s = ServerSocketChannel.open(); + + s.configureBlocking(false); + s.socket().setReuseAddress(true); + s.socket().bind(new InetSocketAddress(port)); // use non-specific IP + String host = s.socket().getInetAddress().getHostName(); + + factories.put(s, factory); + s.register(selector, SelectionKey.OP_ACCEPT); + log.fine("listener " + host + ":" + port); + } + + // ================================================================== + // ================================================================== + // ================================================================== + + + /** + * This is the preferred way of modifying interest ops, giving a + * Connection rather than a SelectionKey as input. This way the + * we can look it up and ensure the correct SelectionKey is always + * used. + * + * @return Returns a <code>this</code> reference for chaining + */ + public Listener modifyInterestOps(Connection connection, + int op, boolean set) { + return modifyInterestOps(connection.socketChannel().keyFor(selector), op, + set); + } + + /** + * Batch version of modifyInterestOps(). + * + * @return Returns a <code>this</code> reference for chaining + */ + public Listener modifyInterestOpsBatch(Connection connection, + int op, boolean set) { + return modifyInterestOpsBatch( + connection.socketChannel().keyFor(selector), op, set); + } + + /** + * Enqueue change to interest set of SelectionKey. This is a workaround + * for an NIO design error that makes it impossible to update interest + * sets for a SelectionKey while a select is in progress -- and sometimes + * you actually want to do this from other threads, which will then + * block. Hence, we make it possible to enqueue requests for + * SelectionKey modification in the thread where select runs. + * + * @return Returns a <code>this</code> reference for chaining + */ + public Listener modifyInterestOps(SelectionKey key, int op, boolean set) { + synchronized (modifyInterestOpsQueue) { + modifyInterestOpsQueue.addLast(new UpdateInterest(key, op, set)); + } + selector.wakeup(); + return this; + } + + /** + * Does the same as modifyInterestOps(), but does not call + * wakeup on the selector. Allows adding more modifications + * before we wake up the selector. + * + * @return Returns a <code>this</code> reference for chaining + */ + public Listener modifyInterestOpsBatch(SelectionKey key, + int op, + boolean set) { + synchronized (modifyInterestOpsQueue) { + modifyInterestOpsQueue.addLast(new UpdateInterest(key, op, set)); + } + return this; + } + + /** + * Signal that a batch update of SelectionKey is done and the + * selector should be awoken. Also see modifyInterestOps(). + * + * @return Returns a <code>this</code> reference for chaining + */ + public Listener modifyInterestOpsDone() { + selector.wakeup(); + return this; + } + + /** + * Process enqueued changes to SelectionKeys. Also see + * modifyInterestOps(). + */ + private void processModifyInterestOps() { + synchronized (modifyInterestOpsQueue) { + while (!modifyInterestOpsQueue.isEmpty()) { + UpdateInterest u = modifyInterestOpsQueue.removeFirst(); + + u.doUpdate(); + } + } + } + + // ================================================================== + // ================================================================== + // ================================================================== + + + /** + * Thread entry point + */ + public void run() { + log.fine("Started listener"); + try { + selectLoop(); + } catch (Throwable t) { + if (fatalErrorHandler != null) { + fatalErrorHandler.handle(t, null); + } + } + } + + /** + * Check channels for readiness and deal with channels that have + * pending operations. + */ + private void selectLoop() { + while (!Thread.currentThread().isInterrupted()) { + processNewConnections(); + processModifyInterestOps(); + + try { + int n = selector.select(); + + if (0 == n) { + continue; + } + } catch (java.io.IOException e) { + log.log(Level.WARNING, "error during select", e); + return; + } + + runSelectLoopPreHooks(); + + Iterator<SelectionKey> i = selector.selectedKeys().iterator(); + + while (i.hasNext()) { + SelectionKey key = i.next(); + + i.remove(); + + if (!key.isValid()) { + continue; + } + + if (key.isReadable()) { + performRead(key); + if (!key.isValid()) { + continue; + } + } + + if (key.isWritable()) { + performWrite(key); + if (!key.isValid()) { + continue; + } + } + + if (key.isConnectable()) { + performConnect(key); + if (!key.isValid()) { + continue; + } + } + + if (key.isAcceptable()) { + performAccept(key); + } + } + + runSelectLoopPostHooks(); + } + } + + /** + * This method is used by the Acceptor to hand off newly accepted + * connections to the Listener. Note that this is run in the + * context of the Acceptor thread, so doing things here versus + * doing them in the acceptNewConnections(), which runs in the context + * of the Listener thread, is a tradeoff that may need to be + * re-evaluated + * + */ + public Connection addNewConnection(Connection newConn) { + + // ensure nonblocking and handle possible errors + // if setting nonblocking fails. this code is really redundant + // but necessary because the older version of this method set + // the connection nonblocking, and clients might still expect + // this behavior. + // + SocketChannel channel = newConn.socketChannel(); + + if (channel.isBlocking()) { + try { + channel.configureBlocking(false); + } catch (java.nio.channels.IllegalBlockingModeException e) { + log.log(Level.SEVERE, "Unable to set nonblocking", e); + try { + channel.close(); + } catch (java.io.IOException ee) { + log.log(Level.WARNING, "channel close failed", ee); + } + return newConn; + } catch (java.io.IOException e) { + log.log(Level.SEVERE, "Unable to set nonblocking", e); + return newConn; + } + } + + synchronized (newConnections) { + newConnections.addLast(newConn); + } + selector.wakeup(); + return newConn; + } + + /** + * This method is called from the selectLoop() method in order to + * process new incoming connections. + */ + private synchronized void processNewConnections() { + synchronized (newConnections) { + while (!newConnections.isEmpty()) { + Connection conn = newConnections.removeFirst(); + + try { + conn.socketChannel().register(selector, conn.selectOps(), + conn); + } catch (ClosedChannelException e) { + log.log(Level.WARNING, "register channel failed", e); + return; + } + } + } + } + + /** + * Accept new connection. This will loop over accept() until + * there are no more new connections to accept. If any error + * occurs after a successful accept, the socket in question will + * be discarded, but we will continue to try to accept new + * connections if available. + * + */ + private void performAccept(SelectionKey key) { + SocketChannel channel; + ServerSocketChannel ssChannel; + + if (Thread.currentThread().isInterrupted()) { + return; + } + + while (true) { + try { + ssChannel = (ServerSocketChannel) key.channel(); + channel = ssChannel.accept(); + + // if for some reason there was no connection we just + // ignore it. + if (null == channel) { + return; + } + } catch (java.io.IOException e) { + log.log(Level.WARNING, "accept failed", e); + return; + } + + // set nonblocking and handle possible errors + try { + channel.configureBlocking(false); + } catch (java.nio.channels.IllegalBlockingModeException e) { + log.log(Level.SEVERE, "Unable to set nonblocking", e); + try { + channel.close(); + } catch (java.io.IOException ee) { + log.log(Level.WARNING, "channel close failed", ee); + continue; + } + continue; + } catch (java.io.IOException e) { + log.log(Level.WARNING, "IO error occurred", e); + try { + channel.close(); + } catch (java.io.IOException ee) { + log.log(Level.WARNING, "channel close failed", ee); + continue; + } + continue; + } + + ConnectionFactory factory = factories.get(ssChannel); + Connection conn = factory.newConnection(channel, this); + + try { + channel.register(selector, conn.selectOps(), conn); + } catch (java.nio.channels.ClosedChannelException e) { + log.log(Level.WARNING, "register channel failed", e); + } + } + } + + /** + * Complete asynchronous connect operation. <em>Note that + * asynchronous connect does not work properly in 1.4, + * so you should not use this if you run anything older + * than 1.5/5.0</em>. + * + */ + private void performConnect(SelectionKey key) { + if (Thread.currentThread().isInterrupted()) { + return; + } + + Connection c = (Connection) key.attachment(); + + try { + c.connect(); + } catch (IOException e) { + log.log(Level.FINE, "connect failed", e); + try { + c.close(); + } catch (IOException e2) { + log.log(Level.FINE, "close failed", e); + } + } + } + + /** + * Perform read operation on channel which is now ready for reading + */ + private void performRead(SelectionKey key) { + if (Thread.currentThread().isInterrupted()) { + return; + } + + Connection c = (Connection) key.attachment(); + + try { + c.read(); + } catch (IOException e) { + log.log(Level.FINE, "read failed", e); + try { + c.close(); + } catch (IOException e2) { + log.log(Level.FINE, "close failed", e); + } + } + } + + /** + * Perform write operation(s) on channel which is now ready for + * writing + */ + private void performWrite(SelectionKey key) { + if (Thread.currentThread().isInterrupted()) { + return; + } + + Connection c = (Connection) key.attachment(); + + try { + c.write(); + } catch (IOException e) { + log.log(Level.FINE, " write failed", e); + try { + c.close(); + } catch (IOException e2) {// ignore + } + } + } + + // ============================================================ + // ==== connections made outside listener + // ============================================================ + + /** + * Register a connection that was set up outside the listener. + * Typically what we do when we actively reach out and connect + * somewhere. + */ + public void registerConnection(Connection connection) { + synchronized (newConnections) { + newConnections.addLast(connection); + } + selector.wakeup(); + } + + /** + * Perform clean shutdown of Listener. + * + * TODO: implement + */ + public void shutdown() {// make writing impossible + // make listening on new ports impossible + // close all listening connections (kill all listener threads) + // flush outbound data if the connection wants it + // close all connections + // have some sort of grace-period before forcibly shutting down + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/ReadLine.java b/vespajlib/src/main/java/com/yahoo/io/ReadLine.java new file mode 100644 index 00000000000..aba67c0c8ca --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/ReadLine.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.charset.Charset; +import java.nio.ByteBuffer; + + +/** + * Conventient utility for reading lines from ByteBuffers. Please + * read the method documentation for readLine() carefully. The NIO + * ByteBuffer abstraction is somewhat clumsy and thus usage of this + * code requires that you understand the semantics clearly. + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * + */ +public class ReadLine { + static private Charset charset = Charset.forName("latin1"); + + /** + * Extract next line from a byte buffer. Looks for EOL characters + * between start and limit, and returns a string between start and + * the EOL charachers. It skips ahead past any remaining EOL + * characters and sets position to the first non-EOL character. + * + * If it doesn't find an EOL characher between start and limit + */ + public static String readLine(ByteBuffer buffer) { + int start = buffer.position(); + + for (int i = start; i < buffer.limit(); i++) { + + if (isEolChar(buffer.get(i))) { + + // detect and skip EOL at beginning. Also, update + // position so we compact the buffer if we exit the + // for loop without having found a proper string + if (i == start) { + for (; (i < buffer.limit()) && isEolChar(buffer.get(i)); i++) { + ; + } + start = i; + buffer.position(i); + continue; + } + + // extract string between start and i. limit() returns + // a buffer so we have to up-cast again + String line = charset.decode((ByteBuffer) buffer.slice().limit(i - start)).toString(); + + // skip remaining + for (; (i < buffer.limit()) && isEolChar(buffer.get(i)); i++) { + ; + } + + buffer.position(i); + return line; + } + } + + // if we get here we didn't find any string. this may be + // because the buffer has no more content, ie. limit == position. + // if that is the case we clear the buffer. + // + // if we have content, but no more EOL characters we compact the + // buffer. + // + if (buffer.hasRemaining()) { + buffer.compact(); + } else { + buffer.clear(); + } + + return null; + } + + static boolean isEolChar(byte b) { + return ((10 == b) || (13 == b)); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/SelectLoopHook.java b/vespajlib/src/main/java/com/yahoo/io/SelectLoopHook.java new file mode 100644 index 00000000000..bcc3c0f3e1d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/SelectLoopHook.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + + +/** + * This interface defines a callback hook which applications can + * use to get work done before or after the select loop finishes + * its tasks. + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * + */ +public interface SelectLoopHook { + + /** + * Callback which can be called before or after + * select loop has done its work, depending on + * how you register the hook. + * + * @param before is <code>true</code> if the hook + * was called before the channels in the ready + * set have been processed, and <code>false</code> + * if called after. + */ + public void selectLoopHook(boolean before); +} diff --git a/vespajlib/src/main/java/com/yahoo/io/SlowInflate.java b/vespajlib/src/main/java/com/yahoo/io/SlowInflate.java new file mode 100644 index 00000000000..b25591aa5b7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/SlowInflate.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + + +import java.util.zip.Inflater; + + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class SlowInflate { + private Inflater inflater = new Inflater(); + + public byte[] unpack(byte[] compressed, int inflatedLen) { + byte[] decompressed = new byte[inflatedLen]; + + inflater.reset(); + inflater.setInput(compressed); + inflater.finished(); + try { + inflater.inflate(decompressed); + } catch (java.util.zip.DataFormatException e) { + throw new RuntimeException("Decompression failure: " + e); + } + return decompressed; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/UpdateInterest.java b/vespajlib/src/main/java/com/yahoo/io/UpdateInterest.java new file mode 100644 index 00000000000..bb718218aeb --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/UpdateInterest.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.channels.SelectionKey; + + +/** + * Command object to perform interest set updates. Workaround for NIO + * design flaw which makes it impossible to update the interest set of + * a SelectionKey while select() is in progress. There should be a + * more elegant way around this, but if it turns out to be performant + * enough we leave it like this. + * + * <P> + * Of course, the ideal would be to have NIO fixed. + * + * @author <a href="mailto:travisb@yahoo-inc.com">Bob Travis</a> + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class UpdateInterest { + private SelectionKey key; + private int operation; + private boolean set; + + /** + * Make sure this can't be run + */ + @SuppressWarnings("unused") + private UpdateInterest() {} + + /** + * Create an object for encapsulating a interest set change + * request. + * + * @param key The key we wish to update + * @param operation The operation we wish to set or remove + * @param set Whether we want to set (true) or clear (false) the + * operation in the interest set + */ + public UpdateInterest(SelectionKey key, int operation, boolean set) { + this.key = key; + this.operation = operation; + this.set = set; + } + + /** + * This method is used for actually applying the updates to the + * SelectionKey in question at a time when it is safe to do so. + * If the SelectionKey has been invalidated in the meanwhile we + * do nothing. + */ + public void doUpdate() { + // bail if this key isn't valid anymore + if ((key == null) || (!key.isValid())) { + return; + } + + if (set) { + key.interestOps(key.interestOps() | operation); + } else { + key.interestOps(key.interestOps() & (~operation)); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/io/WritableByteTransmitter.java b/vespajlib/src/main/java/com/yahoo/io/WritableByteTransmitter.java new file mode 100644 index 00000000000..a7a6a9a1410 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/WritableByteTransmitter.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Marker interface for use with the BufferChain data store. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public interface WritableByteTransmitter { + public void send(ByteBuffer src) throws IOException; +} diff --git a/vespajlib/src/main/java/com/yahoo/io/package-info.java b/vespajlib/src/main/java/com/yahoo/io/package-info.java new file mode 100644 index 00000000000..db0caeb29d7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.io; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java b/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java new file mode 100644 index 00000000000..d0d52a8c619 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io.reader; + +import com.google.common.annotations.Beta; + +import java.io.IOException; +import java.io.Reader; +import java.util.List; + +/** + * A reader with a name. All reader methods are delegated to the wrapped reader. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +@Beta +public class NamedReader extends Reader { + + private final String name; + private final Reader reader; + + public NamedReader(String name, Reader reader) { + this.name = name; + this.reader = reader; + } + + public String getName() { return name; } + + public Reader getReader() { return reader; } + + /** Returns the name */ + public @Override String toString() { + return name; + } + + // The rest is reader method implementations which delegates to the wrapped reader + public @Override int read(java.nio.CharBuffer charBuffer) throws java.io.IOException { return reader.read(charBuffer); } + public @Override int read() throws java.io.IOException { return reader.read(); } + public @Override int read(char[] chars) throws java.io.IOException { return reader.read(chars); } + public @Override int read(char[] chars, int i, int i1) throws java.io.IOException { return reader.read(chars,i,i1); } + public @Override long skip(long l) throws java.io.IOException { return reader.skip(l); } + public @Override boolean ready() throws java.io.IOException { return reader.ready(); } + public @Override boolean markSupported() { return reader.markSupported(); } + public @Override void mark(int i) throws java.io.IOException { reader.mark(i); } + public @Override void reset() throws java.io.IOException { reader.reset(); } + public @Override void close() throws java.io.IOException { reader.close(); } + + /** Convenience method for closing a list of readers. Does nothing if the given reader list is null. */ + public static void closeAll(List<NamedReader> readers) { + if (readers==null) return; + for (Reader reader : readers) { + try { + reader.close(); + } + catch (IOException e) { + // Nothing to do about it + } + } + } + +}
\ No newline at end of file diff --git a/vespajlib/src/main/java/com/yahoo/io/reader/package-info.java b/vespajlib/src/main/java/com/yahoo/io/reader/package-info.java new file mode 100644 index 00000000000..34f57b61a55 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/reader/package-info.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * The classes in this package are not intended for external use. + */ +@PublicApi +@ExportPackage +package com.yahoo.io.reader; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/java7compat/Util.java b/vespajlib/src/main/java/com/yahoo/java7compat/Util.java new file mode 100644 index 00000000000..8a838308fbb --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/java7compat/Util.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.java7compat; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ + +public class Util { + private static final int javaVersion = Integer.valueOf(System.getProperty("java.version").substring(2,3)); + public static boolean isJava7Compatible() { return javaVersion >= 7; } + /** + * Takes the double value and prints it in a way that is compliant with the way java7 prints them. + * This is due to java7 finally fixing the trailing zero problem + * @param d the double value + * @return string representation of the double value + */ + public static String toJava7String(double d) { + String s = String.valueOf(d); + if ( ! isJava7Compatible() ) { + s = nonJava7CompatibleString(s); + } + return s; + } + + static String nonJava7CompatibleString(String s) { + if ((s.length() >= 3) && s.contains(".")) { + int l = s.length(); + for(; l > 2 && (s.charAt(l-1) == '0') && (s.charAt(l-2) >= '0') && (s.charAt(l-1) <= '9'); l--); + if (l != s.length()) { + s = s.substring(0, l); + } + } + return s; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/javacc/FastCharStream.java b/vespajlib/src/main/java/com/yahoo/javacc/FastCharStream.java new file mode 100644 index 00000000000..892240ce253 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/javacc/FastCharStream.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.javacc; + +import java.io.IOException; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class FastCharStream { + + private static final String JAVACC_EXCEPTION_FORMAT = "line -1, column "; + private static final IOException EOF = new IOException(); + private final String inputStr; + private final char[] inputArr; + private int tokenPos = 0; + private int readPos = 0; + + public FastCharStream(String input) { + this.inputStr = input; + this.inputArr = input.toCharArray(); + } + + public char readChar() throws IOException { + if (readPos >= inputArr.length) { + throw EOF; + } + return inputArr[readPos++]; + } + + @SuppressWarnings("deprecation") + public int getColumn() { + return getEndColumn(); + } + + @SuppressWarnings("deprecation") + public int getLine() { + return getEndLine(); + } + + public int getEndColumn() { + return readPos + 1; + } + + public int getEndLine() { + return -1; // indicate unset + } + + public int getBeginColumn() { + return tokenPos + 1; + } + + public int getBeginLine() { + return -1; // indicate unset + } + + public void backup(int amount) { + readPos -= amount; + } + + public char BeginToken() throws IOException { + tokenPos = readPos; + return readChar(); + } + + public String GetImage() { + return inputStr.substring(tokenPos, readPos); + } + + @SuppressWarnings("UnusedParameters") + public char[] GetSuffix(int len) { + throw new UnsupportedOperationException(); + } + + public void Done() { + + } + + public String formatException(String parseException) { + int errPos = findErrPos(parseException); + if (errPos < 0 || errPos > inputArr.length + 1) { + return parseException; + } + int errLine = 0; + int errColumn = 0; + for (int i = 0; i < errPos - 1; ++i) { + if (inputStr.charAt(i) == '\n') { + ++errLine; + errColumn = 0; + } else { + ++errColumn; + } + } + StringBuilder out = new StringBuilder(); + out.append(parseException.replace(JAVACC_EXCEPTION_FORMAT + errPos, + "line " + (errLine + 1) + ", column " + (errColumn + 1))); + out.append("\nAt position:\n"); + appendErrorPosition(errLine, out); + for (int i = 0; i < errColumn; ++i) { + out.append(" "); + } + out.append("^"); + return out.toString(); + } + + private void appendErrorPosition(int errLine, StringBuilder out) { + String[] inputStrLines = inputStr.split("\n"); + if (inputStrLines.length<errLine+1) { + out.append("EOF\n"); + } else { + out.append(inputStrLines[errLine]).append("\n"); + } + } + + private static int findErrPos(String str) { + int from = str.indexOf(JAVACC_EXCEPTION_FORMAT); + if (from < 0) { + return -1; + } + from = from + JAVACC_EXCEPTION_FORMAT.length(); + + int to = from; + while (to < str.length() && Character.isDigit(str.charAt(to))) { + ++to; + } + if (to == from) { + return -1; + } + + return Integer.valueOf(str.substring(from, to)); + } +}
\ No newline at end of file diff --git a/vespajlib/src/main/java/com/yahoo/javacc/UnicodeUtilities.java b/vespajlib/src/main/java/com/yahoo/javacc/UnicodeUtilities.java new file mode 100644 index 00000000000..45099a6855e --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/javacc/UnicodeUtilities.java @@ -0,0 +1,181 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.javacc; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class UnicodeUtilities { + + /** + * Adds a leading and trailing double quotation mark to the given string. This will escape whatever content is + * within the string literal. + * + * @param str The string to quote. + * @param quote The quote character. + * @return The quoted string. + */ + public static String quote(String str, char quote) { + StringBuilder ret = new StringBuilder(); + ret.append(quote); + for (int i = 0; i < str.length(); ++i) { + char c = str.charAt(i); + if (c == quote) { + ret.append("\\").append(c); + } else { + ret.append(escape(c)); + } + } + ret.append(quote); + return ret.toString(); + } + + /** + * Removes leading and trailing quotation mark from the given string. This method will properly unescape whatever + * content is withing the string literal as well. + * + * @param str The string to unquote. + * @return The unquoted string. + */ + public static String unquote(String str) { + if (str.length() == 0) { + return str; + } + char quote = str.charAt(0); + if (quote != '"' && quote != '\'') { + return str; + } + if (str.charAt(str.length() - 1) != quote) { + return str; + } + StringBuilder ret = new StringBuilder(); + for (int i = 1; i < str.length() - 1; ++i) { + char c = str.charAt(i); + if (c == '\\') { + if (++i == str.length() - 1) { + break; // done + } + c = str.charAt(i); + if (c == 'f') { + ret.append("\f"); + } else if (c == 'n') { + ret.append("\n"); + } else if (c == 'r') { + ret.append("\r"); + } else if (c == 't') { + ret.append("\t"); + } else if (c == 'u') { + if (++i > str.length() - 4) { + break; // done + } + try { + ret.append((char)Integer.parseInt(str.substring(i, i + 4), 16)); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(e); + } + i += 3; + } else { + ret.append(c); + } + } else if (c == quote) { + throw new IllegalArgumentException(); + } else { + ret.append(c); + } + } + return ret.toString(); + } + + private static String escape(char c) { + switch (c) { + case '\b': + return "\\b"; + case '\t': + return "\\t"; + case '\n': + return "\\n"; + case '\f': + return "\\f"; + case '\r': + return "\\r"; + case '\\': + return "\\\\"; + } + if (c < 0x20 || c > 0x7e) { + String unicode = Integer.toString(c, 16); + return "\\u" + "0000".substring(0, 4 - unicode.length()) + unicode + ""; + } + return "" + c; + } + + public static String generateToken(Predicate predicate) { + TokenBuilder builder = new TokenBuilder(); + for (int c = 0; c <= 0xffff; ++c) { + if (!predicate.accepts((char)c)) { + continue; + } + builder.add(c); + } + return builder.build(); + } + + public static interface Predicate { + + public boolean accepts(char c); + } + + private static class TokenBuilder { + + final StringBuilder token = new StringBuilder(); + int prevC = -1; + int fromC = 0; + int charCnt = 0; + + void add(int c) { + if (prevC + 1 == c) { + // in range + } else { + flushRange(); + fromC = c; + } + prevC = c; + } + + void flushRange() { + if (fromC > prevC) { + return; // handle initial condition + } + append(fromC); + if (fromC < prevC) { + token.append('-'); + append(prevC); + ++charCnt; + } + token.append(','); + if (++charCnt > 16) { + token.append('\n'); + charCnt = 0; + } + } + + void append(int c) { + token.append("\""); + if (c == '\n') { + token.append("\\n"); + } else if (c == '\r') { + token.append("\\r"); + } else if (c == '"') { + token.append("\\\""); + } else if (c == '\\') { + token.append("\\\\"); + } else { + token.append("\\u").append(String.format("%04x", c & 0xffff)); + } + token.append("\""); + } + + String build() { + flushRange(); + return token.toString(); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/javacc/package-info.java b/vespajlib/src/main/java/com/yahoo/javacc/package-info.java new file mode 100644 index 00000000000..c80e05df51f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/javacc/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.javacc; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/lang/MutableInteger.java b/vespajlib/src/main/java/com/yahoo/lang/MutableInteger.java new file mode 100644 index 00000000000..1dfe8bf5e88 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/lang/MutableInteger.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.lang; + +/** + * A mutable integer + * + * @author bratseth + */ +public class MutableInteger { + + private int value; + + public MutableInteger(int value) { + this.value = value; + } + + public int get() { return value; } + + public void set(int value) { this.value = value; } + + /** Adds the increment to the current value and returns the resulting value */ + public int add(int increment) { + value += increment; + return value; + } + + /** Adds the increment to the current value and returns the resulting value */ + public int subtract(int increment) { + value -= increment; + return value; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/lang/package-info.java b/vespajlib/src/main/java/com/yahoo/lang/package-info.java new file mode 100644 index 00000000000..08c86572029 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/lang/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.lang; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/net/HostName.java b/vespajlib/src/main/java/com/yahoo/net/HostName.java new file mode 100644 index 00000000000..3fb1fe49efd --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/HostName.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import java.io.BufferedReader; +import java.io.InputStreamReader; + +/** + * Utilities for getting the hostname on a system running with the JVM. This is moved here from the old + * HostSystem#getHostName in config-model. + * + * @author lulf + */ +public class HostName { + + private static String myHost = null; + + /** + * Static method that returns the name of localhost using shell + * command "hostname". + * + * @return the name of localhost. + * @throws RuntimeException if executing the command 'hostname' fails. + */ + public static synchronized String getLocalhost() { + if (myHost == null) { + try { + Process p = Runtime.getRuntime().exec("hostname"); + BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream())); + myHost = in.readLine(); + p.waitFor(); + if (p.exitValue() != 0) { + throw new RuntimeException("Command 'hostname' failed: exit("+p.exitValue()+")"); + } + } catch (Exception e) { + throw new RuntimeException("Failed when executing command 'hostname'", e); + } + } + return myHost; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java b/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java new file mode 100644 index 00000000000..540f8300f95 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import java.net.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Utilities for returning localhost addresses on Linux. + * See + * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4665037 + * on why this is necessary. + * + * @author bratseth + */ +public class LinuxInetAddress { + + private static Logger log = Logger.getLogger(LinuxInetAddress.class.getName()); + + /** + * Returns an InetAddress representing the address of the localhost. + * A non-loopback address is preferred if available. + * IPv4 is preferred over IPv6 if available. + * + * @return a localhost address + * @throws UnknownHostException if an address could not be determined + */ + public static InetAddress getLocalHost() throws UnknownHostException { + InetAddress localAddress; + try { + localAddress = InetAddress.getLocalHost(); + } catch (UnknownHostException e) { + return InetAddress.getLoopbackAddress(); + } + + if ( ! localAddress.isLoopbackAddress()) return localAddress; + + List<InetAddress> nonLoopbackAddresses = + getAllLocalFromNetwork().stream().filter(a -> ! a.isLoopbackAddress()).collect(Collectors.toList()); + if (nonLoopbackAddresses.isEmpty()) return localAddress; + + List<InetAddress> ipV4NonLoopbackAddresses = + nonLoopbackAddresses.stream().filter(a -> a instanceof Inet4Address).collect(Collectors.toList()); + if ( ! ipV4NonLoopbackAddresses.isEmpty()) return ipV4NonLoopbackAddresses.get(0); + + return nonLoopbackAddresses.get(0); + } + + /** + * Returns all local addresses of this host. + * + * @return an array of the addresses of this + * @throws UnknownHostException if we cannot access the network + */ + public static InetAddress[] getAllLocal() throws UnknownHostException { + InetAddress[] localInetAddresses = InetAddress.getAllByName("127.0.0.1"); + if ( ! localInetAddresses[0].isLoopbackAddress()) return localInetAddresses; + return getAllLocalFromNetwork().toArray(new InetAddress[0]); + } + + /** + * Returns all local addresses of this host. + * + * @return a list of the addresses of this + * @throws UnknownHostException if we cannot access the network + */ + private static List<InetAddress> getAllLocalFromNetwork() throws UnknownHostException { + try { + List<InetAddress> addresses = new ArrayList<>(); + for (NetworkInterface networkInterface : Collections.list(NetworkInterface.getNetworkInterfaces())) + addresses.addAll(Collections.list(networkInterface.getInetAddresses())); + return addresses; + } + catch (SocketException ex) { + throw new UnknownHostException("127.0.0.1"); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/net/URI.java b/vespajlib/src/main/java/com/yahoo/net/URI.java new file mode 100644 index 00000000000..1f9baa36c06 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/URI.java @@ -0,0 +1,819 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +import static com.yahoo.text.Lowercase.toLowerCase; + +/** + * <p>An URI. This is a pure (immutable) value object.</p> + * + * <p>This does more normalization of hierarchical URIs (URLs) than + * described in the RFC and allows hosts with underscores.</p> + * + * @author <a href="mailto:bratseth@fast.no">Jon S Bratseth</a> + */ +public class URI implements Cloneable, java.io.Serializable, Comparable<URI> { + + /** + * + */ + private static final long serialVersionUID = 2271558213498856909L; + + /** The uri string */ + private String uri; + + /** The scheme of the uri */ + private String scheme = null; + + /** The host part of the uri */ + private String host = null; + + /** The port number of the uri, or -1 if no port is explicitly given */ + private int port = -1; + + /** The part of the uri following the host (host and port) */ + private String rest = null; + + private static final Pattern tokenizePattern = Pattern.compile("[^\\w\\-]"); + + private boolean parsedDomain = false; + private String domain = null; + + private boolean parsedMainTld = false; + private String mainTld = null; + + private boolean parsedPath = false; + private String path = null; + + private boolean parsedParams = false; + private String params = null; + + private boolean parsedFilename = false; + private String filename = null; + + private boolean parsedExtension = false; + private String extension = null; + + private boolean parsedQuery = false; + private String query = null; + + private boolean parsedFragment = false; + private String fragment = null; + + + /** The explanation of why this uri is invalid, or null if it is valid */ + private String invalidExplanation = null; + + /** True if this uri is opaque, false if it is hierarchical */ + private boolean opaque = true; + + /** + * <p>Creates an URI without keeping the fragment (the part starting by #). + * If the uri is hierarchical, it is normalized and incorrect hierarchical uris + * which looks like urls are attempted repaired.</p> + * + * <p>Relative uris are not supported.</p> + * + * @param uriString the uri string + * @throws NullPointerException if the given uriString is null + */ + public URI(String uriString) { + this(uriString, false); + } + + /** + * Creates an URI, optionaly keeping the fragment (the part starting by #). + * If the uri is hierarchical, it is normalized and incorrect hierarchical uris + * which looks like urls are attempted repaired. + * + * <p>Relative uris are not supported.</p> + * + * @param uriString the uri string + * @param keepFragment true to keep the fragment + * @throws NullPointerException if the given uriString is null + */ + public URI(String uriString, boolean keepFragment) { + this(uriString, keepFragment, false); + } + + /** + * Creates an URI, optionaly keeping the fragment (the part starting by #). + * If the uri is hierarchical, it is normalized and incorrect hierarchical uris + * which looks like urls are attempted repaired. + * + * <p>Relative uris are not supported.</p> + * + * @param uriString the uri string + * @param keepFragment true to keep the fragment + * @param hierarchicalOnly will force any uri string given to be parsed as + * a hierarchical one, causing the uri to be invalid if it isn't + * @throws NullPointerException if the given uriString is null + */ + public URI(String uriString, boolean keepFragment, boolean hierarchicalOnly) { + if (uriString == null) { + throw new NullPointerException("Can not create an uri from null"); + } + + if (!keepFragment) { + int fragmentIndex = uriString.indexOf("#"); + + if (fragmentIndex >= 0) { + uriString = uriString.substring(0, fragmentIndex); + } + } + + try { + this.uri = uriString.trim(); + opaque = isOpaque(uri); + + // No further parsing of opaque uris + if (isOpaque() && !hierarchicalOnly) { + return; + } + opaque = false; + normalizeHierarchical(); + } catch (IllegalArgumentException e) { + if (e.getMessage() != null) { + invalidExplanation = e.getMessage(); + } else { + Throwable t = e.getCause(); + if (t != null && t.getMessage() != null) { + invalidExplanation = t.getMessage(); + } else { + invalidExplanation = "Invalid uri: " + e; + } + } + } + } + + /** Creates an url type uri */ + public URI(String scheme, String host, int port, String rest) { + this.scheme = scheme; + this.host = host; + this.port = port; + this.rest = rest; + recombine(); + normalizeHierarchical(); + opaque = false; + } + + /** Returns whether an url is opaque or hierarchical */ + private boolean isOpaque(String uri) { + int colonIndex = uri.indexOf(":"); + + if (colonIndex < 0) { + return true; + } else { + return !(uri.length() > colonIndex + 1 + && uri.charAt(colonIndex + 1) == '/'); + } + } + + /** + * Returns whether this is a valid URI (after normalizing). + * All non-hierarchical uri's containing a scheme is valid. + */ + public boolean isValid() { + return invalidExplanation == null; + } + + /** + * Normalizes this hierarchical uri according to FRC 2396 and the Overture + * standard. Before normalizing, some simple heuritics are use to make + * the uri complete if needed. After normalizing, the scheme, + * host, port and rest of this uri is set if defined. + * + * @throws IllegalArgumentException if this uri can not be normalized into a legal uri + */ + private void normalizeHierarchical() { + complete(); + escapeNonAscii(); + unescapeHtmlEntities(); + decompose(); + lowCaseHost(); + removeDefaultPortNumber(); + removeTrailingHostDot(); + makeDoubleSlashesSingle(); + recombine(); + } + + /** Applies simple heuristics to complete this uri if needed */ + private void complete() { + if (uri.startsWith("www.")) { + uri = "http://" + uri; + } else if (uri.startsWith("WWW")) { + uri = "http://" + uri; + } else if (uri.startsWith("/http:")) { + uri = uri.substring(1); + } else if (isFileURIShortHand(uri)) { + uri = "file://" + uri; + } + } + + private boolean isFileURIShortHand(String uri) { + if (uri.indexOf(":\\") == 1) { + return true; + } + if (uri.indexOf("c:/") == 0) { + return true; + } + if (uri.indexOf("d:/") == 0) { + return true; + } + return false; + } + + /** + * Decomposes this uri into scheme, host, port and rest. + */ + private void decompose() { + java.net.URI neturi = java.net.URI.create(uri).normalize(); + + scheme = neturi.getScheme(); + + host = neturi.getHost(); + boolean portAlreadyParsed = false; + + // No host if the host contains underscores + if (host == null) { + host = neturi.getAuthority(); + if (host != null) { + int colonPos = host.lastIndexOf(":"); + if (!scheme.equals("file") && colonPos > -1) { + //we probably have an (illegal) URI of type http://under_score.com:5000/ + try { + port = Integer.parseInt(host.substring(colonPos + 1, host.length())); + host = host.substring(0, colonPos); + portAlreadyParsed = true; + } catch (NumberFormatException nfe) { + //empty + } + } + } + } + + if ("file".equalsIgnoreCase(scheme)) { + if (host == null) { + host = "localhost"; + } else { + host = repairWindowsDrive(host, uri); + } + } + if (host == null) { + throw new IllegalArgumentException( + "A complete uri must specify a host"); + } + if (!portAlreadyParsed) { + port = neturi.getPort(); + } + rest = (neturi.getRawPath() != null ? neturi.getRawPath() : "") + + (neturi.getRawQuery() != null + ? ("?" + neturi.getRawQuery()) + : "") + + (neturi.getRawFragment() != null + ? ("#" + neturi.getRawFragment()) + : ""); + } + + /** c: turns to c when interpreted by URI. Repair it */ + private String repairWindowsDrive(String host, String uri) { + if (host.length() != 1) { + return host; + } + int driveIndex = uri.indexOf(host + ":"); + + if (driveIndex == 5 || driveIndex == 7) { // file:<drive> or file://<drive> + return host + ":"; + } else { + return host; + } + } + + /** "http://a/\u00E6" → "http://a/%E6;" */ + private void escapeNonAscii() { + char[] uriChars = uri.toCharArray(); + StringBuilder result = new StringBuilder(uri.length()); + + for (char uriChar : uriChars) { + if (uriChar >= 0x80 || uriChar == 0x22) { + result.append("%"); + result.append(Integer.toHexString(uriChar)); + result.append(";"); + } else { + result.append(uriChar); + } + } + uri = result.toString(); + } + + /** "http://a/&amp;" → "http://a/&" Currently ampersand only */ + private void unescapeHtmlEntities() { + int ampIndex = uri.indexOf("&"); + + if (ampIndex < 0) { + return; + } + + StringBuilder result = new StringBuilder(uri.substring(0, ampIndex)); + + while (ampIndex >= 0) { + result.append("&"); + int nextAmpIndex = uri.indexOf("&", ampIndex + 5); + + result.append( + uri.substring(ampIndex + 5, + nextAmpIndex > 0 ? nextAmpIndex : uri.length())); + ampIndex = nextAmpIndex; + } + uri = result.toString(); + } + + /** "HTTP://a" → "http://a" */ + private void lowCaseHost() { + host = toLowerCase(host); + } + + /** "http://a:80" → "http://a" and "https://a:443" → https//a */ + private void removeDefaultPortNumber() { + if (port == 80 && scheme.equals("http")) { + port = -1; + } else if (port == 443 && scheme.equals("https")) { + port = -1; + } + } + + /** "http://a./b" → "http://a/b" */ + private void removeTrailingHostDot() { + if (host.endsWith(".")) { + host = host.substring(0, host.length() - 1); + } + } + + /** "http://a//b" → "http://a/b" */ + private void makeDoubleSlashesSingle() { + StringBuilder result = new StringBuilder(rest.length()); + char[] restChars = rest.toCharArray(); + + for (int i = 0; i < restChars.length; i++) { + if (!(i + 1 < restChars.length && restChars[i] == '/' + && restChars[i + 1] == '/')) { + result.append(restChars[i]); + } + } + rest = result.toString(); + } + + /** Recombines the uri from the scheme, host, port and rest */ + private void recombine() { + StringBuilder recombined = new StringBuilder(100); + + recombined.append(scheme); + recombined.append("://"); + recombined.append(host); + if (port > -1) { + recombined.append(":").append(port); + } + if (rest != null) { + if (!rest.startsWith("/")) { + recombined.append("/"); + } + recombined.append(rest); + } else { + recombined.append("/"); // RFC 2396 violation, as required by search + } + uri = recombined.toString(); + } + + /** + * Returns the normalized scheme of this URI. + * + * @return the normalized scheme (protocol), or null if there is none, + * which may only be the case with non-hierarchical URIs + */ + public String getScheme() { + return scheme; + } + + /** + * Returns whether this URI is hierarchical or opaque. + * A typical example of an hierarchical URI is an URL, + * while URI's are mailto, news and such. + * + * @return true if the url is opaque, false if it is hierarchical + */ + public boolean isOpaque() { + return opaque; + } + + /** + * Returns the normalized host of this URI. + * + * @return the normalized host, or null if there is none, which may + * only be the case if this is a non-hierarchical uri + */ + public String getHost() { + return host; + } + + /** Returns the port number of this scheme if set explicitly, or -1 otherwise */ + public int getPort() { + return port; + } + + /** + * Returns the <i>rest</i> of this uri, that is what is following the host or port. + * This is path, query and fragment as defined in RFC 2396. Returns an empty string + * if this uri has no rest. + */ + public String getRest() { + if (rest == null) { + return null; + } else if (rest.equals("/")) { + return ""; + } else { + return rest; + } + } + + public String getDomain() { + if (parsedDomain) { + return domain; + } + String host = getHost(); + if (host == null) return null; + + int firstDotPos = host.indexOf("."); + int lastDotPos = host.lastIndexOf("."); + + String domain; + if (firstDotPos < 0) { + // "." was not found at all + domain = host; + } else if (firstDotPos == lastDotPos) { + //there is only one "." in the host + domain = host; + } else { + //for www.host.com return host.com + //TODO: Must be corrected when implementing tldlist + domain = host.substring(firstDotPos + 1, host.length()); + } + + this.parsedDomain = true; + this.domain = domain; + return domain; + } + + public String getMainTld() { + if (parsedMainTld) { + return mainTld; + } + String host = getHost(); + if (host == null) return null; + + int lastDotPos = host.lastIndexOf("."); + + String mainTld; + if (lastDotPos < 0) { + //no ".", no TLD + mainTld = null; + } else if (lastDotPos == host.length() - 1) { + //the "." is the last character + mainTld = null; + } else { + //for www.yahoo.co.uk return uk + //TODO: Implement list of TLDs from config? + mainTld = host.substring(lastDotPos + 1, host.length()); + } + this.parsedMainTld = true; + this.mainTld = mainTld; + return mainTld; + } + + public String getPath() { + if (parsedPath) { + return path; + } + String rest = this.rest; + if (rest == null) return null; + + rest = removeFragment(rest); + + int queryPos = rest.lastIndexOf("?"); + if (queryPos > -1) { + rest = rest.substring(0, queryPos); + } + this.parsedPath = true; + this.path = rest; + return this.path; + } + + private String removeFragment(String path) { + int fragmentPos = path.lastIndexOf("#"); + return (fragmentPos > -1) ? path.substring(0, fragmentPos) : path; + } + + public String getFilename() { + if (parsedFilename) { + return filename; + } + String path = getPath(); + if (path == null) return null; + + path = removeParams(path); + + int lastSlash = path.lastIndexOf("/"); + + String filename; + if (lastSlash < 0) { + //there is no slash, return the path, excluding params + filename = path; + } else if (lastSlash == path.length() - 1) { + //the slash is the last character, there is no filename here + filename = ""; + } else { + filename = path.substring(lastSlash + 1, path.length()); + } + this.parsedFilename = true; + this.filename = filename; + return filename; + } + + private String removeParams(String filename) { + int firstSemicolon = filename.indexOf(";"); + + if (firstSemicolon < 0) { + //there are no params + return filename; + } + return filename.substring(0, firstSemicolon); + } + + public String getExtension() { + if (parsedExtension) { + return extension; + } + String filename = getFilename(); + if (filename == null) return null; + + int lastDotPos = filename.lastIndexOf("."); + + String extension; + if (lastDotPos < 0) { + //there is no ".", there is no extension + extension = null; + } else if (lastDotPos == filename.length() - 1) { + //the "." is the last character, there is no extension + extension = null; + } else { + extension = filename.substring(lastDotPos + 1, filename.length()); + } + this.parsedExtension = true; + this.extension = extension; + return extension; + } + + public String getQuery() { + if (parsedQuery) { + return query; + } + String rest = this.rest; + if (rest == null) return null; + + rest = removeFragment(rest); + + int queryPos = rest.lastIndexOf("?"); + String query = null; + if (queryPos > -1) { + //we have a query + query = rest.substring(queryPos+1, rest.length()); + } + this.parsedQuery = true; + this.query = query; + return query; + } + + public String getFragment() { + if (parsedFragment) { + return fragment; + } + String path = this.rest; + if (path == null) return null; + + int fragmentPos = path.lastIndexOf("#"); + String fragment = null; + if (fragmentPos > -1) { + //we have a fragment + fragment = path.substring(fragmentPos+1, path.length()); + } + this.parsedFragment = true; + this.fragment = fragment; + return fragment; + } + + public String getParams() { + if (parsedParams) { + return params; + } + String path = getPath(); + if (path == null) return null; + + int semicolonPos = path.indexOf(";"); + String params; + if (semicolonPos < 0) { + //there is no semicolon, there are no params here + params = null; + } else if (semicolonPos == path.length() - 1) { + //the semicolon is the last character, there are no params here + params = null; + } else { + params = path.substring(semicolonPos + 1, path.length()); + } + this.parsedParams = true; + this.params = params; + return params; + } + + public static String[] tokenize(String item) { + return tokenizePattern.split(item); + } + + public List<Token> tokenize() { + List<Token> tokens = new ArrayList<>(); + + tokens.addAll(tokenize(URLContext.URL_SCHEME, getScheme())); + tokens.addAll(tokenize(URLContext.URL_HOST, getHost())); + tokens.addAll(tokenize(URLContext.URL_PORT, getPort() > -1 ? "" + getPort() : null)); + tokens.addAll(tokenize(URLContext.URL_PATH, getPath())); + tokens.addAll(tokenize(URLContext.URL_QUERY, getQuery())); + tokens.addAll(tokenize(URLContext.URL_FRAGMENT, getFragment())); + + return tokens; + } + + private List<Token> tokenize(URLContext context, String item) { + if (item == null) { + return new ArrayList<>(0); + } + String[] tokenStrings = tokenize(item); + List<Token> tokens = new ArrayList<>(tokenStrings.length); + for (String tokenString : tokenStrings) { + if (tokenString.length() > 0) { + tokens.add(new Token(context, tokenString)); + } + } + return tokens; + } + + /** Returns an explanation of why this uri is invalid, or null if it is valid */ + public String getInvalidExplanation() { + return invalidExplanation; + } + + public int hashCode() { + return uri.hashCode(); + } + + public boolean equals(Object object) { + if (!(object instanceof URI)) { + return false; + } + return (toString().equals(object.toString())); + } + + public int compareTo(URI object) { + return toString().compareTo(object.toString()); + } + + public Object clone() { + try { + return super.clone(); + } catch (CloneNotSupportedException e) { + throw new RuntimeException("Someone made me unclonable!", e); + } + } + + /** Returns a new URI with a changed scheme */ + public URI setScheme(String scheme) { + return new URI(scheme, host, port, rest); + } + + /** Returns a new URI with a changed host (or authority) */ + public URI setHost(String host) { + return new URI(scheme, host, port, rest); + } + + /** Returns a new URI with a changed port */ + public URI setPort(int port) { + return new URI(scheme, host, port, rest); + } + + /** Returns a new URI with a changed rest */ + public URI setRest(String rest) { + return new URI(scheme, host, port, rest); + } + + /** Returns a new uri with the an additional parameter */ + public URI addParameter(String name, String value) { + String newRest = rest; + + if (newRest == null) { + newRest = ""; + } + if (newRest.indexOf("?") < 0) { + newRest += "?"; + } else { + newRest += "&"; + } + newRest += name + "=" + value; + return new URI(scheme, host, port, newRest); + } + + /** Returns this uri as a string */ + public String stringValue() { + return uri; + } + + /** Returns this URI as a string */ + public String toString() { + return uri; + } + + /** + * Returns the depth of this uri. + * The depth of an hierarchical uri equals the number of slashes + * which are not separating the protocol and the host, and not at the end. + * + * @return the depth of this uri if it is hierarchical, or 0 if it is opaque + */ + public int getDepth() { + int colonIndex = uri.indexOf(':'); + + // count number of slashes in the Uri + int currentIndex = colonIndex; + int depth = 0; + + while (currentIndex != -1) { + currentIndex = uri.indexOf('/', currentIndex); + if (currentIndex != -1) { + depth++; + currentIndex++; + } + } + + if (uri.charAt(colonIndex + 1) == '/') { + depth--; + } + if (uri.charAt(colonIndex + 2) == '/') { + depth--; + } + if ((uri.charAt(uri.length() - 1) == '/') + && ((uri.length() - 1) > (colonIndex + 2))) { + depth--; + } + return depth; + } + + + public static class Token { + private final URLContext context; + private final String token; + + private Token(URLContext context, String token) { + this.context = context; + this.token = token; + } + + public URLContext getContext() { + return context; + } + + public String getToken() { + return token; + } + } + + public static enum URLContext { + URL_SCHEME(0, "scheme"), + URL_HOST(1, "host"), + URL_DOMAIN(2, "domain"), + URL_MAINTLD(3, "maintld"), + URL_PORT(4, "port"), + URL_PATH(5, "path"), + URL_FILENAME(6, "filename"), + URL_EXTENSION(7, "extension"), + URL_PARAMS(8, "params"), + URL_QUERY(9, "query"), + URL_FRAGMENT(10, "fragment"); + + public final int id; + public final String name; + + private URLContext(int id, String name) { + this.id = id; + this.name = name; + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/net/UriTools.java b/vespajlib/src/main/java/com/yahoo/net/UriTools.java new file mode 100644 index 00000000000..34d88713274 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/UriTools.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import java.net.URI; + +/** + * Utility methods for working with URIs. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class UriTools { + private UriTools() { + } + + /** + * Build a string representation of the normalized form of the given URI, + * containg the path and optionally query and fragment parts. The query part + * will be delimeted from the preceding data with "?" and the fragment with + * "#". + * + * @param uri + * source for path, query and fragment in returned data + * @return a string containing path, and optionally query and fragment, + * delimited by question mark and hash + */ + public static String rawRequest(final URI uri) { + final String rawQuery = uri.getRawQuery(); + final String rawFragment = uri.getRawFragment(); + final StringBuilder rawRequest = new StringBuilder(); + + rawRequest.append(uri.getRawPath()); + if (rawQuery != null) { + rawRequest.append("?").append(rawQuery); + } + + if (rawFragment != null) { + rawRequest.append("#").append(rawFragment); + } + + return rawRequest.toString(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/net/Url.java b/vespajlib/src/main/java/com/yahoo/net/Url.java new file mode 100644 index 00000000000..33571f9eb34 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/Url.java @@ -0,0 +1,253 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Url { + + private static final Pattern pattern = Pattern.compile( + //12 3 456 7 8 9ab c d e f g h i j + // 2 1 6 87 5 c b ed a4 f hg ji + "^(([^:/?#]+):)?(//((([^:@/?#]+)(:([^@/?#]+))?@))?(((\\[([^\\]]+)\\]|[^:/?#]+)(:([^/?#]+))?)))?([^?#]+)?(\\?([^#]*))?(#(.*))?"); + private final String image; + private final int schemeBegin; + private final int schemeEnd; + private final int userInfoBegin; + private final int userInfoEnd; + private final int passwordBegin; + private final int passwordEnd; + private final int hostBegin; + private final int hostEnd; + private final int portBegin; + private final int portEnd; + private final int pathBegin; + private final int pathEnd; + private final int queryBegin; + private final int queryEnd; + private final int fragmentBegin; + private final int fragmentEnd; + + public Url(String scheme, String user, String password, String host, Integer port, String path, String query, + String fragment) + { + StringBuilder image = new StringBuilder(); + schemeBegin = image.length(); + if (scheme != null) { + image.append(scheme); + schemeEnd = image.length(); + image.append(':'); + } else { + schemeEnd = schemeBegin; + } + if (host != null) { + image.append("//"); + } + userInfoBegin = image.length(); + if (user != null) { + image.append(user); + userInfoEnd = image.length(); + } else { + userInfoEnd = userInfoBegin; + } + if (password != null) { + image.append(':'); + passwordBegin = image.length(); + image.append(password); + passwordEnd = image.length(); + } else { + passwordBegin = image.length(); + passwordEnd = passwordBegin; + } + if (user != null || password != null) { + image.append('@'); + } + if (host != null) { + boolean esc = host.indexOf(':') >= 0; + if (esc) { + image.append('['); + } + hostBegin = image.length(); + image.append(host); + hostEnd = image.length(); + if (esc) { + image.append(']'); + } + } else { + hostBegin = image.length(); + hostEnd = hostBegin; + } + if (port != null) { + image.append(':'); + portBegin = image.length(); + image.append(port); + portEnd = image.length(); + } else { + portBegin = image.length(); + portEnd = portBegin; + } + pathBegin = image.length(); + if (path != null) { + image.append(path); + pathEnd = image.length(); + } else { + pathEnd = pathBegin; + } + if (query != null) { + image.append('?'); + queryBegin = image.length(); + image.append(query); + queryEnd = image.length(); + } else { + queryBegin = image.length(); + queryEnd = queryBegin; + } + if (fragment != null) { + image.append("#"); + fragmentBegin = image.length(); + image.append(fragment); + fragmentEnd = image.length(); + } else { + fragmentBegin = image.length(); + fragmentEnd = fragmentBegin; + } + this.image = image.toString(); + } + + public static Url fromString(String image) { + Matcher matcher = pattern.matcher(image); + if (!matcher.matches()) { + throw new IllegalArgumentException("Malformed URL."); + } + String host = matcher.group(12); + if (host == null) { + host = matcher.group(11); + } + if (host == null) { + host = matcher.group(9); + } + String port = matcher.group(14); + return new Url(matcher.group(2), matcher.group(6), matcher.group(8), host, + port != null ? Integer.valueOf(port) : null, matcher.group(15), matcher.group(17), + matcher.group(19)); + } + + public int getSchemeBegin() { + return schemeBegin; + } + + public int getSchemeEnd() { + return schemeEnd; + } + + public int getUserInfoBegin() { + return userInfoBegin; + } + + public int getUserInfoEnd() { + return userInfoEnd; + } + + public int getPasswordBegin() { + return passwordBegin; + } + + public int getPasswordEnd() { + return passwordEnd; + } + + public int getHostBegin() { + return hostBegin; + } + + public int getHostEnd() { + return hostEnd; + } + + public int getPortBegin() { + return portBegin; + } + + public int getPortEnd() { + return portEnd; + } + + public int getPathBegin() { + return pathBegin; + } + + public int getPathEnd() { + return pathEnd; + } + + public int getQueryBegin() { + return queryBegin; + } + + public int getQueryEnd() { + return queryEnd; + } + + public int getFragmentBegin() { + return fragmentBegin; + } + + public int getFragmentEnd() { + return fragmentEnd; + } + + public String getScheme() { + return schemeBegin < schemeEnd ? image.substring(schemeBegin, schemeEnd) : null; + } + + public String getUserInfo() { + return userInfoBegin < userInfoEnd ? image.substring(userInfoBegin, userInfoEnd) : null; + } + + public String getPassword() { + return passwordBegin < passwordEnd ? image.substring(passwordBegin, passwordEnd) : null; + } + + public String getHost() { + return hostBegin < hostEnd ? image.substring(hostBegin, hostEnd) : null; + } + + public Integer getPort() { + String str = getPortString(); + return str != null ? Integer.valueOf(str) : null; + } + + public String getPortString() { + return portBegin < portEnd ? image.substring(portBegin, portEnd) : null; + } + + public String getPath() { + return pathBegin < pathEnd ? image.substring(pathBegin, pathEnd) : null; + } + + public String getQuery() { + return queryBegin < queryEnd ? image.substring(queryBegin, queryEnd) : null; + } + + public String getFragment() { + return fragmentBegin < fragmentEnd ? image.substring(fragmentBegin, fragmentEnd) : null; + } + + @Override + public int hashCode() { + return image.hashCode(); + } + + @Override + public boolean equals(Object obj) { + return (obj instanceof Url) && image.equals(((Url)obj).image); + } + + @Override + public String toString() { + return image; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/net/UrlToken.java b/vespajlib/src/main/java/com/yahoo/net/UrlToken.java new file mode 100644 index 00000000000..785c3b1fe43 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/UrlToken.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class UrlToken { + + public enum Type { + SCHEME, + USERINFO, + PASSWORD, + HOST, + PORT, + PATH, + QUERY, + FRAGMENT + } + + private final Type type; + private final int offset; + private final String orig; + private final String term; + + public UrlToken(Type type, int offset, String orig, String term) { + if (type == null) { + throw new NullPointerException(); + } + this.type = type; + this.offset = offset; + this.orig = orig; + this.term = term; + } + + public Type getType() { + return type; + } + + public int getOffset() { + return offset; + } + + public int getLength() { + return orig != null ? orig.length() : 0; + } + + public String getOrig() { + return orig; + } + + public String getTerm() { + return term; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof UrlToken)) { + return false; + } + UrlToken rhs = (UrlToken)obj; + if (offset != rhs.offset) { + return false; + } + if (orig != null ? !orig.equals(rhs.orig) : rhs.orig != null) { + return false; + } + if (term != null ? !term.equals(rhs.term) : rhs.term != null) { + return false; + } + if (type != rhs.type) { + return false; + } + return true; + } + + @Override + public int hashCode() { + int result = type != null ? type.hashCode() : 0; + result = 31 * result + offset; + result = 31 * result + (orig != null ? orig.hashCode() : 0); + result = 31 * result + (term != null ? term.hashCode() : 0); + return result; + } + + @Override + public String toString() { + StringBuilder ret = new StringBuilder("UrlToken("); + ret.append("type=").append(type).append(", "); + ret.append("offset=").append(offset).append(", "); + if (orig != null) { + ret.append("orig='").append(orig).append("', "); + } + if (term != null) { + ret.append("term='").append(term).append("', "); + } + ret.setLength(ret.length() - 2); + ret.append(")"); + return ret.toString(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/net/UrlTokenizer.java b/vespajlib/src/main/java/com/yahoo/net/UrlTokenizer.java new file mode 100644 index 00000000000..ec617607b8a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/UrlTokenizer.java @@ -0,0 +1,178 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class UrlTokenizer { + + public static final String TERM_STARTHOST = "StArThOsT"; + public static final String TERM_ENDHOST = "EnDhOsT"; + + private static final Map<String, String> schemeToPort = new HashMap<>(); + private static final Map<String, String> portToScheme = new HashMap<>(); + private static final char TO_LOWER = (char)('A' - 'a'); + private final Url url; + + static { + registerScheme("ftp", 21); + registerScheme("gopher", 70); + registerScheme("http", 80); + registerScheme("https", 443); + registerScheme("imap", 143); + registerScheme("mailto", 25); + registerScheme("news", 119); + registerScheme("nntp", 119); + registerScheme("pop", 110); + registerScheme("rsync", 873); + registerScheme("rtsp", 554); + registerScheme("sftp", 22); + registerScheme("shttp", 443); + registerScheme("sip", 5060); + registerScheme("sips", 5061); + registerScheme("snmp", 161); + registerScheme("ssh", 22); + registerScheme("telnet", 23); + registerScheme("tftp", 69); + } + + public UrlTokenizer(String url) { + this(Url.fromString(url)); + } + + public UrlTokenizer(Url url) { + this.url = url; + } + + private String guessScheme(String port) { + String scheme = portToScheme.get(port); + if (scheme != null) { + return scheme; + } + return "http"; + } + + private String guessPort(String scheme) { + String port = schemeToPort.get(scheme); + if (port != null) { + return port; + } + return null; + } + + public List<UrlToken> tokenize() { + List<UrlToken> lst = new LinkedList<>(); + + int offset = 0; + String port = url.getPortString(); + String scheme = url.getScheme(); + if (scheme == null) { + scheme = guessScheme(port); + addTokens(lst, UrlToken.Type.SCHEME, offset, scheme, false); + } else { + addTokens(lst, UrlToken.Type.SCHEME, url.getSchemeBegin(), scheme, true); + offset = url.getSchemeEnd(); + } + + String userInfo = url.getUserInfo(); + if (userInfo != null) { + addTokens(lst, UrlToken.Type.USERINFO, url.getUserInfoBegin(), userInfo, true); + offset = url.getUserInfoEnd(); + } + + String password = url.getPassword(); + if (password != null) { + addTokens(lst, UrlToken.Type.PASSWORD, url.getPasswordBegin(), password, true); + offset = url.getPasswordEnd(); + } + + String host = url.getHost(); + if (host == null || host.isEmpty()) { + if (host != null) { + offset = url.getHostBegin(); + } + if ("file".equalsIgnoreCase(scheme)) { + addHostTokens(lst, offset, offset, "localhost", false); + } + } else { + addHostTokens(lst, url.getHostBegin(), url.getHostEnd(), host, true); + offset = url.getHostEnd(); + } + + port = url.getPortString(); + if (port == null) { + if ((port = guessPort(scheme)) != null) { + addTokens(lst, UrlToken.Type.PORT, offset, port, false); + } + } else { + addTokens(lst, UrlToken.Type.PORT, url.getPortBegin(), port, true); + } + + String path = url.getPath(); + if (path != null) { + addTokens(lst, UrlToken.Type.PATH, url.getPathBegin(), path, true); + } + + String query = url.getQuery(); + if (query != null) { + addTokens(lst, UrlToken.Type.QUERY, url.getQueryBegin(), query, true); + } + + String fragment = url.getFragment(); + if (fragment != null) { + addTokens(lst, UrlToken.Type.FRAGMENT, url.getFragmentBegin(), fragment, true); + } + + return lst; + } + + public static void addTokens(List<UrlToken> lst, UrlToken.Type type, int offset, String image, boolean orig) { + StringBuilder term = new StringBuilder(); + int prev = 0; + for (int skip, next = 0, len = image.length(); next < len; next += skip) { + char c = image.charAt(next); + if (c == '%') { + c = (char)Integer.parseInt(image.substring(next + 1, next + 3), 16); + skip = 3; + } else { + skip = 1; + } + if ((c >= '0' && c <= '9') || + (c >= 'a' && c <= 'z') || + (c == '-' || c == '_')) + { + term.append(c); + } else if (c >= 'A' && c <= 'Z') { + term.append((char)(c - TO_LOWER)); + } else { + if (prev < next) { + lst.add(new UrlToken(type, offset + (orig ? prev : 0), orig ? image.substring(prev, next) : null, + term.toString())); + term = new StringBuilder(); + } + prev = next + skip; + } + } + if (term.length() > 0) { + lst.add(new UrlToken(type, offset + (orig ? prev : 0), orig ? image.substring(prev) : null, + term.toString())); + } + } + + private static void addHostTokens(List<UrlToken> lst, int begin, int end, String image, boolean orig) { + lst.add(new UrlToken(UrlToken.Type.HOST, begin, null, TERM_STARTHOST)); + addTokens(lst, UrlToken.Type.HOST, begin, image, orig); + lst.add(new UrlToken(UrlToken.Type.HOST, end, null, TERM_ENDHOST)); + } + + private static void registerScheme(String scheme, int port) { + String str = String.valueOf(port); + schemeToPort.put(scheme, str); + portToScheme.put(str, scheme); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/net/package-info.java b/vespajlib/src/main/java/com/yahoo/net/package-info.java new file mode 100644 index 00000000000..ab474304da2 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/net/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.net; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/path/Path.java b/vespajlib/src/main/java/com/yahoo/path/Path.java new file mode 100644 index 00000000000..a15ebacc4cf --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/path/Path.java @@ -0,0 +1,211 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.path; + +import com.google.common.annotations.Beta; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +// TODO: Remove and replace usage by java.nio.file.Path + +/** + * Represents a path represented by a list of elements. Immutable + * + * @author lulf + * @since 5.1 + */ +@Beta +public final class Path { + + private final String delimiter; + private final List<String> elements = new ArrayList<>(); + + /** + * Create an empty path. + */ + private Path(String delimiter) { + this(new ArrayList<>(), delimiter); + } + + /** + * Create a new path as a copy of the provided path. + * @param rhs the path to copy. + */ + private Path(Path rhs) { + this(rhs.elements, rhs.delimiter); + } + + /** + * Create path with given elements. + * @param elements a list of path elements + */ + private Path(List<String> elements, String delimiter) { + this.elements.addAll(elements); + this.delimiter = delimiter; + } + + /** Returns whether this path is an immediate child of the given path */ + public boolean isChildOf(Path parent) { + return toString().startsWith(parent.toString()) && this.elements.size() -1 == parent.elements.size(); + } + + /** + * Add path elements by splitting based on delimiter and appending to elements. + */ + private void addElementsFromString(String path) { + String[] pathElements = path.split(delimiter); + if (pathElements != null) { + for (String elem : pathElements) { + if (!"".equals(elem)) { + elements.add(elem); + } + } + } + } + + /** + * Append an element to the path. Returns a new path with this element appended. + * @param name name of element to append. + * @return this, for chaining + */ + public Path append(String name) { + Path path = new Path(this); + path.addElementsFromString(name); + return path; + } + + /** + * Appends a path to another path, thereby creating a new path with the provided path + * appended to this. + * @param path The path to append. + * @return a new path with argument appended to it. + */ + public Path append(Path path) { + Path newPath = new Path(this); + newPath.elements.addAll(path.elements); + return newPath; + } + + /** + * Get the name of this path element, typically the last element in the path string. + * @return the name + */ + public String getName() { + if (elements.isEmpty()) { + return ""; + } + return elements.get(elements.size() - 1); + } + + /** + * Get a string representation of the path represented by this. + * @return a path string. + */ + public String getRelative() { + if (elements.isEmpty()) { + return ""; + } + StringBuilder sb = new StringBuilder(); + sb.append(elements.get(0)); + for (int i = 1; i < elements.size(); i++) { + sb.append(delimiter); + sb.append(elements.get(i)); + } + return sb.toString(); + } + + /** + * Get the parent path (all elements except last). + * @return the parent path. + */ + public Path getParentPath() { + ArrayList<String> parentElements = new ArrayList<>(); + if (elements.size() > 1) { + for (int i = 0; i < elements.size() - 1; i++) { + parentElements.add(elements.get(i)); + } + } + return new Path(parentElements, delimiter); + } + + /** + * Get string representation of path represented from the root node. + * @return string representation of path + */ + public String getAbsolute() { + return delimiter + getRelative(); + } + + public boolean isRoot() { + return elements.isEmpty(); + } + + public Iterator<String> iterator() { return elements.iterator(); } + + /** + * Convert to string. + * + * @return string representation of relative path + */ + @Override + public String toString() { + // TODO: This and the relative/absolute thing is wrong. The Path either *is* relative or absolute + // and should return accordingly here. getAbsolute/relative should be replaced by an asRelative/absolute + // returning another Path + return getRelative(); + } + + /** + * Create a path from a string. The string is treated as a relative path, and all redundant '/'-characters are + * stripped. + * @param path the relative path that this path should represent. + * @return a path object that may be used with the application package. + */ + public static Path fromString(String path) { + return fromString(path, "/"); + } + + /** + * Create a path from a string. The string is treated as a relative path, and all redundant delimiter-characters are + * stripped. + * @param path the relative path that this path should represent. + * @return a path object that may be used with the application package. + */ + public static Path fromString(String path, String delimiter) { + Path pathObj = new Path(delimiter); + pathObj.addElementsFromString(path); + return pathObj; + } + + /** + * Create an empty root path with '/' delimiter. + * + * @return an empty root path that can be appended + */ + public static Path createRoot() { + return createRoot("/"); + } + + /** + * Create an empty root path with delimiter. + * + * @return an empty root path that can be appended + */ + public static Path createRoot(String delimiter) { + return new Path(delimiter); + } + + @Override + public int hashCode() { + return elements.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other instanceof Path) { + return getRelative().equals(((Path) other).getRelative()); + } + return false; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/path/package-info.java b/vespajlib/src/main/java/com/yahoo/path/package-info.java new file mode 100644 index 00000000000..675e6b64ec2 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/path/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@PublicApi // Mainly because it's imported by config-model-fat +@ExportPackage +package com.yahoo.path; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/protect/ClassValidator.java b/vespajlib/src/main/java/com/yahoo/protect/ClassValidator.java new file mode 100644 index 00000000000..79e9d49c9f5 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/protect/ClassValidator.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.protect; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.List; + +/** + * Static utility methods to validate class properties. + * + * <p> + * Do note, this class will not be a reliable guarantee for correctness if you + * have a forest of methods only differing by return type (as + * contradistinguished from name and argument types), the current implementation + * is minimal. + * </p> + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class ClassValidator { + + /** + * Check all protected, public and package private declared methods of + * maskedClass is implemented in testClass. Note, this will by definition + * blow up on final methods in maskedClass. + * + * @param testClass + * class which wraps or masks another class + * @param maskedClass + * class which is masked or wrapped + * @return the methods which seem to miss from testClass to be complete + */ + public static List<Method> unmaskedMethods(Class<?> testClass, + Class<?> maskedClass) { + List<Method> unmasked = new ArrayList<>(); + Method[] methodsToMask = maskedClass.getDeclaredMethods(); + for (Method m : methodsToMask) { + int modifiers = m.getModifiers(); + if (Modifier.isPrivate(modifiers)) { + continue; + } + try { + testClass.getDeclaredMethod(m.getName(), m.getParameterTypes()); + } catch (NoSuchMethodException e) { + unmasked.add(m); + } + } + return unmasked; + } + + /** + * Check testClass overrides all protected, public and package private + * methods of its immediate super class. See unmaskedMethods(). + * + * @param testClass + * the class to check whether completely masks its super class + * @return the methods missing from testClass to completely override its + * immediate super class + */ + public static List<Method> unmaskedMethodsFromSuperclass(Class<?> testClass) { + return unmaskedMethods(testClass, testClass.getSuperclass()); + } + +}
\ No newline at end of file diff --git a/vespajlib/src/main/java/com/yahoo/protect/ErrorMessage.java b/vespajlib/src/main/java/com/yahoo/protect/ErrorMessage.java new file mode 100644 index 00000000000..c0cdc6017a0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/protect/ErrorMessage.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.protect; + + +/** + * An error message with a code. + * This class should be treated as immutable. + * + * @author bratseth + */ +public class ErrorMessage { + + /** An error code */ + protected int code; + + /** The short message of this error, always set */ + protected String message; + + /** The detailed instance message of this error, not always set */ + protected String detailedMessage = null; + + /** The cause of this error, or null if none is recorded */ + protected Throwable cause = null; + + /** + * Create an invalid instance for a subclass to initialize. + */ + public ErrorMessage() { + } + + public ErrorMessage(int code, String message) { + this.code = code; + this.message = message; + } + + /** + * Create an application specific error message with an application + * specific code + */ + public ErrorMessage(int code, String message, String detailedMessage) { + this(code, message); + this.detailedMessage = detailedMessage; + } + + /** Create an application specific error message with an application specific code */ + public ErrorMessage(int code, String message, String detailedMessage, Throwable cause) { + this(code, message, detailedMessage); + this.cause = cause; + } + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + /** Returns the detailed message, or null if there is no detailed message */ + public String getDetailedMessage() { + return detailedMessage; + } + /** + * Sets the cause of this. This should be set on errors which likely have their origin in plugin component code, + * not on others. + */ + public void setCause(Throwable cause) { this.cause=cause; } + + /** Returns the cause of this, or null if none is set */ + public Throwable getCause() { return cause; } + + public int hashCode() { + return code * 7 + message.hashCode() + (detailedMessage == null ? 0 : 17 * detailedMessage.hashCode()); + } + + /** + * Two error messages are equal if they have the same code and message. + * The cause is ignored in the comparison. + */ + @Override + public boolean equals(Object o) { + if (!(o instanceof ErrorMessage)) return false; + + ErrorMessage other = (ErrorMessage) o; + + if (this.code != other.code) return false; + + if (!this.message.equals(other.message)) return false; + + if (this.detailedMessage==null) return other.detailedMessage==null; + if (other.detailedMessage==null) return false; + + return this.detailedMessage.equals(other.detailedMessage); + } + + @Override + public String toString() { + String details = ""; + + if (detailedMessage != null) { + details = detailedMessage; + } + if (cause !=null) { + if (details.length()>0) + details+=": "; + details+= com.yahoo.yolean.Exceptions.toMessageString(cause); + } + if (details.length()>0) + details=" (" + details + ")"; + + return "error : " + message + details; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/protect/Process.java b/vespajlib/src/main/java/com/yahoo/protect/Process.java new file mode 100644 index 00000000000..6f381b40cd7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/protect/Process.java @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.protect; + + +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + + +/** + * A class for interacting with the global state of the running VM. + * + * @author Steinar Knutsen + */ +public final class Process { + + private static final Logger log = Logger.getLogger(Process.class.getName()); + + /** Die with a message, without dumping thread state */ + public static void logAndDie(String message) { + logAndDie(message, null); + } + + /** Die with a message, optionally dumping thread state */ + public static void logAndDie(String message, boolean dumpThreads) { + logAndDie(message, null, dumpThreads); + } + + /** Die with a message containing an exception, without dumping thread state */ + public static void logAndDie(String message, Throwable thrown) { + logAndDie(message, thrown, false); + } + + /** + * Log message as severe error, then forcibly exit runtime, without running + * exit handlers or otherwise waiting for cleanup. + * + * @param message message to log before exit + * @param thrown the throwable that caused the application to exit. + * @param dumpThreads if true the stack trace of all threads is dumped to the + * log with level info before shutting down + */ + public static void logAndDie(String message, Throwable thrown, boolean dumpThreads) { + try { + if (dumpThreads) + dumpThreads(); + if (thrown != null) + log.log(Level.SEVERE, message, thrown); + else + log.log(Level.SEVERE, message); + } finally { + try { + Runtime.getRuntime().halt(1); + } + catch (Throwable t) { + log.log(Level.SEVERE, "Runtime.halt rejected. Throwing an error."); + throw new ShutdownError("Shutdown requested, but failed to shut down"); + } + } + } + + + private static void dumpThreads() { + try { + log.log(Level.INFO, "About to shut down. Commencing full thread dump for diagnosis."); + Map<Thread, StackTraceElement[]> allStackTraces = Thread.getAllStackTraces(); + for (Map.Entry<Thread, StackTraceElement[]> e : allStackTraces.entrySet()) { + Thread t = e.getKey(); + StackTraceElement[] stack = e.getValue(); + StringBuilder forOneThread = new StringBuilder(); + int initLen; + forOneThread.append("Stack for thread: ").append(t.getName()).append(": "); + initLen = forOneThread.length(); + for (StackTraceElement s : stack) { + if (forOneThread.length() > initLen) { + forOneThread.append(" "); + } + forOneThread.append(s.toString()); + } + log.log(Level.INFO, forOneThread.toString()); + } + log.log(Level.INFO, "End of diagnostic thread dump."); + } catch (Exception e) { + // just give up... + } + } + + @SuppressWarnings("serial") + public static class ShutdownError extends Error { + + public ShutdownError(String message) { + super(message); + } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/protect/Validator.java b/vespajlib/src/main/java/com/yahoo/protect/Validator.java new file mode 100644 index 00000000000..9572fcb2ae4 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/protect/Validator.java @@ -0,0 +1,133 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.protect; + + +/** + * <p>Static utility methods for validating input.</p> + * + * @author bratseth + */ +public abstract class Validator { + + /** Throws NullPointerException if the argument is null */ + public static void ensureNotNull(String argumentDescription, Object argument) { + if (argument == null) + throw new NullPointerException(argumentDescription + " can not be null"); + } + + /** + * Throws an IllegalStateException if the given field value + * is initialized (not null) + */ + public static void ensureNotInitialized(String fieldDescription, Object fieldOwner, Object fieldValue) { + if (fieldValue != null) { + throw new IllegalStateException( + fieldDescription + " of " + fieldOwner + + " cannot be changed, it is already set " + "to " + + fieldValue); + } + } + + /** + * Throws an IllegalArgumentException if the given argument is not + * in the given range + * + * @param argumentDescription a description of the argument + * @param from the range start, inclusive + * @param to the range end, inclusive + * @param argument the argument value to check + */ + public static void ensureInRange(String argumentDescription, int from, int to, int argument) { + if (argument < from || argument > to) { + throw new IllegalArgumentException( + argumentDescription + " is " + argument + + " but must be between " + from + " and " + to); + } + } + + /** + * Throws an IllegalArgumentException if the first argument is not strictly + * smaller than the second argument + * + * @param smallDescription description of the smallest argument + * @param small the smallest argument + * @param largeDescription description of the lergest argument + * @param large the largest argument + */ + public static void ensureSmaller(String smallDescription, int small, String largeDescription, int large) { + if (small >= large) { + throw new IllegalArgumentException( + smallDescription + " is " + small + " but should be " + + "less than " + largeDescription + " " + large); + } + } + + /** + * Throws an IllegalArgumentException if the first argument is not strictly + * smaller than the second argument + * + * @param smallDescription + * description of the smallest argument + * @param small + * the smallest argument + * @param largeDescription + * description of the largest argument + * @param large + * the largest argument + */ + @SuppressWarnings({ "rawtypes", "unchecked" }) + public static void ensureSmaller(String smallDescription, Comparable small, String largeDescription, Comparable large) { + if (small.compareTo(large) >= 0) { + throw new IllegalArgumentException(smallDescription + " is " + + small + " but should be " + "less than " + + largeDescription + " " + large); + } + } + + /** + * Ensures that the given argument is true + * + * @param description of what is the case if the condition is false + * @param condition the condition to ensure is true + * @throws IllegalArgumentException if the given condition was false + */ + public static void ensure(String description, boolean condition) { + if (!condition) { + throw new IllegalArgumentException(description); + } + } + + /** + * Ensure the given argument is true, if not throw IllegalArgumentException + * concatenating the String representation of the description arguments. + */ + public static void ensure(boolean condition, Object... description) { + if (!condition) { + StringBuilder msg = new StringBuilder(); + for (Object part : description) { + msg.append(part.toString()); + } + throw new IllegalArgumentException(msg.toString()); + } + } + + /** + * Ensures that an item is of a particular class + * + * @param description + * a description of the item to be checked + * @param item + * the item to check the type of + * @param type + * the type the given item should be instanceof + * @throws IllegalArgumentException + * if the given item is not of the correct type + */ + public static void ensureInstanceOf(String description, Object item, Class<?> type) { + if (!type.isAssignableFrom(item.getClass())) { + throw new IllegalArgumentException(description + ", " + item + + " should " + "have been an instance of " + type); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/protect/package-info.java b/vespajlib/src/main/java/com/yahoo/protect/package-info.java new file mode 100644 index 00000000000..9260cd2e1e0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/protect/package-info.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Input validators, integrity checkers, error messages + * and similar classes. + */ +@ExportPackage +@PublicApi +package com.yahoo.protect; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/reflection/Casting.java b/vespajlib/src/main/java/com/yahoo/reflection/Casting.java new file mode 100644 index 00000000000..1f5c00bee59 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/reflection/Casting.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.reflection; + +import java.util.Optional; + +/** + * Utility methods for doing casting + * @author tonytv + */ +public class Casting { + /** + * Returns the casted instance if it is assignment-compatible with targetClass, + * or empty otherwise. + * @see Class#isInstance(Object) + */ + public static <T> Optional<T> cast(Class<T> targetClass, Object instance) { + return targetClass.isInstance(instance)? + Optional.of(targetClass.cast(instance)): + Optional.empty(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/reflection/package-info.java b/vespajlib/src/main/java/com/yahoo/reflection/package-info.java new file mode 100644 index 00000000000..43eda3cec94 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/reflection/package-info.java @@ -0,0 +1,9 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Package for reflection utility methods. + * @author tonytv + */ +@ExportPackage +package com.yahoo.reflection; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/rmi/.gitignore b/vespajlib/src/main/java/com/yahoo/rmi/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/rmi/.gitignore diff --git a/vespajlib/src/main/java/com/yahoo/slime/ArrayInserter.java b/vespajlib/src/main/java/com/yahoo/slime/ArrayInserter.java new file mode 100644 index 00000000000..ab1ae28d885 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/ArrayInserter.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Helper class for inserting values into an ArrayValue. + * For justification read Inserter documentation. + **/ +final class ArrayInserter implements Inserter { + private Cursor target; + public final ArrayInserter adjust(Cursor c) { + target = c; + return this; + } + public final Cursor insertNIX() { return target.addNix(); } + public final Cursor insertBOOL(boolean value) { return target.addBool(value); } + public final Cursor insertLONG(long value) { return target.addLong(value); } + public final Cursor insertDOUBLE(double value) { return target.addDouble(value); } + public final Cursor insertSTRING(String value) { return target.addString(value); } + public final Cursor insertSTRING(byte[] utf8) { return target.addString(utf8); } + public final Cursor insertDATA(byte[] value) { return target.addData(value); } + public final Cursor insertARRAY() { return target.addArray(); } + public final Cursor insertOBJECT() { return target.addObject(); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/ArrayTraverser.java b/vespajlib/src/main/java/com/yahoo/slime/ArrayTraverser.java new file mode 100644 index 00000000000..4cd24f15028 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/ArrayTraverser.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Callback interface for traversing arrays. + * Implement this and call Inspector.traverse() + * and you will get one callback for each array entry. + **/ +public interface ArrayTraverser +{ + /** + * Callback function to implement. + * @param idx array index for the current array entry. + * @param inspector accessor for the current array entry's value. + **/ + public void entry(int idx, Inspector inspector); +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/ArrayValue.java b/vespajlib/src/main/java/com/yahoo/slime/ArrayValue.java new file mode 100644 index 00000000000..4e520cde3c4 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/ArrayValue.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + + +final class ArrayValue extends Value { + + private int capacity = 16; + private int used = 0; + private Value[] values = new Value[capacity]; + private final SymbolTable names; + + public ArrayValue(SymbolTable names) { this.names = names; } + public final Type type() { return Type.ARRAY; } + public final int children() { return used; } + public final int entries() { return used; } + public final Value entry(int index) { + return (index < used) ? values[index] : NixValue.invalid(); + } + + public final void accept(Visitor v) { v.visitArray(this); } + + public final void traverse(ArrayTraverser at) { + for (int i = 0; i < used; i++) { + at.entry(i, values[i]); + } + } + + private void grow() { + Value[] v = values; + capacity = (capacity << 1); + values = new Value[capacity]; + System.arraycopy(v, 0, values, 0, used); + } + + protected final Value addLeaf(Value value) { + if (used == capacity) { + grow(); + } + values[used++] = value; + return value; + } + + public final Value addArray() { return addLeaf(new ArrayValue(names)); } + public final Value addObject() { return addLeaf(new ObjectValue(names)); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java new file mode 100644 index 00000000000..70e6892ce9f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import static com.yahoo.slime.BinaryFormat.*; + +final class BinaryDecoder { + BufferedInput in; + + private final SlimeInserter slimeInserter = new SlimeInserter(); + private final ArrayInserter arrayInserter = new ArrayInserter(); + private final ObjectInserter objectInserter = new ObjectInserter(); + + public BinaryDecoder() {} + + public Slime decode(byte[] bytes) { + return decode(bytes, 0, bytes.length); + } + public Slime decode(byte[] bytes, int offset, int length) { + Slime slime = new Slime(); + in = new BufferedInput(bytes, offset, length); + decodeSymbolTable(slime); + decodeValue(slimeInserter.adjust(slime)); + if (in.failed()) { + slime.wrap("partial_result"); + slime.get().setData("offending_input", in.getOffending()); + slime.get().setString("error_message", in.getErrorMessage()); + } + return slime; + } + + long read_cmpr_long() { + long next = in.getByte(); + long value = (next & 0x7f); + int shift = 7; + while ((next & 0x80) != 0) { + next = in.getByte(); + value |= ((next & 0x7f) << shift); + shift += 7; + } + return value; + } + + long read_size(int meta) { + return (meta == 0) ? read_cmpr_long() : (meta - 1); + } + + long read_bytes_le(int bytes) { + long value = 0; + int shift = 0; + for (int i = 0; i < bytes; ++i) { + long b = in.getByte(); + value |= (b & 0xff) << shift; + shift += 8; + } + return value; + } + + long read_bytes_be(int bytes) { + long value = 0; + int shift = 56; + for (int i = 0; i < bytes; ++i) { + long b = in.getByte(); + value |= (b & 0xff) << shift; + shift -= 8; + } + return value; + } + + Cursor decodeNIX(Inserter inserter) { + return inserter.insertNIX(); + } + + Cursor decodeBOOL(Inserter inserter, int meta) { + return inserter.insertBOOL(meta != 0); + } + + Cursor decodeLONG(Inserter inserter, int meta) { + long encoded = read_bytes_le(meta); + return inserter.insertLONG(decode_zigzag(encoded)); + } + + Cursor decodeDOUBLE(Inserter inserter, int meta) { + long encoded = read_bytes_be(meta); + return inserter.insertDOUBLE(decode_double(encoded)); + } + + Cursor decodeSTRING(Inserter inserter, int meta) { + long size = read_size(meta); + int sz = (int)size; // XXX + byte[] image = in.getBytes(sz); + return inserter.insertSTRING(image); + } + + Cursor decodeDATA(Inserter inserter, int meta) { + long size = read_size(meta); + int sz = (int)size; // XXX + byte[] image = in.getBytes(sz); + return inserter.insertDATA(image); + } + + Cursor decodeARRAY(Inserter inserter, int meta) { + Cursor cursor = inserter.insertARRAY(); + long size = read_size(meta); + for (int i = 0; i < size; ++i) { + decodeValue(arrayInserter.adjust(cursor)); + } + return cursor; + } + + Cursor decodeOBJECT(Inserter inserter, int meta) { + Cursor cursor = inserter.insertOBJECT(); + long size = read_size(meta); + for (int i = 0; i < size; ++i) { + long l = read_cmpr_long(); + int symbol = (int)l; // check for overflow? + decodeValue(objectInserter.adjust(cursor, symbol)); + } + return cursor; + } + + Cursor decodeValue(Inserter inserter, Type type, int meta) { + switch (type) { + case NIX: return decodeNIX(inserter); + case BOOL: return decodeBOOL(inserter, meta); + case LONG: return decodeLONG(inserter, meta); + case DOUBLE: return decodeDOUBLE(inserter, meta); + case STRING: return decodeSTRING(inserter, meta); + case DATA: return decodeDATA(inserter, meta); + case ARRAY: return decodeARRAY(inserter, meta); + case OBJECT: return decodeOBJECT(inserter, meta); + } + assert false : "should not be reached"; + return null; + } + + void decodeValue(Inserter inserter) { + byte b = in.getByte(); + Cursor cursor = decodeValue(inserter, + decode_type(b), + decode_meta(b)); + if (!cursor.valid()) { + in.fail("failed to decode value"); + } + } + + void decodeSymbolTable(Slime slime) { + long numSymbols = read_cmpr_long(); + final byte [] backing = in.getBacking(); + for (int i = 0; i < numSymbols; ++i) { + long size = read_cmpr_long(); + int sz = (int)size; // XXX + int offset = in.getPosition(); + in.skip(sz); + int symbol = slime.insert(Utf8Codec.decode(backing, offset, sz)); + if (symbol != i) { + in.fail("duplicate symbols in symbol table"); + return; + } + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java new file mode 100644 index 00000000000..daa926ec45b --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java @@ -0,0 +1,144 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import static com.yahoo.slime.BinaryFormat.*; + +final class BinaryEncoder implements +ArrayTraverser, ObjectSymbolTraverser +{ + BufferedOutput out; + + public BinaryEncoder(int capacity) { + out = new BufferedOutput(capacity); + } + + public BinaryEncoder() { + out = new BufferedOutput(); + } + + public byte[] encode(Slime slime) { + out.reset(); + encodeSymbolTable(slime); + encodeValue(slime.get()); + return out.toArray(); + } + + void encode_cmpr_long(long value) { + byte next = (byte)(value & 0x7f); + value >>>= 7; // unsigned shift + while (value != 0) { + next |= 0x80; + out.put(next); + next = (byte)(value & 0x7f); + value >>>= 7; + } + out.put(next); + } + + void write_type_and_size(int type, long size) { + if (size <= 30) { + out.put(encode_type_and_meta(type, (int)(size + 1))); + } else { + out.put(encode_type_and_meta(type, 0)); + encode_cmpr_long(size); + } + } + + void write_type_and_bytes_le(int type, long bits) { + int pos = out.position(); + byte val = 0; + out.put(val); + while (bits != 0) { + val = (byte)(bits & 0xff); + bits >>>= 8; + out.put(val); + } + val = encode_type_and_meta(type, out.position() - pos - 1); + out.absolutePut(pos, val); + } + + void write_type_and_bytes_be(int type, long bits) { + int pos = out.position(); + byte val = 0; + out.put(val); + while (bits != 0) { + val = (byte)(bits >> 56); + bits <<= 8; + out.put(val); + } + val = encode_type_and_meta(type, out.position() - pos - 1); + out.absolutePut(pos, val); + } + + void encodeNIX() { + out.put(Type.NIX.ID); + } + + void encodeBOOL(boolean value) { + out.put(encode_type_and_meta(Type.BOOL.ID, value ? 1 : 0)); + } + + void encodeLONG(long value) { + write_type_and_bytes_le(Type.LONG.ID, encode_zigzag(value)); + } + + void encodeDOUBLE(double value) { + write_type_and_bytes_be(Type.DOUBLE.ID, encode_double(value)); + } + + void encodeSTRING(byte[] value) { + write_type_and_size(Type.STRING.ID, value.length); + out.put(value); + } + + void encodeDATA(byte[] value) { + write_type_and_size(Type.DATA.ID, value.length); + out.put(value); + } + + void encodeARRAY(Inspector inspector) { + write_type_and_size(Type.ARRAY.ID, inspector.children()); + ArrayTraverser at = this; + inspector.traverse(at); + } + + void encodeOBJECT(Inspector inspector) { + write_type_and_size(Type.OBJECT.ID, inspector.children()); + ObjectSymbolTraverser ot = this; + inspector.traverse(ot); + } + + void encodeValue(Inspector inspector) { + switch(inspector.type()) { + case NIX: encodeNIX(); return; + case BOOL: encodeBOOL(inspector.asBool()); return; + case LONG: encodeLONG(inspector.asLong()); return; + case DOUBLE: encodeDOUBLE(inspector.asDouble()); return; + case STRING: encodeSTRING(inspector.asUtf8()); return; + case DATA: encodeDATA(inspector.asData()); return; + case ARRAY: encodeARRAY(inspector); return; + case OBJECT: encodeOBJECT(inspector); return; + } + assert false : "Should not be reached"; + } + + void encodeSymbolTable(Slime slime) { + int numSymbols = slime.symbols(); + encode_cmpr_long(numSymbols); + for (int i = 0 ; i < numSymbols; ++i) { + String name = slime.inspect(i); + byte[] bytes = Utf8Codec.encode(name); + encode_cmpr_long(bytes.length); + out.put(bytes); + } + } + + public void entry(int idx, Inspector inspector) { + encodeValue(inspector); + } + + public void field(int symbol, Inspector inspector) { + encode_cmpr_long(symbol); + encodeValue(inspector); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryFormat.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryFormat.java new file mode 100644 index 00000000000..4a126932f1e --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryFormat.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Class for serializing Slime data into binary format, or deserializing + * the binary format into a Slime object. + **/ +public class BinaryFormat { + static long encode_zigzag(long x) { + return ((x << 1) ^ (x >> 63)); // note ASR + } + + static long decode_zigzag(long x) { + return ((x >>> 1) ^ (-(x & 0x1))); // note LSR + } + + static long encode_double(double x) { + return Double.doubleToRawLongBits(x); + } + + static double decode_double(long x) { + return Double.longBitsToDouble(x); + } + + static byte encode_type_and_meta(int type, int meta) { + return (byte) ((meta << 3) | (type & 0x7)); + } + + static Type decode_type(byte type_and_meta) { + return Type.asType(type_and_meta & 0x7); + } + + static int decode_meta(byte type_and_meta) { + return ((type_and_meta & 0xff) >>> 3); + } + + /** + * Take a Slime object and serialize it into binary format. + * @param slime the object which is to be serialized. + * @return a new byte array with just the encoded slime. + **/ + public static byte[] encode(Slime slime) { + BinaryEncoder encoder = new BinaryEncoder(); + return encoder.encode(slime); + } + + /** + * Take binary data and deserialize it into a Slime object. + * The data is assumed to be the binary representation + * as if obtained by a call to the @ref encode() method. + * + * If the binary data can't be deserialized without problems + * the returned Slime object will instead only contain the + * three fields "partial_result" (contains anything successfully + * decoded before encountering problems), "offending_input" + * (containing any data that could not be deserialized) and + * "error_message" (a string describing the problem encountered). + * + * @param data the data to be deserialized. + * @return a new Slime object constructed from the data. + **/ + public static Slime decode(byte[] data) { + BinaryDecoder decoder = new BinaryDecoder(); + return decoder.decode(data); + } + + /** + * Take binary data and deserialize it into a Slime object. + * The data is assumed to be the binary representation + * as if obtained by a call to the @ref encode() method. + * + * If the binary data can't be deserialized without problems + * the returned Slime object will instead only contain the + * three fields "partial_result" (contains anything successfully + * decoded before encountering problems), "offending_input" + * (containing any data that could not be deserialized) and + * "error_message" (a string describing the problem encountered). + * + * @param data array containing the data to be deserialized. + * @param offset where in the array to start deserializing. + * @param length how many bytes the deserializer is allowed to consume. + * @return a new Slime object constructed from the data. + **/ + public static Slime decode(byte[] data, int offset, int length) { + BinaryDecoder decoder = new BinaryDecoder(); + return decoder.decode(data, offset, length); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/BoolValue.java b/vespajlib/src/main/java/com/yahoo/slime/BoolValue.java new file mode 100644 index 00000000000..95b22d47d4f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/BoolValue.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +final class BoolValue extends Value { + private static final BoolValue trueValue = new BoolValue(true); + private static final BoolValue falseValue = new BoolValue(false); + private final boolean value; + private BoolValue(boolean value) { this.value = value; } + final public Type type() { return Type.BOOL; } + final public boolean asBool() { return this.value; } + public final void accept(Visitor v) { v.visitBool(value); } + public static BoolValue instance(boolean bit) { return (bit ? trueValue : falseValue); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/BufferedInput.java b/vespajlib/src/main/java/com/yahoo/slime/BufferedInput.java new file mode 100644 index 00000000000..8eecc0d50f2 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/BufferedInput.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +final class BufferedInput { + + private final byte[] source; + private final int end; + private final int start; + private int position; + private String failReason; + private int failPos; + + void fail(String reason) { + if (failed()) { + return; + } + failReason = reason; + failPos = position; + position = end; + } + + public BufferedInput(byte[] bytes) { + this(bytes, 0, bytes.length); + } + + public BufferedInput(byte[] bytes, int offset, int length) { + this.source = bytes; + this.start = offset; + position = offset; + this.end = offset + length; + } + public final byte getByte() { + if (position == end) { + fail("underflow"); + return 0; + } + return source[position++]; + } + + public boolean failed() { + return failReason != null; + } + + public boolean eof() { + return this.position == this.end; + } + + public String getErrorMessage() { + return failReason; + } + + public int getConsumedSize() { + return failed() ? 0 : position - start; + } + + public byte[] getOffending() { + byte[] ret = new byte[failPos-start]; + System.arraycopy(source, start, ret, 0, failPos-start); + return ret; + } + + public final byte [] getBacking() { return source; } + public final int getPosition() { return position; } + public final void skip(int size) { + if (position + size > end) { + fail("underflow"); + } else { + position += size; + } + } + + public final byte[] getBytes(int size) { + if (position + size > end) { + fail("underflow"); + return new byte[0]; + } + byte[] ret = new byte[size]; + for (int i = 0; i < size; i++) { + ret[i] = source[position++]; + } + return ret; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java b/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java new file mode 100644 index 00000000000..ad7d2191130 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +final class BufferedOutput { + + private byte[] buf; + private int capacity; + private int pos; + + public BufferedOutput(int cap) { + capacity = (cap < 64) ? 64 : cap; + buf = new byte[capacity]; + } + + public BufferedOutput() { + this(4096); + } + + public void reset() { + pos = 0; + } + + private void reserve(int bytes) { + if (pos + bytes > capacity) { + while (pos + bytes > capacity) { + capacity = capacity * 2; + } + byte[] tmp = new byte[capacity]; + System.arraycopy(buf, 0, tmp, 0, pos); + buf = tmp; + } + } + + public int position() { return pos; } + + final void put(byte b) { + reserve(1); + buf[pos++] = b; + } + + final void absolutePut(int position, byte b) { + buf[position] = b; + } + + final void put(byte[] bytes) { + reserve(bytes.length); + for (byte b : bytes) { + buf[pos++] = b; + } + } + + public byte[] toArray() { + byte[] ret = new byte[pos]; + System.arraycopy(buf, 0, ret, 0, pos); + return ret; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Cursor.java b/vespajlib/src/main/java/com/yahoo/slime/Cursor.java new file mode 100644 index 00000000000..18d225c97be --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Cursor.java @@ -0,0 +1,285 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Interface for read-write access to any value or object that is part + * of a Slime. All accessors (including meta-data) are inherited from + * the Inspector interface. The navigational methods also work the + * same, except that they return a new Cursor for contained values and + * sub-structures, to permit writes to embedded values. + * + * The write operations are adding a new entry (to arrays), or setting + * a field value (for objects). If adding an entry or setting a field + * cannot be performed for any reason, an invalid Cursor is returned. + * + * This could happen because the current cursor is invalid, or it's + * not connected to an array value (for add methods), or it's not + * connected to an object (for set methods). Also note that you can + * only set() a field once; you cannot overwrite the field in any way. + **/ +public interface Cursor extends Inspector { + + /** + * Access an array entry. + * + * If the current Cursor doesn't connect to an array value, + * or the given array index is out of bounds, the returned + * Cursor will be invalid. + * @param idx array index. + * @return a new Cursor for the entry value. + **/ + @Override + public Cursor entry(int idx); + + /** + * Access an field in an object by symbol id. + * + * If the current Cursor doesn't connect to an object value, or + * the object value does not contain a field with the given symbol + * id, the returned Cursor will be invalid. + * @param sym symbol id. + * @return a new Cursor for the field value. + **/ + @Override + public Cursor field(int sym); + + /** + * Access an field in an object by symbol name. + * + * If the current Cursor doesn't connect to an object value, or + * the object value does not contain a field with the given symbol + * name, the returned Cursor will be invalid. + * @param name symbol name. + * @return a new Cursor for the field value. + **/ + @Override + public Cursor field(String name); + + /** + * Append an array entry containing a new value of NIX type. + * Returns an invalid Cursor if unsuccessful. + * @return a valid Cursor referencing the new entry value if successful. + **/ + public Cursor addNix(); + + /** + * Append an array entry containing a new value of BOOL type. + * Returns an invalid Cursor if unsuccessful. + * @param bit the actual boolean value for initializing a new BoolValue. + * @return a valid Cursor referencing the new entry value if successful. + **/ + public Cursor addBool(boolean bit); + + /** add a new entry of LONG type to an array */ + public Cursor addLong(long l); + + /** add a new entry of DOUBLE type to an array */ + public Cursor addDouble(double d); + + /** add a new entry of STRING type to an array */ + public Cursor addString(String str); + + /** add a new entry of STRING type to an array */ + public Cursor addString(byte[] utf8); + + /** add a new entry of DATA type to an array */ + public Cursor addData(byte[] data); + + /** + * Append an array entry containing a new value of ARRAY type. + * Returns a valid Cursor (thay may again be used for adding new + * sub-array entries) referencing the new entry value if + * successful; otherwise returns an invalid Cursor. + * @return new Cursor for the new entry value + **/ + public Cursor addArray(); + + /** + * Append an array entry containing a new value of OBJECT type. + * Returns a valid Cursor (thay may again be used for setting + * sub-fields inside the new object) referencing the new entry + * value if successful; otherwise returns an invalid Cursor. + * @return new Cursor for the new entry value + **/ + public Cursor addObject(); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of NIX type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @return new Cursor for the new field value + **/ + public Cursor setNix(int sym); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of BOOL type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @param bit the actual boolean value for the new field + * @return new Cursor for the new field value + **/ + public Cursor setBool(int sym, boolean bit); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of BOOL type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @param l the actual long value for the new field + * @return new Cursor for the new field value + **/ + public Cursor setLong(int sym, long l); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of BOOL type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @param d the actual double value for the new field + * @return new Cursor for the new field value + **/ + public Cursor setDouble(int sym, double d); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of BOOL type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @param str the actual string for the new field + * @return new Cursor for the new field value + **/ + public Cursor setString(int sym, String str); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of BOOL type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @param utf8 the actual string (encoded as UTF-8 data) for the new field + * @return new Cursor for the new field value + **/ + public Cursor setString(int sym, byte[] utf8); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of BOOL type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @param data the actual data to be put into the new field + * @return new Cursor for the new field value + **/ + public Cursor setData(int sym, byte[] data); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of ARRAY type. Returns a valid Cursor (thay may again be + * used for adding new array entries) referencing the new field + * value if successful; otherwise returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @return new Cursor for the new field value + **/ + public Cursor setArray(int sym); + + /** + * Set a field (identified with a symbol id) to contain a new + * value of OBJECT type. Returns a valid Cursor (thay may again + * be used for setting sub-fields inside the new object) + * referencing the new field value if successful; otherwise + * returns an invalid Cursor. + * @param sym symbol id for the field to be set + * @return new Cursor for the new field value + **/ + public Cursor setObject(int sym); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of NIX type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @return new Cursor for the new field value + **/ + public Cursor setNix(String name); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of BOOL type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @param bit the actual boolean value for the new field + * @return new Cursor for the new field value + **/ + public Cursor setBool(String name, boolean bit); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of LONG type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @param l the actual long value for the new field + * @return new Cursor for the new field value + **/ + public Cursor setLong(String name, long l); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of DOUBLE type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @param d the actual double value for the new field + * @return new Cursor for the new field value + **/ + public Cursor setDouble(String name, double d); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of STRING type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @param str the actual string for the new field + * @return new Cursor for the new field value + **/ + public Cursor setString(String name, String str); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of STRING type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @param utf8 the actual string (encoded as UTF-8 data) for the new field + * @return new Cursor for the new field value + **/ + public Cursor setString(String name, byte[] utf8); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of DATA type. Returns a valid Cursor referencing the new + * field value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @param data the actual data to be put into the new field + * @return new Cursor for the new field value + **/ + public Cursor setData(String name, byte[] data); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of ARRAY type. Returns a valid Cursor (thay may again be + * used for adding new array entries) referencing the new field + * value if successful; otherwise returns an invalid Cursor. + * @param name symbol name for the field to be set + * @return new Cursor for the new field value + **/ + public Cursor setArray(String name); + + /** + * Set a field (identified with a symbol name) to contain a new + * value of OBJECT type. Returns a valid Cursor (thay may again + * be used for setting sub-fields inside the new object) + * referencing the new field value if successful; otherwise + * returns an invalid Cursor. + * @param name symbol name for the field to be set + * @return new Cursor for the new field value + **/ + public Cursor setObject(String name); +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/DataValue.java b/vespajlib/src/main/java/com/yahoo/slime/DataValue.java new file mode 100644 index 00000000000..abaeac52245 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/DataValue.java @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +final class DataValue extends Value { + private final byte[] value; + public DataValue(byte[] value) { this.value = value; } + public final Type type() { return Type.DATA; } + public final byte[] asData() { return this.value; } + public final void accept(Visitor v) { v.visitData(value); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/DoubleValue.java b/vespajlib/src/main/java/com/yahoo/slime/DoubleValue.java new file mode 100644 index 00000000000..75b5acafb9b --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/DoubleValue.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +final class DoubleValue extends Value { + private final double value; + public DoubleValue(double value) { this.value = value; } + public final Type type() { return Type.DOUBLE; } + public final long asLong() { return (long)this.value; } + public final double asDouble() { return this.value; } + public final void accept(Visitor v) { v.visitDouble(value); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Inserter.java b/vespajlib/src/main/java/com/yahoo/slime/Inserter.java new file mode 100644 index 00000000000..8319efeb4f0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Inserter.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Helper interface for inserting values into any of the container + * classes (ArrayValue, ObjectValue, or Slime). May be useful for + * deserializers where you can use it to decouple the actual value + * decoding from the container where the value should be inserted. + **/ +interface Inserter { + Cursor insertNIX(); + Cursor insertBOOL(boolean value); + Cursor insertLONG(long value); + Cursor insertDOUBLE(double value); + Cursor insertSTRING(String value); + Cursor insertSTRING(byte[] utf8); + Cursor insertDATA(byte[] value); + Cursor insertARRAY(); + Cursor insertOBJECT(); +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Inspector.java b/vespajlib/src/main/java/com/yahoo/slime/Inspector.java new file mode 100644 index 00000000000..c4a98d98627 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Inspector.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Interface for read-only access to any value or object that is part + * of a Slime. You can access meta-data such as validity and actual + * type. You can always convert to any basic type by calling the + * various "as" accessor methods; these return a default value if the + * current Inspector is invalid or the type doesn't match your + * accessor type. If you want to do something exceptional instead + * when the types don't match, you must check using type() first. + **/ +public interface Inspector { + + /** check if this inspector is valid */ + public boolean valid(); + + /** return an enum describing value type */ + public Type type(); + + /** + * Check how many entries or fields are contained in the current value. + * Useful for arrays and objects; anything else always returns 0. + * @return number of entries/fields contained. + **/ + public int children(); + + /** + * Check how many entries are contained in the current value. + * Useful for arrays; anything else always returns 0. + * @return number of entries contained. + **/ + public int entries(); + + /** + * Check how many fields are contained in the current value. + * Useful for objects; anything else always returns 0. + * @return number of fields contained. + **/ + public int fields(); + + /** the current value (for booleans); default: false */ + public boolean asBool(); + + /** the current value (for integers); default: 0 */ + public long asLong(); + + /** the current value (for floating-point values); default: 0.0 */ + public double asDouble(); + + /** the current value (for string values); default: empty string */ + public String asString(); + + /** the current value encoded into UTF-8 (for string values); default: empty array */ + public byte[] asUtf8(); + + /** the current value (for data values); default: empty array */ + public byte[] asData(); + + /** + * Use the visitor pattern to resolve the underlying type of this value. + * @param v the visitor + **/ + public void accept(Visitor v); + + /** + * Traverse an array value, performing callbacks for each entry. + * + * If the current Inspector is connected to an array value, + * perform callbacks to the given traverser for each entry + * contained in the array. + * @param at traverser callback object. + **/ + @SuppressWarnings("overloads") + public void traverse(ArrayTraverser at); + + /** + * Traverse an object value, performing callbacks for each field. + * + * If the current Inspector is connected to an object value, + * perform callbacks to the given traverser for each field + * contained in the object. + * @param ot traverser callback object. + **/ + @SuppressWarnings("overloads") + public void traverse(ObjectSymbolTraverser ot); + + /** + * Traverse an object value, performing callbacks for each field. + * + * If the current Inspector is connected to an object value, + * perform callbacks to the given traverser for each field + * contained in the object. + * @param ot traverser callback object. + **/ + @SuppressWarnings("overloads") + public void traverse(ObjectTraverser ot); + + /** + * Access an array entry. + * + * If the current Inspector doesn't connect to an array value, + * or the given array index is out of bounds, the returned + * Inspector will be invalid. + * @param idx array index. + * @return a new Inspector for the entry value. + **/ + public Inspector entry(int idx); + + /** + * Access an field in an object by symbol id. + * + * If the current Inspector doesn't connect to an object value, or + * the object value does not contain a field with the given symbol + * id, the returned Inspector will be invalid. + * @param sym symbol id. + * @return a new Inspector for the field value. + **/ + public Inspector field(int sym); + + /** + * Access an field in an object by symbol name. + * + * If the current Inspector doesn't connect to an object value, or + * the object value does not contain a field with the given symbol + * name, the returned Inspector will be invalid. + * @param name symbol name. + * @return a new Inspector for the field value. + **/ + public Inspector field(String name); +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java b/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java new file mode 100644 index 00000000000..72837dc3354 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java @@ -0,0 +1,305 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import com.yahoo.text.Utf8; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * A port of the C++ json decoder intended to be fast. + * + * @author lulf + * @since 5.1.21 + */ +public class JsonDecoder { + private BufferedInput in; + private byte c; + + private final SlimeInserter slimeInserter = new SlimeInserter(); + private final ArrayInserter arrayInserter = new ArrayInserter(); + private final JsonObjectInserter objectInserter = new JsonObjectInserter(); + private final ByteArrayOutputStream buf = new ByteArrayOutputStream(); + + private static final byte[] TRUE = {'t', 'r', 'u', 'e'}; + private static final byte[] FALSE = {'f', 'a', 'l', 's', 'e'}; + private static final byte[] NULL = {'n', 'u', 'l', 'l'}; + private static final byte [] SQUARE_BRACKET_OPEN = { '[' }; + private static final byte [] SQUARE_BRACKET_CLOSE = { ']' }; + private static final byte [] CURLY_BRACE_OPEN = { '{' }; + private static final byte [] CURLY_BRACE_CLOSE = { '}' }; + private static final byte [] COLON = { ':' }; + private static final byte COMMA = ','; + + public JsonDecoder() {} + + public Slime decode(Slime slime, byte[] bytes) { + in = new BufferedInput(bytes); + next(); + decodeValue(slimeInserter.adjust(slime)); + if (in.failed()) { + slime.wrap("partial_result"); + slime.get().setData("offending_input", in.getOffending()); + slime.get().setString("error_message", in.getErrorMessage()); + } + return slime; + } + + private void decodeValue(Inserter inserter) { + skipWhiteSpace(); + switch (c) { + case '"': case '\'': decodeString(inserter); return; + case '{': decodeObject(inserter); return; + case '[': decodeArray(inserter); return; + case 't': expect(TRUE); inserter.insertBOOL(true); return; + case 'f': expect(FALSE); inserter.insertBOOL(false); return; + case 'n': expect(NULL); inserter.insertNIX(); return; + case '-': case '0': case '1': case '2': case '3': case '4': case '5': + case '6': case '7': case '8': case '9': decodeNumber(inserter); return; + } + in.fail("invalid initial character for value"); + } + + @SuppressWarnings("fallthrough") + private void decodeNumber(Inserter inserter) { + buf.reset(); + boolean likelyFloatingPoint=false; + for (;;) { + switch (c) { + case '.': case 'e': case 'E': + likelyFloatingPoint = true; + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + case '+': case '-': + buf.write(c); + next(); + break; + default: + if (likelyFloatingPoint) { + double num = Double.parseDouble(Utf8.toString(buf.toByteArray())); + inserter.insertDOUBLE(num); + } else { + long num = Long.parseLong(Utf8.toString(buf.toByteArray())); + inserter.insertLONG(num); + } + return; + } + } + } + + private void expect(byte [] expected) { + int i; + for (i = 0; i < expected.length && skip(expected[i]); i++) + ; + if (i != expected.length) { + in.fail("unexpected character"); + } + + } + + private void decodeArray(Inserter inserter) { + Cursor cursor = inserter.insertARRAY(); + expect(SQUARE_BRACKET_OPEN); + skipWhiteSpace(); + if (c != ']') { + do { + arrayInserter.adjust(cursor); + decodeValue(arrayInserter); + skipWhiteSpace(); + } while (skip(COMMA)); + } + expect(SQUARE_BRACKET_CLOSE); + } + + private void decodeObject(Inserter inserter) { + Cursor cursor = inserter.insertOBJECT(); + expect(CURLY_BRACE_OPEN); + skipWhiteSpace(); + if (c != '}') { + do { + skipWhiteSpace(); + String key = readKey(); + skipWhiteSpace(); + expect(COLON); + objectInserter.adjust(cursor, key); + decodeValue(objectInserter); + skipWhiteSpace(); + } while (skip(COMMA)); + } + expect(CURLY_BRACE_CLOSE); + } + + private String readKey() { + buf.reset(); + switch (c) { + case '"': case '\'': return readString(); + default: + for (;;) { + switch (c) { + case ':': case ' ': case '\t': case '\n': case '\r': case '\0': return Utf8.toString(buf.toByteArray()); + default: + buf.write(c); + next(); + break; + } + } + } + } + + private void decodeString(Inserter inserter) { + String value = readString(); + inserter.insertSTRING(value); + } + + private String readString() { + buf.reset(); + byte quote = c; + assert(quote == '"' || quote == '\''); + next(); + for (;;) { + switch (c) { + case '\\': + next(); + switch (c) { + case '"': case '\\': case '/': case '\'': + buf.write(c); + break; + case 'b': buf.write((byte) '\b'); break; + case 'f': buf.write((byte) '\f'); break; + case 'n': buf.write((byte) '\n'); break; + case 'r': buf.write((byte) '\r'); break; + case 't': buf.write((byte) '\t'); break; + case 'u': writeUtf8(dequoteUtf16(), buf, 0xffffff80); continue; + default: + in.fail("invalid quoted char(" + c + ")"); + break; + } + next(); + break; + case '"': case '\'': + if (c == quote) { + next(); + return Utf8.toString(buf.toByteArray()); + } else { + buf.write(c); + next(); + } + break; + case '\0': + in.fail("unterminated string"); + return Utf8.toString(buf.toByteArray()); + default: + buf.write(c); + next(); + break; + } + } + } + + private static void writeUtf8(long codepoint, ByteArrayOutputStream buf, long mask) { + if ((codepoint & mask) == 0) { + buf.write((byte) ((mask << 1) | codepoint)); + } else { + writeUtf8(codepoint >> 6, buf, mask >> (2 - ((mask >> 6) & 0x1))); + buf.write((byte) (0x80 | (codepoint & 0x3f))); + } + + } + + private static byte[] unicodeStart = {'\\', 'u'}; + private long dequoteUtf16() { + long codepoint = readHexValue(4); + if (codepoint >= 0xd800) { + if (codepoint < 0xdc00) { // high + expect(unicodeStart); + long low = readHexValue(4); + if (low >= 0xdc00 && low < 0xe000) { + codepoint = 0x10000 + ((codepoint - 0xd800) << 10) + (low - 0xdc00); + } else { + in.fail("missing low surrogate"); + } + } else if (codepoint < 0xe000) { // low + in.fail("unexpected low surrogate"); + } + } + return codepoint; + } + + private long readHexValue(int numBytes) { + long ret = 0; + for (long i = 0; i < numBytes; ++i) { + switch (c) { + case '0': ret = (ret << 4); break; + case '1': ret = (ret << 4) | 1; break; + case '2': ret = (ret << 4) | 2; break; + case '3': ret = (ret << 4) | 3; break; + case '4': ret = (ret << 4) | 4; break; + case '5': ret = (ret << 4) | 5; break; + case '6': ret = (ret << 4) | 6; break; + case '7': ret = (ret << 4) | 7; break; + case '8': ret = (ret << 4) | 8; break; + case '9': ret = (ret << 4) | 9; break; + case 'a': case 'A': ret = (ret << 4) | 0xa; break; + case 'b': case 'B': ret = (ret << 4) | 0xb; break; + case 'c': case 'C': ret = (ret << 4) | 0xc; break; + case 'd': case 'D': ret = (ret << 4) | 0xd; break; + case 'e': case 'E': ret = (ret << 4) | 0xe; break; + case 'f': case 'F': ret = (ret << 4) | 0xf; break; + default: + in.fail("invalid hex character"); + return 0; + } + next(); + } + return ret; + } + + + private void next() { + if (!in.eof()) { + c = in.getByte(); + } else { + c = 0; + } + } + + private boolean skip(byte x) { + if (c != x) { + return false; + } + next(); + return true; + } + + private void skipWhiteSpace() { + for (;;) { + switch (c) { + case ' ': case '\t': case '\n': case '\r': + next(); + break; + default: return; + } + } + } + + private static final class JsonObjectInserter implements Inserter { + private Cursor target; + private String key; + public final JsonObjectInserter adjust(Cursor c, String key) { + target = c; + this.key = key; + return this; + } + public final Cursor insertNIX() { return target.setNix(key); } + public final Cursor insertBOOL(boolean value) { return target.setBool(key, value); } + public final Cursor insertLONG(long value) { return target.setLong(key, value); } + public final Cursor insertDOUBLE(double value) { return target.setDouble(key, value); } + public final Cursor insertSTRING(String value) { return target.setString(key, value); } + public final Cursor insertSTRING(byte[] utf8) { return target.setString(key, utf8); } + public final Cursor insertDATA(byte[] value) { return target.setData(key, value); } + public final Cursor insertARRAY() { return target.setArray(key); } + public final Cursor insertOBJECT() { return target.setObject(key); } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java new file mode 100644 index 00000000000..28879311372 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java @@ -0,0 +1,220 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import com.yahoo.io.AbstractByteWriter; +import com.yahoo.io.ByteWriter; +import com.yahoo.text.AbstractUtf8Array; +import com.yahoo.text.Utf8; +import com.yahoo.text.Utf8String; +import com.yahoo.text.DoubleFormatter; + +import java.io.*; + +/** + * Encodes json from a slime object. + * + * @author lulf + */ +public final class JsonFormat implements SlimeFormat +{ + private final static byte [] HEX = Utf8.toBytes("0123456789ABCDEF"); + private final boolean compact; + public JsonFormat(boolean compact) { + this.compact = compact; + } + + @Override + public void encode(OutputStream os, Slime slime) throws IOException { + new Encoder(slime.get(), os, compact).encode(); + } + + public void encode(OutputStream os, Inspector value) throws IOException { + new Encoder(value, os, compact).encode(); + } + + public void encode(AbstractByteWriter os, Slime slime) throws IOException { + new Encoder(slime.get(), os, compact).encode(); + } + + public void encode(AbstractByteWriter os, Inspector value) throws IOException { + new Encoder(value, os, compact).encode(); + } + + @Override + public void decode(InputStream is, Slime slime) throws IOException { + throw new UnsupportedOperationException("Not implemented"); + } + + public static final class Encoder implements ArrayTraverser, ObjectTraverser + { + private final Inspector top; + private final AbstractByteWriter out; + private boolean head = true; + private boolean compact; + private int level = 0; + final static AbstractUtf8Array NULL=new Utf8String("null"); + final static AbstractUtf8Array FALSE=new Utf8String("false"); + final static AbstractUtf8Array TRUE=new Utf8String("true"); + + public Encoder(Inspector value, OutputStream out, boolean compact) { + this.top = value; + this.out = new ByteWriter(out); + this.compact = compact; + } + + public Encoder(Inspector value, AbstractByteWriter out, boolean compact) { + this.top = value; + this.out = out; + this.compact = compact; + } + + public void encode() throws IOException { + encodeValue(top); + if (!compact) { + out.append((byte) '\n'); + } + out.flush(); + } + + private void encodeNIX() throws IOException { + out.write(NULL); + } + + private void encodeBOOL(boolean value) throws IOException { + out.write(value ? TRUE : FALSE); + } + + private void encodeLONG(long value) throws IOException { + out.write(value); + } + + private void encodeDOUBLE(double value) throws IOException { + if (Double.isNaN(value) || Double.isInfinite(value)) { + out.write(NULL); + } else { + out.write(DoubleFormatter.stringValue(value)); + } + } + + private void encodeSTRING(byte[] value) throws IOException { + + byte [] data = new byte[value.length * 6 + 2]; + int len = 2; + int p = 0; + data[p++] = '"'; + for (int pos = 0; pos < value.length; pos++) { + byte c = value[pos]; + switch (c) { + case '"': data[p++] = '\\'; data[p++] = '"'; len += 2; break; + case '\\': data[p++] = '\\'; data[p++] = '\\'; len += 2; break; + case '\b': data[p++] = '\\'; data[p++] = 'b'; len += 2; break; + case '\f': data[p++] = '\\'; data[p++] = 'f'; len += 2; break; + case '\n': data[p++] = '\\'; data[p++] = 'n'; len += 2; break; + case '\r': data[p++] = '\\'; data[p++] = 'r'; len += 2; break; + case '\t': data[p++] = '\\'; data[p++] = 't'; len += 2; break; + default: + if (c > 0x1f || c < 0) { + data[p++] = c; + len++; + } else { // requires escaping according to RFC 4627 + data[p++] = '\\'; data[p++] = 'u'; data[p++] = '0'; data[p++] = '0'; + data[p++] = HEX[(c >> 4) & 0xf]; data[p++] = HEX[c & 0xf]; + len += 6; + } + } + } + data[p] = '"'; + out.append(data, 0, len); + } + + private void encodeDATA(byte[] value) throws IOException { + int len = value.length * 2 + 4; + byte [] data = new byte[len]; + int p = 0; + + data[p++] = '"'; data[p++] = '0'; data[p++] = 'x'; + for (int pos = 0; pos < value.length; pos++) { + data[p++] = HEX[(value[pos] >> 4) & 0xf]; data[p++] = HEX[value[pos] & 0xf]; + } + data[p] = '"'; + out.append(data, 0, len); + } + + private void encodeARRAY(Inspector inspector) throws IOException { + openScope((byte)'['); + ArrayTraverser at = this; + inspector.traverse(at); + closeScope((byte)']'); + } + + private void encodeOBJECT(Inspector inspector) throws IOException { + openScope((byte)'{'); + ObjectTraverser ot = this; + inspector.traverse(ot); + closeScope((byte) '}'); + } + + private void openScope(byte opener) throws IOException { + out.append(opener); + level++; + head = true; + } + + private void closeScope(byte closer) throws IOException { + level--; + separate(false); + out.append(closer); + } + + private void encodeValue(Inspector inspector) throws IOException { + switch(inspector.type()) { + case NIX: encodeNIX(); return; + case BOOL: encodeBOOL(inspector.asBool()); return; + case LONG: encodeLONG(inspector.asLong()); return; + case DOUBLE: encodeDOUBLE(inspector.asDouble()); return; + case STRING: encodeSTRING(inspector.asUtf8()); return; + case DATA: encodeDATA(inspector.asData()); return; + case ARRAY: encodeARRAY(inspector); return; + case OBJECT: encodeOBJECT(inspector); return; + } + assert false : "Should not be reached"; + } + + private void separate(boolean useComma) throws IOException { + if (!head && useComma) { + out.append((byte)','); + } else { + head = false; + } + if (!compact) { + out.append((byte)'\n'); + for (int lvl = 0; lvl < level; lvl++) { out.append((byte)' '); } + } + } + + public void entry(int idx, Inspector inspector) { + try { + separate(true); + encodeValue(inspector); + } catch (Exception e) { + // FIXME: Should we fix ArrayTraverser/ObjectTraverser API or do something more fancy here? + e.printStackTrace(); + } + } + + public void field(String name, Inspector inspector) { + try { + separate(true); + encodeSTRING(Utf8Codec.encode(name)); + out.append((byte)':'); + if (!compact) + out.append((byte)' '); + encodeValue(inspector); + } catch (Exception e) { + // FIXME: Should we fix ArrayTraverser/ObjectTraverser API or do something more fancy here? + e.printStackTrace(); + } + } + } +} + diff --git a/vespajlib/src/main/java/com/yahoo/slime/LongValue.java b/vespajlib/src/main/java/com/yahoo/slime/LongValue.java new file mode 100644 index 00000000000..fd423e178ec --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/LongValue.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +final class LongValue extends Value { + private final long value; + public LongValue(long value) { this.value = value; } + public final Type type() { return Type.LONG; } + public final long asLong() { return this.value; } + public final double asDouble() { return (double)this.value; } + public final void accept(Visitor v) { v.visitLong(value); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/NixValue.java b/vespajlib/src/main/java/com/yahoo/slime/NixValue.java new file mode 100644 index 00000000000..524fd391cdd --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/NixValue.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +final class NixValue extends Value { + private static final NixValue invalidNix = new NixValue(); + private static final NixValue validNix = new NixValue(); + private NixValue() {} + public final Type type() { return Type.NIX; } + public final void accept(Visitor v) { + if (valid()) { + v.visitNix(); + } else { + v.visitInvalid(); + } + } + public static NixValue invalid() { return invalidNix; } + public static NixValue instance() { return validNix; } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/ObjectInserter.java b/vespajlib/src/main/java/com/yahoo/slime/ObjectInserter.java new file mode 100644 index 00000000000..e8651d53702 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/ObjectInserter.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Helper class for inserting values into an ObjectValue. + * For justification read Inserter documentation. + **/ +final class ObjectInserter implements Inserter { + private Cursor target; + private int symbol; + public final ObjectInserter adjust(Cursor c, int sym) { + target = c; + symbol = sym; + return this; + } + public final Cursor insertNIX() { return target.setNix(symbol); } + public final Cursor insertBOOL(boolean value) { return target.setBool(symbol, value); } + public final Cursor insertLONG(long value) { return target.setLong(symbol, value); } + public final Cursor insertDOUBLE(double value) { return target.setDouble(symbol, value); } + public final Cursor insertSTRING(String value) { return target.setString(symbol, value); } + public final Cursor insertSTRING(byte[] utf8) { return target.setString(symbol, utf8); } + public final Cursor insertDATA(byte[] value) { return target.setData(symbol, value); } + public final Cursor insertARRAY() { return target.setArray(symbol); } + public final Cursor insertOBJECT() { return target.setObject(symbol); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/ObjectSymbolTraverser.java b/vespajlib/src/main/java/com/yahoo/slime/ObjectSymbolTraverser.java new file mode 100644 index 00000000000..fe939d15969 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/ObjectSymbolTraverser.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Callback interface for traversing objects. + * Implement this and call Inspector.traverse() + * and you will get one callback for each field in an object. + **/ +public interface ObjectSymbolTraverser +{ + /** + * Callback function to implement. + * @param sym symbol id for the current field. + * @param inspector accessor for the current field's value. + **/ + public void field(int sym, Inspector inspector); +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/ObjectTraverser.java b/vespajlib/src/main/java/com/yahoo/slime/ObjectTraverser.java new file mode 100644 index 00000000000..9d933670363 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/ObjectTraverser.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Callback interface for traversing objects. + * Implement this and call Inspector.traverse() + * and you will get one callback for each field in an object. + **/ +public interface ObjectTraverser +{ + /** + * Callback function to implement. + * @param name symbol name for the current field. + * @param inspector accessor for the current field's value. + **/ + public void field(String name, Inspector inspector); +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/ObjectValue.java b/vespajlib/src/main/java/com/yahoo/slime/ObjectValue.java new file mode 100644 index 00000000000..3d8b54ed294 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/ObjectValue.java @@ -0,0 +1,108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * A Value holding a slime "Object", a dynamic collection of named + * value fields. Fields can be inspected or traversed using the + * {@link Inspector} interface, and you can add new fields by using the + * various "set" methods in the @ref Cursor interface. + **/ +final class ObjectValue extends Value { + + private int capacity = 16; + private int hashSize() { return (capacity + (capacity >> 1) - 1); } + private int used = 0; + private Value[] values = new Value[capacity]; + private int[] hash = new int[capacity + hashSize() + (capacity << 1)]; + private final SymbolTable names; + + private final void rehash() { + capacity = (capacity << 1); + Value[] v = values; + values = new Value[capacity]; + System.arraycopy(v, 0, values, 0, used); + int[] h = hash; + hash = new int[capacity + hashSize() + (capacity << 1)]; + System.arraycopy(h, 0, hash, 0, used); + for (int i = 0; i < used; i++) { + int prev = (capacity + (hash[i] % hashSize())); + int entry = hash[prev]; + while (entry != 0) { + prev = entry + 1; + entry = hash[prev]; + } + final int insertIdx = (capacity + hashSize() + (i << 1)); + hash[prev] = insertIdx; + hash[insertIdx] = i; + } + } + + private final Value put(int sym, Value value) { + if (used == capacity) { + rehash(); + } + int prev = (capacity + (sym % hashSize())); + int entry = hash[prev]; + while (entry != 0) { + final int idx = hash[entry]; + if (hash[idx] == sym) { // found entry + return NixValue.invalid(); + } + prev = entry + 1; + entry = hash[prev]; + } + final int insertIdx = (capacity + hashSize() + (used << 1)); + hash[prev] = insertIdx; + hash[insertIdx] = used; + hash[used] = sym; + values[used++] = value; + return value; + } + + private final Value get(int sym) { + int entry = hash[capacity + (sym % hashSize())]; + while (entry != 0) { + final int idx = hash[entry]; + if (hash[idx] == sym) { // found entry + return values[idx]; + } + entry = hash[entry + 1]; + } + return NixValue.invalid(); + } + + public ObjectValue(SymbolTable names) { this.names = names; } + public ObjectValue(SymbolTable names, int sym, Value value) { + this.names = names; + put(sym, value); + } + + public final Type type() { return Type.OBJECT; } + public final int children() { return used; } + public final int fields() { return used; } + + public final Value field(int sym) { return get(sym); } + public final Value field(String name) { return get(names.lookup(name)); } + + public final void accept(Visitor v) { v.visitObject(this); } + + public final void traverse(ObjectSymbolTraverser ot) { + for (int i = 0; i < used; ++i) { + ot.field(hash[i], values[i]); + } + } + + public final void traverse(ObjectTraverser ot) { + for (int i = 0; i < used; ++i) { + ot.field(names.inspect(hash[i]), values[i]); + } + } + + protected final Cursor setLeaf(int sym, Value value) { return put(sym, value); } + public final Cursor setArray(int sym) { return put(sym, new ArrayValue(names)); } + public final Cursor setObject(int sym) { return put(sym, new ObjectValue(names)); } + + protected final Cursor setLeaf(String name, Value value) { return put(names.insert(name), value); } + public final Cursor setArray(String name) { return put(names.insert(name), new ArrayValue(names)); } + public final Cursor setObject(String name) { return put(names.insert(name), new ObjectValue(names)); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Slime.java b/vespajlib/src/main/java/com/yahoo/slime/Slime.java new file mode 100644 index 00000000000..387a81a7655 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Slime.java @@ -0,0 +1,150 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Top-level value class that contains one Value data object and a + * symbol table (shared between all directly or indirectly contained + * ObjectValue data objects). + **/ +public final class Slime +{ + private final SymbolTable names = new SymbolTable(); + private Value root = NixValue.instance(); + + /** + * Construct an empty Slime with an empty top-level value. + **/ + public Slime() {} + + /** return count of names in the symbol table. */ + public int symbols() { + return names.symbols(); + } + + /** + * Return the symbol name associated with an id. + * @param symbol the id, must be in range [0, symbols()-1] + **/ + public String inspect(int symbol) { + return names.inspect(symbol); + } + + /** + * Add a name to the symbol table; if the name is already + * in the symbol table just returns the id it already had. + * @param name the name to insert + * @return the id now associated with the name + **/ + public int insert(String name) { + return names.insert(name); + } + + /** + * Find the id associated with a symbol name; if the + * name was not in the symbol table returns the + * constant Integer.MAX_VALUE instead. + **/ + public int lookup(String name) { + return names.lookup(name); + } + + /** Get a Cursor connected to the top-level data object. */ + public Cursor get() { return root; } + + /** + * Create a new empty value and make it the new top-level data object. + **/ + public Cursor setNix() { + root = NixValue.instance(); + return root; + } + + /** + * Create a new boolean value and make it the new top-level data object. + * @param bit the actual boolean value for the new value + **/ + public Cursor setBool(boolean bit) { + root = BoolValue.instance(bit); + return root; + } + + /** + * Create a new double value and make it the new top-level data object. + * @param l the actual long value for the new value + **/ + public Cursor setLong(long l) { + root = new LongValue(l); + return root; + } + + /** + * Create a new double value and make it the new top-level data object. + * @param d the actual double value for the new value + **/ + public Cursor setDouble(double d) { + root = new DoubleValue(d); + return root; + } + + /** + * Create a new string value and make it the new top-level data object. + * @param str the actual string for the new value + **/ + public Cursor setString(String str) { + root = new StringValue(str); + return root; + } + + /** + * Create a new string value and make it the new top-level data object. + * @param utf8 the actual string (encoded as UTF-8 data) for the new value + **/ + public Cursor setString(byte[] utf8) { + root = new Utf8Value(utf8); + return root; + } + + /** + * Create a new data value and make it the new top-level data object. + * @param data the actual data to be put into the new value. + **/ + public Cursor setData(byte[] data) { + root = new DataValue(data); + return root; + } + + /** + * Create a new array value and make it the new top-level data object. + **/ + public Cursor setArray() { + root = new ArrayValue(names); + return root; + } + + /** + * Create a new object value and make it the new top-level data object. + **/ + public Cursor setObject() { + root = new ObjectValue(names); + return root; + } + + /** + * Take the current top-level data object and make it a field in a + * new ObjectValue with the given symbol id as field id; the new + * ObjectValue will also become the new top-level data object. + **/ + public Cursor wrap(int sym) { + root = new ObjectValue(names, sym, root); + return root; + } + + /** + * Take the current top-level data object and make it a field in a + * new ObjectValue with the given symbol name as field name; the new + * ObjectValue will also become the new top-level data object. + **/ + public Cursor wrap(String name) { + return wrap(names.insert(name)); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeFormat.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeFormat.java new file mode 100644 index 00000000000..142514c45a8 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeFormat.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * @author lulf + * @since 5.1 + */ +public interface SlimeFormat { + /** + * Encode a slime object into the provided output stream + * @param os The outputstream to write to. + * @param slime The slime object to encode. + */ + public void encode(OutputStream os, Slime slime) throws IOException; + + /** + * Encode a slime object into the provided output stream + * @param is The input stream to read from. + * @param slime The slime object to decode into. + */ + public void decode(InputStream is, Slime slime) throws IOException; +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeInserter.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeInserter.java new file mode 100644 index 00000000000..d6e78873ec7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeInserter.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Helper class for inserting values into a Slime object. + * For justification read Inserter documentation. + **/ +final class SlimeInserter implements Inserter { + private Slime target; + public final SlimeInserter adjust(Slime slime) { + target = slime; + return this; + } + public final Cursor insertNIX() { return target.setNix(); } + public final Cursor insertBOOL(boolean value) { return target.setBool(value); } + public final Cursor insertLONG(long value) { return target.setLong(value); } + public final Cursor insertDOUBLE(double value) { return target.setDouble(value); } + public final Cursor insertSTRING(String value) { return target.setString(value); } + public final Cursor insertSTRING(byte[] utf8) { return target.setString(utf8); } + public final Cursor insertDATA(byte[] value) { return target.setData(value); } + public final Cursor insertARRAY() { return target.setArray(); } + public final Cursor insertOBJECT() { return target.setObject(); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/StringValue.java b/vespajlib/src/main/java/com/yahoo/slime/StringValue.java new file mode 100644 index 00000000000..a5b72578d5d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/StringValue.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * A value holding a String in Java native format. + * See also @ref Utf8Value (for lazy decoding). + **/ +final class StringValue extends Value { + private final String value; + private byte[] utf8; + public StringValue(String value) { this.value = value; } + public final Type type() { return Type.STRING; } + public final String asString() { return this.value; } + public final byte[] asUtf8() { + if (utf8 == null) { + utf8 = Utf8Codec.encode(value); + } + return utf8; + } + public final void accept(Visitor v) { v.visitString(value); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/SymbolTable.java b/vespajlib/src/main/java/com/yahoo/slime/SymbolTable.java new file mode 100644 index 00000000000..133cbd1ba8e --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/SymbolTable.java @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * A mapping from an arbitrary set of unique strings to a range of + * integers. Slime users normally won't need to use this class + * directly. + **/ +final class SymbolTable { + + public static final int INVALID = Integer.MAX_VALUE; + + private static final int[] emptyHash = new int[1]; + + private int capacity = 0; + private int hashSize() { return (capacity + (capacity >> 1) - 1); } + private int used = 0; + private String[] names; + private int[] hash = emptyHash; + + private final void rehash() { + if (capacity == 0) { + capacity = 32; + names = new String[capacity]; + hash = new int[hashSize() + (capacity << 1)]; + return; + } + capacity = (capacity << 1); + String[] n = names; + names = new String[capacity]; + System.arraycopy(n, 0, names, 0, used); + hash = new int[hashSize() + (capacity << 1)]; + for (int i = 0; i < used; i++) { + int prev = Math.abs(names[i].hashCode() % hashSize()); + int entry = hash[prev]; + while (entry != 0) { + prev = entry + 1; + entry = hash[prev]; + } + final int insertIdx = (hashSize() + (i << 1)); + hash[prev] = insertIdx; + hash[insertIdx] = i; + } + } + + /** Return count of contained symbol names. */ + final int symbols() { return used; } + + /** + * Return the symbol name associated with an id. + * @param symbol the id, must be in range [0, symbols()-1] + **/ + final String inspect(int symbol) { return names[symbol]; } + + /** + * Add a name to the symbol table; if the name is already + * in the symbol table just returns the id it already had. + * @param name the name to insert + * @return the id now associated with the name + **/ + final int insert(String name) { + if (used == capacity) { + rehash(); + } + int prev = Math.abs(name.hashCode() % hashSize()); + int entry = hash[prev]; + while (entry != 0) { + final int sym = hash[entry]; + if (names[sym].equals(name)) { // found entry + return sym; + } + prev = entry + 1; + entry = hash[prev]; + } + final int insertIdx = (hashSize() + (used << 1)); + hash[prev] = insertIdx; + hash[insertIdx] = used; + names[used++] = name; + return (used - 1); + } + + /** + * Find the id associated with a symbol name; if the + * name was not in the symbol table returns the + * INVALID constant instead. + **/ + final int lookup(String name) { + int entry = hash[Math.abs(name.hashCode() % hashSize())]; + while (entry != 0) { + final int sym = hash[entry]; + if (names[sym].equals(name)) { // found entry + return sym; + } + entry = hash[entry + 1]; + } + return INVALID; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Type.java b/vespajlib/src/main/java/com/yahoo/slime/Type.java new file mode 100644 index 00000000000..036d577e106 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Type.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Enumeration of all possibly Slime data types. + **/ +public enum Type { + NIX(0), + BOOL(1), + LONG(2), + DOUBLE(3), + STRING(4), + DATA(5), + ARRAY(6), + OBJECT(7); + + public final byte ID; + private Type(int id) { this.ID = (byte)id; } + + private static final Type[] types = values(); + static Type asType(int id) { return types[id]; } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Utf8Codec.java b/vespajlib/src/main/java/com/yahoo/slime/Utf8Codec.java new file mode 100644 index 00000000000..c9e86b73073 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Utf8Codec.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import com.yahoo.text.Utf8; + +/** + * Helper class for conversion between String and UTF-8 representations. + **/ +class Utf8Codec { + public static String decode(byte[] data, int pos, int len) { + return Utf8.toString(data, pos, len); + } + public static byte[] encode(String str) { + return Utf8.toBytes(str); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Utf8Value.java b/vespajlib/src/main/java/com/yahoo/slime/Utf8Value.java new file mode 100644 index 00000000000..6aa95310b86 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Utf8Value.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * A value type encapsulating a String in its UTF-8 representation. + * Useful for lazy decoding; if the data is just passed through in + * UTF-8 it will never be converted at all. + **/ +final class Utf8Value extends Value { + private final byte[] value; + private String string; + public Utf8Value(byte[] value) { this.value = value; } + public final Type type() { return Type.STRING; } + public final String asString() { + if (string == null) { + string = Utf8Codec.decode(value, 0, value.length); + } + return string; + } + public final byte[] asUtf8() { return value; } + public final void accept(Visitor v) { v.visitString(value); } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Value.java b/vespajlib/src/main/java/com/yahoo/slime/Value.java new file mode 100644 index 00000000000..d86bf8607bd --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Value.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + + +import java.io.ByteArrayOutputStream; + +/** + * Common implementation for all value types. + * All default behavior is here, so specific types only + * need override their actually useful parts. + **/ + +abstract class Value implements Cursor { + + private static final String emptyString = ""; + private static final byte[] emptyData = new byte[0]; + + public final boolean valid() { return this != NixValue.invalid(); } + public int children() { return 0; } + public int entries() { return 0; } + public int fields() { return 0; } + + public boolean asBool() { return false; } + public long asLong() { return 0; } + public double asDouble() { return 0.0; } + public String asString() { return emptyString; } + public byte[] asUtf8() { return emptyData; } + public byte[] asData() { return emptyData; } + + public void traverse(ArrayTraverser at) {} + public void traverse(ObjectSymbolTraverser ot) {} + public void traverse(ObjectTraverser ot) {} + + public Value entry(int idx) { return NixValue.invalid(); } + public Value field(String name) { return NixValue.invalid(); } + public Value field(int sym) { return NixValue.invalid(); } + + protected Cursor addLeaf(Value value) { return NixValue.invalid(); } + public Cursor addArray() { return NixValue.invalid(); } + public Cursor addObject() { return NixValue.invalid(); } + + public final Cursor addNix() { return addLeaf(NixValue.instance()); } + public final Cursor addBool(boolean bit) { return addLeaf(BoolValue.instance(bit)); } + public final Cursor addLong(long l) { return addLeaf(new LongValue(l)); } + public final Cursor addDouble(double d) { return addLeaf(new DoubleValue(d)); } + public final Cursor addString(String str) { return addLeaf(new StringValue(str)); } + public final Cursor addString(byte[] utf8) { return addLeaf(new Utf8Value(utf8)); } + public final Cursor addData(byte[] data) { return addLeaf(new DataValue(data)); } + + protected Cursor setLeaf(int sym, Value value) { return NixValue.invalid(); } + public Cursor setArray(int sym) { return NixValue.invalid(); } + public Cursor setObject(int sym) { return NixValue.invalid(); } + + public final Cursor setNix(int sym) { return setLeaf(sym, NixValue.instance()); } + public final Cursor setBool(int sym, boolean bit) { return setLeaf(sym, BoolValue.instance(bit)); } + public final Cursor setLong(int sym, long l) { return setLeaf(sym, new LongValue(l)); } + public final Cursor setDouble(int sym, double d) { return setLeaf(sym, new DoubleValue(d)); } + public final Cursor setString(int sym, String str) { return setLeaf(sym, new StringValue(str)); } + public final Cursor setString(int sym, byte[] utf8) { return setLeaf(sym, new Utf8Value(utf8)); } + public final Cursor setData(int sym, byte[] data) { return setLeaf(sym, new DataValue(data)); } + + protected Cursor setLeaf(String name, Value value) { return NixValue.invalid(); } + public Cursor setArray(String name) { return NixValue.invalid(); } + public Cursor setObject(String name) { return NixValue.invalid(); } + + public final Cursor setNix(String name) { return setLeaf(name, NixValue.instance()); } + public final Cursor setBool(String name, boolean bit) { return setLeaf(name, BoolValue.instance(bit)); } + public final Cursor setLong(String name, long l) { return setLeaf(name, new LongValue(l)); } + public final Cursor setDouble(String name, double d) { return setLeaf(name, new DoubleValue(d)); } + public final Cursor setString(String name, String str) { return setLeaf(name, new StringValue(str)); } + public final Cursor setString(String name, byte[] utf8) { return setLeaf(name, new Utf8Value(utf8)); } + public final Cursor setData(String name, byte[] data) { return setLeaf(name, new DataValue(data)); } + + public final String toString() { + try { + // should produce non-compact json, but we need compact + // json for slime summaries until we have a more generic + // json rendering pipeline in place. + ByteArrayOutputStream a = new ByteArrayOutputStream(); + new JsonFormat(true).encode(a, this); + byte[] utf8 = a.toByteArray(); + return Utf8Codec.decode(utf8, 0, utf8.length); + } catch (Exception e) { + return "null"; + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/Visitor.java b/vespajlib/src/main/java/com/yahoo/slime/Visitor.java new file mode 100644 index 00000000000..d36a7da9078 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/Visitor.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +/** + * Visitor interface used to resolve the underlying type of a value + * represented by an Inspector. + **/ +public interface Visitor { + /** + * Called when the visited Inspector is not valid. + **/ + public void visitInvalid(); + public void visitNix(); + public void visitBool(boolean bit); + public void visitLong(long l); + public void visitDouble(double d); + public void visitString(String str); + public void visitString(byte[] utf8); + public void visitData(byte[] data); + public void visitArray(Inspector arr); + public void visitObject(Inspector obj); +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/package-info.java b/vespajlib/src/main/java/com/yahoo/slime/package-info.java new file mode 100644 index 00000000000..92a37a2ca37 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/slime/package-info.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * SLIME: 'Schema-Less Interface/Model/Exchange'. Slime is a way to + * handle schema-less structured data to be used as part of interfaces + * between components (RPC signatures), internal models + * (config/parameters) and data exchange between components + * (documents). The goal for Slime is to be flexible and lightweight + * and at the same time limit the extra overhead in space and time + * compared to schema-oriented approaches like protocol buffers and + * avro. The data model is inspired by JSON and associative arrays + * typically used in programming languages with dynamic typing. + **/ +@ExportPackage +package com.yahoo.slime; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/system/CatchSigTerm.java b/vespajlib/src/main/java/com/yahoo/system/CatchSigTerm.java new file mode 100644 index 00000000000..f58c161941a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/system/CatchSigTerm.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +import java.lang.reflect.*; + +// import sun.misc.Signal; +// import sun.misc.SignalHandler; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class CatchSigTerm { + /** + * Sets up a signal handler for SIGTERM, where a given AtomicBoolean + * gets a true value when the TERM signal is caught. + * + * Callers basically have two options for acting on the TERM signal: + * + * They may choose to synchronize and wait() on this variable, + * and they will be notified when it changes state to true. To avoid + * problems with spurious wakeups, use a while loop and wait() + * again if the state is still false. As soon as the caller has been + * woken up and the state is true, the application should exit as + * soon as possible. + * + * They may also choose to poll the state of this variable. As soon + * as its state becomes true, the signal has been received, and the + * application should exit as soon as possible. + * + * @param signalCaught set to false initially, will be set to true when SIGTERM is caught. + */ + @SuppressWarnings("rawtypes") + public static void setup(final AtomicBoolean signalCaught) { + signalCaught.set(false); + try { + Class shc = Class.forName("sun.misc.SignalHandler"); + Class ssc = Class.forName("sun.misc.Signal"); + + InvocationHandler ihandler = new InvocationHandler() { + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + synchronized (signalCaught) { + signalCaught.set(true); + signalCaught.notifyAll(); + } + return null; + } + }; + Object shandler = Proxy.newProxyInstance(CatchSigTerm.class.getClassLoader(), + new Class[] { shc }, + ihandler); + Constructor[] c = ssc.getDeclaredConstructors(); + assert c.length == 1; + Object sigterm = c[0].newInstance("TERM"); + Method m = findMethod(ssc, "handle"); + assert m != null; // "NoSuchMethodException" + m.invoke(null, sigterm, shandler); + } catch (ClassNotFoundException | InvocationTargetException | InstantiationException | IllegalAccessException e) { + System.err.println("FAILED setting up signal catching: "+e); + } + } + + @SuppressWarnings("rawtypes") + private static Method findMethod(Class c, String name) { + for (Method m : c.getDeclaredMethods()) { + if (m.getName().equals(name)) { + return m; + } + } + return null; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/system/CommandLineParser.java b/vespajlib/src/main/java/com/yahoo/system/CommandLineParser.java new file mode 100644 index 00000000000..9f6922e5084 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/system/CommandLineParser.java @@ -0,0 +1,216 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +import java.util.*; + +/** + * Simple command line parser, handling multiple arguments and multiple unary and binary switches starting with -. + * + * Terms used: + * + * progname -binaryswitch foo -unaryswitch argument1 argument2 + * + * @author vegardh + * + */ +public class CommandLineParser { + private List<String> inputStrings = new ArrayList<>(); + private Map<String, String> legalUnarySwitches = new HashMap<>(); + private Map<String, String> legalBinarySwitches = new HashMap<>(); + private List<String> unarySwitches = new ArrayList<>(); + private Map<String, String> binarySwitches = new HashMap<>(); + private List<String> arguments = new ArrayList<>(); + private Map<String, String> requiredUnarySwitches = new HashMap<>(); + private Map<String, String> requiredBinarySwitches = new HashMap<>(); + private String progname = "progname"; + private String argumentExplanation; + private int minArguments=0; + private int maxArguments=Integer.MAX_VALUE; + private String helpText; + private static HashSet<String> helpSwitches = new HashSet<>(); + private boolean helpSwitchUsed = false; + + static { + helpSwitches.add("-h"); + helpSwitches.add("-help"); + helpSwitches.add("--help"); + helpSwitches.add("-?"); + } + + public CommandLineParser(String[] cmds) { + inputStrings = Arrays.asList(cmds); + } + + public CommandLineParser(String progname, String[] cmds) { + this.progname=progname; + inputStrings = Arrays.asList(cmds); + } + + /** + * Parses the command line + * @throws IllegalArgumentException if a parse error occured + */ + public void parse() { + for (Iterator<String> it = inputStrings.iterator() ; it.hasNext() ; ) { + String i = it.next(); + if (isHelpSwitch(i)) { + helpSwitchUsed = true; + usageAndThrow(); + } + if (i.startsWith("-")) { + if (!isLegalSwitch(i)) { + usageAndThrow(); + } else if (legalUnarySwitches.keySet().contains(i)) { + unarySwitches.add(i); + } else if (legalBinarySwitches.keySet().contains(i)) { + if (!it.hasNext()) { + throw new IllegalArgumentException(i+ " requires value"); + } else { + String val = it.next(); + binarySwitches.put(i, val); + } + } + } else { + arguments.add(i); + } + } + if (!requiredUnarySwitches.isEmpty() && !getUnarySwitches().containsAll(requiredUnarySwitches.keySet())) { + usageAndThrow(); + } + if (!requiredBinarySwitches.isEmpty() && !getBinarySwitches().keySet().containsAll(requiredBinarySwitches.keySet())) { + usageAndThrow(); + } + if (getArguments().size()<minArguments || getArguments().size()>maxArguments) { + usageAndThrow(); + } + } + + private boolean isHelpSwitch(String i) { + return helpSwitches.contains(i); + } + + void usageAndThrow() { + StringBuffer error_sb = new StringBuffer(); + error_sb.append("\nusage: ").append(progname).append(" "); + if (argumentExplanation!=null) { + error_sb.append(argumentExplanation); + } + if (!legalUnarySwitches.isEmpty()) error_sb.append("\nSwitches:\n"); + error_sb.append("-h This help text\n"); + for (Map.Entry<String, String> e : legalUnarySwitches.entrySet()) { + error_sb.append(e.getKey()).append(" ").append(e.getValue()).append("\n"); + } + for (Map.Entry<String, String> e : legalBinarySwitches.entrySet()) { + error_sb.append(e.getKey()).append(" <").append(e.getValue()).append(">\n"); + } + if (helpText!=null) { + error_sb.append("\n").append(helpText).append("\n"); + } + throw new IllegalArgumentException(error_sb.toString()); + } + + private boolean isLegalSwitch(String s) { + return (legalUnarySwitches.containsKey(s) || legalBinarySwitches.containsKey(s)); + } + + /** + * Add a legal unary switch such as "-d" + */ + public void addLegalUnarySwitch(String s, String explanation) { + if (legalBinarySwitches.containsKey(s)) { + throw new IllegalArgumentException(s +" already added as a binary switch"); + } + legalUnarySwitches.put(s, explanation); + } + + public void addLegalUnarySwitch(String s) { + addLegalUnarySwitch(s, null); + } + + /** + * Adds a required switch, such as -p + */ + public void addRequiredUnarySwitch(String s, String explanation) { + addLegalUnarySwitch(s, explanation); + requiredUnarySwitches.put(s, explanation); + } + + /** + * Add a legal binary switch such as "-f /foo/bar" + */ + public void addLegalBinarySwitch(String s, String explanation) { + if (legalUnarySwitches.containsKey(s)) { + throw new IllegalArgumentException(s +" already added as a unary switch"); + } + legalBinarySwitches.put(s, explanation); + } + + /** + * Adds a legal binary switch without explanation + */ + public void addLegalBinarySwitch(String s) { + addLegalBinarySwitch(s, null); + } + + /** + * Adds a required binary switch + */ + public void addRequiredBinarySwitch(String s, String explanation) { + addLegalBinarySwitch(s, explanation); + requiredBinarySwitches.put(s, explanation); + } + + /** + * The unary switches that were given on the command line + */ + public List<String> getUnarySwitches() { + return unarySwitches; + } + + /** + * The binary switches that were given on the command line + */ + public Map<String, String> getBinarySwitches() { + return binarySwitches; + } + + /** + * All non-switch strings that were given on the command line + */ + public List<String> getArguments() { + return arguments; + } + + /** + * Sets the argument explanation used in printing method, i.e. "names,..." + */ + public void setArgumentExplanation(String argumentExplanation) { + this.argumentExplanation = argumentExplanation; + } + + public void setExtendedHelpText(String text) { + this.helpText=text; + } + + public String getHelpText() { + return helpText; + } + + /** + * Sets minimum number of required arguments + */ + public void setMinArguments(int minArguments) { + this.minArguments = minArguments; + } + + /** + * Sets the maximum number of allowed arguments + */ + public void setMaxArguments(int maxArguments) { + this.maxArguments = maxArguments; + } + + public boolean helpSwitchUsed() { + return helpSwitchUsed; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/system/ForceLoad.java b/vespajlib/src/main/java/com/yahoo/system/ForceLoad.java new file mode 100644 index 00000000000..f924740321f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/system/ForceLoad.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +/** + * Utility class used to force the loading of other classes. + **/ +public class ForceLoad { + + /** + * Force the loading of the given classes. If any of the named + * classes can not be loaded, an error will be thrown. + * + * @param packageName the name of the package for which + * we want to forceload classes. + * @param classNames array of names of classes (without package prefix) + * to force load. + **/ + public static void forceLoad(String packageName, String[] classNames) + throws ForceLoadError + { + String fullClassName = ""; + try { + for (String className : classNames) { + fullClassName = packageName + "." + className; + Class.forName(fullClassName); + } + } catch (Exception e) { + throw new ForceLoadError(fullClassName, e); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/system/ForceLoadError.java b/vespajlib/src/main/java/com/yahoo/system/ForceLoadError.java new file mode 100644 index 00000000000..376374a510d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/system/ForceLoadError.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +/** + * Special error to be propagated when force-loading a class fails. + **/ +@SuppressWarnings("serial") +public class ForceLoadError extends java.lang.Error { + + /** + * Create a new force load error + * + * @param className full name of offending class + * @param cause what caused the failure + **/ + public ForceLoadError(String className, Throwable cause) { + super("Force loading class '" + className + "' failed", cause); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java b/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java new file mode 100644 index 00000000000..bb2909b346a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.StringTokenizer; + +import com.yahoo.collections.Pair; + +/** + * Executes a system command synchronously. + * + * @author <a href="mailto:bratseth@yahoo-inc">Jon S Bratseth</a> + */ +public class ProcessExecuter { + + /** + * Executes the given command synchronously without timeout. + * @return Retcode and stdout/stderr merged + */ + public Pair<Integer, String> exec(String command) throws IOException { + StringTokenizer tok = new StringTokenizer(command); + List<String> tokens = new ArrayList<>(); + while (tok.hasMoreElements()) tokens.add(tok.nextToken()); + return exec(tokens.toArray(new String[0])); + } + + /** + * Executes the given command synchronously without timeout. + * @param command tokens + * @return Retcode and stdout/stderr merged + */ + public Pair<Integer, String> exec(String[] command) throws IOException { + ProcessBuilder pb = new ProcessBuilder(command); + StringBuilder ret = new StringBuilder(); + pb.environment().remove("VESPA_LOG_TARGET"); + pb.redirectErrorStream(true); + Process p = pb.start(); + InputStream is = p.getInputStream(); + while (true) { + int b = is.read(); + if (b==-1) break; + ret.append((char)b); + } + int rc=0; + try { + rc = p.waitFor(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return new Pair<>(rc, ret.toString()); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/system/package-info.java b/vespajlib/src/main/java/com/yahoo/system/package-info.java new file mode 100644 index 00000000000..397f0f5d791 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/system/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.system; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MapTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MapTensor.java new file mode 100644 index 00000000000..3bda4159ca6 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/MapTensor.java @@ -0,0 +1,136 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.annotations.Beta; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.UnaryOperator; + +/** + * A sparse implementation of a tensor backed by a Map of cells to values. + * + * @author bratseth + */ +@Beta +public class MapTensor implements Tensor { + + private final ImmutableSet<String> dimensions; + + private final ImmutableMap<TensorAddress, Double> cells; + + /** Creates a sparse tensor where the dimensions are determined by the cells */ + public MapTensor(Map<TensorAddress, Double> cells) { + this(dimensionsOf(cells.keySet()), cells); + } + + /** Creates a sparse tensor */ + MapTensor(Set<String> dimensions, Map<TensorAddress, Double> cells) { + ensureValidDimensions(cells, dimensions); + this.dimensions = ImmutableSet.copyOf(dimensions); + this.cells = ImmutableMap.copyOf(cells); + } + + private void ensureValidDimensions(Map<TensorAddress, Double> cells, Set<String> dimensions) { + for (TensorAddress address : cells.keySet()) + if ( ! dimensions.containsAll(address.dimensions())) + throw new IllegalArgumentException("Cell address " + address + " is outside this tensors dimensions " + + dimensions); + } + + /** + * Creates a tensor from the string form returned by the {@link #toString} of this. + * + * @param s the tensor string + * @throws IllegalArgumentException if the string is not in the correct format + */ + public static MapTensor from(String s) { + s = s.trim(); + if ( s.startsWith("(")) + return fromTensorWithEmptyDimensions(s); + else if ( s.startsWith("{")) + return fromTensor(s, Collections.emptySet()); + else + throw new IllegalArgumentException("Excepted a string starting by { or (, got '" + s + "'"); + } + + private static MapTensor fromTensorWithEmptyDimensions(String s) { + s = s.substring(1).trim(); + int multiplier = s.indexOf("*"); + if (multiplier < 0 || ! s.endsWith(")")) + throw new IllegalArgumentException("Expected a tensor on the form ({dimension:-,...}*{{cells}}), got '" + s + "'"); + MapTensor dimensionTensor = fromTensor(s.substring(0, multiplier).trim(), Collections.emptySet()); + return fromTensor(s.substring(multiplier + 1, s.length() - 1), dimensionTensor.dimensions()); + } + + private static MapTensor fromTensor(String s, Set<String> additionalDimensions) { + s = s.trim().substring(1).trim(); + ImmutableMap.Builder<TensorAddress, Double> cells = new ImmutableMap.Builder<>(); + while (s.length() > 1) { + int keyEnd = s.indexOf('}'); + TensorAddress address = TensorAddress.from(s.substring(0, keyEnd+1)); + s = s.substring(keyEnd + 1).trim(); + if ( ! s.startsWith(":")) + throw new IllegalArgumentException("Expecting a ':' after " + address + ", got '" + s + "'"); + int valueEnd = s.indexOf(','); + if (valueEnd < 0) { // last value + valueEnd = s.indexOf("}"); + if (valueEnd < 0) + throw new IllegalArgumentException("A tensor string must end by '}'"); + } + Double value = asDouble(address, s.substring(1, valueEnd).trim()); + cells.put(address, value); + s = s.substring(valueEnd+1).trim(); + } + + ImmutableMap<TensorAddress, Double> cellMap = cells.build(); + Set<String> dimensions = dimensionsOf(cellMap.keySet()); + dimensions.addAll(additionalDimensions); + return new MapTensor(dimensions, cellMap); + } + + private static Double asDouble(TensorAddress address, String s) { + try { + return Double.valueOf(s); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("At " + address + ": Expected a floating point number, got '" + s + "'"); + } + } + + private static Set<String> dimensionsOf(Set<TensorAddress> addresses) { + Set<String> dimensions = new HashSet<>(); + for (TensorAddress address : addresses) + for (TensorAddress.Element element : address.elements()) + dimensions.add(element.dimension()); + return dimensions; + } + + @Override + public Set<String> dimensions() { return dimensions; } + + @Override + public Map<TensorAddress, Double> cells() { return cells; } + + @Override + public double get(TensorAddress address) { return cells.getOrDefault(address, Double.NaN); } + + @Override + public int hashCode() { return cells.hashCode(); } + + @Override + public String toString() { return Tensor.toStandardString(this); } + + @Override + public boolean equals(Object o) { + if ( ! (o instanceof Tensor)) return false; + return Tensor.equals(this, (Tensor)o); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MapTensorBuilder.java b/vespajlib/src/main/java/com/yahoo/tensor/MapTensorBuilder.java new file mode 100644 index 00000000000..f46f000d1ee --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/MapTensorBuilder.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.annotations.Beta; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Builder class for a MapTensor. + * + * The set of dimensions of the resulting tensor is the union of + * the dimensions specified explicitly and the ones specified in the + * tensor cell addresses. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +@Beta +public class MapTensorBuilder { + + private final Set<String> dimensions = new HashSet<>(); + private final Map<TensorAddress, Double> cells = new HashMap<>(); + + public class CellBuilder { + + private final TensorAddress.Builder addressBuilder = new TensorAddress.Builder(); + + public CellBuilder label(String dimension, String label) { + dimensions.add(dimension); + addressBuilder.add(dimension, label); + return this; + } + public MapTensorBuilder value(double cellValue) { + cells.put(addressBuilder.build(), cellValue); + return MapTensorBuilder.this; + } + } + + public MapTensorBuilder() { + } + + public MapTensorBuilder dimension(String dimension) { + dimensions.add(dimension); + return this; + } + + public CellBuilder cell() { + return new CellBuilder(); + } + + public Tensor build() { + return new MapTensor(dimensions, cells); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MatchProduct.java b/vespajlib/src/main/java/com/yahoo/tensor/MatchProduct.java new file mode 100644 index 00000000000..074742acee1 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/MatchProduct.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; +import java.util.Set; + +/** + * Computes a <i>match product</i>, see {@link Tensor#match} + * + * @author bratseth + */ +class MatchProduct { + + private final Set<String> dimensions; + private final ImmutableMap.Builder<TensorAddress, Double> cells = new ImmutableMap.Builder<>(); + + public MatchProduct(Tensor a, Tensor b) { + this.dimensions = TensorOperations.combineDimensions(a, b); + for (Map.Entry<TensorAddress, Double> aCell : a.cells().entrySet()) { + Double sameValueInB = b.cells().get(aCell.getKey()); + if (sameValueInB != null) + cells.put(aCell.getKey(), aCell.getValue() * sameValueInB); + } + } + + /** Returns the result of taking this product */ + public MapTensor result() { + return new MapTensor(dimensions, cells.build()); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java new file mode 100644 index 00000000000..41f4d6c0b3d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -0,0 +1,247 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.annotations.Beta; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.UnaryOperator; + +/** +/** + * A multidimensional array which can be used in computations. + * <p> + * A tensor consists of a set of <i>dimension</i> names and a set of <i>cells</i> containing scalar <i>values</i>. + * Each cell is is identified by its <i>address</i>, which consists of a set of dimension-label pairs which defines + * the location of that cell. Both dimensions and labels are string on the form of an identifier or integer. + * Any dimension in an address may be assigned the special label "undefined", represented in string form as "-". + * <p> + * The size of the set of dimensions of a tensor is called its <i>order</i>. + * <p> + * In contrast to regular mathematical formulations of tensors, this definition of a tensor allows <i>sparseness</i> + * as there is no built-in notion of a contiguous space, and even in cases where a space is implied (such as when + * address labels are integers), there is no requirement that every implied cell has a defined value. + * Undefined values have no define representation as they are never observed. + * <p> + * Tensors can be read and serialized to and from a string form documented in the {@link #toString} method. + * + * @author bratseth + */ +@Beta +public interface Tensor { + + /** + * Returns the immutable set of dimensions of this tensor. + * The size of this set is the tensor's <i>order</i>. + */ + Set<String> dimensions(); + + /** Returns an immutable map of the cells of this */ + Map<TensorAddress, Double> cells(); + + /** Returns the value of a cell, or NaN if this cell does not exist/have no value */ + double get(TensorAddress address); + + /** + * Returns the <i>sparse tensor product</i> of this tensor and the argument tensor. + * This is the all-to-all combinations of cells in the argument tenors, except the combinations + * which have conflicting labels for the same dimension. The value of each combination is the product + * of the values of the two input cells. The dimensions of the tensor product is the set union of the + * dimensions of the argument tensors. + * <p> + * If there are no overlapping dimensions this is the regular tensor product. + * If the two tensors have exactly the same dimensions this is the Hadamard product. + * <p> + * The sparse tensor product is associative and commutative. + * + * @param argument the tensor to multiply by this + * @return the resulting tensor. + */ + default Tensor multiply(Tensor argument) { + return new TensorProduct(this, argument).result(); + } + + /** + * Returns the <i>match product</i> of two tensors. + * This returns a tensor which contains the <i>matching</i> cells in the two tensors, with their + * values multiplied. + * <p> + * Two cells are matching if they have the same labels for all dimensions shared between the two argument tensors, + * and have the value undefined for any non-shared dimension. + * <p> + * The dimensions of the resulting tensor is the set intersection of the two argument tensors. + * <p> + * If the two tensors have exactly the same dimensions, this is the Hadamard product. + */ + default Tensor match(Tensor argument) { + return new MatchProduct(this, argument).result(); + } + + /** + * Returns a tensor which contains the cells of both argument tensors, where the value for + * any <i>matching</i> cell is the min of the two possible values. + * <p> + * Two cells are matching if they have the same labels for all dimensions shared between the two argument tensors, + * and have the value undefined for any non-shared dimension. + */ + default Tensor min(Tensor argument) { + return new TensorMin(this, argument).result(); + } + + /** + * Returns a tensor which contains the cells of both argument tensors, where the value for + * any <i>matching</i> cell is the max of the two possible values. + * <p> + * Two cells are matching if they have the same labels for all dimensions shared between the two argument tensors, + * and have the value undefined for any non-shared dimension. + */ + default Tensor max(Tensor argument) { + return new TensorMax(this, argument).result(); + } + + /** + * Returns a tensor which contains the cells of both argument tensors, where the value for + * any <i>matching</i> cell is the sum of the two possible values. + * <p> + * Two cells are matching if they have the same labels for all dimensions shared between the two argument tensors, + * and have the value undefined for any non-shared dimension. + */ + default Tensor add(Tensor argument) { + return new TensorSum(this, argument).result(); + } + + /** + * Returns a tensor which contains the cells of both argument tensors, where the value for + * any <i>matching</i> cell is the difference of the two possible values. + * <p> + * Two cells are matching if they have the same labels for all dimensions shared between the two argument tensors, + * and have the value undefined for any non-shared dimension. + */ + default Tensor subtract(Tensor argument) { + return new TensorDifference(this, argument).result(); + } + + /** + * Returns a tensor with the same cells as this and the given function is applied to all its cell values. + * + * @param function the function to apply to all cells + * @return the tensor with the function applied to all the cells of this + */ + default Tensor apply(UnaryOperator<Double> function) { + return new TensorFunction(this, function).result(); + } + + /** + * Returns a tensor with the given dimension removed and cells which contains the sum of the values + * in the removed dimension. + */ + default Tensor sum(String dimension) { + return new TensorDimensionSum(dimension, this).result(); + } + + /** + * Returns the sum of all the cells of this tensor. + */ + default double sum() { + double sum = 0; + for (Map.Entry<TensorAddress, Double> cell : cells().entrySet()) + sum += cell.getValue(); + return sum; + } + + /** + * Returns true if the given tensor is mathematically equal to this: + * Both are of type Tensor and have the same content. + */ + @Override + boolean equals(Object o); + + /** Returns true if the two given tensors are mathematically equivalent, that is whether both have the same content */ + static boolean equals(Tensor a, Tensor b) { + if (a == b) return true; + if ( ! a.dimensions().equals(b.dimensions())) return false; + if ( ! a.cells().equals(b.cells())) return false; + return true; + } + + /** + * Returns this tensor on the form + * <code>{address1:value1,address2:value2,...}</code> + * where each address is on the form <code>{dimension1:label1,dimension2:label2,...}</code>, + * and values are numbers. + * <p> + * Cells are listed in the natural order of tensor addresses: Increasing size primarily + * and by element lexical order secondarily. + * <p> + * Note that while this is suggestive of JSON, it is not JSON. + */ + @Override + String toString(); + + /** Returns a tensor instance containing the given data on the standard string format returned by toString */ + static Tensor from(String tensorString) { + return MapTensor.from(tensorString); + } + + /** + * Returns a tensor instance containing the given data on the standard string format returned by toString + * + * @param tensorType the type of the tensor to return, as a string on the tensor type format, given in + * {@link TensorType#fromSpec} + * @param tensorString the tensor on the standard tensor string format + */ + static Tensor from(String tensorType, String tensorString) { + TensorType.fromSpec(tensorType); // Just validate type spec for now, as we only have one, generic implementation + return MapTensor.from(tensorString); + } + + /** + * Call this from toString in implementations to return the standard string format. + * (toString cannot be a default method because default methods cannot override super methods). + * + * @param tensor the tensor to return the standard string format of + * @return the tensor on the standard string format + */ + static String toStandardString(Tensor tensor) { + Set<String> emptyDimensions = emptyDimensions(tensor); + if (emptyDimensions.size() > 0) // explicitly list empty dimensions + return "( " + unitTensorWithDimensions(emptyDimensions) + " * " + contentToString(tensor) + " )"; + else + return contentToString(tensor); + } + + static String contentToString(Tensor tensor) { + List<Map.Entry<TensorAddress, Double>> cellEntries = new ArrayList<>(tensor.cells().entrySet()); + Collections.sort(cellEntries, Map.Entry.<TensorAddress, Double>comparingByKey()); + + StringBuilder b = new StringBuilder("{"); + for (Map.Entry<TensorAddress, Double> cell : cellEntries) { + b.append(cell.getKey()).append(":").append(cell.getValue()); + b.append(","); + } + if (b.length() > 1) + b.setLength(b.length() - 1); + b.append("}"); + return b.toString(); + } + + /** + * Returns the dimensions of this which have no values. + * This is a possibly empty subset of the dimensions of this tensor. + */ + static Set<String> emptyDimensions(Tensor tensor) { + Set<String> emptyDimensions = new HashSet<>(tensor.dimensions()); + for (TensorAddress address : tensor.cells().keySet()) + emptyDimensions.removeAll(address.dimensions()); + return emptyDimensions; + } + + static String unitTensorWithDimensions(Set<String> dimensions) { + return new MapTensor(Collections.singletonMap(TensorAddress.emptyWithDimensions(dimensions), 1.0)).toString(); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java new file mode 100644 index 00000000000..11c6a5f6685 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -0,0 +1,207 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.annotations.Beta; +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * An immutable address to a tensor cell. + * This is sparse: Only dimensions which have a different label than "undefined" are + * explicitly included. + * <p> + * Tensor addresses are ordered by increasing size primarily, and by the natural order of the elements in sorted + * order secondarily. + * + * @author bratseth + */ +@Beta +public final class TensorAddress implements Comparable<TensorAddress> { + + public static final TensorAddress empty = new TensorAddress.Builder().build(); + + private final ImmutableList<Element> elements; + + /** Note that the elements list MUST be sorted before calling this */ + private TensorAddress(List<Element> elements) { + this.elements = ImmutableList.copyOf(elements); + } + + public static TensorAddress fromSorted(List<Element> elements) { + return new TensorAddress(elements); + } + + /** + * Creates a tensor address from an unsorted list of elements. + * This call assigns ownership of the elements list to this class. + */ + public static TensorAddress fromUnsorted(List<Element> elements) { + Collections.sort(elements); + return new TensorAddress(elements); + } + + /** Creates a tenor address from a string on the form {dimension1:label1,dimension2:label2,...} */ + public static TensorAddress from(String address) { + address = address.trim(); + if ( ! (address.startsWith("{") && address.endsWith("}"))) + throw new IllegalArgumentException("Expecting a tensor address to be enclosed in {}, got '" + address + "'"); + + String addressBody = address.substring(1, address.length() - 1).trim(); + if (addressBody.isEmpty()) return TensorAddress.empty; + + List<Element> elements = new ArrayList<>(); + for (String elementString : addressBody.split(",")) { + String[] pair = elementString.split(":"); + if (pair.length != 2) + throw new IllegalArgumentException("Expecting argument elements to be on the form dimension:label, " + + "got '" + elementString + "'"); + elements.add(new Element(pair[0].trim(), pair[1].trim())); + } + Collections.sort(elements); + return TensorAddress.fromSorted(elements); + } + + /** Creates an empty address with a set of dimensions */ + public static TensorAddress emptyWithDimensions(Set<String> dimensions) { + List<Element> elements = new ArrayList<>(dimensions.size()); + for (String dimension : dimensions) + elements.add(new Element(dimension, Element.undefinedLabel)); + return TensorAddress.fromUnsorted(elements); + } + + /** Returns an immutable list of the elements of this address in sorted order */ + public List<Element> elements() { return elements; } + + /** Returns true if this address has a value (other than implicit "undefined") for the given dimension */ + public boolean hasDimension(String dimension) { + for (TensorAddress.Element element : elements) + if (element.dimension().equals(dimension)) + return true; + return false; + } + + /** Returns a possibly immutable set of the dimensions of this */ + public Set<String> dimensions() { + Set<String> dimensions = new HashSet<>(); + for (Element e : elements) + dimensions.add(e.dimension()); + return dimensions; + } + + @Override + public int compareTo(TensorAddress other) { + int sizeComparison = Integer.compare(this.elements.size(), other.elements.size()); + if (sizeComparison != 0) return sizeComparison; + + for (int i = 0; i < elements.size(); i++) { + int elementComparison = this.elements.get(i).compareTo(other.elements.get(i)); + if (elementComparison != 0) return elementComparison; + } + + return 0; + } + + @Override + public int hashCode() { + return elements.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof TensorAddress)) return false; + return ((TensorAddress)other).elements.equals(this.elements); + } + + /** Returns this on the form {dimension1:label1,dimension2:label2,... */ + @Override + public String toString() { + StringBuilder b = new StringBuilder("{"); + for (TensorAddress.Element element : elements) { + //if (element.label() == Element.undefinedLabel) continue; + b.append(element.toString()); + b.append(","); + } + if (b.length() > 1) + b.setLength(b.length() - 1); + b.append("}"); + return b.toString(); + } + + /** A tensor address element. Elements have the lexical order of the dimensions as natural order. */ + public static class Element implements Comparable<Element> { + + static final String undefinedLabel = "-"; + + private final String dimension; + private final String label; + private final int hashCode; + + public Element(String dimension, String label) { + this.dimension = dimension; + if (label.equals(undefinedLabel)) + this.label = undefinedLabel; + else + this.label = label; + this.hashCode = dimension.hashCode() + label.hashCode(); + } + + public String dimension() { return dimension; } + + public String label() { return label; } + + @Override + public int compareTo(Element other) { + return this.dimension.compareTo(other.dimension); + } + + @Override + public int hashCode() { return hashCode; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof Element)) return false; + Element other = (Element)o; + if ( ! other.dimension.equals(this.dimension)) return false; + if ( ! other.label.equals(this.label)) return false; + return true; + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append(dimension).append(":").append(label); + return b.toString(); + } + + } + + /** Supports building of a tensor address */ + public static class Builder { + + private final List<Element> elements = new ArrayList<>(); + + /** + * Adds a label in a dimension to this. + * + * @return this for convenience + */ + public Builder add(String dimension, String label) { + elements.add(new Element(dimension, label)); + return this; + } + + public TensorAddress build() { + Collections.sort(elements); // Consistent order to get a consistent hash + return TensorAddress.fromSorted(elements); + } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorDifference.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorDifference.java new file mode 100644 index 00000000000..ceb003b1615 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorDifference.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Takes the difference between two tensors, see {@link Tensor#subtract} + * + * @author bratseth + */ +class TensorDifference { + + private final Set<String> dimensions; + private final Map<TensorAddress, Double> cells = new HashMap<>(); + + public TensorDifference(Tensor a, Tensor b) { + this.dimensions = TensorOperations.combineDimensions(a, b); + cells.putAll(a.cells()); + for (Map.Entry<TensorAddress, Double> bCell : b.cells().entrySet()) + cells.put(bCell.getKey(), a.cells().getOrDefault(bCell.getKey(), 0d) - bCell.getValue()); + } + + /** Returns the result of taking this sum */ + public Tensor result() { + return new MapTensor(dimensions, cells); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorDimensionSum.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorDimensionSum.java new file mode 100644 index 00000000000..3cd791fc60e --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorDimensionSum.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Returns a tensor with the given dimension removed and the cell values in that dimension summed + * + * @author bratseth + */ +class TensorDimensionSum { + + private final Set<String> dimensions; + private final Map<TensorAddress, Double> cells = new HashMap<>(); + + public TensorDimensionSum(String dimension, Tensor t) { + dimensions = new HashSet<>(t.dimensions()); + dimensions.remove(dimension); + + for (Map.Entry<TensorAddress, Double> cell : t.cells().entrySet()) { + TensorAddress reducedAddress = removeDimension(dimension, cell.getKey()); + Double newValue = cell.getValue(); + Double existingValue = cells.get(reducedAddress); + if (existingValue != null) + newValue += existingValue; + cells.put(reducedAddress, newValue); + } + } + + private TensorAddress removeDimension(String dimension, TensorAddress address) { + List<TensorAddress.Element> reducedAddress = new ArrayList<>(); + for (TensorAddress.Element element : address.elements()) + if ( ! element.dimension().equals(dimension)) + reducedAddress.add(element); + return TensorAddress.fromSorted(reducedAddress); + } + + /** Returns the result of taking this sum */ + public MapTensor result() { return new MapTensor(dimensions, cells); } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorFunction.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorFunction.java new file mode 100644 index 00000000000..db73626d6d0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorFunction.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; +import java.util.Set; +import java.util.function.UnaryOperator; + +/** + * Computes the tensor with some function to all the cells of the input tensor + * + * @author bratseth + */ +class TensorFunction { + + private final Set<String> dimensions; + private final ImmutableMap.Builder<TensorAddress, Double> cells = new ImmutableMap.Builder<>(); + + public TensorFunction(Tensor t, UnaryOperator<Double> f) { + dimensions = t.dimensions(); + for (Map.Entry<TensorAddress, Double> cell : t.cells().entrySet()) { + cells.put(cell.getKey(), f.apply(cell.getValue())); + } + } + + /** Returns the result of taking this sum */ + public MapTensor result() { + return new MapTensor(dimensions, cells.build()); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorMax.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorMax.java new file mode 100644 index 00000000000..d15e5092476 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorMax.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Takes the max of each cell of two tensors, see {@link Tensor#max} + * + * @author bratseth + */ +class TensorMax { + + private final Set<String> dimensions; + private final Map<TensorAddress, Double> cells = new HashMap<>(); + + public TensorMax(Tensor a, Tensor b) { + dimensions = TensorOperations.combineDimensions(a, b); + cells.putAll(a.cells()); + for (Map.Entry<TensorAddress, Double> bCell : b.cells().entrySet()) { + Double aValue = a.cells().get(bCell.getKey()); + if (aValue == null) + cells.put(bCell.getKey(), bCell.getValue()); + else + cells.put(bCell.getKey(), Math.max(aValue, bCell.getValue())); + } + } + + /** Returns the result of taking this sum */ + public Tensor result() { + return new MapTensor(dimensions, cells); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorMin.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorMin.java new file mode 100644 index 00000000000..e389dea3883 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorMin.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Takes the min of each cell of two tensors, see {@link Tensor#min} + * + * @author bratseth + */ +class TensorMin { + + private final Set<String> dimensions; + private final Map<TensorAddress, Double> cells = new HashMap<>(); + + public TensorMin(Tensor a, Tensor b) { + dimensions = TensorOperations.combineDimensions(a, b); + cells.putAll(a.cells()); + for (Map.Entry<TensorAddress, Double> bCell : b.cells().entrySet()) { + Double aValue = a.cells().get(bCell.getKey()); + if (aValue == null) + cells.put(bCell.getKey(), bCell.getValue()); + else + cells.put(bCell.getKey(), Math.min(aValue, bCell.getValue())); + } + } + + /** Returns the result of taking this sum */ + public Tensor result() { return new MapTensor(dimensions, cells); } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorOperations.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorOperations.java new file mode 100644 index 00000000000..aca306b914c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorOperations.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.collect.ImmutableSet; + +import java.util.Set; + +/** + * Functions on tensors + * + * @author bratseth + */ +class TensorOperations { + + /** + * A utility method which returns an ummutable set of the union of the dimensions + * of the two argument tensors. + * + * @return the combined dimensions as an unmodifiable set + */ + static Set<String> combineDimensions(Tensor a, Tensor b) { + ImmutableSet.Builder<String> setBuilder = new ImmutableSet.Builder<>(); + setBuilder.addAll(a.dimensions()); + setBuilder.addAll(b.dimensions()); + return setBuilder.build(); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorProduct.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorProduct.java new file mode 100644 index 00000000000..221bd985380 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorProduct.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.collect.ImmutableMap; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; + +/** + * Computes a <i>sparse tensor product</i>, see {@link Tensor#multiply} + * + * @author bratseth + */ +class TensorProduct { + + private final Set<String> dimensionsA, dimensionsB; + + private final Set<String> dimensions; + private final ImmutableMap.Builder<TensorAddress, Double> cells = new ImmutableMap.Builder<>(); + + public TensorProduct(Tensor a, Tensor b) { + dimensionsA = a.dimensions(); + dimensionsB = b.dimensions(); + + // Dimension product + dimensions = TensorOperations.combineDimensions(a, b); + + // Cell product (slow baseline implementation) + for (Map.Entry<TensorAddress, Double> aCell : a.cells().entrySet()) { + for (Map.Entry<TensorAddress, Double> bCell : b.cells().entrySet()) { + TensorAddress combinedAddress = combine(aCell.getKey(), bCell.getKey()); + if (combinedAddress == null) continue; // not combinable + cells.put(combinedAddress, aCell.getValue() * bCell.getValue()); + } + } + } + + private TensorAddress combine(TensorAddress a, TensorAddress b) { + List<TensorAddress.Element> combined = new ArrayList<>(); + combined.addAll(dense(a, dimensionsA)); + combined.addAll(dense(b, dimensionsB)); + Collections.sort(combined); + TensorAddress.Element previous = null; + for (ListIterator<TensorAddress.Element> i = combined.listIterator(); i.hasNext(); ) { + TensorAddress.Element current = i.next(); + if (previous != null && previous.dimension().equals(current.dimension())) { // an overlapping dimension + if (previous.label().equals(current.label())) + i.remove(); // a match: remove the duplicate + else + return null; // no match: a combination isn't viable + } + previous = current; + } + return TensorAddress.fromSorted(sparse(combined)); + } + + /** + * Returns a set of tensor elements which contains an entry for each dimension including "undefined" values + * (which are not present in the sparse elements list). + */ + private List<TensorAddress.Element> dense(TensorAddress sparse, Set<String> dimensions) { + if (sparse.elements().size() == dimensions.size()) return sparse.elements(); + + List<TensorAddress.Element> dense = new ArrayList<>(sparse.elements()); + for (String dimension : dimensions) { + if ( ! sparse.hasDimension(dimension)) + dense.add(new TensorAddress.Element(dimension, TensorAddress.Element.undefinedLabel)); + } + return dense; + } + + /** + * Removes any "undefined" entries from the given elements. + */ + private List<TensorAddress.Element> sparse(List<TensorAddress.Element> dense) { + List<TensorAddress.Element> sparse = new ArrayList<>(); + for (TensorAddress.Element element : dense) { + if ( ! element.label().equals(TensorAddress.Element.undefinedLabel)) + sparse.add(element); + } + return sparse; + } + + /** Returns the result of taking this product */ + public Tensor result() { + return new MapTensor(dimensions, cells.build()); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorSum.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorSum.java new file mode 100644 index 00000000000..85dfa289bd3 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorSum.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Takes the sum of two tensors, see {@link Tensor#add} + * + * @author bratseth + */ +class TensorSum { + + private final Set<String> dimensions; + private final Map<TensorAddress, Double> cells = new HashMap<>(); + + public TensorSum(Tensor a, Tensor b) { + dimensions = TensorOperations.combineDimensions(a, b); + cells.putAll(a.cells()); + for (Map.Entry<TensorAddress, Double> bCell : b.cells().entrySet()) { + cells.put(bCell.getKey(), a.cells().getOrDefault(bCell.getKey(), 0d) + bCell.getValue()); + } + } + + /** Returns the result of taking this sum */ + public Tensor result() { return new MapTensor(dimensions, cells); } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java new file mode 100644 index 00000000000..507a2f9f612 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -0,0 +1,195 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.annotations.Beta; +import com.google.common.collect.ImmutableList; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * A tensor type with its dimensions. This is immutable. + * <p> + * A dimension can be indexed (bound or unbound) or mapped. + * Currently, we only support tensor types where all dimensions have the same type. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +@Beta +public class TensorType { + + public static abstract class Dimension { + + public enum Type { indexedBound, indexedUnbound, mapped } + + private final String name; + + private Dimension(String name) { this.name = name; } + + public final String name() { return name; } + + /** Returns the size of this dimension if it is indexedUnbound, empty otherwise */ + public abstract Optional<Integer> size(); + + public abstract Type type(); + + @Override + public abstract String toString(); + + } + + public static class IndexedBoundDimension extends TensorType.Dimension { + + private final Optional<Integer> size; + + private IndexedBoundDimension(String name, int size) { + super(name); + if (size < 1) + throw new IllegalArgumentException("Size of bound dimension '" + name + "' must be at least 1"); + this.size = Optional.of(size); + } + + @Override + public Optional<Integer> size() { return size; } + + @Override + public Type type() { return Type.indexedBound; } + + @Override + public String toString() { return name() + "[" + size.get() + "]"; } + + } + + public static class IndexedUnboundDimension extends TensorType.Dimension { + + private IndexedUnboundDimension(String name) { + super(name); + } + + @Override + public Optional<Integer> size() { return Optional.empty(); } + + @Override + public Type type() { return Type.indexedUnbound; } + + @Override + public String toString() { return name() + "[]"; } + + } + + public static class MappedDimension extends TensorType.Dimension { + + private MappedDimension(String name) { + super(name); + } + + @Override + public Optional<Integer> size() { return Optional.empty(); } + + @Override + public Type type() { return Type.mapped; } + + @Override + public String toString() { return name() + "{}"; } + + } + + public static class Builder { + + private final Map<String, Dimension> dimensions = new LinkedHashMap<>(); + private Dimension prevDimension = null; + + private Builder add(Dimension dimension) { + if (!dimensions.isEmpty()) { + validateDimensionName(dimension); + validateDimensionType(dimension); + } + + dimensions.put(dimension.name(), dimension); + prevDimension = dimension; + return this; + } + + private void validateDimensionName(Dimension newDimension) { + Dimension prevDimension = dimensions.get(newDimension.name()); + if (prevDimension != null) { + throw new IllegalArgumentException("Expected all dimensions to have unique names, " + + "but '" + prevDimension + "' and '" + newDimension + "' have the same name"); + } + } + + private void validateDimensionType(Dimension newDimension) { + if (prevDimension.type() != newDimension.type()) { + throw new IllegalArgumentException("Expected all dimensions to have the same type, " + + "but '" + prevDimension + "' does not have the same type as '" + newDimension + "'"); + } + } + + public Builder indexedBound(String name, int size) { + return add(new IndexedBoundDimension(name, size)); + } + + public Builder indexedUnbound(String name) { + return add(new IndexedUnboundDimension(name)); + } + + public Builder mapped(String name) { + return add(new MappedDimension(name)); + } + + public TensorType build() { + return new TensorType(dimensions.values()); + } + } + + private final List<Dimension> dimensions; + + private TensorType(Collection<Dimension> dimensions) { + this.dimensions = ImmutableList.copyOf(dimensions); + } + + /** + * Returns a tensor type instance from a string on the format + * <code>tensor(dimension1, dimension2, ...)</code> + * where each dimension is either + * <ul> + * <li><code>dimension-name[]</code> - an unbound indexed dimension + * <li><code>dimension-name[int]</code> - an bound indexed dimension + * <li><code>dimension-name{}</code> - a mapped dimension + * </ul> + * Example: <code>tensor(x[10],y[20])</code> (a matrix) + */ + public static TensorType fromSpec(String specString) { + return TensorTypeParser.fromSpec(specString); + } + + /** Returns an immutable list of the dimensions of this */ + public List<Dimension> dimensions() { return dimensions; } + + @Override + public String toString() { + return "tensor(" + dimensions.stream().map(Dimension::toString).collect(Collectors.joining(",")) + ")"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TensorType that = (TensorType) o; + + if (!dimensions.equals(that.dimensions)) return false; + + return true; + } + + @Override + public int hashCode() { + return dimensions.hashCode(); + } +} + diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java new file mode 100644 index 00000000000..3d2e1663971 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.annotations.Beta; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Class for parsing a tensor type spec. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +@Beta +class TensorTypeParser { + + private final static String START_STRING = "tensor("; + private final static String END_STRING = ")"; + + private static final Pattern indexedPattern = Pattern.compile("(\\w+)\\[(\\d*)\\]"); + private static final Pattern mappedPattern = Pattern.compile("(\\w+)\\{\\}"); + + static TensorType fromSpec(String specString) { + if (!specString.startsWith(START_STRING) || !specString.endsWith(END_STRING)) { + throw new IllegalArgumentException("Tensor type spec must start with '" + START_STRING + "'" + + " and end with '" + END_STRING + "', but was '" + specString + "'"); + } + TensorType.Builder builder = new TensorType.Builder(); + String dimensionsSpec = specString.substring(START_STRING.length(), specString.length() - END_STRING.length()); + if (dimensionsSpec.isEmpty()) { + return builder.build(); + } + for (String element : dimensionsSpec.split(",")) { + String trimmedElement = element.trim(); + if (tryParseIndexedDimension(trimmedElement, builder)) { + } else if (tryParseMappedDimension(trimmedElement, builder)) { + } else { + throw new IllegalArgumentException("Failed parsing element '" + element + + "' in type spec '" + specString + "'"); + } + } + return builder.build(); + } + + private static boolean tryParseIndexedDimension(String element, TensorType.Builder builder) { + Matcher matcher = indexedPattern.matcher(element); + if (matcher.matches()) { + String dimensionName = matcher.group(1); + String dimensionSize = matcher.group(2); + if (dimensionSize.isEmpty()) { + builder.indexedUnbound(dimensionName); + } else { + builder.indexedBound(dimensionName, Integer.valueOf(dimensionSize)); + } + return true; + } + return false; + } + + private static boolean tryParseMappedDimension(String element, TensorType.Builder builder) { + Matcher matcher = mappedPattern.matcher(element); + if (matcher.matches()) { + String dimensionName = matcher.group(1); + builder.mapped(dimensionName); + return true; + } + return false; + } +} + diff --git a/vespajlib/src/main/java/com/yahoo/tensor/package-info.java b/vespajlib/src/main/java/com/yahoo/tensor/package-info.java new file mode 100644 index 00000000000..13ca7fa8a13 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/package-info.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Tensor data types + * + * @author bratseth + */ +@ExportPackage +@PublicApi +package com.yahoo.tensor; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/BinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/BinaryFormat.java new file mode 100644 index 00000000000..97d62d5169a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/BinaryFormat.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor.serialization; + +import com.google.common.annotations.Beta; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.Tensor; + +/** + * Representation of a specific binary format with functions for serializing a Tensor object into + * this format or de-serializing binary data into a Tensor object. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +@Beta +interface BinaryFormat { + + /** + * Serialize the given tensor into binary format. + */ + public void encode(GrowableByteBuffer buffer, Tensor tensor); + + /** + * Deserialize the given binary data into a Tensor object. + */ + public Tensor decode(GrowableByteBuffer buffer); +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/CompactBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/CompactBinaryFormat.java new file mode 100644 index 00000000000..0c1f04552f4 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/CompactBinaryFormat.java @@ -0,0 +1,113 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor.serialization; + +import com.google.common.annotations.Beta; +import com.google.common.collect.Sets; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.MapTensorBuilder; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorAddress; +import com.yahoo.text.Utf8; + +import java.util.*; + +/** + * Implementation of a compact binary format for a tensor on the form: + * + * Sorted dimensions = num_dimensions [dimension_str_len dimension_str_bytes]* + * Cells = num_cells [label_1_str_len label_1_str_bytes ... label_N_str_len label_N_str_bytes cell_value]* + * + * Note that the dimensions are sorted and the tensor address labels are given in the same sorted order. + * Unspecified labels are encoded as the empty string "". + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +@Beta +class CompactBinaryFormat implements BinaryFormat { + + @Override + public void encode(GrowableByteBuffer buffer, Tensor tensor) { + List<String> sortedDimensions = new ArrayList<>(tensor.dimensions()); + Collections.sort(sortedDimensions); + encodeDimensions(buffer, sortedDimensions); + encodeCells(buffer, tensor.cells(), sortedDimensions); + } + + private static void encodeDimensions(GrowableByteBuffer buffer, List<String> sortedDimensions) { + buffer.putInt1_4Bytes(sortedDimensions.size()); + for (String dimension : sortedDimensions) { + encodeString(buffer, dimension); + } + } + + private static void encodeCells(GrowableByteBuffer buffer, Map<TensorAddress, Double> cells, + List<String> sortedDimensions) { + buffer.putInt1_4Bytes(cells.size()); + for (Map.Entry<TensorAddress, Double> cellEntry : cells.entrySet()) { + encodeAddress(buffer, cellEntry.getKey(), sortedDimensions); + buffer.putDouble(cellEntry.getValue().doubleValue()); + } + } + + private static void encodeAddress(GrowableByteBuffer buffer, TensorAddress address, List<String> sortedDimensions) { + for (String dimension : sortedDimensions) { + Optional<TensorAddress.Element> element = + address.elements().stream().filter(elem -> elem.dimension().equals(dimension)).findFirst(); + String label = (element.isPresent() ? element.get().label() : ""); + encodeString(buffer, label); + } + } + + private static void encodeString(GrowableByteBuffer buffer, String value) { + byte[] stringBytes = Utf8.toBytes(value); + buffer.putInt1_4Bytes(stringBytes.length); + buffer.put(stringBytes); + } + + @Override + public Tensor decode(GrowableByteBuffer buffer) { + List<String> sortedDimensions = decodeDimensions(buffer); + MapTensorBuilder builder = new MapTensorBuilder(); + for (String dimension : sortedDimensions) { + builder.dimension(dimension); + } + decodeCells(buffer, builder, sortedDimensions); + return builder.build(); + } + + private static List<String> decodeDimensions(GrowableByteBuffer buffer) { + int numDimensions = buffer.getInt1_4Bytes(); + List<String> sortedDimensions = new ArrayList<>(); + for (int i = 0; i < numDimensions; ++i) { + sortedDimensions.add(decodeString(buffer)); + } + return sortedDimensions; + } + + private static void decodeCells(GrowableByteBuffer buffer, MapTensorBuilder builder, + List<String> sortedDimensions) { + int numCells = buffer.getInt1_4Bytes(); + for (int i = 0; i < numCells; ++i) { + MapTensorBuilder.CellBuilder cellBuilder = builder.cell(); + decodeAddress(buffer, cellBuilder, sortedDimensions); + cellBuilder.value(buffer.getDouble()); + } + } + + private static void decodeAddress(GrowableByteBuffer buffer, MapTensorBuilder.CellBuilder builder, + List<String> sortedDimensions) { + for (String dimension : sortedDimensions) { + String label = decodeString(buffer); + if (!label.isEmpty()) { + builder.label(dimension, label); + } + } + } + + private static String decodeString(GrowableByteBuffer buffer) { + int stringLength = buffer.getInt1_4Bytes(); + byte[] stringBytes = new byte[stringLength]; + buffer.get(stringBytes); + return Utf8.toString(stringBytes); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java new file mode 100644 index 00000000000..cdd26a11ac2 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor.serialization; + +import com.google.common.annotations.Beta; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.Tensor; + +/** + * Class used by clients for serializing a Tensor object into binary format or + * de-serializing binary data into a Tensor object. + * + * The actual binary format used is not a concern for the client and + * is hidden in this class and in the binary data. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +@Beta +public class TypedBinaryFormat { + + private static final int COMPACT_BINARY_FORMAT_TYPE = 1; + + public static byte[] encode(Tensor tensor) { + GrowableByteBuffer buffer = new GrowableByteBuffer(); + buffer.putInt1_4Bytes(COMPACT_BINARY_FORMAT_TYPE); + new CompactBinaryFormat().encode(buffer, tensor); + buffer.flip(); + byte[] result = new byte[buffer.remaining()]; + buffer.get(result); + return result; + } + + public static Tensor decode(byte[] data) { + GrowableByteBuffer buffer = GrowableByteBuffer.wrap(data); + int formatType = buffer.getInt1_4Bytes(); + switch (formatType) { + case COMPACT_BINARY_FORMAT_TYPE: + return new CompactBinaryFormat().decode(buffer); + default: + throw new IllegalArgumentException("Binary format type " + formatType + " is not a known format"); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/package-info.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/package-info.java new file mode 100644 index 00000000000..72027284bc1 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.tensor.serialization; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/text/AbstractUtf8Array.java b/vespajlib/src/main/java/com/yahoo/text/AbstractUtf8Array.java new file mode 100644 index 00000000000..1a11e30dd9d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/AbstractUtf8Array.java @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.nio.ByteBuffer; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public abstract class AbstractUtf8Array implements Comparable<AbstractUtf8Array> { + /** + * This will write the utf8 sequence to the given target. + */ + final public void writeTo(ByteBuffer target) { + target.put(getBytes(), getByteOffset(), getByteLength()); + } + + /** + * This will return the byte at the given position. + */ + public byte getByte(int index) { return getBytes()[getByteOffset() + index]; } + + /** + * + * @return Length in bytes of the utf8 sequence. + */ + public abstract int getByteLength(); + + /** + * Wraps the utf8 sequence in a ByteBuffer + * @return The wrapping buffer. + */ + public ByteBuffer wrap() { return ByteBuffer.wrap(getBytes(), getByteOffset(), getByteLength()); } + + /** + * + * @return The backing byte array. + */ + protected abstract byte [] getBytes(); + + public boolean isEmpty() { return getByteLength() == 0; } + + /** + * + * @return The offset in the backing array where the utf8 sequence starts. + */ + protected abstract int getByteOffset(); + @Override + public int hashCode() { + final int l = getByteLength(); + final int c = getByteOffset(); + final byte [] b = getBytes(); + int h = 0; + for (int i=0; i < l; i++) { + int v = b[c+i]; + h ^= v << ((i%4)*8); + } + return h; + } + @Override + public boolean equals(Object o) { + if (o instanceof AbstractUtf8Array) { + AbstractUtf8Array other = (AbstractUtf8Array)o; + return compareTo(other) == 0; + } else if (o instanceof String) { + return toString().equals(o); + } + return false; + } + + /** + * Will convert the utf8 sequence to a Java string + * @return The converted Java String + */ + @Override + public String toString() { + return Utf8.toString(getBytes(), getByteOffset(), getByteLength()); + } + + @Override + public int compareTo(AbstractUtf8Array rhs) { + final int l = getByteLength(); + final int rl = rhs.getByteLength(); + if (l < rl) { + return -1; + } else if (l > rl) { + return 1; + } else { + final byte [] b = getBytes(); + final byte [] rb = rhs.getBytes(); + final int c = getByteOffset(); + final int rc = rhs.getByteOffset(); + for (int i=0; i < l; i++) { + if (b[c+i] < rb[rc+i]) { + return -1; + } else if (b[c+i] > rb[rc+i]) { + return 1; + } + } + return 0; + } + } + + public Utf8Array ascii7BitLowerCase() { + byte [] upper = new byte[getByteLength()]; + + for (int i=0; i< upper.length; i++ ) { + byte b = getByte(i); + if ((b >= 0x41) && (b < (0x41+26))) { + b |= 0x20; // Lowercase + } + upper[i] = b; + } + return new Utf8Array(upper); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Ascii.java b/vespajlib/src/main/java/com/yahoo/text/Ascii.java new file mode 100644 index 00000000000..552a46fd36f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Ascii.java @@ -0,0 +1,226 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.util.Iterator; +import java.util.Set; +import java.util.TreeSet; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class Ascii { + + public final static char ESCAPE_CHAR = '\\'; + + public static String encode(String str, Charset charset, int... requiresEscape) { + return newEncoder(charset, requiresEscape).encode(str); + } + + public static String decode(String str, Charset charset) { + return newDecoder(charset).decode(str); + } + + public static Encoder newEncoder(Charset charset, int... requiresEscape) { + switch (requiresEscape.length) { + case 0: + return new Encoder(charset, new EmptyPredicate()); + case 1: + return new Encoder(charset, new SingletonPredicate(requiresEscape[0])); + default: + return new Encoder(charset, new ArrayPredicate(requiresEscape)); + } + } + + public static Decoder newDecoder(Charset charset) { + return new Decoder(charset); + } + + public static class Encoder { + + private final Charset charset; + private final EncodePredicate predicate; + + private Encoder(Charset charset, EncodePredicate predicate) { + this.charset = charset; + this.predicate = predicate; + } + + public String encode(String str) { + StringBuilder out = new StringBuilder(); + for (int c : new CodePointSequence(str)) { + if (c < 0x20 || c >= 0x7F || c == ESCAPE_CHAR || predicate.requiresEscape(c)) { + escape(c, out); + } else { + out.appendCodePoint(c); + } + } + return out.toString(); + } + + private void escape(int c, StringBuilder out) { + switch (c) { + case ESCAPE_CHAR: + out.append(ESCAPE_CHAR).append(ESCAPE_CHAR); + break; + case '\f': + out.append(ESCAPE_CHAR).append("f"); + break; + case '\n': + out.append(ESCAPE_CHAR).append("n"); + break; + case '\r': + out.append(ESCAPE_CHAR).append("r"); + break; + case '\t': + out.append(ESCAPE_CHAR).append("t"); + break; + default: + ByteBuffer buf = charset.encode(CharBuffer.wrap(Character.toChars(c))); + while (buf.hasRemaining()) { + out.append(ESCAPE_CHAR).append(String.format("x%02X", buf.get())); + } + break; + } + } + } + + public static class Decoder { + + private final Charset charset; + + private Decoder(Charset charset) { + this.charset = charset; + } + + public String decode(String str) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (Iterator<Integer> it = new CodePointIterator(str); it.hasNext(); ) { + int c = it.next(); + if (c == ESCAPE_CHAR) { + unescape(it, out); + } else { + ByteBuffer buf = charset.encode(CharBuffer.wrap(Character.toChars(c))); + while (buf.hasRemaining()) { + out.write(buf.get()); + } + } + } + return new String(out.toByteArray(), charset); + } + + private void unescape(Iterator<Integer> it, ByteArrayOutputStream out) { + int c = it.next(); + switch (c) { + case 'f': + out.write('\f'); + break; + case 'n': + out.write('\n'); + break; + case 'r': + out.write('\r'); + break; + case 't': + out.write('\t'); + break; + case 'x': + int x1 = it.next(); + int x2 = it.next(); + out.write((Character.digit(x1, 16) << 4) + + (Character.digit(x2, 16))); + break; + default: + out.write(c); + break; + } + } + } + + private static interface EncodePredicate { + + boolean requiresEscape(int codePoint); + } + + private static class EmptyPredicate implements EncodePredicate { + + @Override + public boolean requiresEscape(int codePoint) { + return false; + } + } + + private static class SingletonPredicate implements EncodePredicate { + + final int requiresEscape; + + private SingletonPredicate(int requiresEscape) { + this.requiresEscape = requiresEscape; + } + + @Override + public boolean requiresEscape(int codePoint) { + return codePoint == requiresEscape; + } + } + + private static class ArrayPredicate implements EncodePredicate { + + final Set<Integer> requiresEscape = new TreeSet<>(); + + private ArrayPredicate(int[] requiresEscape) { + for (int codePoint : requiresEscape) { + this.requiresEscape.add(codePoint); + } + } + + @Override + public boolean requiresEscape(int codePoint) { + return requiresEscape.contains(codePoint); + } + } + + private static class CodePointSequence implements Iterable<Integer> { + + final String str; + + CodePointSequence(String str) { + this.str = str; + } + + @Override + public Iterator<Integer> iterator() { + return new CodePointIterator(str); + } + } + + private static class CodePointIterator implements Iterator<Integer> { + + final String str; + int idx = 0; + + CodePointIterator(String str) { + this.str = str; + } + + @Override + public boolean hasNext() { + return idx < str.length(); + } + + @Override + public Integer next() { + int c = str.codePointAt(idx); + idx += Character.charCount(c); + return c; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/BooleanParser.java b/vespajlib/src/main/java/com/yahoo/text/BooleanParser.java new file mode 100644 index 00000000000..a17d821ff9d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/BooleanParser.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * Utility class parsing a string into a boolean. + * In contrast to Boolean.parseBoolean in the Java API this parser is strict. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class BooleanParser { + + /** + * Returns true if the input string is case insensitive equal to "true" and + * false if it is case insensitive equal to "false". + * In any other case an exception is thrown. + * + * @param s the string to parse + * @return true if s is "true", false if it is "false" + * @throws IllegalArgumentException if s is not null but neither "true" or "false" + * @throws NullPointerException if s is null + */ + public static boolean parseBoolean(String s) { + if (s==null) + throw new NullPointerException("Expected 'true' or 'false', got NULL"); + if (s.equalsIgnoreCase("false")) + return false; + if (s.equalsIgnoreCase("true")) + return true; + throw new IllegalArgumentException("Expected 'true' or 'false', got '" + s + "'"); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/CaseInsensitiveIdentifier.java b/vespajlib/src/main/java/com/yahoo/text/CaseInsensitiveIdentifier.java new file mode 100644 index 00000000000..258f5f74d14 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/CaseInsensitiveIdentifier.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 11.11.12 + * Time: 11:25 + * To change this template use File | Settings | File Templates. + */ +public class CaseInsensitiveIdentifier extends Identifier { + private final Identifier original; + + public CaseInsensitiveIdentifier(String s) { + this(new Utf8String(s)); + } + public CaseInsensitiveIdentifier(byte [] utf8) { + this(new Utf8Array(utf8)); + } + public CaseInsensitiveIdentifier(AbstractUtf8Array utf8) { + super(utf8.ascii7BitLowerCase()); + original = new Identifier(utf8); + } + public String toString() { return original.toString(); } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/DataTypeIdentifier.java b/vespajlib/src/main/java/com/yahoo/text/DataTypeIdentifier.java new file mode 100644 index 00000000000..364cb87d6f7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/DataTypeIdentifier.java @@ -0,0 +1,134 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 11.11.12 + * Time: 21:11 + * To change this template use File | Settings | File Templates. + */ +public class DataTypeIdentifier { + private static final byte [] ARRAY = {'a', 'r', 'r', 'a', 'y'}; + private static final byte [] ANNOTATIONREFERENCE = {'a','n','n','o','t','a','t','i','o','n','r','e','f','e','r','e','n','c','e'}; + private static final byte [] MAP = { 'm', 'a', 'p'}; + private static final byte [] WSET = {'w', 'e', 'i', 'g', 'h', 't', 'e', 'd', 's', 'e', 't'}; + private static final byte [] CREATEIFNONEXISTENT = {';','a', 'd', 'd'}; + private static final byte [] REMOVEIFZERO = {';','r', 'e', 'm', 'o', 'v', 'e'}; + private static final byte [] CREATANDREMOVE = {';','a', 'd', 'd',';','r', 'e', 'm', 'o', 'v', 'e'}; + private static final byte [] EMPTY = {}; + private Utf8String utf8; + public DataTypeIdentifier(String s) { + utf8 = new Utf8String(s); + verify(utf8.wrap().array()); + } + public DataTypeIdentifier(AbstractUtf8Array utf8) { + this.utf8 = new Utf8String(utf8); + verify(utf8.wrap().array()); + } + public DataTypeIdentifier(byte [] utf8) { + this(new Utf8Array(utf8)); + } + + private DataTypeIdentifier(final byte [] prefix, DataTypeIdentifier nested, final byte [] postfix) { + utf8 = new Utf8String(new Utf8Array(createPrefixDataType(prefix, nested, postfix))); + } + private DataTypeIdentifier(final byte [] prefix, DataTypeIdentifier key, DataTypeIdentifier value) { + utf8 = new Utf8String(new Utf8Array(createMapDataType(prefix, key, value))); + } + + public static DataTypeIdentifier createArrayDataTypeIdentifier(DataTypeIdentifier nested) { + return new DataTypeIdentifier(ARRAY, nested, EMPTY); + } + public static DataTypeIdentifier createAnnotationReferenceDataTypeIdentifier(DataTypeIdentifier nested) { + return new DataTypeIdentifier(ANNOTATIONREFERENCE, nested, EMPTY); + } + public static DataTypeIdentifier createMapDataTypeIdentifier(DataTypeIdentifier key, DataTypeIdentifier value) { + return new DataTypeIdentifier(MAP, key, value); + } + public static DataTypeIdentifier createWeightedSetTypeIdentifier(DataTypeIdentifier nested, boolean createIfNonExistent, boolean removeIfZero) { + return new DataTypeIdentifier(WSET, nested, createPostfix(createIfNonExistent, removeIfZero)); + } + @Override + public int hashCode() { + return utf8.hashCode(); + } + @Override + public boolean equals(Object obj) { + if (obj instanceof DataTypeIdentifier) { + return utf8.equals(((DataTypeIdentifier)obj).utf8); + } + return false; + } + @Override + public String toString() { + return utf8.toString(); + } + public final Utf8String getUtf8() { + return utf8; + } + private static byte [] createPostfix(boolean createIfNonExistent, boolean removeIfZero) { + if (createIfNonExistent && removeIfZero) { + return CREATANDREMOVE; + } else if (createIfNonExistent) { + return CREATEIFNONEXISTENT; + } else if (removeIfZero) { + return REMOVEIFZERO; + } + return EMPTY; + } + private static byte [] createPrefixDataType(final byte [] prefix, final DataTypeIdentifier nested, final byte [] postfix) { + byte [] whole = new byte[prefix.length + 2 + nested.utf8.getByteLength() + postfix.length]; + for (int i=0; i < prefix.length; i++) { + whole[i] = prefix[i]; + } + whole[prefix.length] = '<'; + for (int i = 0, m=nested.utf8.getByteLength(); i < m; i++ ) { + whole[prefix.length+1+i] = nested.utf8.getByte(i); + } + whole[prefix.length + 1 + nested.utf8.getByteLength()] = '>'; + for (int i = 0; i < postfix.length; i++) { + whole[prefix.length + 1 + nested.utf8.length() + 1 + i] = postfix[i]; + } + return whole; + } + private static byte [] createMapDataType(final byte [] prefix, final DataTypeIdentifier key, final DataTypeIdentifier value) { + byte [] whole = new byte[prefix.length + 3 + key.utf8.getByteLength() + value.utf8.getByteLength()]; + for (int i=0; i < prefix.length; i++) { + whole[i] = prefix[i]; + } + whole[prefix.length] = '<'; + for (int i = 0, m=key.utf8.getByteLength(); i < m; i++ ) { + whole[prefix.length+1+i] = key.utf8.getByte(i); + } + whole[prefix.length + 1 + key.utf8.getByteLength()] = ','; + for (int i = 0; i < value.utf8.getByteLength(); i++) { + whole[prefix.length + 1 + key.utf8.getByteLength() + 1 + i] = value.utf8.getByte(i); + } + whole[whole.length-1] = '>'; + return whole; + } + private static byte [] verify(final byte [] utf8) { + if (utf8.length > 0) { + verifyFirst(utf8[0], utf8); + for (int i=1; i < utf8.length; i++) { + verifyAny(utf8[i], utf8); + } + } + return utf8; + + } + private static boolean verifyFirst(byte c, byte [] identifier) { + if (!((c == '_') || ((c >= 'a') && (c <= 'z')))) { + throw new IllegalArgumentException("Illegal starting character '" + (char)c + "' of identifier '" + new Utf8String(new Utf8Array(identifier)).toString() +"'."); + } + return true; + } + private static boolean verifyAny(byte c, byte [] identifier) { + if (!((c == '_') || (c == '.') || ((c >= 'a') && (c <= 'z')) || ((c >= '0') && (c <= '9')))) { + throw new IllegalArgumentException("Illegal character '" + (char)c + "' of identifier '" + new Utf8String(new Utf8Array(identifier)).toString() +"'."); + } + return true; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/DoubleFormatter.java b/vespajlib/src/main/java/com/yahoo/text/DoubleFormatter.java new file mode 100644 index 00000000000..a11694f466c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/DoubleFormatter.java @@ -0,0 +1,532 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import com.google.common.annotations.Beta; + +/** + * Utility class to format a double into a String. + * <p> + * This is intended as a lower-cost replacement for the standard + * String.valueOf(double) since that method will cause lock + * contention if it's used too often. + * <p> + * Note that this implementation won't always produce the same results + * as java.lang.* formatting. + * <p> + * Also, this implementation is very poorly tested at the moment, so + * it should be used carefully, only in cases where you know the input + * will be well-defined and you don't need full precision. + * + * @author arnej27959 + */ + +@Beta +public final class DoubleFormatter { + + private static void tooSmall(StringBuilder target, long mantissa, int exponent) { + double carry = 0; + int prExp = 0; + while (exponent < 0) { + while (mantissa < (1L << 53)) { + carry *= 10.0; + mantissa *= 10; + ++prExp; + } + carry *= 0.5; + carry += 0.5*(mantissa & 1); + mantissa >>= 1; + ++exponent; + } + while (mantissa < (1L << 53)) { + carry *= 10.0; + mantissa *= 10; + ++prExp; + } + mantissa += (long)(carry+0.5); + + int[] befor = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + int b = 0; + for (int i = 0; mantissa > 0; i++) { + befor[i] = (int)(mantissa % 10); + mantissa /= 10; + ++b; + } + --b; + target.append((char)('0'+befor[b])); + target.append('.'); + if (b == 0) { + target.append('0'); + } else { + for (int i = b; i-- > 0; ) { + target.append((char)('0'+befor[i])); + } + } + prExp -= b; + target.append("E-"); + target.append(String.valueOf(prExp)); + } + + public static StringBuilder fmt(StringBuilder target, double v) { + append(target, v); + return target; + } + + public static String stringValue(double v) { + return fmt(new StringBuilder(), v).toString(); + } + + + //Hardcode some byte arrays to make them quickly available + public static final char[] INFINITY = {'I','n','f','i','n','i','t','y'}; + public static final char[] NaN = {'N','a','N'}; + public static final char[][] ZEROS = { + {}, + {'0'}, + {'0','0'}, + {'0','0','0'}, + {'0','0','0','0'}, + {'0','0','0','0','0'}, + {'0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0'}, + {'0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0','0'}, + }; + + private static final char[] charForDigit = { + '0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f','g','h', + 'i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z' + }; + + //And required double related constants. + private static final long DoubleSignMask = 0x8000000000000000L; + private static final long DoubleExpMask = 0x7ff0000000000000L; + private static final long DoubleFractMask= ~(DoubleSignMask|DoubleExpMask); + private static final int DoubleExpShift = 52; + private static final int DoubleExpBias = 1023; + + private static final double[] d_tenthPowers = { + 1e-323D, 1e-322D, 1e-321D, 1e-320D, 1e-319D, 1e-318D, 1e-317D, 1e-316D, 1e-315D, 1e-314D, + 1e-313D, 1e-312D, 1e-311D, 1e-310D, 1e-309D, 1e-308D, 1e-307D, 1e-306D, 1e-305D, 1e-304D, + 1e-303D, 1e-302D, 1e-301D, 1e-300D, 1e-299D, 1e-298D, 1e-297D, 1e-296D, 1e-295D, 1e-294D, + 1e-293D, 1e-292D, 1e-291D, 1e-290D, 1e-289D, 1e-288D, 1e-287D, 1e-286D, 1e-285D, 1e-284D, + 1e-283D, 1e-282D, 1e-281D, 1e-280D, 1e-279D, 1e-278D, 1e-277D, 1e-276D, 1e-275D, 1e-274D, + 1e-273D, 1e-272D, 1e-271D, 1e-270D, 1e-269D, 1e-268D, 1e-267D, 1e-266D, 1e-265D, 1e-264D, + 1e-263D, 1e-262D, 1e-261D, 1e-260D, 1e-259D, 1e-258D, 1e-257D, 1e-256D, 1e-255D, 1e-254D, + 1e-253D, 1e-252D, 1e-251D, 1e-250D, 1e-249D, 1e-248D, 1e-247D, 1e-246D, 1e-245D, 1e-244D, + 1e-243D, 1e-242D, 1e-241D, 1e-240D, 1e-239D, 1e-238D, 1e-237D, 1e-236D, 1e-235D, 1e-234D, + 1e-233D, 1e-232D, 1e-231D, 1e-230D, 1e-229D, 1e-228D, 1e-227D, 1e-226D, 1e-225D, 1e-224D, + 1e-223D, 1e-222D, 1e-221D, 1e-220D, 1e-219D, 1e-218D, 1e-217D, 1e-216D, 1e-215D, 1e-214D, + 1e-213D, 1e-212D, 1e-211D, 1e-210D, 1e-209D, 1e-208D, 1e-207D, 1e-206D, 1e-205D, 1e-204D, + 1e-203D, 1e-202D, 1e-201D, 1e-200D, 1e-199D, 1e-198D, 1e-197D, 1e-196D, 1e-195D, 1e-194D, + 1e-193D, 1e-192D, 1e-191D, 1e-190D, 1e-189D, 1e-188D, 1e-187D, 1e-186D, 1e-185D, 1e-184D, + 1e-183D, 1e-182D, 1e-181D, 1e-180D, 1e-179D, 1e-178D, 1e-177D, 1e-176D, 1e-175D, 1e-174D, + 1e-173D, 1e-172D, 1e-171D, 1e-170D, 1e-169D, 1e-168D, 1e-167D, 1e-166D, 1e-165D, 1e-164D, + 1e-163D, 1e-162D, 1e-161D, 1e-160D, 1e-159D, 1e-158D, 1e-157D, 1e-156D, 1e-155D, 1e-154D, + 1e-153D, 1e-152D, 1e-151D, 1e-150D, 1e-149D, 1e-148D, 1e-147D, 1e-146D, 1e-145D, 1e-144D, + 1e-143D, 1e-142D, 1e-141D, 1e-140D, 1e-139D, 1e-138D, 1e-137D, 1e-136D, 1e-135D, 1e-134D, + 1e-133D, 1e-132D, 1e-131D, 1e-130D, 1e-129D, 1e-128D, 1e-127D, 1e-126D, 1e-125D, 1e-124D, + 1e-123D, 1e-122D, 1e-121D, 1e-120D, 1e-119D, 1e-118D, 1e-117D, 1e-116D, 1e-115D, 1e-114D, + 1e-113D, 1e-112D, 1e-111D, 1e-110D, 1e-109D, 1e-108D, 1e-107D, 1e-106D, 1e-105D, 1e-104D, + 1e-103D, 1e-102D, 1e-101D, 1e-100D, 1e-99D, 1e-98D, 1e-97D, 1e-96D, 1e-95D, 1e-94D, + 1e-93D, 1e-92D, 1e-91D, 1e-90D, 1e-89D, 1e-88D, 1e-87D, 1e-86D, 1e-85D, 1e-84D, + 1e-83D, 1e-82D, 1e-81D, 1e-80D, 1e-79D, 1e-78D, 1e-77D, 1e-76D, 1e-75D, 1e-74D, + 1e-73D, 1e-72D, 1e-71D, 1e-70D, 1e-69D, 1e-68D, 1e-67D, 1e-66D, 1e-65D, 1e-64D, + 1e-63D, 1e-62D, 1e-61D, 1e-60D, 1e-59D, 1e-58D, 1e-57D, 1e-56D, 1e-55D, 1e-54D, + 1e-53D, 1e-52D, 1e-51D, 1e-50D, 1e-49D, 1e-48D, 1e-47D, 1e-46D, 1e-45D, 1e-44D, + 1e-43D, 1e-42D, 1e-41D, 1e-40D, 1e-39D, 1e-38D, 1e-37D, 1e-36D, 1e-35D, 1e-34D, + 1e-33D, 1e-32D, 1e-31D, 1e-30D, 1e-29D, 1e-28D, 1e-27D, 1e-26D, 1e-25D, 1e-24D, + 1e-23D, 1e-22D, 1e-21D, 1e-20D, 1e-19D, 1e-18D, 1e-17D, 1e-16D, 1e-15D, 1e-14D, + 1e-13D, 1e-12D, 1e-11D, 1e-10D, 1e-9D, 1e-8D, 1e-7D, 1e-6D, 1e-5D, 1e-4D, + 1e-3D, 1e-2D, 1e-1D, 1e0D, 1e1D, 1e2D, 1e3D, 1e4D, + 1e5D, 1e6D, 1e7D, 1e8D, 1e9D, 1e10D, 1e11D, 1e12D, 1e13D, 1e14D, + 1e15D, 1e16D, 1e17D, 1e18D, 1e19D, 1e20D, 1e21D, 1e22D, 1e23D, 1e24D, + 1e25D, 1e26D, 1e27D, 1e28D, 1e29D, 1e30D, 1e31D, 1e32D, 1e33D, 1e34D, + 1e35D, 1e36D, 1e37D, 1e38D, 1e39D, 1e40D, 1e41D, 1e42D, 1e43D, 1e44D, + 1e45D, 1e46D, 1e47D, 1e48D, 1e49D, 1e50D, 1e51D, 1e52D, 1e53D, 1e54D, + 1e55D, 1e56D, 1e57D, 1e58D, 1e59D, 1e60D, 1e61D, 1e62D, 1e63D, 1e64D, + 1e65D, 1e66D, 1e67D, 1e68D, 1e69D, 1e70D, 1e71D, 1e72D, 1e73D, 1e74D, + 1e75D, 1e76D, 1e77D, 1e78D, 1e79D, 1e80D, 1e81D, 1e82D, 1e83D, 1e84D, + 1e85D, 1e86D, 1e87D, 1e88D, 1e89D, 1e90D, 1e91D, 1e92D, 1e93D, 1e94D, + 1e95D, 1e96D, 1e97D, 1e98D, 1e99D, 1e100D, 1e101D, 1e102D, 1e103D, 1e104D, + 1e105D, 1e106D, 1e107D, 1e108D, 1e109D, 1e110D, 1e111D, 1e112D, 1e113D, 1e114D, + 1e115D, 1e116D, 1e117D, 1e118D, 1e119D, 1e120D, 1e121D, 1e122D, 1e123D, 1e124D, + 1e125D, 1e126D, 1e127D, 1e128D, 1e129D, 1e130D, 1e131D, 1e132D, 1e133D, 1e134D, + 1e135D, 1e136D, 1e137D, 1e138D, 1e139D, 1e140D, 1e141D, 1e142D, 1e143D, 1e144D, + 1e145D, 1e146D, 1e147D, 1e148D, 1e149D, 1e150D, 1e151D, 1e152D, 1e153D, 1e154D, + 1e155D, 1e156D, 1e157D, 1e158D, 1e159D, 1e160D, 1e161D, 1e162D, 1e163D, 1e164D, + 1e165D, 1e166D, 1e167D, 1e168D, 1e169D, 1e170D, 1e171D, 1e172D, 1e173D, 1e174D, + 1e175D, 1e176D, 1e177D, 1e178D, 1e179D, 1e180D, 1e181D, 1e182D, 1e183D, 1e184D, + 1e185D, 1e186D, 1e187D, 1e188D, 1e189D, 1e190D, 1e191D, 1e192D, 1e193D, 1e194D, + 1e195D, 1e196D, 1e197D, 1e198D, 1e199D, 1e200D, 1e201D, 1e202D, 1e203D, 1e204D, + 1e205D, 1e206D, 1e207D, 1e208D, 1e209D, 1e210D, 1e211D, 1e212D, 1e213D, 1e214D, + 1e215D, 1e216D, 1e217D, 1e218D, 1e219D, 1e220D, 1e221D, 1e222D, 1e223D, 1e224D, + 1e225D, 1e226D, 1e227D, 1e228D, 1e229D, 1e230D, 1e231D, 1e232D, 1e233D, 1e234D, + 1e235D, 1e236D, 1e237D, 1e238D, 1e239D, 1e240D, 1e241D, 1e242D, 1e243D, 1e244D, + 1e245D, 1e246D, 1e247D, 1e248D, 1e249D, 1e250D, 1e251D, 1e252D, 1e253D, 1e254D, + 1e255D, 1e256D, 1e257D, 1e258D, 1e259D, 1e260D, 1e261D, 1e262D, 1e263D, 1e264D, + 1e265D, 1e266D, 1e267D, 1e268D, 1e269D, 1e270D, 1e271D, 1e272D, 1e273D, 1e274D, + 1e275D, 1e276D, 1e277D, 1e278D, 1e279D, 1e280D, 1e281D, 1e282D, 1e283D, 1e284D, + 1e285D, 1e286D, 1e287D, 1e288D, 1e289D, 1e290D, 1e291D, 1e292D, 1e293D, 1e294D, + 1e295D, 1e296D, 1e297D, 1e298D, 1e299D, 1e300D, 1e301D, 1e302D, 1e303D, 1e304D, + 1e305D, 1e306D, 1e307D, 1e308D + }; + + + /** + * Assumes i is positive. Returns the magnitude of i in base 10. + */ + private static long tenthPower(long i) + { + if (i < 10L) return 1; + else if (i < 100L) return 10L; + else if (i < 1000L) return 100L; + else if (i < 10000L) return 1000L; + else if (i < 100000L) return 10000L; + else if (i < 1000000L) return 100000L; + else if (i < 10000000L) return 1000000L; + else if (i < 100000000L) return 10000000L; + else if (i < 1000000000L) return 100000000L; + else if (i < 10000000000L) return 1000000000L; + else if (i < 100000000000L) return 10000000000L; + else if (i < 1000000000000L) return 100000000000L; + else if (i < 10000000000000L) return 1000000000000L; + else if (i < 100000000000000L) return 10000000000000L; + else if (i < 1000000000000000L) return 100000000000000L; + else if (i < 10000000000000000L) return 1000000000000000L; + else if (i < 100000000000000000L) return 10000000000000000L; + else if (i < 1000000000000000000L) return 100000000000000000L; + else return 1000000000000000000L; + } + + + private static int magnitude(double d) + { + //It works. What else can I say. + long doubleToLongBits = Double.doubleToLongBits(d); + int magnitude = + (int) ((((doubleToLongBits & DoubleExpMask) >> DoubleExpShift) - DoubleExpBias) * 0.301029995663981); + + if (magnitude < -323) + magnitude = -323; + else if (magnitude > 308) + magnitude = 308; + + if (d >= d_tenthPowers[magnitude+323]) + { + while(magnitude < 309 && d >= d_tenthPowers[magnitude+323]) + magnitude++; + magnitude--; + return magnitude; + } + else + { + while(magnitude > -324 && d < d_tenthPowers[magnitude+323]) + magnitude--; + return magnitude; + } + } + + static long[] l_tenthPowers = { + 1, + 10L, + 100L, + 1000L, + 10000L, + 100000L, + 1000000L, + 10000000L, + 100000000L, + 1000000000L, + 10000000000L, + 100000000000L, + 1000000000000L, + 10000000000000L, + 100000000000000L, + 1000000000000000L, + 10000000000000000L, + 100000000000000000L, + 1000000000000000000L, + }; + + public static long getNthDigit(long l, int n) + { + return (l/(tenthPower(l)/l_tenthPowers[n-1]))%10; + } + + + + + public static void append(StringBuilder s, double d) + { + if (d == Double.NEGATIVE_INFINITY) + s.append(NEGATIVE_INFINITY); + else if (d == Double.POSITIVE_INFINITY) + s.append(POSITIVE_INFINITY); + else if (d != d) + s.append(NaN); + else if (d == 0.0) + { + if ( (Double.doubleToLongBits(d) & DoubleSignMask) != 0) + s.append('-'); + s.append(DOUBLE_ZERO); + } + else + { + if (d < 0) + { + s.append('-'); + d = -d; + } + + if (d >= 0.001 && d < 0.01) + { + long i = (long) (d * 1E20); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + s.append(DOUBLE_ZERO2); + appendFractDigits(s, i,-1); + } + else if (d >= 0.01 && d < 0.1) + { + long i = (long) (d * 1E19); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + s.append(DOUBLE_ZERO); + appendFractDigits(s, i,-1); + } + else if (d >= 0.1 && d < 1) + { + long i = (long) (d * 1E18); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + s.append(DOUBLE_ZERO0); + appendFractDigits(s, i,-1); + } + else if (d >= 1 && d < 10) + { + long i = (long) (d * 1E17); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i,1); + } + else if (d >= 10 && d < 100) + { + long i = (long) (d * 1E16); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i,2); + } + else if (d >= 100 && d < 1000) + { + long i = (long) (d * 1E15); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i,3); + } + else if (d >= 1000 && d < 10000) + { + long i = (long) (d * 1E14); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i,4); + } + else if (d >= 10000 && d < 100000) + { + long i = (long) (d * 1E13); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i,5); + } + else if (d >= 100000 && d < 1000000) + { + long i = (long) (d * 1E12); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i,6); + } + else if (d >= 1000000 && d < 10000000) + { + long i = (long) (d * 1E11); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i,7); + } + else + { + int magnitude = magnitude(d); + long i; + if (magnitude < -280) { + long valueBits = Double.doubleToRawLongBits(d); + long mantissa = valueBits & 0x000fffffffffffffL; + int exponent = (int)((valueBits >> 52) & 0x7ff); + if (exponent == 0) { + tooSmall(s, mantissa, -1074); + } else { + mantissa |= 0x0010000000000000L; + exponent -= 1075; + tooSmall(s, mantissa, exponent); + } + } else { + i = (long) (d / d_tenthPowers[magnitude + 323 - 17]); + i = i%100 >= 50 ? (i/100) + 1 : i/100; + appendFractDigits(s, i, 1); + s.append('E'); + append(s, magnitude); + } + } + } + } + public static void append(StringBuilder s, int i) + { + if (i < 0) + { + if (i == Integer.MIN_VALUE) + { + //cannot make this positive due to integer overflow + s.append("-2147483648"); + } + s.append('-'); + i = -i; + } + int mag; + int c; + if (i < 10) + { + //one digit + s.append(charForDigit[i]); + } + else if (i < 100) + { + //two digits + s.append(charForDigit[i/10]); + s.append(charForDigit[i%10]); + } + else if (i < 1000) + { + //three digits + s.append(charForDigit[i/100]); + s.append(charForDigit[(c=i%100)/10]); + s.append(charForDigit[c%10]); + } + else if (i < 10000) + { + //four digits + s.append(charForDigit[i/1000]); + s.append(charForDigit[(c=i%1000)/100]); + s.append(charForDigit[(c%=100)/10]); + s.append(charForDigit[c%10]); + } + else if (i < 100000) + { + //five digits + s.append(charForDigit[i/10000]); + s.append(charForDigit[(c=i%10000)/1000]); + s.append(charForDigit[(c%=1000)/100]); + s.append(charForDigit[(c%=100)/10]); + s.append(charForDigit[c%10]); + } + else if (i < 1000000) + { + //six digits + s.append(charForDigit[i/100000]); + s.append(charForDigit[(c=i%100000)/10000]); + s.append(charForDigit[(c%=10000)/1000]); + s.append(charForDigit[(c%=1000)/100]); + s.append(charForDigit[(c%=100)/10]); + s.append(charForDigit[c%10]); + } + else if (i < 10000000) + { + //seven digits + s.append(charForDigit[i/1000000]); + s.append(charForDigit[(c=i%1000000)/100000]); + s.append(charForDigit[(c%=100000)/10000]); + s.append(charForDigit[(c%=10000)/1000]); + s.append(charForDigit[(c%=1000)/100]); + s.append(charForDigit[(c%=100)/10]); + s.append(charForDigit[c%10]); + } + else if (i < 100000000) + { + //eight digits + s.append(charForDigit[i/10000000]); + s.append(charForDigit[(c=i%10000000)/1000000]); + s.append(charForDigit[(c%=1000000)/100000]); + s.append(charForDigit[(c%=100000)/10000]); + s.append(charForDigit[(c%=10000)/1000]); + s.append(charForDigit[(c%=1000)/100]); + s.append(charForDigit[(c%=100)/10]); + s.append(charForDigit[c%10]); + } + else if (i < 1000000000) + { + //nine digits + s.append(charForDigit[i/100000000]); + s.append(charForDigit[(c=i%100000000)/10000000]); + s.append(charForDigit[(c%=10000000)/1000000]); + s.append(charForDigit[(c%=1000000)/100000]); + s.append(charForDigit[(c%=100000)/10000]); + s.append(charForDigit[(c%=10000)/1000]); + s.append(charForDigit[(c%=1000)/100]); + s.append(charForDigit[(c%=100)/10]); + s.append(charForDigit[c%10]); + } + else + { + //ten digits + s.append(charForDigit[i/1000000000]); + s.append(charForDigit[(c=i%1000000000)/100000000]); + s.append(charForDigit[(c%=100000000)/10000000]); + s.append(charForDigit[(c%=10000000)/1000000]); + s.append(charForDigit[(c%=1000000)/100000]); + s.append(charForDigit[(c%=100000)/10000]); + s.append(charForDigit[(c%=10000)/1000]); + s.append(charForDigit[(c%=1000)/100]); + s.append(charForDigit[(c%=100)/10]); + s.append(charForDigit[c%10]); + } + } + private static void appendFractDigits(StringBuilder s, long i, int decimalOffset) + { + long mag = tenthPower(i); + long c; + while ( i > 0 ) + { + c = i/mag; + s.append(charForDigit[(int) c]); + decimalOffset--; + if (decimalOffset == 0) + s.append('.'); + c *= mag; + if ( c <= i) + i -= c; + mag = mag/10; + } + if (i != 0) + s.append(charForDigit[(int) i]); + else if (decimalOffset > 0) + { + s.append(ZEROS[decimalOffset]); + decimalOffset = 1; + } + + decimalOffset--; + if (decimalOffset == 0) + s.append(DOT_ZERO); + else if (decimalOffset == -1) + s.append('0'); + } + + public static final char[] NEGATIVE_INFINITY = {'-','I','n','f','i','n','i','t','y'}; + public static final char[] POSITIVE_INFINITY = {'I','n','f','i','n','i','t','y'}; + public static final char[] DOUBLE_ZERO = {'0','.','0'}; + public static final char[] DOUBLE_ZERO2 = {'0','.','0','0'}; + public static final char[] DOUBLE_ZERO0 = {'0','.'}; + public static final char[] DOT_ZERO = {'.','0'}; + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/DoubleParser.java b/vespajlib/src/main/java/com/yahoo/text/DoubleParser.java new file mode 100644 index 00000000000..8dfe8f012f7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/DoubleParser.java @@ -0,0 +1,199 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * Utility class to parse a String into a double. + * <p> + * This is intended as a lower-cost replacement for the standard + * Double.parseDouble(String) since that method will cause lock + * contention if it's used too often. + * <p> + * Note that this implementation won't always produce the same results + * as java.lang.Double (low-order bits may differ), and it doesn't + * properly support denormalized numbers. + * <p> + * Also, this implementation is very poorly tested at the moment, so + * it should be used carefully, only in cases where you know the input + * will be well-defined and you don't need full precision. + * + * @author arnej27959 + */ +public final class DoubleParser { + + /** + * Utility method that parses a String and returns a double. + * + * @param data the String to parse + * @return double parsed value of the string + * @throws NumberFormatException if the string is not a well-formatted number + * @throws NullPointerException if the string is a null pointer + */ + public static double parse(String data) { + final int len = data.length(); + double result = 0; + boolean negative = false; + int beforePoint = 0; + int exponent = 0; + byte[] digits = new byte[25]; + int numDigits = 0; + + int i = 0; + while (i < len && Character.isWhitespace(data.charAt(i))) { + i++; + } + if (data.charAt(i) == '+') { + i++; + } else if (data.charAt(i) == '-') { + negative = true; + i++; + } + if (i + 3 <= len && data.substring(i, i+3).equals("NaN")) { + i += 3; + result = Double.NaN; + } else if (i + 8 <= len && data.substring(i, i+8).equals("Infinity")) { + i += 8; + if (negative) { + result = Double.NEGATIVE_INFINITY; + } else { + result = Double.POSITIVE_INFINITY; + } + } else { + while (i < len && Character.isDigit(data.charAt(i))) { + int dval = Character.digit(data.charAt(i), 10); + assert dval >= 0; + assert dval < 10; + if (numDigits < 25) { + digits[numDigits++] = (byte)dval; + } + ++beforePoint; + i++; + } + if (i < len && data.charAt(i) == '.') { + i++; + while (i < len && Character.isDigit(data.charAt(i))) { + int dval = Character.digit(data.charAt(i), 10); + assert dval >= 0; + assert dval < 10; + if (numDigits < 25) { + digits[numDigits++] = (byte)dval; + } + i++; + } + } + if (numDigits == 0) { + throw new NumberFormatException("No digits in number: '"+data+"'"); + } + if (i < len && (data.charAt(i) == 'e' || data.charAt(i) == 'E')) { + i++; + boolean expNeg = false; + int expDigits = 0; + if (data.charAt(i) == '+') { + i++; + } else if (data.charAt(i) == '-') { + expNeg = true; + i++; + } + while (i < len && Character.isDigit(data.charAt(i))) { + int dval = Character.digit(data.charAt(i), 10); + assert dval >= 0; + assert dval < 10; + exponent *= 10; + exponent += dval; + ++expDigits; + i++; + } + if (expDigits == 0) { + throw new NumberFormatException("Missing digits in exponent part: "+data); + } + if (expNeg) { + exponent = -exponent; + } + } + // System.out.println("parsed exp: "+exponent); + // System.out.println("before pt: "+beforePoint); + exponent += beforePoint; + exponent -= numDigits; + // System.out.println("adjusted exp: "+exponent); + for (int d = numDigits; d > 0; d--) { + double dv = digits[d-1]; + dv *= powTen(numDigits - d); + result += dv; + } + // System.out.println("digits sum: "+result); + while (exponent < -99) { + result *= powTen(-99); + exponent += 99; + } + while (exponent > 99) { + result *= powTen(99); + exponent -= 99; + } + // System.out.println("digits sum: "+result); + // System.out.println("exponent multiplier: "+powTen(exponent)); + result *= powTen(exponent); + + if (negative) { + result = -result; + } + } + while (i < len && Character.isWhitespace(data.charAt(i))) { + i++; + } + if (i < len) { + throw new NumberFormatException("Extra characters after number: "+data.substring(i)); + } + return result; + } + + private static double[] tens = { + 1.0e00, 1.0e01, 1.0e02, 1.0e03, 1.0e04, 1.0e05, 1.0e06, + 1.0e07, 1.0e08, 1.0e09, 1.0e10, 1.0e11, 1.0e12, 1.0e13, + 1.0e14, 1.0e15, 1.0e16, 1.0e17, 1.0e18, 1.0e19, 1.0e20, + 1.0e21, 1.0e22, 1.0e23, 1.0e24, 1.0e25, 1.0e26, 1.0e27, + 1.0e28, 1.0e29, 1.0e30, 1.0e31, 1.0e32, 1.0e33, 1.0e34, + 1.0e35, 1.0e36, 1.0e37, 1.0e38, 1.0e39, 1.0e40, 1.0e41, + 1.0e42, 1.0e43, 1.0e44, 1.0e45, 1.0e46, 1.0e47, 1.0e48, + 1.0e49, 1.0e50, 1.0e51, 1.0e52, 1.0e53, 1.0e54, 1.0e55, + 1.0e56, 1.0e57, 1.0e58, 1.0e59, 1.0e60, 1.0e61, 1.0e62, + 1.0e63, 1.0e64, 1.0e65, 1.0e66, 1.0e67, 1.0e68, 1.0e69, + 1.0e70, 1.0e71, 1.0e72, 1.0e73, 1.0e74, 1.0e75, 1.0e76, + 1.0e77, 1.0e78, 1.0e79, 1.0e80, 1.0e81, 1.0e82, 1.0e83, + 1.0e84, 1.0e85, 1.0e86, 1.0e87, 1.0e88, 1.0e89, 1.0e90, + 1.0e91, 1.0e92, 1.0e93, 1.0e94, 1.0e95, 1.0e96, 1.0e97, + 1.0e98, 1.0e99 + }; + + private static double[] tenths = { + 1.0e-00, 1.0e-01, 1.0e-02, 1.0e-03, 1.0e-04, 1.0e-05, + 1.0e-06, 1.0e-07, 1.0e-08, 1.0e-09, 1.0e-10, 1.0e-11, + 1.0e-12, 1.0e-13, 1.0e-14, 1.0e-15, 1.0e-16, 1.0e-17, + 1.0e-18, 1.0e-19, 1.0e-20, 1.0e-21, 1.0e-22, 1.0e-23, + 1.0e-24, 1.0e-25, 1.0e-26, 1.0e-27, 1.0e-28, 1.0e-29, + 1.0e-30, 1.0e-31, 1.0e-32, 1.0e-33, 1.0e-34, 1.0e-35, + 1.0e-36, 1.0e-37, 1.0e-38, 1.0e-39, 1.0e-40, 1.0e-41, + 1.0e-42, 1.0e-43, 1.0e-44, 1.0e-45, 1.0e-46, 1.0e-47, + 1.0e-48, 1.0e-49, 1.0e-50, 1.0e-51, 1.0e-52, 1.0e-53, + 1.0e-54, 1.0e-55, 1.0e-56, 1.0e-57, 1.0e-58, 1.0e-59, + 1.0e-60, 1.0e-61, 1.0e-62, 1.0e-63, 1.0e-64, 1.0e-65, + 1.0e-66, 1.0e-67, 1.0e-68, 1.0e-69, 1.0e-70, 1.0e-71, + 1.0e-72, 1.0e-73, 1.0e-74, 1.0e-75, 1.0e-76, 1.0e-77, + 1.0e-78, 1.0e-79, 1.0e-80, 1.0e-81, 1.0e-82, 1.0e-83, + 1.0e-84, 1.0e-85, 1.0e-86, 1.0e-87, 1.0e-88, 1.0e-89, + 1.0e-90, 1.0e-91, 1.0e-92, 1.0e-93, 1.0e-94, 1.0e-95, + 1.0e-96, 1.0e-97, 1.0e-98, 1.0e-99 + }; + + private static double powTen(int exponent) { + if (exponent > 0) { + assert exponent < 100; + return tens[exponent]; + } + if (exponent < 0) { + exponent = -exponent; + assert exponent < 100; + return tenths[exponent]; + } + return 1.0; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/ForwardWriter.java b/vespajlib/src/main/java/com/yahoo/text/ForwardWriter.java new file mode 100644 index 00000000000..a7876d620b9 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/ForwardWriter.java @@ -0,0 +1,157 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.IOException; + +/** + * Wraps another writer and also converting IOException to Exceptions. + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public class ForwardWriter extends GenericWriter { + + private final GenericWriter out; + + public ForwardWriter(GenericWriter writer) { + super(); + this.out = writer; + } + + @Override + public void write(char[] c, int offset, int bytes) { + try { + out.write(c, offset, bytes); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + @Override + public GenericWriter write(AbstractUtf8Array v) { + try { + out.write(v); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + @Override + public void write(String v) { + try { + out.write(v); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + @Override + public GenericWriter write(CharSequence c) { + try { + out.write(c); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + + @Override + public GenericWriter write(double d) { + try { + out.write(d); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public GenericWriter write(float f) { + try { + out.write(f); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public GenericWriter write(long v) { + try { + out.write(v); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + @Override + public void write(int v) { + try { + out.write(v); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + @Override + public GenericWriter write(short v) { + try { + out.write(v); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + @Override + public GenericWriter write(char c) { + try { + out.write(c); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + @Override + public GenericWriter write(byte b) { + try { + out.write(b); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + + @Override + public GenericWriter write(boolean v) { + try { + out.write(v); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public void flush() { + try { + out.flush(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() { + try { + out.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Gives access to the wrapped writer. + * @return wrapped writer. + */ + public GenericWriter getWriter() { return out; } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/GenericWriter.java b/vespajlib/src/main/java/com/yahoo/text/GenericWriter.java new file mode 100644 index 00000000000..0a07b617352 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/GenericWriter.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.IOException; +import java.io.Writer; + +/** + * This is a basic writer for presenting text. Its has the pattern as + * java.io.Writer, but it allows for more overrides for speed. + * This introduces additional interfaces in addition to the java.lang.Writer. + * The purpose is to allow for optimizations. + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ + +public abstract class GenericWriter extends Writer { +/* + public abstract void write(char [] c, int offset, int bytes); + public abstract void flush(); + public abstract void close(); + +*/ + public GenericWriter write(char c) throws java.io.IOException { + char t[] = new char[1]; + t[0] = c; + try { + write(t, 0, 1); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + public GenericWriter write(CharSequence s) throws java.io.IOException { + for (int i=0, m=s.length(); i < m; i++) { + write(s.charAt(i)); + } + return this; + } + + public GenericWriter write(long i) throws java.io.IOException { + write(String.valueOf(i)); + return this; + } + + public GenericWriter write(short i) throws java.io.IOException { + write(String.valueOf(i)); + return this; + } + public GenericWriter write(byte i) throws java.io.IOException { + write(String.valueOf(i)); + return this; + } + public GenericWriter write(double i) throws java.io.IOException { + write(String.valueOf(i)); + return this; + } + public GenericWriter write(float i) throws java.io.IOException { + write(String.valueOf(i)); + return this; + } + public GenericWriter write(boolean i) throws java.io.IOException { + write(String.valueOf(i)); + return this; + } + + public GenericWriter write(AbstractUtf8Array v) throws java.io.IOException { + write(v.toString()); + return this; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/HTML.java b/vespajlib/src/main/java/com/yahoo/text/HTML.java new file mode 100644 index 00000000000..e76a2f54d1f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/HTML.java @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + + +import java.util.Map; +import java.util.HashMap; + + +/** + * Static HTML escaping stuff + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class HTML { + static Object[][] entities = { + // {"#39", new Integer(39)}, // ' - apostrophe + {"quot", 34}, // " - double-quote + {"amp", 38}, // & - ampersand + {"lt", 60}, // < - less-than + {"gt", 62}, // > - greater-than + {"nbsp", 160}, // non-breaking space + {"copy", 169}, // \u00A9 - copyright + {"reg", 174}, // \u00AE - registered trademark + {"Agrave", 192}, // \u00C0 - uppercase A, grave accent + {"Aacute", 193}, // \u00C1 - uppercase A, acute accent + {"Acirc", 194}, // \u00C2 - uppercase A, circumflex accent + {"Atilde", 195}, // \u00C3 - uppercase A, tilde + {"Auml", 196}, // \u00C4 - uppercase A, umlaut + {"Aring", 197}, // \u00C5 - uppercase A, ring + {"AElig", 198}, // \u00C6 - uppercase AE + {"Ccedil", 199}, // \u00C7 - uppercase C, cedilla + {"Egrave", 200}, // \u00C8 - uppercase E, grave accent + {"Eacute", 201}, // \u00C9 - uppercase E, acute accent + {"Ecirc", 202}, // \u00CA - uppercase E, circumflex accent + {"Euml", 203}, // \u00CB - uppercase E, umlaut + {"Igrave", 204}, // \u00CC - uppercase I, grave accent + {"Iacute", 205}, // \u00CD - uppercase I, acute accent + {"Icirc", 206}, // \u00CE - uppercase I, circumflex accent + {"Iuml", 207}, // \u00CF - uppercase I, umlaut + {"ETH", 208}, // \u00D0 - uppercase Eth, Icelandic + {"Ntilde", 209}, // \u00D1 - uppercase N, tilde + {"Ograve", 210}, // \u00D2 - uppercase O, grave accent + {"Oacute", 211}, // \u00D3 - uppercase O, acute accent + {"Ocirc", 212}, // \u00D4 - uppercase O, circumflex accent + {"Otilde", 213}, // \u00D5 - uppercase O, tilde + {"Ouml", 214}, // \u00D6 - uppercase O, umlaut + {"Oslash", 216}, // \u00D8 - uppercase O, slash + {"Ugrave", 217}, // \u00D9 - uppercase U, grave accent + {"Uacute", 218}, // \u00DA - uppercase U, acute accent + {"Ucirc", 219}, // \u00DB - uppercase U, circumflex accent + {"Uuml", 220}, // \u00DC - uppercase U, umlaut + {"Yacute", 221}, // \u00DD - uppercase Y, acute accent + {"THORN", 222}, // \u00DE - uppercase THORN, Icelandic + {"szlig", 223}, // \u00DF - lowercase sharps, German + {"agrave", 224}, // \u00E0 - lowercase a, grave accent + {"aacute", 225}, // \u00E1 - lowercase a, acute accent + {"acirc", 226}, // \u00E2 - lowercase a, circumflex accent + {"atilde", 227}, // \u00E3 - lowercase a, tilde + {"auml", 228}, // \u00E4 - lowercase a, umlaut + {"aring", 229}, // \u00E5 - lowercase a, ring + {"aelig", 230}, // \u00E6 - lowercase ae + {"ccedil", 231}, // \u00E7 - lowercase c, cedilla + {"egrave", 232}, // \u00E8 - lowercase e, grave accent + {"eacute", 233}, // \u00E9 - lowercase e, acute accent + {"ecirc", 234}, // \u00EA - lowercase e, circumflex accent + {"euml", 235}, // \u00EB - lowercase e, umlaut + {"igrave", 236}, // \u00EC - lowercase i, grave accent + {"iacute", 237}, // \u00ED - lowercase i, acute accent + {"icirc", 238}, // \u00EE - lowercase i, circumflex accent + {"iuml", 239}, // \u00EF - lowercase i, umlaut + {"igrave", 236}, // \u00EC - lowercase i, grave accent + {"iacute", 237}, // \u00ED - lowercase i, acute accent + {"icirc", 238}, // \u00EE - lowercase i, circumflex accent + {"iuml", 239}, // \u00EF - lowercase i, umlaut + {"eth", 240}, // \u00F0 - lowercase eth, Icelandic + {"ntilde", 241}, // \u00F1 - lowercase n, tilde + {"ograve", 242}, // \u00F2 - lowercase o, grave accent + {"oacute", 243}, // \u00F3 - lowercase o, acute accent + {"ocirc", 244}, // \u00F4 - lowercase o, circumflex accent + {"otilde", 245}, // \u00F5 - lowercase o, tilde + {"ouml", 246}, // \u00F6 - lowercase o, umlaut + {"oslash", 248}, // \u00F8 - lowercase o, slash + {"ugrave", 249}, // \u00F9 - lowercase u, grave accent + {"uacute", 250}, // \u00FA - lowercase u, acute accent + {"ucirc", 251}, // \u00FB - lowercase u, circumflex accent + {"uuml", 252}, // \u00FC - lowercase u, umlaut + {"yacute", 253}, // \u00FD - lowercase y, acute accent + {"thorn", 254}, // \u00FE - lowercase thorn, Icelandic + {"yuml", 255}, // \u00FF - lowercase y, umlaut + {"euro", 8364}, // Euro symbol + }; + + static Map<String, Integer> e2i = new HashMap<>(); + static Map<Integer, String> i2e = new HashMap<>(); + + static { + for (Object[] entity : entities) { + e2i.put((String) entity[0], (Integer) entity[1]); + i2e.put((Integer) entity[1], (String) entity[0]); + } + } + + public static String htmlescape(String s1) { + if (s1 == null) return ""; + + int len = s1.length(); + // about 20% guess + StringBuilder buf = new StringBuilder((int) (len * 1.2)); + int i; + + for (i = 0; i < len; ++i) { + char ch = s1.charAt(i); + String entity = i2e.get((int) ch); + + if (entity == null) { + if (((int) ch) > 128) buf.append("&#").append((int) ch).append(";"); + else buf.append(ch); + } else { + buf.append("&").append(entity).append(";"); + } + } + return buf.toString(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Identifier.java b/vespajlib/src/main/java/com/yahoo/text/Identifier.java new file mode 100644 index 00000000000..2d74b61e1e0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Identifier.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 11.11.12 + * Time: 10:37 + * This class is used to represent a legal identifier of [a-zA-Z_][a-zA-Z_0-9]* + */ +public class Identifier extends Utf8Array { + public Identifier(String s) { + this(Utf8.toBytes(s)); + } + public Identifier(AbstractUtf8Array utf8) { + this(utf8.getBytes()); + } + public Identifier(byte [] utf8) { + super(verify(utf8)); + } + private static byte [] verify(final byte [] utf8) { + if (utf8.length > 0) { + verifyFirst(utf8[0], utf8); + for (int i=1; i < utf8.length; i++) { + verifyAny(utf8[i], utf8); + } + } + return utf8; + + } + private static boolean verifyFirst(byte c, byte [] identifier) { + if (!((c == '_') || ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z')))) { + throw new IllegalArgumentException("Illegal starting character '" + (char)c + "' of identifier '" + new Utf8String(new Utf8Array(identifier)).toString() +"'."); + } + return true; + } + private static boolean verifyAny(byte c, byte [] identifier) { + if (!((c == '_') || ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z')) || ((c >= '0') && (c <= '9')))) { + throw new IllegalArgumentException("Illegal character '" + (char)c + "' of identifier '" + new Utf8String(new Utf8Array(identifier)).toString() +"'."); + } + return true; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/JSON.java b/vespajlib/src/main/java/com/yahoo/text/JSON.java new file mode 100644 index 00000000000..33af017b81d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/JSON.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.util.Map; + +/** + * Static methods for working with the map textual format which is parsed by {@link MapParser} + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public final class JSON { + + /** No instances */ + private JSON() {} + + /** + * Outputs a map as a JSON 'object' string, provided that the map values + * are either + * <ul> + * <li>String + * <li>Number + * <li>Any object whose toString returns JSON + * </ul> + */ + public static String encode(Map<String, ?> map) { + StringBuilder b = new StringBuilder("{"); + for (Map.Entry<String,?> entry : map.entrySet()) { + b.append("\"").append(escape(entry.getKey())).append("\":"); + if (entry.getValue() instanceof String) + b.append("\"").append(escape(entry.getValue().toString())).append("\""); + else // Number, or some other object which returns JSON + b.append(entry.getValue()); + b.append(","); + } + if (b.length()>1) + b.setLength(b.length()-1); // remove last comma + b.append("}"); + return b.toString(); + } + + /** Returns the given string as a properly json escaped string */ + public static String escape(String s) { + StringBuilder b = null; // lazy create to optimize for "nothing to do" case + + for (int i=0; i < s.length(); i = s.offsetByCodePoints(i, 1)) { + final int codepoint = s.codePointAt(i); + if (codepoint == '"') { + if (b == null) + b = new StringBuilder(s.substring(0, i)); + b.append('\\'); + } + + if (b != null) + b.appendCodePoint(codepoint); + } + return b != null ? b.toString() : s; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/JSONWriter.java b/vespajlib/src/main/java/com/yahoo/text/JSONWriter.java new file mode 100644 index 00000000000..0df2b2d2d3a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/JSONWriter.java @@ -0,0 +1,202 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayDeque; +import java.util.Deque; + +/** + * A class which knows how to write JSON markup. All methods return this to + * enable chaining of method calls. + * Consider using the Jackson generator API instead, as that may be faster. + * + * @author bratseth + */ +public final class JSONWriter { + + /** A stack maintaining the "needs comma" state at the current level */ + private Deque<Boolean> needsComma=new ArrayDeque<>(); + + private static final char[] DIGITS = new char[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + + private final OutputStream stream; + + public JSONWriter(OutputStream stream) { + this.stream = stream; + } + + /** Called on the start of a field or array value */ + private void beginFieldOrArrayValue() throws IOException { + if (needsComma.getFirst()) { + write(","); + } + } + + /** Called on the end of a field or array value */ + private void endFieldOrArrayValue() { + setNeedsComma(); + } + + /** Begins an object field */ + public JSONWriter beginField(String fieldName) throws IOException { + beginFieldOrArrayValue(); + write("\"" + fieldName + "\":"); + return this; + } + + /** Ends an object field */ + public JSONWriter endField() throws IOException { + endFieldOrArrayValue(); + return this; + } + + /** Begins an array value */ + public JSONWriter beginArrayValue() throws IOException { + beginFieldOrArrayValue(); + return this; + } + + /** Ends an array value */ + public JSONWriter endArrayValue() throws IOException { + endFieldOrArrayValue(); + return this; + } + + /** Begin an object value */ + public JSONWriter beginObject() throws IOException { + write("{"); + needsComma.addFirst(Boolean.FALSE); + return this; + } + + /** End an object value */ + public JSONWriter endObject() throws IOException { + write("}"); + needsComma.removeFirst(); + return this; + } + + /** Begin an array value */ + public JSONWriter beginArray() throws IOException { + write("["); + needsComma.addFirst(Boolean.FALSE); + return this; + } + + /** End an array value */ + public JSONWriter endArray() throws IOException { + write("]"); + needsComma.removeFirst(); + return this; + } + + /** Writes a string value */ + public JSONWriter value(String value) throws IOException { + write("\"").write(escape(value)).write("\""); + return this; + } + + /** Writes a numeric value */ + public JSONWriter value(Number value) throws IOException { + write(value.toString()); + return this; + } + + /** Writes a boolean value */ + public JSONWriter value(boolean value) throws IOException { + write(Boolean.toString(value)); + return this; + } + + /** Writes a null value */ + public JSONWriter value() throws IOException { + write("null"); + return this; + } + + private void setNeedsComma() { + if (level() == 0) return; + needsComma.removeFirst(); + needsComma.addFirst(Boolean.TRUE); + } + + /** Returns the current nested level */ + private int level() { return needsComma.size(); } + + /** + * Writes a string directly as-is to the stream of this. + * + * @return this for convenience + */ + private JSONWriter write(String string) throws IOException { + if (string.length() == 0) return this; + stream.write(Utf8.toBytes(string)); + return this; + } + + /** + * Do JSON escaping of a string. + * + * @param in a string to escape + * @return a String suitable for use in JSON strings + */ + private String escape(final String in) { + final StringBuilder quoted = new StringBuilder((int) (in.length() * 1.2)); + return escape(in, quoted).toString(); + } + + /** + * Do JSON escaping of the incoming string to the "quoted" buffer. The + * buffer returned is the same as the one given in the "quoted" parameter. + * + * @param in a string to escape + * @param escaped the target buffer for escaped data + * @return the same buffer as given in the "quoted" parameter + */ + private StringBuilder escape(final String in, final StringBuilder escaped) { + for (final char c : in.toCharArray()) { + switch (c) { + case ('"'): + escaped.append("\\\""); + break; + case ('\\'): + escaped.append("\\\\"); + break; + case ('\b'): + escaped.append("\\b"); + break; + case ('\f'): + escaped.append("\\f"); + break; + case ('\n'): + escaped.append("\\n"); + break; + case ('\r'): + escaped.append("\\r"); + break; + case ('\t'): + escaped.append("\\t"); + break; + default: + if (c < 32) { + escaped.append("\\u").append(fourDigitHexString(c)); + } else { + escaped.append(c); + } + } + } + return escaped; + } + + private static char[] fourDigitHexString(final char c) { + final char[] hex = new char[4]; + int in = ((c) & 0xFFFF); + for (int i = 3; i >= 0; --i) { + hex[i] = DIGITS[in & 0xF]; + in >>>= 4; + } + return hex; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/JavaWriterWriter.java b/vespajlib/src/main/java/com/yahoo/text/JavaWriterWriter.java new file mode 100644 index 00000000000..b89092e9780 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/JavaWriterWriter.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.IOException; +import java.io.Writer; + +/** + * Wraps a simple java.lang.Writer. Of course you loose the possible optimizations. + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public final class JavaWriterWriter extends GenericWriter { + + final Writer out; + + public JavaWriterWriter(Writer writer) { + out = writer; + } + + @Override + public void write(char[] c, int offset, int bytes) { + try { + out.write(c, offset, bytes); + } catch (IOException e) { + throw new RuntimeException("Caught exception in Java writer.write.", e); + } + } + + @Override + public void flush() { + try { + out.flush(); + } catch (IOException e) { + throw new RuntimeException("Caught exception in Java writer.flush.", e); + } + } + + @Override + public void close() { + try { + out.close(); + } catch (IOException e) { + throw new RuntimeException("Caught exception in Java writer.close.", e); + } + } + public final Writer getWriter() { return out; } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/LanguageHacks.java b/vespajlib/src/main/java/com/yahoo/text/LanguageHacks.java new file mode 100644 index 00000000000..37fa01ccfa0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/LanguageHacks.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import static com.yahoo.text.Lowercase.toLowerCase; + +/** + * Language helper functions. + * + * @deprecated do not use + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +@Deprecated +public class LanguageHacks { + + /** + * Whether a language is in the CJK group. + */ + public static boolean isCJK(String language) { + if (language == null) return false; + + language = toLowerCase(language); + return "ja".equals(language) + || "ko".equals(language) + || language.startsWith("zh") + || language.startsWith("tw"); // TODO: tw is a bogus value? + } + + /** + * Whether there is desegmenting in this language. + */ + public static boolean yellDesegments(String language) { + if (language == null) return false; + + language = toLowerCase(language); + return "de".equals(language) || isCJK(language); + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Lowercase.java b/vespajlib/src/main/java/com/yahoo/text/Lowercase.java new file mode 100644 index 00000000000..4babba29ab3 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Lowercase.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.util.Locale; + +/** + * The lower casing method to use in Vespa when doing string processing of data + * which is not to be handled as natural language data, e.g. field names or + * configuration paramaters. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class Lowercase { + + private static final char[] lowercase = new char[123]; + + static { + lowercase[0x41] = 'a'; + lowercase[0x42] = 'b'; + lowercase[0x43] = 'c'; + lowercase[0x44] = 'd'; + lowercase[0x45] = 'e'; + lowercase[0x46] = 'f'; + lowercase[0x47] = 'g'; + lowercase[0x48] = 'h'; + lowercase[0x49] = 'i'; + lowercase[0x4A] = 'j'; + lowercase[0x4B] = 'k'; + lowercase[0x4C] = 'l'; + lowercase[0x4D] = 'm'; + lowercase[0x4E] = 'n'; + lowercase[0x4F] = 'o'; + lowercase[0x50] = 'p'; + lowercase[0x51] = 'q'; + lowercase[0x52] = 'r'; + lowercase[0x53] = 's'; + lowercase[0x54] = 't'; + lowercase[0x55] = 'u'; + lowercase[0x56] = 'v'; + lowercase[0x57] = 'w'; + lowercase[0x58] = 'x'; + lowercase[0x59] = 'y'; + lowercase[0x5A] = 'z'; + + lowercase[0x61] = 'a'; + lowercase[0x62] = 'b'; + lowercase[0x63] = 'c'; + lowercase[0x64] = 'd'; + lowercase[0x65] = 'e'; + lowercase[0x66] = 'f'; + lowercase[0x67] = 'g'; + lowercase[0x68] = 'h'; + lowercase[0x69] = 'i'; + lowercase[0x6A] = 'j'; + lowercase[0x6B] = 'k'; + lowercase[0x6C] = 'l'; + lowercase[0x6D] = 'm'; + lowercase[0x6E] = 'n'; + lowercase[0x6F] = 'o'; + lowercase[0x70] = 'p'; + lowercase[0x71] = 'q'; + lowercase[0x72] = 'r'; + lowercase[0x73] = 's'; + lowercase[0x74] = 't'; + lowercase[0x75] = 'u'; + lowercase[0x76] = 'v'; + lowercase[0x77] = 'w'; + lowercase[0x78] = 'x'; + lowercase[0x79] = 'y'; + lowercase[0x7A] = 'z'; + } + + /** + * Return a lowercased version of the given string. Since this is language + * independent, this is more of a case normalization operation than + * lowercasing. Vespa code should <i>never</i> do lowercasing with implicit + * locale. + * + * @param in + * a string to lowercase + * @return a string containing only lowercase character + */ + public static String toLowerCase(String in) { + // def is picked from http://docs.oracle.com/javase/6/docs/api/java/lang/String.html#toLowerCase%28%29 + String lower = toLowerCasePrintableAsciiOnly(in); + return (lower == null) ? in.toLowerCase(Locale.ENGLISH) : lower; + } + public static String toUpperCase(String in) { + // def is picked from http://docs.oracle.com/javase/6/docs/api/java/lang/String.html#toLowerCase%28%29 + return in.toUpperCase(Locale.ENGLISH); + } + + private static String toLowerCasePrintableAsciiOnly(String in) { + boolean anyUpper = false; + for (int i = 0; i < in.length(); i++) { + char c = in.charAt(i); + if (c < 0x41) { //lower than A-Z + return null; + } + if (c > 0x5A && c < 0x61) { //between A-Z and a-z + return null; + } + if (c > 0x7A) { //higher than a-z + return null; + } + if (c != lowercase[c]) { + anyUpper = true; + } + } + if (!anyUpper) { + return in; + } + StringBuilder builder = new StringBuilder(in.length()); + for (int i = 0; i < in.length(); i++) { + builder.append((char) (in.charAt(i) | ((char) 0x20))); + } + return builder.toString(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/LowercaseIdentifier.java b/vespajlib/src/main/java/com/yahoo/text/LowercaseIdentifier.java new file mode 100644 index 00000000000..b0f5b023a38 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/LowercaseIdentifier.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 11.11.12 + * Time: 20:50 + * To change this template use File | Settings | File Templates. + */ +public class LowercaseIdentifier extends Identifier { + public LowercaseIdentifier(String s) { + this(Utf8.toBytes(s)); + } + public LowercaseIdentifier(AbstractUtf8Array utf8) { + this(utf8.getBytes()); + } + public LowercaseIdentifier(byte [] utf8) { + super(verify(utf8)); + } + private static byte [] verify(final byte [] utf8) { + for (int i=0; i < utf8.length; i++) { + verifyAny(utf8[i], utf8); + } + + return utf8; + + } + private static boolean verifyAny(byte c, byte [] identifier) { + if ((c >= 'A') && (c <= 'Z')) { + throw new IllegalArgumentException("Illegal uppercase character '" + (char)c + "' of identifier '" + new Utf8String(new Utf8Array(identifier)).toString() +"'."); + } + return true; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/MapParser.java b/vespajlib/src/main/java/com/yahoo/text/MapParser.java new file mode 100644 index 00000000000..e627ba9bb11 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/MapParser.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.util.HashMap; +import java.util.Map; + +/** + * <p>Superclasses of parsers of a map represented textually as + * <code>{key1:value1,"anystringkey":value2,'anystringkey2':value3 ...}</code>. + * This parser must be extended to override the way values are parsed and constructed.</p> + * + * <p>Example: To create a Double map parser:</p> + * <pre> + * public static final class DoubleMapParser extends MapParser<Double> { + * + * @Override + * protected Double parseValue(String value) { + * return Double.parseDouble(value); + * } + * + * } + * </pre> + * + * <p>Map parsers are NOT multithread safe, but are cheap to construct.</p> + * + * @author bratseth + * @since 5.1.15 + */ +public abstract class MapParser<VALUETYPE> extends SimpleMapParser { + + private Map<String, VALUETYPE> map; + + /** + * Convenience method doing return parse(s,new HashMap<String,VALUETYPE>()) + */ + public Map<String,VALUETYPE> parseToMap(String s) { + return parse(s,new HashMap<>()); + } + + /** + * Parses a map on the form <code>{key1:value1,key2:value2 ...}</code> + * + * @param string the textual representation of the map + * @param map the map to which the values will be added + * @return the input map instance for convenience + */ + public Map<String,VALUETYPE> parse(String string,Map<String,VALUETYPE> map) { + this.map = map; + parse(string); + return this.map; + } + + protected void handleKeyValue(String key, String value) { + map.put(key, parseValue(value)); + } + + protected abstract VALUETYPE parseValue(String value); + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/PositionedString.java b/vespajlib/src/main/java/com/yahoo/text/PositionedString.java new file mode 100644 index 00000000000..de2e349ef82 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/PositionedString.java @@ -0,0 +1,137 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * A string which has a current position. + * Useful for writing simple single-pass parsers. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.1.15 + */ +public class PositionedString { + + private String s; + private int p; + + /** + * Creates this from a given string. + */ + public PositionedString(String string) { + this.s=string; + } + + /** The complete string value of this */ + public String string() { return s; } + + /** The current position into this string */ + public int position() { return p; } + + /** Assigns the current position in the string */ + public void setPosition(int position) { p=position; } + + /** + * Consumes the character at this position. + * <br>Precondition: The character at this position is c. + * <br>Postcondition: The position is increased by 1 + * + * @param c the expected character at this + * @throws IllegalArgumentException if the character at this position is not c + */ + public void consume(char c) { + if (s.charAt(p++)!=c) + throw new IllegalArgumentException("Expected '" + c + "' " + at(p -1)); + } + + /** + * Consumes zero or more whitespace characters starting at the current position + */ + public void consumeSpaces() { + while (Character.isWhitespace(s.charAt(p))) + p++; + } + + /** + * Advances the position by 1 if the character at the current position is c. + * Does nothing otherwise. + * + * @return whether this consumed a c at the current position, or if it did nothing + */ + public boolean consumeOptional(char c) { + if (s.charAt(p)!=c) return false; + p++; + return true; + } + + /** + * Returns whether the character at the current position is c. + */ + public boolean peek(char c) { + return s.charAt(p)==c; + } + + /** + * Returns the position of the next occurrence of c, + * or -1 if there are no occurrences of c in the string after the current position. + */ + public int indexOf(char c) { + return s.indexOf(c,p); + } + + /** Adds n to the current position */ + public void skip(int n) { + p = p +n; + } + + /** + * Sets the position of this to the next occurrence of c after the current position. + * + * @param c the char to move the position to + * @return the substring between the current position and the new position at c + * @throws IllegalArgumentException if there was no occurrence of c after the current position + */ + public String consumeTo(char c) { + int nextC=indexOf(c); + if (nextC<0) + throw new IllegalArgumentException("Expected a string terminated by '" + c + "' " + at()); + String value=substring(nextC); + p=nextC; + return value; + } + + /** + * Returns the substring between the current position and <code>position</code> + * and advances the current position to <code>position</code> + */ + public String consumeToPosition(int position) { + String consumed=substring(position); + p=position; + return consumed; + } + + /** Returns a substring of this from the current position to the end argument */ + public String substring(int end) { + return string().substring(position(),end); + } + + /** Returns the substring of this string from the current position to the end */ + public String substring() { + return string().substring(position()); + } + + /** Returns a textual description of the current position, useful for appending to error messages. */ + public String at() { + return at(p); + } + + /** Returns a textual description of a given position, useful for appending to error messages. */ + public String at(int position) { + return "starting at position " + position + " but was '" + s.charAt(position) + "'"; + } + + /** Returns the string */ + @Override + public String toString() { + return s; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/SimpleMapParser.java b/vespajlib/src/main/java/com/yahoo/text/SimpleMapParser.java new file mode 100644 index 00000000000..a27563ebea1 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/SimpleMapParser.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.util.HashMap; +import java.util.Map; + +/** + * <p>Superclasses of parsers of a map represented textually as + * <code>{key1:value1,"anystringkey":value2,'anystringkey2':value3 ...}</code>. + * This parser must be extended to specify how to handle the key/value pairs.</p> + * + * <p>Example: To create a Double map parser:</p> + * <pre> + * public static final class DoubleMapParser extends MapParser<Double> { + * private Map<String, Double> map; + * + * ... + * + * @Override + * protected Double handleKeyValue(String key, String value) { + * map.put(key, Double.parseDouble(value)); + * } + * + * } + * </pre> + * + * <p>Map parsers are NOT multithread safe, but are cheap to construct.</p> + * + * @author bratseth + * @since 5.1.15 + */ +public abstract class SimpleMapParser { + + private PositionedString s; + + /** + * Parses a map on the form <code>{key1:value1,key2:value2 ...}</code> + * + * @param string the textual representation of the map + */ + public void parse(String string) { + try { + this.s=new PositionedString(string); + + s.consumeSpaces(); + s.consume('{'); + while ( ! s.peek('}')) { + s.consumeSpaces(); + String key=consumeKey(); + s.consume(':'); + s.consumeSpaces(); + consumeValue(key); + s.consumeOptional(','); + s.consumeSpaces(); + } + s.consume('}'); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("'" + s + "' is not a legal sparse vector string",e); + } + } + + private String consumeKey() { + if (s.consumeOptional('"')) { + String key=s.consumeTo('"'); + s.consume('"'); + return key; + } + else if (s.consumeOptional('\'')) { + String key=s.consumeTo('\''); + s.consume('\''); + return key; + } + else { + int keyEnd=findEndOfKey(); + if (keyEnd<0) + throw new IllegalArgumentException("Expected a key followed by ':' " + s.at()); + return s.consumeToPosition(keyEnd); + } + } + + protected int findEndOfKey() { + for (int peekI=s.position(); peekI<s.string().length(); peekI++) { + if (s.string().charAt(peekI)==':' || s.string().charAt(peekI)==',') + return peekI; + } + return -1; + } + + protected int findEndOfValue() { + for (int peekI=s.position(); peekI<s.string().length(); peekI++) { + if (s.string().charAt(peekI)==',' || s.string().charAt(peekI)=='}') + return peekI; + } + return -1; + } + + protected void consumeValue(String key) { + // find the next comma or bracket, whichever is next + int endOfValue=findEndOfValue(); + if (endOfValue<0) { + throw new IllegalArgumentException("Expected a value followed by ',' or '}' " + s.at()); + } + try { + handleKeyValue(key, s.substring(endOfValue)); + s.setPosition(endOfValue); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Expected a legal value from position " + s.position() + " to " + endOfValue + + " but was '" + s.substring(endOfValue) + "'", e); + } + } + + /** Returns the string being parsed along with its current position */ + public PositionedString string() { return s; } + + protected abstract void handleKeyValue(String key, String value); + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java new file mode 100644 index 00000000000..d65681e8f5b --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java @@ -0,0 +1,204 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.nio.charset.Charset; +import java.util.List; +import java.io.ByteArrayOutputStream; + +/** + * Escapes strings into and out of a format where they only contain printable characters. + * + * Need to duplicate escape / unescape of strings as we have in C++ for java version of system states. + * + * @author <a href="mailto:humbe@yahoo-inc.com">Haakon Humberset</a> + */ +public class StringUtilities { + private static Charset UTF8 = Charset.forName("utf8"); + + private static byte toHex(int val) { return (byte) (val < 10 ? '0' + val : 'a' + (val - 10)); } + + private static class ReplacementCharacters { + public byte needEscape[] = new byte[256]; + public byte replacement1[] = new byte[256]; + public byte replacement2[] = new byte[256]; + + public ReplacementCharacters() { + for (int i=0; i<256; ++i) { + if (i >= 32 && i <= 126) { + needEscape[i] = 0; + } else { + needEscape[i] = 3; + replacement1[i] = toHex((i >> 4) & 0xF); + replacement2[i] = toHex(i & 0xF); + } + } + makeSimpleEscape('"', '"'); + makeSimpleEscape('\\', '\\'); + makeSimpleEscape('\t', 't'); + makeSimpleEscape('\n', 'n'); + makeSimpleEscape('\r', 'r'); + makeSimpleEscape('\f', 'f'); + } + + private void makeSimpleEscape(char source, char dest) { + needEscape[source] = 1; + replacement1[source] = '\\'; + replacement2[source] = (byte) dest; + } + } + + private final static ReplacementCharacters replacementCharacters = new ReplacementCharacters(); + + public static String escape(String source) { return escape(source, '\0'); } + + /** + * Escapes strings into a format with only printable ASCII characters. + * + * @param source The string to escape + * @param delimiter Escape this character too, even if it is printable. + * @return The escaped string + */ + public static String escape(String source, char delimiter) { + byte bytes[] = source.getBytes(UTF8); + ByteArrayOutputStream result = new ByteArrayOutputStream(); + for (byte b : bytes) { + int val = b; + if (val < 0) val += 256; + if (b == delimiter) { + result.write('\\'); + result.write('x'); + result.write(toHex((val >> 4) & 0xF)); + result.write(toHex(val & 0xF)); + } else if (replacementCharacters.needEscape[val] == 0) { + result.write(b); + } else { + if (replacementCharacters.needEscape[val] == 3) { + result.write('\\'); + result.write('x'); + } + result.write(replacementCharacters.replacement1[val]); + result.write(replacementCharacters.replacement2[val]); + } + } + return new String(result.toByteArray(), UTF8); + } + + public static String unescape(String source) { + byte bytes[] = source.getBytes(UTF8); + ByteArrayOutputStream result = new ByteArrayOutputStream(); + for (int i=0; i<bytes.length; ++i) { + if (bytes[i] != '\\') { + result.write(bytes[i]); + continue; + } + if (i + 1 == bytes.length) throw new IllegalArgumentException("Found backslash at end of input"); + + if (bytes[i + 1] != (byte) 'x') { + switch (bytes[i + 1]) { + case '\\': result.write('\\'); break; + case '"': result.write('"'); break; + case 't': result.write('\t'); break; + case 'n': result.write('\n'); break; + case 'r': result.write('\r'); break; + case 'f': result.write('\f'); break; + default: + throw new IllegalArgumentException("Illegal escape sequence \\" + ((char) bytes[i+1]) + " found"); + } + ++i; + continue; + } + + if (i + 3 >= bytes.length) throw new IllegalArgumentException("Found \\x at end of input"); + + String hexdigits = "" + ((char) bytes[i + 2]) + ((char) bytes[i + 3]); + result.write((byte) Integer.parseInt(hexdigits, 16)); + i += 3; + } + return new String(result.toByteArray(), UTF8); + } + + /** + * Returns the given array flattened to string, with the given separator string + * @param array the array + * @param sepString or null + * @return imploded array + */ + public static String implode(String[] array, String sepString) { + if (array==null) return null; + StringBuilder ret = new StringBuilder(); + if (sepString==null) sepString=""; + for (int i = 0 ; i<array.length ; i++) { + ret.append(array[i]); + if (!(i==array.length-1)) ret.append(sepString); + } + return ret.toString(); + } + + /** + * Returns the given list flattened to one with newline between + * + * @return flattened string + */ + public static String implodeMultiline(List<String> lines) { + if (lines==null) return null; + return implode(lines.toArray(new String[0]), "\n"); + } + + /** + * This will truncate sequences in a string of the same character that exceed the maximum + * allowed length. + * + * @return The same string or a new one if truncation is done. + */ + public static String truncateSequencesIfNecessary(String text, int maxConsecutiveLength) { + char prev = 0; + int sequenceCount = 1; + for (int i = 0, m = text.length(); i < m ; i++) { + char curr = text.charAt(i); + if (prev == curr) { + sequenceCount++; + if (sequenceCount > maxConsecutiveLength) { + return truncateSequences(text, maxConsecutiveLength, i); + } + } else { + sequenceCount = 1; + prev = curr; + } + } + return text; + } + + private static String truncateSequences(String text, int maxConsecutiveLength, int firstTruncationPos) { + char [] truncated = text.toCharArray(); + char prev = truncated[firstTruncationPos]; + int sequenceCount = maxConsecutiveLength + 1; + int wp=firstTruncationPos; + for (int rp=wp+1; rp < truncated.length; rp++) { + char curr = truncated[rp]; + if (prev == curr) { + sequenceCount++; + if (sequenceCount <= maxConsecutiveLength) { + truncated[wp++] = curr; + } + } else { + truncated[wp++] = curr; + sequenceCount = 1; + prev = curr; + } + } + return String.copyValueOf(truncated, 0, wp); + } + + public static String stripSuffix(String string, String suffix) { + int index = string.lastIndexOf(suffix); + return index == -1 ? string : string.substring(0, index); + } + + /** + * Adds single quotes around object.toString + * Example: '12' + */ + public static String quote(Object object) { + return "'" + object.toString() + "'"; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8.java b/vespajlib/src/main/java/com/yahoo/text/Utf8.java new file mode 100644 index 00000000000..9126870117e --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8.java @@ -0,0 +1,595 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.ReadOnlyBufferException; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; + +/** + * utility class with functions for handling UTF-8 + * + * @author arnej27959 + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * + */ +public final class Utf8 { + + private static final byte [] TRUE = {(byte) 't', (byte) 'r', (byte) 'u', (byte) 'e'}; + private static final byte [] FALSE = {(byte) 'f', (byte) 'a', (byte) 'l', (byte) 's', (byte) 'e'}; + private static final byte[] LONG_MIN_VALUE_BYTES = String.valueOf(Long.MIN_VALUE).getBytes(StandardCharsets.UTF_8); + + /** Returns the Charset instance for UTF-8 */ + public static Charset getCharset() { + return StandardCharsets.UTF_8; + } + + /** To be used instead of String.String(byte[] bytes) */ + public static String toStringStd(byte[] data) { + return new String(data, StandardCharsets.UTF_8); + } + + /** + * Utility method as toString(byte[]). + * + * @param data + * bytes to decode + * @param offset + * index of first byte to decode + * @param length + * number of bytes to decode + * @return String decoded from UTF-8 + */ + public static String toString(byte[] data, int offset, int length) { + String s = toStringAscii(data, offset, length); + return s != null ? s : toString(ByteBuffer.wrap(data, offset, length)); + } + + /** + * Fetch a string from a ByteBuffer instance. ByteBuffer instances are + * stateful, so it is assumed to caller manipulates the instance's limit if + * the entire buffer is not a string. + * + * @param data + * The UTF-8 data source + * @return a decoded String + */ + public static String toString(ByteBuffer data) { + CharBuffer c = StandardCharsets.UTF_8.decode(data); + return c.toString(); + } + + /** + * Uses String.getBytes directly. + */ + public static byte[] toBytesStd(String str) { + return str.getBytes(StandardCharsets.UTF_8); + } + + /** + * Encode a long as its decimal representation, i.e. toAsciiBytes(15L) will + * return "15" encoded as UTF-8. In other words it is an optimized version + * of String.valueOf() followed by UTF-8 encoding. Avoid going through + * string in order to get a simple UTF-8 sequence. + * + * @param l + * value to represent as a decimal number encded as utf8 + * @return byte array + */ + public static byte[] toAsciiBytes(long l) { + // Handle Long.MIN_VALUE specifically, since it breaks all the assumptions + if (Long.MIN_VALUE == l) { + return LONG_MIN_VALUE_BYTES; + } + int count=1; + for (long v= l<0 ? -l : l; v >= 10; v=v/10, count++); + byte [] buf = new byte [count + ((l<0) ? 1 : 0)]; + int offset = 0; + if (l < 0) { + buf[offset++] = (byte) '-'; + l = -l; + } + for (count--; count >= 0; l=l/10, count--) { + buf[count+offset] = (byte)(0x30 + l%10); + } + return buf; + } + + public static byte [] toAsciiBytes(boolean v) { + return v ? TRUE : FALSE; + } + + /** + * Will try an optimistic approach to utf8 encoding. + * That is 4.6x faster that the brute encode for ascii, not accounting for reduced memory footprint and GC. + * @param str The string to encode. + * @return Utf8 encoded array + */ + public static byte[] toBytes(String str) { + byte [] utf8 = toBytesAscii(str); + return utf8 != null ? utf8 : str.getBytes(StandardCharsets.UTF_8); + } + /** + * Will try an optimistic approach to utf8 decoding. + * + * @param utf8 The string to encode. + * @return Utf8 encoded array + */ + public static String toString(byte [] utf8) { + String s = toStringAscii(utf8, 0, utf8.length); + return s != null ? s : new String(utf8, StandardCharsets.UTF_8); + } + + /** + * If String is purely ascii 7bit it will encode it as a byte array. + * @param str The string to encode + * @return Utf8 encoded array + */ + private static byte[] toBytesAscii(final CharSequence str) { + byte [] utf8 = new byte[str.length()]; + for (int i=0; i < utf8.length; i++) { + char c = str.charAt(i); + if ((c < 0) || (c >= 0x80)) { + return null; + } + utf8[i] = (byte)c; + } + return utf8; + } + + private static String toStringAscii(byte [] b, int offset, int length) { + if (length > 0) { + char [] s = new char[length]; + for (int i=0; i < length; i++) { + if (b[offset + i] >= 0) { + s[i] = (char)b[offset+i]; + } else { + return null; + } + } + return new String(s); + } else { + return ""; + } + } + + /** + * Utility method as toBytes(String). + * + * @param str + * String to encode + * @param offset + * index of first character to encode + * @param length + * number of characters to encode + * @return substring encoded as UTF-8 + */ + public static byte[] toBytes(String str, int offset, int length) { + CharBuffer c = CharBuffer.wrap(str, offset, offset + length); + ByteBuffer b = StandardCharsets.UTF_8.encode(c); + byte[] result = new byte[b.remaining()]; + b.get(result); + return result; + } + + /** + * Direct encoding of a String into an array. + * + * @param str + * string to encode + * @param srcOffset + * index of first character in string to encode + * @param srcLen + * number of characters in string to encode + * @param dst + * destination for encoded data + * @param dstOffset + * index of first position to write data + * @return the number of bytes written to the array. + */ + public static int toBytes(String str, int srcOffset, int srcLen, byte[] dst, int dstOffset) { + CharBuffer c = CharBuffer.wrap(str, srcOffset, srcOffset + srcLen); + ByteBuffer b = StandardCharsets.UTF_8.encode(c); + int encoded = b.remaining(); + b.get(dst, dstOffset, encoded); + return encoded; + } + + /** + * Encode a string directly into a ByteBuffer instance. + * + * <p> + * This method is somewhat more cumbersome than the rest of the helper + * methods in this library, as it is intended for use cases in the following + * style, if extraneous copying is highly undesirable: + * + * <pre> + * String[] a = {"abc", "def", "ghi\u00e8"}; + * int[] aLens = {3, 3, 5}; + * CharsetEncoder ce = Utf8.getNewEncoder(); + * ByteBuffer forWire = ByteBuffer.allocate(someNumber); + * + * for (int i = 0; i < a.length; i++) { + * forWire.putInt(aLens[i]); + * Utf8.toBytes(a[i], 0, a[i].length(), forWire, ce); + * } + * </pre> + * + * @see Utf8#getNewEncoder() + * + * @param src the string to encode + * @param srcOffset index of first character to encode + * @param srcLen number of characters to encode + * @param dst the destination ByteBuffer + * @param encoder the character encoder to use + */ + public static void toBytes(String src, int srcOffset, int srcLen, ByteBuffer dst, CharsetEncoder encoder) { + CharBuffer c = CharBuffer.wrap(src, srcOffset, srcOffset + srcLen); + encoder.encode(c, dst, true); + } + + /** + * Create a new UTF-8 encoder. + * + * @see Utf8#toBytes(String, int, int, ByteBuffer, CharsetEncoder) + */ + public static CharsetEncoder getNewEncoder() { + return StandardCharsets.UTF_8.newEncoder().onMalformedInput(CodingErrorAction.REPLACE) + .onUnmappableCharacter(CodingErrorAction.REPLACE); + } + + /** + * Count the number of bytes needed to represent a given sequence of 16-bit + * char values as a UTF-8 encoded array. This method is written to be cheap + * to invoke. + * + * Note: It is strongly assumed to character sequence is valid. + */ + public static int byteCount(CharSequence str) { return byteCount(str, 0, str.length()); } + + /** + * Count the number of bytes needed to represent a given sequence of 16-bit + * char values as a UTF-8 encoded array. This method is written to be cheap + * to invoke. + * + * Note: It is strongly assumed to character sequence is valid. + */ + public static int byteCount(CharSequence str, int offset, int length) { + int count = 0; + int barrier = offset + length; + int i = offset; + while (i < barrier) { + int codePoint = (int) str.charAt(i); + if (codePoint < 0x800) { + if (codePoint < 0x80) { + ++count; + } else { + count += 2; + } + ++i; + } else { + // bit masking to check (codePoint >= 0xd800 && codePoint < + // 0xe000) + if ((codePoint & 0xF800) == 0xD800) { + count += 4; + i += 2; + } else { + count += 3; + ++i; + } + } + } + return count; + } + + /** + * Count the number of Unicode code units ("UTF-16 characters") needed to + * represent a given array of UTF-8 characters. This method is written to be + * cheap to invoke. + * + * Note: It is strongly assumed the sequence is valid. + */ + public static int unitCount(byte[] utf8) { return unitCount(utf8, 0, utf8.length); } + + /** + * Count the number of Unicode code units ("UTF-16 characters") needed to + * represent a given array of UTF-8 characters. This method is written to be + * cheap to invoke. + * + * Note: It is strongly assumed the sequence is valid. + * + * @param utf8 + * raw data + * @param offset + * index of first byte of UTF-8 sequence to check + * @param length + * number of bytes in the UTF-8 sequence to check + */ + public static int unitCount(byte[] utf8, int offset, int length) { + int units = 0; + int barrier = offset + length; + int i = offset; + while (i < barrier) { + byte firstByte = utf8[i]; + if (firstByte >= -16) { + if (firstByte >= 0) { + ++units; + ++i; + } else { + units += 2; + i += 4; + } + } else { + if (firstByte >= -32) { + ++units; + i += 3; + } else { + ++units; + i += 2; + } + } + } + return units; + } + + /** + * Calculate the number of Unicode code units ("UTF-16 characters") needed + * to represent a given UTF-8 encoded code point. + * + * @param firstByte + * the first byte of a character encoded as UTF-8 + * @return the number of UTF-16 code units needed to represent the given + * code point + */ + public static int unitCount(byte firstByte) { + int units = 0; + if (firstByte >= -16) { + if (firstByte >= 0) { + units = 1; + } else { + units = 2; + } + } else { + units = 1; + } + return units; + } + + /** + * Inspects a byte assumed to be the first byte in a UTF8 to check how many + * bytes in total the sequence of bytes will use. + * + * @param firstByte + * the first byte of a UTF8 encoded character + * @return the number of bytes used to encode the character + */ + // To avoid code duplication, this function should be used by unitCount(), + // but then unitCount(byte[], int, int) would not be as tight. This class is in general + // meant to be safe to use in performance sensitive code. + public static int totalBytes(byte firstByte) { + if (firstByte >= -16) { + if (firstByte >= 0) { + return 1; + } else { + return 4; + } + } else { + if (firstByte >= -32) { + return 3; + } else { + return 2; + } + } + } + + /** + * Returns an integer array the length as the input string plus one. For + * every index in the array, the corresponding value gives the index into + * the UTF-8 byte sequence that can be created from the input. + * + * @param value + * a String to generate UTF-8 byte indexes from + * @return an array containing corresponding UTF-8 byte indexes + */ + public static int[] calculateBytePositions(CharSequence value) { + int[] positions = new int[value.length() + 1]; + + int bytePos = 0; + int barrier = value.length(); + int i = 0; + int codepointNo = 0; + positions[codepointNo++] = bytePos; + while (i < barrier) { + int codePoint = (int) value.charAt(i); + if (codePoint < 0x800) { + if (codePoint < 0x80) { + ++bytePos; + } else { + bytePos += 2; + } + ++i; + } else { + // bit masking to check (codePoint >= 0xd800 && codePoint < + // 0xe000) + if ((codePoint & 0xF800) == 0xD800) { + // double position write, as we have a surrogate pair + positions[codepointNo++] = bytePos; + bytePos += 4; + i += 2; + } else { + bytePos += 3; + ++i; + } + } + positions[codepointNo++] = bytePos; + } + return positions; + } + + /** + * Returns an array of the same length as the input array plus one. For + * every index in the array, the corresponding value gives the index into + * the Java string (UTF-16 sequence) that can be created from the input. + * + * @param utf8 + * a byte array containing a string encoded as UTF-8. Note: It is + * strongly assumed that this sequence is correct. + * @return an array containing corresponding UTF-16 character indexes. If input + * array is empty, returns an array containg a single zero. + */ + public static int[] calculateStringPositions(byte[] utf8) { + if (utf8.length == 0) { + return new int[] { 0 }; + } + int[] positions = new int[utf8.length + 1]; + int utf8BytePos = 0; + int charPos = 0; + int lastUtf8SequencePos = 0; + int utf8SequenceLen = 0; + while (utf8BytePos < utf8.length) { + utf8SequenceLen = totalBytes(utf8[utf8BytePos]); + lastUtf8SequencePos = utf8BytePos; + for (int utf8SequenceCnt = 0; utf8SequenceCnt < utf8SequenceLen; utf8SequenceCnt++) { + positions[utf8BytePos + utf8SequenceCnt] = charPos; + } + utf8BytePos += utf8SequenceLen; + charPos++; + } + //we need to check if the last UTF-8 sequence resulted in a surrogate pair: + int lastCharLen = unitCount(utf8, lastUtf8SequencePos, utf8SequenceLen); + positions[utf8.length] = charPos + lastCharLen - 1; + return positions; + } + + + /** + * Encode a valid Unicode codepoint as a sequence of UTF-8 bytes into a new allocated array. + * + * @param codepoint Unicode codepoint to encode + * @return number of bytes written + * @throws IndexOutOfBoundsException if there is insufficient room for the encoded data in the given array + */ + public static byte[] encode(int codepoint) { + byte[] destination = new byte[codePointAsUtf8Length(codepoint)]; + encode(codepoint, destination, 0); + return destination; + } + + /** + * Encode a valid Unicode codepoint as a sequence of UTF-8 bytes into an array. + * + * @param codepoint Unicode codepoint to encode + * @param destination array to write into + * @param offset index of first byte written + * @return index of the first byte after the last byte written (i.e. offset plus number of bytes written) + * @throws IndexOutOfBoundsException if there is insufficient room for the encoded data in the given array + */ + public static int encode(int codepoint, byte[] destination, int offset) { + int writeOffset = offset; + byte firstByte = firstByte(codepoint); + int leftToWrite = codePointAsUtf8Length(codepoint) - 1; + destination[writeOffset++] = firstByte; + while (leftToWrite-- > 0) { + destination[writeOffset++] = trailingOctet(codepoint, leftToWrite); + } + return writeOffset; + } + + /** + * Encode a valid Unicode codepoint as a sequence of UTF-8 bytes into a + * ByteBuffer. + * + * @param codepoint + * Unicode codepoint to encode + * @param destination + * buffer to write into + * @throws BufferOverflowException + * if the buffer's limit is met while writing (propagated from + * the ByteBuffer) + * @throws ReadOnlyBufferException + * if the buffer is read only (propagated from the ByteBuffer) + */ + public static void encode(int codepoint, ByteBuffer destination) { + byte firstByte = firstByte(codepoint); + int leftToWrite = codePointAsUtf8Length(codepoint) - 1; + destination.put(firstByte); + while (leftToWrite-- > 0) { + destination.put(trailingOctet(codepoint, leftToWrite)); + } + } + + /** + * Encode a valid Unicode codepoint as a sequence of UTF-8 bytes into an + * OutputStream. + * + * @param codepoint + * Unicode codepoint to encode + * @param destination + * buffer to write into + * @return number of bytes written + * @throws IOException + * propagated from stream + */ + public static int encode(int codepoint, OutputStream destination) throws IOException { + byte firstByte = firstByte(codepoint); + int toWrite = codePointAsUtf8Length(codepoint); + int leftToWrite = toWrite - 1; + destination.write(firstByte); + while (leftToWrite-- > 0) { + destination.write(trailingOctet(codepoint, leftToWrite)); + } + return toWrite; + } + + + private static byte trailingOctet(int codepoint, int leftToWrite) { + return (byte) (0x80 | ((codepoint >> (6 * leftToWrite)) & 0x3F)); + } + + private static byte firstByte(int codepoint) { + if (codepoint < 0x800) { + if (codepoint < 0x80) { + return (byte) codepoint; + } else { + return (byte) (0xC0 | codepoint >> 6); + } + } else { + if (codepoint < 0x10000) { + return (byte) (0xE0 | codepoint >> 12); + } else { + return (byte) (0xF0 | codepoint >> 18); + } + } + + } + + /** + * Return the number of octets needed to encode a valid Unicode codepoint as UTF-8. + * + * @param codepoint the Unicode codepoint to inspect + * @return the number of bytes needed for UTF-8 representation + */ + public static int codePointAsUtf8Length(int codepoint) { + if (codepoint < 0x800) { + if (codepoint < 0x80) { + return 1; + } else { + return 2; + } + } else { + if (codepoint < 0x10000) { + return 3; + } else { + return 4; + } + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8Array.java b/vespajlib/src/main/java/com/yahoo/text/Utf8Array.java new file mode 100644 index 00000000000..30b2e665392 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8Array.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + + +import java.nio.ByteBuffer; + +/** + * This is a primitive class that owns an array of utf8 encoded string. + * This is a class that has speed as its primary purpose. + * If you have a string, consider Utf8String + * If you have a large backing array consider Utf8PartialArray + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ + +public class Utf8Array extends AbstractUtf8Array { + + protected final byte[] utf8; + + /** + * This will simply wrap the given array assuming it is valid utf8. + * Note that the immutability of this primitive class depends on that the buffer + * is not modified after ownership has been transferred. + * @param utf8data The utf8 byte sequence. + */ + public Utf8Array(final byte[] utf8data) { + utf8 = utf8data; + } + + /** + * This will create a new array from the window given. No validation done. + * Note that this will copy data. You might also want to consider Utf8PartialArray + * @param utf8data The base array. + * @param offset The offset from where to copy from + * @param length The number of bytes that should be copied. + */ + public Utf8Array(byte[] utf8data, int offset, int length) { + this.utf8 = new byte[length]; + System.arraycopy(utf8data, offset, this.utf8, 0, length); + } + + /** + * This will fetch length bytes from the given buffer. + * @param buf The ByteBuffer to read from + * @param length number of bytes to read + */ + public Utf8Array(ByteBuffer buf, int length) { + this.utf8 = new byte[length]; + buf.get(this.utf8, 0, length); + } + + @Override + public byte[] getBytes() { + return utf8; + } + + @Override + public int getByteLength() { + return utf8.length; + } + + @Override + protected int getByteOffset() { + return 0; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8PartialArray.java b/vespajlib/src/main/java/com/yahoo/text/Utf8PartialArray.java new file mode 100644 index 00000000000..c6032e751b7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8PartialArray.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +/** + * This wraps a window in a backing byte array. Without doing any copying. + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public class Utf8PartialArray extends Utf8Array { + final int offset; + final int length; + + /** + * Takes ownership of the given byte array. And keeps note of where + * the interesting utf8 sequence start and its length. + * @param utf8data The backing byte array. + * @param offset The start of the utf8 sequence. + * @param bytes The length of the utf8 sequence. + */ + public Utf8PartialArray(byte[] utf8data, int offset, int bytes) { + super(utf8data); + this.offset = offset; + this.length = bytes; + } + @Override + public int getByteLength() { + return length; + } + + @Override + protected int getByteOffset() { + return offset; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8String.java b/vespajlib/src/main/java/com/yahoo/text/Utf8String.java new file mode 100644 index 00000000000..1f4dfc0d4f6 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8String.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + + +/** + * String with Utf8 backing. + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public final class Utf8String extends Utf8Array implements CharSequence +{ + private final String s; + + /** + * This will construct a utf8 backing of the given string. + * @param str The string that will be utf8 encoded + */ + public Utf8String(String str) { + super(Utf8.toBytes(str)); + s = str; + } + + /** + * This will create a string based on the utf8 sequence. + * @param utf8 The backing array + */ + public Utf8String(AbstractUtf8Array utf8) { + super(utf8.getBytes(), utf8.getByteOffset(), utf8.getByteLength()); + s = utf8.toString(); + } + + @Override + public char charAt(int index) { + return toString().charAt(index); + } + @Override + public int length() { + return toString().length(); + } + @Override + public CharSequence subSequence(int start, int end) { + return toString().subSequence(start, end); + } + + @Override + public boolean equals(Object o) { + if (o instanceof Utf8String) { + return s.equals(o.toString()); + } + return super.equals(o); + } + + @Override + public String toString() { + return s; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/XML.java b/vespajlib/src/main/java/com/yahoo/text/XML.java new file mode 100644 index 00000000000..c688d5f9722 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/XML.java @@ -0,0 +1,636 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.SAXParseException; + +/** + * Static XML utility methods + * + * @author Bjorn Borud + * @author Vegard Havdal + * @author bratseth + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class XML { + /** + * The point of this weird class and the jumble of abstract methods is + * linking the scan for characters that must be quoted into the quoting + * table, and making it actual work to make them go out of sync again. + */ + private static abstract class LegalCharacters { + // To quote http://www.w3.org/TR/REC-xml/ : + // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | + // [#x10000-#x10FFFF] + final boolean isLegal(final int codepoint, final boolean escapeLow, + final int stripCodePoint, final boolean isAttribute) { + if (codepoint == stripCodePoint) { + return removeCodePoint(); + } else if (codepoint < ' ') { + if (!escapeLow) { + return true; + } + switch (codepoint) { + case 0x09: + case 0x0a: + case 0x0d: + return true; + default: + return ctrlEscapeCodePoint(codepoint); + } + } else if (codepoint >= 0x20 && codepoint <= 0xd7ff) { + switch (codepoint) { + case '&': + return ampCodePoint(); + case '<': + return ltCodePoint(); + case '>': + return gtCodePoint(); + case '"': + return quotCodePoint(isAttribute); + default: + return true; + } + } else if ((codepoint >= 0xe000 && codepoint <= 0xfffd) + || (codepoint >= 0x10000 && codepoint <= 0x10ffff)) { + return true; + } else { + return filterCodePoint(codepoint); + + } + } + + private boolean quotCodePoint(final boolean isAttribute) { + if (isAttribute) { + quoteQuot(); + return false; + } else { + return true; + } + } + + private boolean filterCodePoint(final int codepoint) { + replace(codepoint); + return false; + } + + private boolean gtCodePoint() { + quoteGt(); + return false; + } + + private boolean ltCodePoint() { + quoteLt(); + return false; + } + + private boolean ampCodePoint() { + quoteAmp(); + return false; + } + + private boolean ctrlEscapeCodePoint(final int codepoint) { + ctrlEscape(codepoint); + return false; + } + + private boolean removeCodePoint() { + remove(); + return false; + } + + protected abstract void quoteQuot(); + + protected abstract void quoteGt(); + + protected abstract void quoteLt(); + + protected abstract void quoteAmp(); + + protected abstract void remove(); + + protected abstract void ctrlEscape(int codepoint); + + protected abstract void replace(int codepoint); + } + + private static final class Quote extends LegalCharacters { + + char[] lastQuoted; + private static final char[] EMPTY = new char[0]; + private static final char[] REPLACEMENT_CHARACTER = "\ufffd".toCharArray(); + private static final char[] AMP = "&".toCharArray(); + private static final char[] LT = "<".toCharArray(); + private static final char[] GT = ">".toCharArray(); + private static final char[] QUOT = """.toCharArray(); + + @Override + protected void remove() { + lastQuoted = EMPTY; + } + + @Override + protected void replace(final int codepoint) { + lastQuoted = REPLACEMENT_CHARACTER; + } + + @Override + protected void quoteQuot() { + lastQuoted = QUOT; + } + + @Override + protected void quoteGt() { + lastQuoted = GT; + } + + @Override + protected void quoteLt() { + lastQuoted = LT; + } + + @Override + protected void quoteAmp() { + lastQuoted = AMP; + } + + @Override + protected void ctrlEscape(final int codepoint) { + lastQuoted = REPLACEMENT_CHARACTER; + } + } + + private static final class Scan extends LegalCharacters { + + @Override + protected void quoteQuot() { + } + + @Override + protected void quoteGt() { + } + + @Override + protected void quoteLt() { + } + + @Override + protected void quoteAmp() { + } + + @Override + protected void remove() { + } + + @Override + protected void ctrlEscape(final int codepoint) { + } + + @Override + protected void replace(final int codepoint) { + } + } + + private static final Scan scanner = new Scan(); + + /** + * Replaces the characters that need to be escaped with their corresponding + * character entities. + * + * @param s1 + * String possibly containing characters that need to be escaped + * in XML + * + * @return Returns the input string with special characters that need to be + * escaped replaced by character entities. + */ + public static String xmlEscape(String s1) { + return xmlEscape(s1, true, true, null, -1); + } + + /** + * Replaces the characters that need to be escaped with their corresponding + * character entities. + * + * @param s1 + * String possibly containing characters that need to be escaped + * in XML + * @param isAttribute + * Is the input string to be used as an attribute? + * + * @return Returns the input string with special characters that need to be + * escaped replaced by character entities + */ + public static String xmlEscape(String s1, boolean isAttribute) { + return xmlEscape(s1, isAttribute, true, null, -1); + } + + /** + * Replaces the characters that need to be escaped with their corresponding + * character entities. + * + * @param s1 + * String possibly containing characters that need to be escaped + * in XML + * @param isAttribute + * Is the input string to be used as an attribute? + * + * + * @param stripCharacter + * any occurrence of this character is removed from the string + * + * @return Returns the input string with special characters that need to be + * escaped replaced by character entities + */ + public static String xmlEscape(String s1, boolean isAttribute, char stripCharacter) { + return xmlEscape(s1, isAttribute, true, null, (int) stripCharacter); + } + + /** + * Replaces the characters that need to be escaped with their corresponding + * character entities. + * + * @param s1 + * String possibly containing characters that need to be escaped + * in XML + * @param isAttribute + * Is the input string to be used as an attribute? + * + * @param escapeLowAscii + * Should ascii characters below 32 be escaped as well + * + * @return Returns the input string with special characters that need to be + * escaped replaced by character entities + */ + public static String xmlEscape(String s1, boolean isAttribute, boolean escapeLowAscii) { + return xmlEscape(s1, isAttribute, escapeLowAscii, null, -1); + } + + /** + * Replaces the characters that need to be escaped with their corresponding + * character entities. + * + * @param s1 + * String possibly containing characters that need to be escaped + * in XML + * @param isAttribute + * Is the input string to be used as an attribute? + * + * @param escapeLowAscii + * Should ascii characters below 32 be escaped as well + * + * @param stripCharacter + * any occurrence of this character is removed from the string + * + * @return Returns the input string with special characters that need to be + * escaped replaced by character entities + */ + public static String xmlEscape(String s1, boolean isAttribute, boolean escapeLowAscii, char stripCharacter) { + return xmlEscape(s1, isAttribute, escapeLowAscii, null, (int) stripCharacter); + } + + /** + * Replaces the following: + * <ul> + * <li>all ascii codes less than 32 except 9 (tab), 10 (nl) and 13 (cr) + * <li>ampersand (&) + * <li>less than (<) + * <li>larger than (>) + * <li>double quotes (") if isAttribute is <code>true</code> + * </ul> + * with character entities. + * + */ + public static String xmlEscape(String string, boolean isAttribute, StringBuilder buffer) { + return xmlEscape(string, isAttribute, true, buffer, -1); + } + + /** + * Replaces the following: + * <ul> + * <li>all ascii codes less than 32 except 9 (tab), 10 (nl) and 13 (cr) if + * escapeLowAscii is <code>true</code> + * <li>ampersand (&) + * <li>less than (<) + * <li>larger than (>) + * <li>double quotes (") if isAttribute is <code>true</code> + * </ul> + * with character entities. + * + */ + public static String xmlEscape(String string, boolean isAttribute, boolean escapeLowAscii, StringBuilder buffer) { + return xmlEscape(string, isAttribute, escapeLowAscii, buffer, -1); + } + + /** + * Replaces the following: + * <ul> + * <li>all ascii codes less than 32 except 9 (tab), 10 (nl) and 13 (cr) if + * escapeLowAscii is <code>true</code> + * <li>ampersand (&) + * <li>less than (<) + * <li>larger than (>) + * <li>double quotes (") if isAttribute is <code>true</code> + * </ul> + * with character entities. + * + * @param stripCodePoint + * any occurrence of this character is removed from the string + */ + public static String xmlEscape(String string, boolean isAttribute, boolean escapeLowAscii, + StringBuilder buffer, int stripCodePoint) { + // buffer and stripCodePoint changed order in the signature compared to + // the char based API to avoid wrong method being called + + // This is inner loop stuff, so we sacrifice a little for speed - + // no copying will occur until a character needing escaping is found + boolean legalCharacter = true; + Quote escaper; + int i = 0; + + for (i = 0; i < string.length() && legalCharacter; i = string.offsetByCodePoints(i, 1)) { + legalCharacter = scanner.isLegal(string.codePointAt(i), escapeLowAscii, stripCodePoint, isAttribute); + } + if (legalCharacter) { + return string; + } + + i = string.offsetByCodePoints(i, -1); // Back to the char needing escaping + escaper = new Quote(); + + if (buffer == null) { + buffer = new StringBuilder((int) (string.length() * 1.2)); + } + + // ugly appending zero length strings + if (i > 0) { + buffer.append(string.substring(0, i)); + } + + // i is at the first codepoint which needs replacing + // Don't guard against double-escaping, as: + // don't try to be clever (LCJ). + for (; i < string.length(); i = string.offsetByCodePoints(i, 1)) { + int codepoint = string.codePointAt(i); + if (escaper.isLegal(codepoint, escapeLowAscii, stripCodePoint, isAttribute)) { + buffer.appendCodePoint(codepoint); + } else { + buffer.append(escaper.lastQuoted); + } + } + return buffer.toString(); + } + + /** + * Returns the Document of an XML file reader + * + * @throws RuntimeException + * if the root Document cannot be returned + */ + public static Document getDocument(Reader reader) { + try { + return getDocumentBuilder().parse(new InputSource(reader)); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read '" + reader + "'", e); + } catch (SAXParseException e) { + throw new IllegalArgumentException("Could not parse '" + reader + "', error at line " + e.getLineNumber() + ", column " + e.getColumnNumber(), e); + } catch (SAXException e) { + throw new IllegalArgumentException("Could not parse '" + reader + "'", e); + } + } + + /** + * Returns the Document of the string XML payload + */ + public static Document getDocument(String string) { + return getDocument(new StringReader(string)); + } + + /** + * Creates a new XML DocumentBuilder + * + * @return a DocumentBuilder + * @throws RuntimeException + * if we fail to create one + */ + public static DocumentBuilder getDocumentBuilder() { + return getDocumentBuilder("com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl", null); + } + + /** + * Creates a new XML DocumentBuilder + * + * @param implementation + * which jaxp implementation should be used + * @param classLoader + * which class loader should be used when getting a new + * DocumentBuilder + * @throws RuntimeException + * if we fail to create one + * @return a DocumentBuilder + */ + public static DocumentBuilder getDocumentBuilder(String implementation, ClassLoader classLoader) { + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(implementation, classLoader); + factory.setNamespaceAware(true); + factory.setXIncludeAware(true); + return factory.newDocumentBuilder(); + } catch (ParserConfigurationException e) { + throw new RuntimeException("Could not create an XML builder"); + } + } + + /** + * Returns the child Element objects from a w3c dom spec + * + * @return List of elements. Empty list (never null) if none found or if the + * given element is null + */ + public static List<Element> getChildren(Element spec) { + List<Element> children = new ArrayList<>(); + if (spec == null) { + return children; + } + + NodeList childNodes = spec.getChildNodes(); + for (int i = 0; i < childNodes.getLength(); i++) { + Node child = childNodes.item(i); + if (child instanceof Element) { + children.add((Element) child); + } + } + return children; + } + + /** + * Returns the child Element objects with given name from a w3c dom spec + * + * @return List of elements. Empty list (never null) if none found or the + * given element is null + */ + public static List<Element> getChildren(Element spec, String name) { + List<Element> ret = new ArrayList<>(); + if (spec == null) { + return ret; + } + + NodeList children = spec.getChildNodes(); + if (children == null) { + return ret; + } + for (int i = 0; i < children.getLength(); i++) { + Node child = children.item(i); + if (child != null && child instanceof Element) { + if (child.getNodeName().equals(name)) { + ret.add((Element) child); + } + } + } + return ret; + } + + /** + * Gets the string contents of the given Element. Returns "", never null if + * the element is null, or has no content + */ + public static String getValue(Element e) { + if (e == null) { + return ""; + } + Node child = e.getFirstChild(); + if (child == null) { + return ""; + } + return child.getNodeValue(); + } + + /** Returns the first child with the given name, or null if none */ + public static Element getChild(Element e, String name) { + return (getChildren(e, name).size() >= 1) ? getChildren(e, name).get(0) : null; + } + + /** + * Returns the path to the given xml node, where each node name is separated + * by the given separator string. + * + * @param n + * The xml node to find path to + * @param sep + * The separator string + * @return The path to the xml node as a String + */ + public static String getNodePath(Node n, String sep) { + if (n == null) { + return ""; + } + StringBuffer ret = new StringBuffer(n.getNodeName()); + while ((n.getParentNode() != null) && !(n.getParentNode() instanceof Document)) { + n = n.getParentNode(); + ret.insert(0, sep).insert(0, n.getNodeName()); + } + return ret.toString(); + } + + + private static boolean inclusiveWithin(int x, int low, int high) { + return low <= x && x <= high; + } + + private static boolean nameStartSet(int codepoint) { + // NameStartChar ::= ":" | [A-Z] | "_" | [a-z] | [#xC0-#xD6] | + // [#xD8-#xF6] | [#xF8-#x2FF] | [#x370-#x37D] | [#x37F-#x1FFF] | + // [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] | [#x3001-#xD7FF] + // | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] | [#x10000-#xEFFFF] + + boolean valid; + if (codepoint < 0xC0) { + valid = inclusiveWithin(codepoint, 'a', 'z') + || inclusiveWithin(codepoint, 'A', 'Z') || codepoint == '_' + || codepoint == ':'; + } else { + valid = inclusiveWithin(codepoint, 0xC0, 0xD6) + || inclusiveWithin(codepoint, 0xD8, 0xF6) + || inclusiveWithin(codepoint, 0xF8, 0x2FF) + || inclusiveWithin(codepoint, 0x370, 0x37D) + || inclusiveWithin(codepoint, 0x37F, 0x1FFF) + || inclusiveWithin(codepoint, 0x200C, 0x200D) + || inclusiveWithin(codepoint, 0x2070, 0x218F) + || inclusiveWithin(codepoint, 0x2C00, 0x2FEF) + || inclusiveWithin(codepoint, 0x3001, 0xD7FF) + || inclusiveWithin(codepoint, 0xF900, 0xFDCF) + || inclusiveWithin(codepoint, 0xFDF0, 0xFFFD) + || inclusiveWithin(codepoint, 0x10000, 0xEFFFF); + } + return valid; + } + + private static boolean nameSetExceptStart(int codepoint) { + // "-" | "." | [0-9] | #xB7 | [#x0300-#x036F] | [#x203F-#x2040] + boolean valid; + if (codepoint < 0xB7) { + valid = inclusiveWithin(codepoint, '0', '9') || codepoint == '-' + || codepoint == '.'; + } else { + + valid = codepoint == '\u00B7' + || inclusiveWithin(codepoint, 0x300, 0x36F) + || inclusiveWithin(codepoint, 0x023F, 0x2040); + } + return valid; + } + + private static boolean nameChar(int codepoint, boolean first) { + // NameChar ::= NameStartChar | "-" | "." | [0-9] | #xB7 | [#x0300-#x036F] | [#x203F-#x2040] + boolean valid = nameStartSet(codepoint); + return first ? valid : valid || nameSetExceptStart(codepoint); + } + + + /** + * Check whether the name of a tag or attribute conforms to <a + * href="http://www.w3.org/TR/2006/REC-xml11-20060816/#sec-common-syn">XML + * 1.1 (Second Edition)</a>. This does not check against reserved names, it + * only checks the set of characters used. + * + * @param possibleName + * a possibly valid XML name + * @return true if the name may be used as an XML tag or attribute name + */ + public static boolean isName(CharSequence possibleName) { + final int barrier = possibleName.length(); + int i = 0; + boolean valid = true; + boolean first = true; + + if (barrier < 1) { + valid = false; + } + + while (valid && i < barrier) { + char c = possibleName.charAt(i++); + if (Character.isHighSurrogate(c)) { + valid = nameChar(Character.toCodePoint(c, possibleName.charAt(i++)), first); + } else { + valid = nameChar((int) c, first); + } + first = false; + } + return valid; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/text/XMLWriter.java b/vespajlib/src/main/java/com/yahoo/text/XMLWriter.java new file mode 100644 index 00000000000..ee5ff753c57 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/XMLWriter.java @@ -0,0 +1,410 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.io.Writer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * A stream wrapper which contains utility methods for writing xml. + * All methods return this for convenience. + * <p> + * The methods of this writer can be used in conjunction with writing tags in raw form directly to the writer + * if some care is taken to close start tags and insert line breaks explicitly. If all content is written + * using these methods, start tags are closed and newlines inserted automatically as appropriate. + * + * @author bratseth + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class XMLWriter extends ForwardWriter { + + /** Configuration */ + private final int maxIndentLevel, maxLineSeparatorLevel; + + /** The current list of parent tags */ + private final List<Utf8String> openTags = new ArrayList<>(); + private final List<Utf8String> unmodifiableOpenTags = Collections.unmodifiableList(openTags); + + /** Control state */ + private boolean inOpenStartTag, currentIsMultiline, isFirstInParent; + + /** Write markup directly to this with no encoding if it is non-null (an optimization) */ + private final boolean markupIsAscii; + static private final Utf8String SPACE = new Utf8String(" "); + static private final Utf8String INDENT = new Utf8String(" "); + static private final Utf8String ATTRIBUTE_START = new Utf8String("=\""); + static private final Utf8String ATTRIBUTE_END = new Utf8String("\""); + static private final Utf8String ENCODING_START = new Utf8String("<?xml version=\"1.0\" encoding=\""); + static private final Utf8String ENCODING_END = new Utf8String("\" ?>\n"); + static private final Utf8String LF = new Utf8String("\n"); + static private final Utf8String LT = new Utf8String("<"); + static private final Utf8String GT = new Utf8String(">"); + static private final Utf8String ELT = new Utf8String("</"); + static private final Utf8String EGT = new Utf8String("/>"); + /** + * Creates an XML wrapper of a writer having maxIndentLevel=10 and maxLineSeparatorLevel=1 + * + * @param writer the writer to which this writers (accessible from this by getWrapped) + */ + public XMLWriter(Writer writer) { + this(writer,10); + } + + /** + * Creates an XML wrapper of a writer having maxIndentLevel=10 and maxLineSeparatorLevel=1 + * + * @param writer the writer to which this writers (accessible from this by getWrapped) + * @param markupIsAscii set to false to make this encode markup (tags, attributes). By default encoding + * is skipped if the underlying stream uses utf encoding for performance (yes, this matters) + */ + public XMLWriter(Writer writer,boolean markupIsAscii) { + this(writer,10,markupIsAscii); + } + + /** + * Creates an XML wrapper of a writer having maxLineSeparatorLevel=1 + * + * @param writer the writer to which this writers (accessible from this by getWrapped) + * @param maxIndentLevel the max number of tag levels for which we'll continue to indent, or -1 to + * never indent. The top level tag is level 0, etc. + */ + public XMLWriter(Writer writer,int maxIndentLevel) { + this(writer,maxIndentLevel,1); + } + + /** + * Creates an XML wrapper of a writer having maxLineSeparatorLevel=1 + * + * @param writer the writer to which this writers (accessible from this by getWrapped) + * @param maxIndentLevel the max number of tag levels for which we'll continue to indent, or -1 to + * never indent. The top level tag is level 0, etc. + * @param markupIsAscii set to false to make this encode markup (tags, attributes). By default encoding + * is skipped if the underlying stream uses utf encoding for performance (yes, this matters) + */ + public XMLWriter(Writer writer,int maxIndentLevel,boolean markupIsAscii) { + this(writer,maxIndentLevel,1,markupIsAscii); + } + + /** + * Creates an XML wrapper of a writer + * + * @param writer the writer to which this writers (accessible from this by getWrapped) + * @param maxIndentLevel the max number of tag levels for which we'll continue to indent, or -1 to + * never indent. The top level tag is level 0, etc. + * @param maxLineSeparatorLevel the max number of tag levels for which we'll add a blank line separator, + * or -1 to never add line separators. + * The top level tag is level 0, etc. + */ + public XMLWriter(Writer writer,int maxIndentLevel,int maxLineSeparatorLevel) { + this(writer,maxIndentLevel,maxLineSeparatorLevel,true); + } + + /** + * Creates an XML wrapper of a writer + * + * @param writer the writer to which this writers (accessible from this by getWrapped) + * @param maxIndentLevel the max number of tag levels for which we'll continue to indent, or -1 to + * never indent. The top level tag is level 0, etc. + * @param maxLineSeparatorLevel the max number of tag levels for which we'll add a blank line separator, + * or -1 to never add line separators. + * The top level tag is level 0, etc. + * @param markupIsAscii set to false to make this encode markup (tags, attributes). By default encoding + * is skipped if the underlying stream uses utf encoding for performance (yes, this matters) + */ + public XMLWriter(Writer writer,int maxIndentLevel,int maxLineSeparatorLevel,boolean markupIsAscii) { + super(writer instanceof GenericWriter ? (GenericWriter)writer : new JavaWriterWriter(writer)); + this.maxIndentLevel=maxIndentLevel; + this.maxLineSeparatorLevel=maxLineSeparatorLevel; + this.markupIsAscii = markupIsAscii; + } + + /** Returns the input writer as-is if it is an XMLWriter instance. Returns new XMLWriter(writer) otherwise */ + @SuppressWarnings("resource") + public static XMLWriter from(Writer writer, int maxIndentLevel,int maxLineSeparatorLevel) { + return (writer instanceof XMLWriter) + ? (XMLWriter)writer + : new XMLWriter(writer, maxIndentLevel, maxLineSeparatorLevel); + } + + /** Returns the input writer as-is if it is an XMLWriter instance. Returns new XMLWriter(writer) otherwise */ + @SuppressWarnings("resource") + public static XMLWriter from(Writer writer) { + return (writer instanceof XMLWriter) + ? (XMLWriter)writer + : new XMLWriter(writer); + } + + public Writer getWrapped() { + return (getWriter() instanceof JavaWriterWriter) ? ((JavaWriterWriter)getWriter()).getWriter() : getWriter(); + } + + /** Writes the first line of an XML file */ + public void xmlHeader(String encoding) { + w(ENCODING_START).w(encoding).w(ENCODING_END); + } + + public XMLWriter openTag(String s) { + return openTag(new Utf8String(s)); + } + public XMLWriter openTag(Utf8String tag) { + closeStartTag(); + if (openTags.size()>0) { + w(LF); + if (isFirstInParent && openTags.size()<=maxLineSeparatorLevel) { + w(LF); + } + indent(); + } + w(LT).w(tag); + openTags.add(tag); + inOpenStartTag=true; + currentIsMultiline=false; + isFirstInParent=true; + return this; + } + + public XMLWriter closeTag() { + if (openTags.size()<=0) { + throw new RuntimeException("Called closeTag() when no tag was open"); + } + Utf8String lastOpenTag=openTags.remove(openTags.size()-1); + + if (inOpenStartTag) {// this tag has no content - use short form + w(EGT); + } + else { + if (currentIsMultiline) { + w(LF).indent(); + } + w(ELT).w(lastOpenTag).w(GT); + } + if (openTags.size()==0 || openTags.size()<=maxLineSeparatorLevel) { + w(LF); + } + inOpenStartTag=false; + currentIsMultiline=true; // When we go up from a subtag we are at a multiline tag (because it contains subtags) + isFirstInParent=false; // the next opened tag will not be first + return this; + } + + private XMLWriter indent() { + for (int i=0; i<openTags.size() && i<maxIndentLevel; i++) { + w(INDENT); + } + return this; + } + + /** + * Closes the start tag. Usually, it is not necessary to call this, as the other methods in this will do + * the right thing as needed. However, this can be called explicitly to allow content or subtags to be written + * by a regular write call which bypasses the logic in this. + * If a start tag is not currently open this has no effect. + */ + public XMLWriter closeStartTag() { + if (!inOpenStartTag) return this; + w(GT); + inOpenStartTag=false; + return this; + } + + /** + * Writes an attribute by XML.xmlEscape(value.toString(),false) + * + * @param name the name of the attribute. An exception is thrown if this is null + * @param value the value of the attribute. The empty string if the attribute is null or empty + */ + public XMLWriter forceAttribute(Utf8String name, Object value) { + String stringValue = value!=null ? value.toString() : ""; + allowAttribute(); + return w(SPACE).w(name).w(ATTRIBUTE_START).wTranscode(XML.xmlEscape(stringValue,true)).w(ATTRIBUTE_END); + } + + public XMLWriter forceAttribute(String name, Object value) { + return forceAttribute(new Utf8String(name), value); + } + + private void allowAttribute() { + if (!inOpenStartTag) { + throw new RuntimeException("Called writeAttribute() while not in an open start tag"); + } + } + /** + * Writes an attribute by its utf8 value + * + * @param name the name of the attribute. An exception is thrown if this is null + * @param value the value of the attribute. This method does nothing if the value is null or empty + */ + public XMLWriter attribute(Utf8String name, AbstractUtf8Array value) { + if (value.isEmpty()) return this; + allowAttribute(); + return w(SPACE).w(name).w(ATTRIBUTE_START).w(value).w(ATTRIBUTE_END); + } + + /** + * Writes an attribute by its utf8 value + * + * @param name the name of the attribute. An exception is thrown if this is null + * @param value the value of the attribute. This method does nothing if the value is null. + */ + public XMLWriter attribute(Utf8String name, Number value) { + if (value == null) return this; + allowAttribute(); + return w(SPACE).w(name).w(ATTRIBUTE_START).w(value).w(ATTRIBUTE_END); + } + + /** + * Writes an attribute by its utf8 value + * + * @param name the name of the attribute. An exception is thrown if this is null + * @param value the value of the attribute. + */ + public XMLWriter attribute(Utf8String name, long value) { + allowAttribute(); + return w(SPACE).w(name).w(ATTRIBUTE_START).w(value).w(ATTRIBUTE_END); + } + + /** + * Writes an attribute by its utf8 value + * + * @param name the name of the attribute. An exception is thrown if this is null + * @param value the value of the attribute. + */ + public XMLWriter attribute(Utf8String name, double value) { + allowAttribute(); + return w(SPACE).w(name).w(ATTRIBUTE_START).w(value).w(ATTRIBUTE_END); + } + + /** + * Writes an attribute by its utf8 value + * + * @param name the name of the attribute. An exception is thrown if this is null + * @param value the value of the attribute. This method does nothing if the value is null or empty + */ + public XMLWriter attribute(Utf8String name, boolean value) { + allowAttribute(); + return w(SPACE).w(name).w(ATTRIBUTE_START).w(value).w(ATTRIBUTE_END); + } + + /** + * Writes an attribute by XML.xmlEscape(value.toString(),false) + * + * @param name the name of the attribute. An exception is thrown if this is null + * @param value the value of the attribute. This method does nothing if the value is null or empty + */ + public XMLWriter attribute(Utf8String name, String value) { + if ((value == null) || value.isEmpty()) return this; + allowAttribute(); + return w(SPACE).w(name).w(ATTRIBUTE_START).wTranscode(XML.xmlEscape(value, true)).w(ATTRIBUTE_END); + } + + public XMLWriter attribute(String name, Object value) { + if (value==null) return this; + return attribute(new Utf8String(name), value.toString()); + } + + /** + * XML escapes and writes the content.toString(). If the content is null this does nothing but closing the start tag. + * + * @param content the content - output by XML.xmlEscape(content.toString()) + * @param multiline whether the content should be treated as multiline, + * such that the following end tag should appear on a new line + */ + public XMLWriter content(Object content,boolean multiline) { + closeStartTag(); + return (content==null) + ? this + : escapedContent(XML.xmlEscape(content.toString(),false),multiline); + } + + /** + * Writes the given string as-is. The content string <i>must</i> be XML escaped before calling this. + * If the content is null this does nothing but closing the start tag. + * + * @param content the content - output by XML.xmlEscape(content.toString()) + * @param multiline whether the content should be treated as multiline, + * such that the following end tag should appear on a new line + */ + public XMLWriter escapedContent(String content,boolean multiline) { + closeStartTag(); + if (content==null) return this; + if (multiline) currentIsMultiline=true; + return wTranscode(content); + } + + /** + * Writes the given US-ASCII only string as-is. + * If the content is <b>not</b> US-ASCII <i>only</i> this <i>may</i> cause + * incorrectly encoded content to be written. + * This is faster than using escapedContent as transcoding is skipped. + * <p> + * The content string <i>must</i> be XML escaped before calling this. + * If the content is null this does nothing but closing the start tag. + * + * @param content the content - output by XML.xmlEscape(content.toString()) + * @param multiline whether the content should be treated as multiline, + * such that the following end tag should appear on a new line + */ + public XMLWriter escapedAsciiContent(String content,boolean multiline) { + closeStartTag(); + if (content==null) return this; + if (multiline) currentIsMultiline=true; + return w(content); + } + + /** + * Writes the given string. If markup is us ascii (default), and the wrapped writer encodes in UTF, this will write + * the string <b>as is, with no transcoding</b> (for speed). Hence, this should never be used for just any content. + * + * @return this for consistency + */ + private final XMLWriter w(String s) { + return markupIsAscii ? w(new Utf8String(s)) : w(s); + } + + private final XMLWriter w(AbstractUtf8Array utf8) { + write(utf8); + return this; + } + private final XMLWriter w(long v) { + write(v); + return this; + } + private final XMLWriter w(boolean v) { + write(v); + return this; + } + private final XMLWriter w(double v) { + write(v); + return this; + } + private final XMLWriter w(Number v) { + write(v.toString()); + return this; + } + + /** Calls write(s) and returns this. Use this for general content which must be transcoded */ + private final XMLWriter wTranscode(String s) { + write(s); + return this; + } + + /** + * Returns a read only view of the currently open tags we are within, sorted by highest to + * lowest in the hierarchy + * Only used for testing. + */ + public List<Utf8String> openTags() { return unmodifiableOpenTags; } + + /** + * Returns true if the immediate parent (i.e the last element in openTags) + * is the tag with the given name + */ + public boolean isIn(Utf8String containingTag) { + return (openTags.size()!=0) && openTags.get(openTags.size()-1).equals(containingTag); + } + + public boolean isIn(String containingTag) { + return (openTags.size()!=0) && openTags.get(openTags.size()-1).equals(containingTag); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/text/package-info.java b/vespajlib/src/main/java/com/yahoo/text/package-info.java new file mode 100644 index 00000000000..f0322acbcc6 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.text; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/main/java/com/yahoo/time/WallClockSource.java b/vespajlib/src/main/java/com/yahoo/time/WallClockSource.java new file mode 100644 index 00000000000..5fef1f94879 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/time/WallClockSource.java @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.time; + +import com.google.common.annotations.Beta; + +/** + * A source for high-resolution timestamps. + * + * @author arnej27959 + */ + +@Beta +public class WallClockSource { + + private volatile long offset; + + /** + * Obtain the current time in nanoseconds. + * The epoch is January 1, 1970 UTC just as for System.currentTimeMillis(), + * but with greater resolution. Note that the absolute accuracy may be + * no better than currentTimeMills(). + * @return nanoseconds since the epoch. + **/ + public final long currentTimeNanos() { + return System.nanoTime() + offset; + } + + /** + * Create a source with 1 millisecond accuracy at start. + **/ + WallClockSource() { + long actual = System.currentTimeMillis(); + actual *= 1000000; + this.offset = actual - System.nanoTime(); + initialAdjust(); + } + + /** adjust the clock source from currentTimeMillis() + * to ensure that it is no more than 1 milliseconds off. + * @return true if we want adjust called again soon + **/ + boolean adjust() { + long nanosB = System.nanoTime(); + long actual = System.currentTimeMillis(); + long nanosA = System.nanoTime(); + if (nanosA - nanosB > 100000) { + return true; // not a good time to adjust, try again soon + } + return adjustOffset(nanosB, actual, nanosA); + } + + private boolean adjustOffset(long before, long actual, long after) { + actual *= 1000000; // convert millis to nanos + if (actual > after + offset) { + // System.out.println("WallClockSource adjust UP "+(actual-after-offset)); + offset = actual - after; + return true; + } + if (actual + 999999 < before + offset) { + // System.out.println("WallClockSource adjust DOWN "+(before+offset-actual-999999)); + offset = actual + 999999 - before; + return true; + } + return false; + } + + private void initialAdjust() { + for (int i = 0; i < 100; i++) { + long nanosB = System.nanoTime(); + long actual = System.currentTimeMillis(); + long nanosA = System.nanoTime(); + adjustOffset(nanosB, actual, nanosA); + } + } + + + static private WallClockSource autoAdjustingInstance = new WallClockSource(); + + /** + * Get a WallClockSource which auto adjusts to wall clock time. + **/ + static public WallClockSource get() { + autoAdjustingInstance.startAdjuster(); + return autoAdjustingInstance; + } + + private Thread adjuster; + + private synchronized void startAdjuster() { + if (adjuster == null) { + adjuster = new AdjustThread(); + adjuster.setDaemon(true); + adjuster.start(); + } + } + + private class AdjustThread extends Thread { + public void run() { + int millis = 0; + int nanos = 313373; // random number + while (true) { + try { + sleep(millis, nanos); + if (++millis > 4321) { + millis = 1000; // do not sleep too long + } + } catch (InterruptedException e) { + return; + } + if (adjust()) { + // adjust more often in case clock jumped + millis = 0; + } + } + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/transaction/Mutex.java b/vespajlib/src/main/java/com/yahoo/transaction/Mutex.java new file mode 100644 index 00000000000..3c4168b77f3 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/transaction/Mutex.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.transaction; + +/** + * An auto closeable mutex + * + * @author bratseth + */ +public interface Mutex extends AutoCloseable { + + public void close(); + +} diff --git a/vespajlib/src/main/java/com/yahoo/transaction/NestedTransaction.java b/vespajlib/src/main/java/com/yahoo/transaction/NestedTransaction.java new file mode 100644 index 00000000000..4be0a32ffe8 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/transaction/NestedTransaction.java @@ -0,0 +1,200 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.transaction; + +import java.util.ArrayList; +import java.util.List; +import java.util.ListIterator; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * A transaction which may contain a list of transactions, typically to represent a distributed transaction + * over multiple systems. + * + * @author bratseth + */ +public final class NestedTransaction implements AutoCloseable { + + private static final Logger log = Logger.getLogger(NestedTransaction.class.getName()); + + /** Nested transactions with ordering constraints, in the order they are added */ + private final List<ConstrainedTransaction> transactions = new ArrayList<>(2); + + /** Transaction ordering pairs */ + //private final List<OrderingConstraint> transactionOrders = new ArrayList<>(2); + + /** A list of (non-transactional) operations to execute after this transaction has committed successfully */ + private final List<Runnable> onCommitted = new ArrayList<>(2); + + /** + * Adds a transaction to this. + * + * @param transaction the transaction to add + * @param before transaction classes which should commit after this, if present. It is beneficial + * to order transaction types from the least to most reliable. If conflicting ordering constraints are + * given this will not be detected at add time but the transaction will fail to commit + * @return this for convenience + */ + @SafeVarargs // don't warn on 'before' argument + @SuppressWarnings("varargs") // don't warn on passing 'before' to the nested class constructor + public final NestedTransaction add(Transaction transaction, Class<? extends Transaction> ... before) { + transactions.add(new ConstrainedTransaction(transaction, before)); + return this; + } + + /** Returns the transactions nested in this, as they will be committed. */ + public List<Transaction> transactions() { return organizeTransactions(transactions); } + + /** Perform a 2 phase commit */ + public void commit() { + List<Transaction> organizedTransactions = organizeTransactions(transactions); + + // First phase + for (Transaction transaction : organizedTransactions) + transaction.prepare(); + + // Second phase + for (ListIterator<Transaction> i = organizedTransactions.listIterator(); i.hasNext(); ) { + Transaction transaction = i.next(); + try { + transaction.commit(); + } + catch (Exception e) { + // Clean up committed part or log that we can't + i.previous(); + while (i.hasPrevious()) + i.previous().rollbackOrLog(); + throw new IllegalStateException("Transaction failed during commit", e); + } + } + + // After commit: Execute completion tasks + for (Runnable task : onCommitted) { + try { + task.run(); + } + catch (Exception e) { // Don't throw from here as that indicates transaction didn't complete + log.log(Level.WARNING, "A committed task in " + this + " caused an exception", e); + } + } + } + + public void onCommitted(Runnable runnable) { + onCommitted.add(runnable); + } + + /** Free up any temporary resources held by this */ + @Override + public void close() { + for (ConstrainedTransaction transaction : transactions) + transaction.transaction.close(); + } + + private List<Transaction> organizeTransactions(List<ConstrainedTransaction> transactions) { + return orderTransactions(combineTransactions(transactions), findOrderingConstraints(transactions)); + } + + /** Combines all transactions of the same type to one */ + private List<Transaction> combineTransactions(List<ConstrainedTransaction> transactions) { + List<Transaction> combinedTransactions = new ArrayList<>(transactions.size()); + for (List<Transaction> combinableTransactions : + transactions.stream().map(ConstrainedTransaction::transaction). + collect(Collectors.groupingBy(Transaction::getClass)).values()) { + Transaction combinedTransaction = combinableTransactions.get(0); + for (int i = 1; i < combinableTransactions.size(); i++) + combinedTransaction = combinedTransaction.add(combinableTransactions.get(i).operations()); + combinedTransactions.add(combinedTransaction); + } + return combinedTransactions; + } + + private List<OrderingConstraint> findOrderingConstraints(List<ConstrainedTransaction> transactions) { + List<OrderingConstraint> orderingConstraints = new ArrayList<>(1); + for (ConstrainedTransaction transaction : transactions) { + for (Class<? extends Transaction> afterThis : transaction.before()) + orderingConstraints.add(new OrderingConstraint(transaction.transaction().getClass(), afterThis)); + } + return orderingConstraints; + } + + /** Orders combined transactions consistent with the ordering constraints */ + private List<Transaction> orderTransactions(List<Transaction> transactions, List<OrderingConstraint> constraints) { + if (transactions.size() == 1) return transactions; + + List<Transaction> orderedTransactions = new ArrayList<>(); + for (Transaction transaction : transactions) + orderedTransactions.add(findSuitablePositionFor(transaction, orderedTransactions, constraints), transaction); + return orderedTransactions; + } + + private int findSuitablePositionFor(Transaction transaction, List<Transaction> orderedTransactions, + List<OrderingConstraint> constraints) { + for (int i = 0; i < orderedTransactions.size(); i++) { + Transaction candidateNextTransaction = orderedTransactions.get(i); + if ( ! mustBeAfter(candidateNextTransaction.getClass(), transaction.getClass(), constraints)) return i; + + // transaction must be after this: continue to next position + if (mustBeAfter(transaction.getClass(), candidateNextTransaction.getClass(), constraints)) // must be after && must be before + throw new IllegalStateException("Conflicting transaction ordering constraints between" + + transaction + " and " + candidateNextTransaction); + } + return orderedTransactions.size(); // add last as it must be after everything + } + + /** + * Returns whether transaction type B must be after type A according to the ordering constraints. + * This is the same as asking whether there is a path between node a and b in the bi-directional + * graph defined by the ordering constraints. + */ + private boolean mustBeAfter(Class<? extends Transaction> a, Class<? extends Transaction> b, + List<OrderingConstraint> constraints) { + for (OrderingConstraint fromA : findAllOrderingConstraintsFrom(a, constraints)) { + if (fromA.after().equals(b)) return true; + if (mustBeAfter(fromA.after(), b, constraints)) return true; + } + return false; + } + + private List<OrderingConstraint> findAllOrderingConstraintsFrom(Class<? extends Transaction> transactionType, + List<OrderingConstraint> constraints) { + return constraints.stream().filter(c -> c.before().equals(transactionType)).collect(Collectors.toList()); + } + + private static class ConstrainedTransaction { + + private final Transaction transaction; + + private final Class<? extends Transaction>[] before; + + public ConstrainedTransaction(Transaction transaction, Class<? extends Transaction>[] before) { + this.transaction = transaction; + this.before = before; + } + + public Transaction transaction() { return transaction; } + + /** Returns transaction types which should commit after this */ + public Class<? extends Transaction>[] before() { return before; } + + } + + private static class OrderingConstraint { + + private final Class<? extends Transaction> before, after; + + public OrderingConstraint(Class<? extends Transaction> before, Class<? extends Transaction> after) { + this.before = before; + this.after = after; + } + + public Class<? extends Transaction> before() { return before; } + + public Class<? extends Transaction> after() { return after; } + + @Override + public String toString() { return before + " -> " + after; } + + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/transaction/Transaction.java b/vespajlib/src/main/java/com/yahoo/transaction/Transaction.java new file mode 100644 index 00000000000..642438dda0a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/transaction/Transaction.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.transaction; + +import java.util.List; + +/** + * An interface for building a transaction and committing it. Implementations are required to atomically apply changes + * in the commit step or throw an exception if it fails. + * + * @author lulf + * @author bratseth + */ +public interface Transaction extends AutoCloseable { + + /** + * Adds an operation to this transaction. Return self for chaining. + * + * @param operation {@link Operation} to append + * @return self, for chaining + */ + Transaction add(Operation operation); + + /** + * Adds multiple operations to this transaction. Return self for chaining. + * + * @param operation {@link Operation} to append + * @return self, for chaining + */ + Transaction add(List<Operation> operation); + + /** + * Returns the operations of this. + * Ownership of the returned list is transferred to the caller. The ist may be ready only. + */ + List<Operation> operations(); + + /** + * Checks whether or not the transaction is able to commit in its current state and do any transient preparatory + * work to commit. + * + * @throws IllegalStateException if the transaction cannot be committed + */ + void prepare(); + + /** + * Commit this transaction. If this method returns, all operations in this transaction was committed + * successfully. Implementations of this must be exception safe or log a message of type severe if they partially + * alter state. + * + * @throws IllegalStateException if transaction failed. + */ + void commit(); + + /** + * This is called if the transaction should be rolled back after commit. If a rollback is not possible or + * supported. This must log a message of type severe with detailed information about the resulting state. + */ + void rollbackOrLog(); + + /** + * Closes and frees any resources allocated by this transaction. The transaction instance cannot be reused once + * closed. + */ + void close(); + + /** + * Operations that a transaction supports should implement this interface. + */ + public interface Operation { + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/transaction/package-info.java b/vespajlib/src/main/java/com/yahoo/transaction/package-info.java new file mode 100644 index 00000000000..72ac10d13d0 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/transaction/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.transaction; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/vespajlib/src/main/java/com/yahoo/vespa/VersionTagger.java b/vespajlib/src/main/java/com/yahoo/vespa/VersionTagger.java new file mode 100644 index 00000000000..556bcc0e90d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/VersionTagger.java @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa; + +import java.io.*; +import java.util.HashMap; +import java.util.Map; + +/** + * This class generates a java class based on the vtag.map file generated by dist/getversion.pl + */ +public class VersionTagger { + public static final String V_TAG_PKG = "V_TAG_PKG"; + + VersionTagger() throws IOException { + } + + private static void printUsage(PrintStream out) { + out.println("Usage: java VersionTagger vtagmap pkgname outputdir"); + } + + public static void main(String[] args) { + if (args.length < 3) { + printUsage(System.err); + throw new RuntimeException("bad arguments to main(): vtag.map packageName outputDirectory [outputFormat (simple or vtag)]"); + } + try { + VersionTagger me = new VersionTagger(); + me.runProgram(args); + } catch (Exception e) { + System.err.println(e); + printUsage(System.err); + throw new RuntimeException(e); + } + } + + private Map<String, String> readVtagMap(String path) { + Map<String, String> map = new HashMap<>(); + try { + BufferedReader in = new BufferedReader(new FileReader(path)); + String line; + while ((line = in.readLine()) != null) { + String elements[] = line.split("\\s+", 2); + map.put(elements[0], elements[1]); + } + } catch (FileNotFoundException e) { + // Use default values + map.put("V_TAG", "NOTAG"); + map.put("V_TAG_DATE", "NOTAG"); + map.put("V_TAG_PKG", "6.9999.0"); + map.put("V_TAG_ARCH", "NOTAG"); + map.put("V_TAG_SYSTEM", "NOTAG"); + map.put("V_TAG_SYSTEM_REV", "NOTAG"); + map.put("V_TAG_BUILDER", "NOTAG"); + map.put("V_TAG_COMPONENT", "6.9999.0"); + } catch (IOException e) { + throw new RuntimeException(e); + } + return map; + } + private enum Format { + SIMPLE, + VTAG + } + + void runProgram(String[] args) throws IOException, InterruptedException { + + String vtagmapPath = args[0]; + String packageName = args[1]; + String dirName = args[2] + "/" + packageName.replaceAll("\\.", "/"); + Format format = args.length >= 4 ? Format.valueOf(args[3].toUpperCase()) : Format.SIMPLE; + File outDir = new File(dirName); + if (!outDir.isDirectory() && !outDir.mkdirs()) { + throw new IOException("could not create directory " + outDir); + } + + String className = format == Format.SIMPLE ? "VespaVersion" : "Vtag"; + String outFile = dirName + "/" + className +".java"; + FileOutputStream out = new FileOutputStream(outFile); + OutputStreamWriter writer = new OutputStreamWriter(out); + System.err.println("generating: " + outFile); + + Map<String, String> vtagMap = readVtagMap(vtagmapPath); + writer.write(String.format("package %s;\n\n", packageName)); + + if (format == Format.VTAG) { + writer.write("import com.yahoo.component.Version;\n"); + } + + writer.write(String.format("public class %s {\n", className)); + if (!vtagMap.containsKey(V_TAG_PKG)) { + throw new RuntimeException("V_TAG_PKG not present in map file"); + } + switch (format) { + case SIMPLE: + String version = vtagMap.get(V_TAG_PKG); + String elements[] = version.split("\\."); + writer.write(String.format(" public static final int major = %s;\n", elements[0])); + writer.write(String.format(" public static final int minor = %s;\n", elements[1])); + writer.write(String.format(" public static final int micro = %s;\n", elements[2])); + break; + case VTAG: + vtagMap.forEach((key, value) -> { + try { + writer.write(String.format(" public static final String %s = \"%s\";\n", key, value)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + writer.write(" public static final Version currentVersion = new Version(V_TAG_COMPONENT);\n"); + break; + } + writer.write("}\n"); + writer.close(); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/BufferSerializer.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/BufferSerializer.java new file mode 100644 index 00000000000..cf5d2e28af3 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/BufferSerializer.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.text.Utf8; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * @author balder + */ +public class BufferSerializer implements Serializer, Deserializer { + protected GrowableByteBuffer buf; + + public BufferSerializer(GrowableByteBuffer buf) { this.buf = buf; } + public BufferSerializer(ByteBuffer buf) { this(new GrowableByteBuffer(buf)); } + public BufferSerializer(byte [] buf) { this(ByteBuffer.wrap(buf)); } + public BufferSerializer() { this(new GrowableByteBuffer()); } + public static BufferSerializer wrap(byte [] buf) { return new BufferSerializer(buf); } + public final GrowableByteBuffer getBuf() { return buf; } + protected final void setBuf(GrowableByteBuffer buf) { this.buf = buf; } + public Serializer putByte(FieldBase field, byte value) { buf.put(value); return this; } + public Serializer putShort(FieldBase field, short value) { buf.putShort(value); return this; } + public Serializer putInt(FieldBase field, int value) { buf.putInt(value); return this; } + public Serializer putLong(FieldBase field, long value) { buf.putLong(value); return this; } + public Serializer putFloat(FieldBase field, float value) { buf.putFloat(value); return this; } + public Serializer putDouble(FieldBase field, double value) { buf.putDouble(value); return this; } + public Serializer put(FieldBase field, byte[] value) { buf.put(value); return this; } + public Serializer put(FieldBase field, String value) { + byte [] utf8 = createUTF8CharArray(value); + putInt(null, utf8.length+1); + put(null, utf8); + putByte(null, (byte) 0); + return this; + } + public Serializer put(FieldBase field, ByteBuffer value) { buf.put(value); return this; } + public Serializer putInt1_4Bytes(FieldBase field, int value) { buf.putInt1_4Bytes(value); return this; } + public Serializer putInt2_4_8Bytes(FieldBase field, long value) { buf.putInt2_4_8Bytes(value); return this; } + public int position() { return buf.position(); } + public ByteOrder order() { return buf.order(); } + public void position(int pos) { buf.position(pos); } + public void order(ByteOrder v) { buf.order(v); } + public void flip() { buf.flip(); } + + public byte getByte(FieldBase field) { return buf.getByteBuffer().get(); } + public short getShort(FieldBase field) { return buf.getByteBuffer().getShort(); } + public int getInt(FieldBase field) { return buf.getByteBuffer().getInt(); } + public long getLong(FieldBase field) { return buf.getByteBuffer().getLong(); } + public float getFloat(FieldBase field) { return buf.getByteBuffer().getFloat(); } + public double getDouble(FieldBase field) { return buf.getByteBuffer().getDouble(); } + public byte [] getBytes(FieldBase field, int length) { + if (buf.remaining() < length) { + throw new IllegalArgumentException("Wanted " + length + " bytes, but I only had " + buf.remaining()); + } + byte [] bbuf =new byte [length]; + buf.getByteBuffer().get(bbuf); + return bbuf; + } + public String getString(FieldBase field) { + int length = getInt(null); + byte[] stringArray = new byte[length-1]; + buf.get(stringArray); + getByte(null); + return Utf8.toString(stringArray); + } + public int getInt1_4Bytes(FieldBase field) { return buf.getInt1_4Bytes(); } + public int getInt1_2_4Bytes(FieldBase field) { return buf.getInt1_2_4Bytes(); } + public long getInt2_4_8Bytes(FieldBase field) { return buf.getInt2_4_8Bytes(); } + public int remaining() { return buf.remaining(); } + + public static byte[] createUTF8CharArray(String input) { + if (input == null || input.length() < 1) { + return new byte[0]; + } + return Utf8.toBytes(input); + } + +} + diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Deserializer.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Deserializer.java new file mode 100644 index 00000000000..abd82f6b251 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Deserializer.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +/** + * @author balder + */ +public interface Deserializer { + byte getByte(FieldBase field); + short getShort(FieldBase field); + int getInt(FieldBase field); + long getLong(FieldBase field); + float getFloat(FieldBase field); + double getDouble(FieldBase field); + byte [] getBytes(FieldBase field, int length); + String getString(FieldBase field); +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/FieldBase.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/FieldBase.java new file mode 100644 index 00000000000..2a7f9cbff7a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/FieldBase.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class FieldBase { + private final String name; + + public FieldBase(String name) { + this.name = name; + } + + public final String getName() { + return name; + } + + @Override + public boolean equals(Object o) { + return this == o || o instanceof FieldBase && name.equalsIgnoreCase(((FieldBase) o).name); + } + + @Override + public int hashCode() { + return name.toLowerCase(java.util.Locale.US).hashCode(); + } + + public String toString() { + return "field " + name; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java new file mode 100644 index 00000000000..7bc9c2f8d6b --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java @@ -0,0 +1,368 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +import com.yahoo.collections.Pair; +import com.yahoo.text.Utf8; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; + +/** + * This is the base class to do cross-language serialization and deserialization of complete object structures without + * the need for a separate protocol. Each subclass needs to register itself using the {@link #registerClass(int, Class)} + * method, and override {@link #onGetClassId()} to return the same classId as the one registered. Creating an instance + * of an identifiable object is done through the {@link #create(Deserializer)} or {@link #createFromId(int)} factory + * methods. + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Identifiable extends Selectable implements Cloneable { + + private static Registry registry = null; + public static int classId = registerClass(1, Identifiable.class); + + /** + * Returns the class identifier of this class. This proxies the {@link #onGetClassId()} method that must be + * implemented by every subclass. + * + * @return The class identifier. + */ + public final int getClassId() { + return onGetClassId(); + } + + /** + * Returns the class identifier for which this class is registered. It is important that all subclasses match the + * return value of this with their call to {@link #registerClass(int, Class)}. + * + * @return The class identifier. + */ + protected int onGetClassId() { + return classId; + } + + /** + * Serializes the content of this class into the given byte buffer. This method serializes its own identifier into + * the buffer before invoking the {@link #serialize(Serializer)} method. + * + * @param buf The buffer to serialize to. + * @return The buffer argument, to allow chaining. + */ + public final Serializer serializeWithId(Serializer buf) { + buf.putInt(null, getClassId()); + return serialize(buf); + } + + /** + * Serializes the content (excluding the identifier) of this class into the given byte buffer. If you need the + * identifier serialized, use the {@link #serializeWithId(Serializer)} method instead. This method invokes the + * {@link #onSerialize(Serializer)} method. + * + * @param buf The buffer to serialize to. + * @return The buffer argument, to allow chaining. + */ + public final Serializer serialize(Serializer buf) { + onSerialize(buf); + return buf; + } + + /** + * Serializes the content of this class into the given + * buffer. This method must be implemented by all subclasses that + * have content. If the subclass has no other content than the + * semantics of its class type, this method does not need to be + * overloaded. + * + * @param buf The buffer to serialize to. + */ + protected void onSerialize(Serializer buf) { + // empty + } + + /** + * Deserializes the content of this class from the given byte buffer. This method deserialize a class identifier + * first, and asserts that this identifier matches the identifier of this class. This is usable if you have an + * instance of a class whose content you wish to retrieve from a buffer. + * + * @param buf The buffer to deserialize from. + * @return The buffer argument, to allow chaining. + * @throws IllegalArgumentException Thrown if the deserialized class identifier does not match this class. + */ + public final Deserializer deserializeWithId(Deserializer buf) { + int id = buf.getInt(null); + if (id != getClassId()) { + Class<?> spec = registry.get(id); + if (spec != null) { + throw new IllegalArgumentException( + "Can not deserialize class '" + getClass().getName() + "' (id " + getClassId() + ") from " + + "buffer containing class '" + spec.getName() + "' (id " + id + ")."); + } else { + throw new IllegalArgumentException( + "Can not deserialize class '" + getClass().getName() + "' (id " + getClassId() + ") from " + + "buffer containing unknown class id " + id + "."); + } + } + return deserialize(buf); + } + + /** + * Deserializes the content (excluding the identifier) of this class from the given byte buffer. If you need the + * identifier deserialized and verified, use the {@link #deserializeWithId(Deserializer)} method instead. This + * method invokes the {@link #onDeserialize(Deserializer)} method. + * + * @param buf The buffer to deserialize from. + * @return The buffer argument, to allow chaining. + */ + public final Deserializer deserialize(Deserializer buf) { + onDeserialize(buf); + return buf; + } + + /** + * Deserializes the content of this class from the given byte + * buffer. This method must be implemented by all subclasses that + * have content. If the subclass has no other content than the + * semantics of its class type, this method does not need to be + * overloaded. + * + * @param buf The buffer to deserialize from. + */ + protected void onDeserialize(Deserializer buf) { + // empty + } + + /** + * Declares that all subclasses of Identifiable supports clone() by _not_ throwing CloneNotSupported exceptions. + * + * @return A cloned instance of this. + * @throws AssertionError Thrown if a subclass does not implement clone(). + */ + @Override + public Identifiable clone() { + try { + return (Identifiable)super.clone(); + } catch (CloneNotSupportedException e) { + throw (AssertionError)new AssertionError("The cloneable structure has been broken.").initCause(e); + } + } + + @Override + public int hashCode() { + return getClassId(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Identifiable)) { + return false; + } + Identifiable rhs = (Identifiable)obj; + return (getClassId() == rhs.getClassId()); + } + + @Override + public String toString() { + ObjectDumper ret = new ObjectDumper(); + ret.visit("", this); + return ret.toString(); + } + + /** + * Registers the given class specification for the given identifier in the class registry. This method returns the + * supplied identifier, so that subclasses can declare a static classId member like so: + * + * <code>public static int classId = registerClass(<id>, <ClassName>.class);</code> + * + * @param id The class identifier to register with. + * @param spec The class to register. + * @return The identifier argument. + */ + protected static int registerClass(int id, Class<? extends Identifiable> spec) { + if (registry == null) { + registry = new Registry(); + } + registry.add(id, spec); + return id; + } + + /** + * Deserializes a single {@link Identifiable} object from the given byte buffer. The object itself may perform + * recursive deserialization of {@link Identifiable} objects, but there is no requirement that this method consumes + * the whole content of the buffer. + * + * @param buf The buffer to deserialize from. + * @return The instantiated object. + * @throws IllegalArgumentException Thrown if an unknown class is contained in the buffer. + */ + public static Identifiable create(Deserializer buf) { + int classId = buf.getInt(null); + Identifiable obj = createFromId(classId); + if (obj != null) { + obj.deserialize(buf); + } else { + throw new IllegalArgumentException("Failed creating class for classId " + classId); + } + return obj; + } + + /** + * Creates an instance of the class registered with the given identifier. If the indentifier is unknown, this method + * returns null. + * + * @param id The identifier of the class to instantiate. + * @return The instantiated object. + */ + public static Identifiable createFromId(int id) { + return registry.createFromId(id); + } + + /** + * This is a convenience method to allow serialization of an optional field. A single byte is added to the buffer + * indicating whether or not an object follows. If the object is not null, it is serialized following this flag. + * + * @param buf The buffer to serialize to. + * @param obj The object to serialize, may be null. + * @return The buffer, to allow chaining. + */ + protected static Serializer serializeOptional(Serializer buf, Identifiable obj) { + if (obj != null) { + buf.putByte(null, (byte)1); + obj.serializeWithId(buf); + } else { + buf.putByte(null, (byte)0); + } + return buf; + } + + /** + * This is a convenience method to allow deserialization of an optional field. See {@link + * #serializeOptional(Serializer, Identifiable)} for notes on this. + * + * @param buf The buffer to deserialize from. + * @return The instantiated object, or null. + */ + protected static Identifiable deserializeOptional(Deserializer buf) { + byte hasObject = buf.getByte(null); + if (hasObject == 1) { + return create(buf); + } + return null; + } + + /** + * Returns whether or not two objects are equal, taking into account that either can be null. + * + * @param lhs The left hand side of the comparison. + * @param rhs The right hand side of the comparison. + * @return True if both arguments are null or equal. + */ + protected static boolean equals(Object lhs, Object rhs) { + return !(lhs == null && rhs != null) && + !(lhs != null && rhs == null) && + ((lhs == null || lhs.equals(rhs))); + } + + /** + * This function needs to be implemented in such a way that it visits all its members. This is done by invoking the + * {@link com.yahoo.vespa.objects.ObjectVisitor#visit(String, Object)} on the visitor argument for all members. + * + * @param visitor The visitor that is to access the member data. + */ + public void visitMembers(ObjectVisitor visitor) { + visitor.visit("classId", getClassId()); + } + + /** + * This class implements the class registry used by {@link Identifiable} to allow for creation of classes from + * shared class identifiers. It's methods are proxied through {@link Identifiable#registerClass(int, Class)}, {@link + * Identifiable#createFromId(int)} and {@link Identifiable#create(Deserializer)}. + */ + private static class Registry { + + // The map from class id to class descriptor. + private HashMap<Integer, Pair<Class<? extends Identifiable>, Constructor<? extends Identifiable>>> typeMap = + new HashMap<>(); + + /** + * Adds an entry in the type map, pairing the given identifier with the given class specification. + * + * @param id The class identifier to register with. + * @param spec The class to register. + * @throws IllegalArgumentException Thrown if two classes attempt to register with the same identifier. + */ + private void add(int id, Class<? extends Identifiable> spec) { + Class<?> old = get(id); + if (old == null) { + Constructor<? extends Identifiable> constructor; + try { + constructor = spec.getConstructor(); + } catch (NoSuchMethodException e) { + constructor = null; + } + typeMap.put(id, new Pair<Class<? extends Identifiable>, Constructor<? extends Identifiable>>(spec, constructor)); + } else if (!spec.equals(old)) { + throw new IllegalArgumentException("Can not register class '" + spec.toString() + "' with id " + id + + ", because it already maps to class '" + old.toString() + "'."); + } + } + + /** + * Returns the class registered for the given identifier. + * + * @param id The identifer whose class to return. + * @return The class specification, may be null. + */ + private Class<? extends Identifiable> get(int id) { + Pair<Class<? extends Identifiable>, Constructor<? extends Identifiable>> pair = typeMap.get(id); + return (pair != null) ? pair.getFirst() : null; + } + + /** + * Creates an instance of the class mapped to by the given identifier. This method proxies {@link + * #createFromClass(Constructor)}. + * + * @param id The id of the class to create. + * @return The instantiated object. + */ + private Identifiable createFromId(int id) { + Pair<Class<? extends Identifiable>, Constructor<? extends Identifiable>> pair = typeMap.get(id); + return createFromClass((pair != null) ? pair.getSecond() : null); + } + + /** + * Creates an instance of a given class specification. All instantiation-type exceptions are consumed and + * wrapped inside a runtime exception so that calling methods can let this propagate without declaring them + * thrown. + * + * @param spec The class to instantiate. + * @return The instantiated object. + * @throws IllegalArgumentException Thrown if instantiation failed. + */ + private Identifiable createFromClass(Constructor<? extends Identifiable> spec) { + Identifiable obj = null; + if (spec != null) { + try { + obj = spec.newInstance(); + } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) { + throw new IllegalArgumentException("Failed to create object from class '" + + spec.getName() + "'.", e); + } + } + return obj; + } + } + + protected String getUtf8(Deserializer buf) { + int len = buf.getInt(null); + byte[] arr = buf.getBytes(null, len); + return Utf8.toString(arr); + } + + protected void putUtf8(Serializer buf, String val) { + byte[] raw = Utf8.toBytes(val); + buf.putInt(null, raw.length); + buf.put(null, raw); + } +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Ids.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Ids.java new file mode 100644 index 00000000000..85647c58744 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Ids.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +/** + * This is a class containing the global ids that are given out. + * Must be in sync with version for c++ in staging_vespalib/src/vespalib/objects/ids.h + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public interface Ids { + public static int document = 0x1000; + public static int searchlib = 0x4000; + public static int vespa_configmodel = 0x7000; + public static int annotation = 0x10000; +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectDumper.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectDumper.java new file mode 100755 index 00000000000..42c9a09550d --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectDumper.java @@ -0,0 +1,134 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +import com.yahoo.vespa.objects.ObjectVisitor; + +import java.lang.reflect.Array; +import java.util.List; + +/** + * This is a concrete object visitor that will build up a structured human-readable string representation of an object. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ObjectDumper extends ObjectVisitor { + + // The current string being written to. + private final StringBuilder str = new StringBuilder(); + + // The number of spaces to indent each level. + private final int indent; + + // The current indent level. + private int currIndent = 0; + + /** + * Create an object dumper with the default indent size. + */ + public ObjectDumper() { + this(4); + } + + /** + * Create an object dumper with the given indent size. + * + * @param indent indent size in number of spaces + */ + public ObjectDumper(int indent) { + this.indent = indent; + } + + /** + * Add a number of spaces equal to the current indent to the string we are building. + */ + private void addIndent() { + int n = currIndent; + for (int i = 0; i < n; ++i) { + str.append(' '); + } + } + + /** + * Add a complete line of output. Appropriate indentation will be added before the given string and a newline will + * be added after it. + * + * @param line the line we want to add + */ + private void addLine(String line) { + addIndent(); + str.append(line); + str.append('\n'); + } + + /** + * Open a subscope by increasing the current indent level + */ + private void openScope() { + currIndent += indent; + } + + /** + * Close a subscope by decreasing the current indent level + */ + private void closeScope() { + currIndent -= indent; + } + + /** + * Obtain the created object string representation. This object should be invoked after the complete object + * structure has been visited. + * + * @return object string representation + */ + @Override + public String toString() { + return str.toString(); + } + + // Inherit doc from ObjectVisitor. + @Override + public void openStruct(String name, String type) { + if (name == null || name.isEmpty()) { + addLine(type + " {"); + } else { + addLine(name + ": " + type + " {"); + } + openScope(); + } + + // Inherit doc from ObjectVisitor. + @Override + public void closeStruct() { + closeScope(); + addLine("}"); + } + + // Inherit doc from ObjectVisitor. + @Override + public void visit(String name, Object obj) { + if (obj == null) { + addLine(name + ": <NULL>"); + } else if (obj instanceof Identifiable) { + openStruct(name, obj.getClass().getSimpleName()); + ((Identifiable)obj).visitMembers(this); + closeStruct(); + } else if (obj instanceof String) { + addLine(name + ": '" + obj + "'"); + } else if (obj.getClass().isArray()) { + openStruct(name, obj.getClass().getComponentType().getSimpleName() + "[]"); + for (int i = 0, len = Array.getLength(obj); i < len; ++i) { + visit("[" + i + "]", Array.get(obj, i)); + } + closeStruct(); + } else if (obj instanceof List) { + openStruct(name, "List"); + List<?> lst = (List<?>) obj; + for (int i = 0; i < lst.size(); ++i) { + visit("[" + i + "]", lst.get(i)); + } + closeStruct(); + } else { + addLine(name + ": " + obj); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectOperation.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectOperation.java new file mode 100755 index 00000000000..7e652aa588c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectOperation.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +/** + * An operation that is able to operate on a generic object. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface ObjectOperation { + + /** + * Apply this operation to the given object. + * + * @param obj The object to operate on. + */ + public void execute(Object obj); +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectPredicate.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectPredicate.java new file mode 100755 index 00000000000..adc918ae696 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectPredicate.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +/** + * A predicate that is able to say either true or false when presented with a + * generic object. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface ObjectPredicate { + + /** + * Apply this predicate to the given object. + * + * @param obj The object to check. + * @return True or false. + */ + public boolean check(Object obj); +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectVisitor.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectVisitor.java new file mode 100755 index 00000000000..07c8e90f4b7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectVisitor.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +/** + * This is an abstract class used to visit structured objects. It contains a basic interface that is intended to be + * overridden by subclasses. As an extension to this class, the visit.hpp file contains various versions of the visit + * method that maps visitation of various types into invocations of the basic interface defined by this class. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class ObjectVisitor { + + /** + * Open a (sub-)structure + * + * @param name name of structure + * @param type type of structure + */ + public abstract void openStruct(String name, String type); + + /** + * Close a (sub-)structure + */ + public abstract void closeStruct(); + + /** + * Visits some object. + * + * @param name variable name + * @param obj object to visit + */ + public abstract void visit(String name, Object obj); +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Selectable.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Selectable.java new file mode 100644 index 00000000000..a49d09a212b --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Selectable.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * + * This class acts as an interface for traversing a tree, or a graph. + * Every non leaf Object implements {@link #selectMembers(ObjectPredicate, ObjectOperation)} implementing + * the actual traversal. You can then implement an {@link ObjectPredicate} to select which nodes you want to look at with + * your {@link ObjectOperation} + */ +public class Selectable { + + /** + * Apply the predicate to this object. If the predicate returns true, pass this object to the operation, otherwise + * invoke the {@link #selectMembers(ObjectPredicate, ObjectOperation)} method to locate sub-elements that might + * trigger the predicate. + * + * @param predicate component used to select (sub-)objects + * @param operation component performing some operation on the selected (sub-)objects + */ + public final void select(ObjectPredicate predicate, ObjectOperation operation) { + if (predicate.check(this)) { + operation.execute(this); + } else { + selectMembers(predicate, operation); + } + } + + /** + * Invoke {@link #select(ObjectPredicate, ObjectOperation)} on any member objects this object wants to expose + * through the selection mechanism. Overriding this method is optional, and which objects to expose is determined by + * the application logic of the object itself. + * + * @param predicate component used to select (sub-)objects + * @param operation component performing some operation on the selected (sub-)objects + */ + public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { + // empty + } + + public static void select(Selectable selectable, ObjectPredicate predicate, ObjectOperation operation) { + if (selectable != null) { + selectable.select(predicate, operation); + } + } +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Serializer.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Serializer.java new file mode 100644 index 00000000000..a50252fc70c --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Serializer.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +import java.nio.ByteBuffer; + +/** + * @author balder + */ +public interface Serializer { + Serializer putByte(FieldBase field, byte value); + Serializer putShort(FieldBase field, short value); + Serializer putInt(FieldBase field, int value); + Serializer putLong(FieldBase field, long value); + Serializer putFloat(FieldBase field, float value); + Serializer putDouble(FieldBase field, double value); + Serializer put(FieldBase field, byte[] value); + Serializer put(FieldBase field, ByteBuffer value); + Serializer put(FieldBase field, String value); +} diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/package-info.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/package-info.java new file mode 100644 index 00000000000..bb4b11d182a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.vespa.objects; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespajlib/src/test/java/com/yahoo/binaryprefix/BinaryScaledAmountTestCase.java b/vespajlib/src/test/java/com/yahoo/binaryprefix/BinaryScaledAmountTestCase.java new file mode 100644 index 00000000000..bfb36ec80ce --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/binaryprefix/BinaryScaledAmountTestCase.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.binaryprefix; + +import junit.framework.TestCase; + +/** + * @author tonytv + */ +public class BinaryScaledAmountTestCase extends TestCase { + public void testConversion() { + BinaryScaledAmount oneMeg = new BinaryScaledAmount(1024, BinaryPrefix.kilo); + + assertEquals(1, oneMeg.as(BinaryPrefix.mega)); + assertEquals(1024, oneMeg.as(BinaryPrefix.kilo)); + assertEquals(1024*1024, oneMeg.as(BinaryPrefix.unit)); + assertEquals(1 << 20, oneMeg.hashCode()); + + Object v = this; + assertEquals(false, oneMeg.equals(v)); + v = new BinaryScaledAmount(1, BinaryPrefix.mega); + assertEquals(true, oneMeg.equals(v)); + } + + public void testSymbols() { + BinaryScaledAmount oneMeg = new BinaryScaledAmount(1024, BinaryPrefix.kilo); + + assertEquals(1, oneMeg.as(BinaryPrefix.fromSymbol('M'))); + assertEquals(1024, oneMeg.as(BinaryPrefix.fromSymbol('K'))); + + boolean ex = false; + try { + BinaryPrefix invalid = BinaryPrefix.fromSymbol('q'); + } catch (RuntimeException e) { + ex = true; + } + assertEquals(true, ex); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/cache/CacheTestCase.java b/vespajlib/src/test/java/com/yahoo/cache/CacheTestCase.java new file mode 100644 index 00000000000..992974eeb63 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/cache/CacheTestCase.java @@ -0,0 +1,197 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.cache; + +import junit.framework.TestCase; + +import java.util.ArrayList; +import java.util.Collection; + +public class CacheTestCase extends TestCase { + + public void testBasicGet() { + Cache<String, String> cache = new Cache<>(100 * 1024 * 1024, 3600, 10000); + String q = "/std_xmls_a00?hits=5&offset=5&query=flowers+shop&tracelevel=4&objid=ffffffffffffffff"; + String q2 = "/std_xmls_a00?hits=5&offset=5&query=flowers+shop&tracelevel=4&objid=ffffffffffffffff"; + String r = "result"; + String r2 = "result2"; + assertNull(cache.get(q)); + cache.put(q, r); + assertNotNull(cache.get(q)); + assertEquals(cache.get(q), r); + cache.put(q2, r); + assertEquals(cache.get(q2), r); + cache.put(q, r2); + assertEquals(cache.get(q), r2); + } + + public void testPutTooLarge() { + byte[] tenMB = new byte[10*1024*1024]; + for (int i = 0 ; i <10*1024*1024 ; i++) { + tenMB[i]=127; + } + byte[] sevenMB = new byte[7*1024*1024]; + for (int i = 0 ; i <7*1024*1024 ; i++) { + sevenMB[i]=127; + } + Cache<String, byte[]> cache=new Cache<>(9*1024*1024,3600, 100*1024*1024); // 9 MB + assertFalse(cache.put("foo", tenMB)); + assertTrue(cache.put("foo", sevenMB)); + assertEquals(cache.get("foo"), sevenMB); + } + + public void testInvalidate() { + byte[] tenMB = new byte[10*1024*1024]; + for (int i = 0 ; i <10*1024*1024 ; i++) { + tenMB[i]=127; + } + byte[] sevenMB = new byte[7*1024*1024]; + for (int i = 0 ; i <7*1024*1024 ; i++) { + sevenMB[i]=127; + } + //log.info("10 MB: "+calc.sizeOf(tenMB)); + //log.info("7 MB: "+calc.sizeOf(sevenMB)); + Cache<String, byte[]> cache=new Cache<>(11*1024*1024,3600, 100*1024*1024); // 11 MB + assertTrue(cache.put("foo", sevenMB)); + assertTrue(cache.put("bar", tenMB)); + assertNull(cache.get("foo")); + assertEquals(cache.get("bar"), tenMB); + } + + public void testInvalidateLRU() { + Cache<String, byte[]> cache=new Cache<>(10*1024*1024,3600, 100*1024*1024); // 10 MB + byte[] fiveMB = new byte[5*1024*1024]; + for (int i = 0 ; i <5*1024*1024 ; i++) { + fiveMB[i]=127; + } + + byte[] twoMB = new byte[2*1024*1024]; + for (int i = 0 ; i <2*1024*1024 ; i++) { + twoMB[i]=127; + } + + byte[] fourMB = new byte[4*1024*1024]; + for (int i = 0 ; i <4*1024*1024 ; i++) { + fourMB[i]=127; + } + assertTrue(cache.put("five", fiveMB)); + assertTrue(cache.put("two", twoMB)); + Object dummy = cache.get("five"); // Makes two LRU + assertEquals(dummy, fiveMB); + assertTrue(cache.put("four", fourMB)); + assertNull(cache.get("two")); + assertEquals(cache.get("five"), fiveMB); + assertEquals(cache.get("four"), fourMB); + + // Same, without the access, just to check + cache=new Cache<>(10*1024*1024,3600, 100*1024*1024); // 10 MB + assertTrue(cache.put("five", fiveMB)); + assertTrue(cache.put("two", twoMB)); + assertTrue(cache.put("four", fourMB)); + assertEquals(cache.get("two"), twoMB); + assertNull(cache.get("five")); + assertEquals(cache.get("four"), fourMB); + } + + public void testPutSameKey() { + Cache<String, byte[]> cache=new Cache<>(10*1024*1024,3600, 100*1024*1024); // 10 MB + byte[] fiveMB = new byte[5*1024*1024]; + for (int i = 0 ; i <5*1024*1024 ; i++) { + fiveMB[i]=127; + } + + byte[] twoMB = new byte[2*1024*1024]; + for (int i = 0 ; i <2*1024*1024 ; i++) { + twoMB[i]=127; + } + + byte[] fourMB = new byte[4*1024*1024]; + for (int i = 0 ; i <4*1024*1024 ; i++) { + fourMB[i]=127; + } + assertTrue(cache.put("five", fiveMB)); + assertTrue(cache.put("two", twoMB)); + assertEquals(cache.get("two"), twoMB); + assertEquals(cache.get("five"), fiveMB); + assertTrue(cache.put("five", twoMB)); + assertEquals(cache.get("five"), twoMB); + assertEquals(cache.get("two"), twoMB); + } + + public void testExpire() throws InterruptedException { + Cache<String, String> cache=new Cache<>(10*1024*1024,400, 10000); // 10 MB, .4 sec expire + cache.put("foo", "bar"); + cache.put("hey", "ho"); + assertEquals(cache.get("foo"), "bar"); + assertEquals(cache.get("hey"), "ho"); + Thread.sleep(600); + assertNull(cache.get("foo")); + assertNull(cache.get("hey")); + } + + public void testInsertSame() { + Cache<String, String> cache=new Cache<>(10*1024*1024,500, 10000); // 10 MB, .5 sec expire + String k = "foo"; + String r = "bar"; + cache.put(k, r); + assertEquals(cache.size(), 1); + cache.put(k, r); + assertEquals(cache.size(), 1); + } + + public void testMaxSize() { + Cache<String, byte[]> cache=new Cache<>(20*1024*1024,500, 3*1024*1024); + byte[] fourMB = new byte[4*1024*1024]; + for (int i = 0 ; i <4*1024*1024 ; i++) { + fourMB[i]=127; + } + byte[] twoMB = new byte[2*1024*1024]; + for (int i = 0 ; i <2*1024*1024 ; i++) { + twoMB[i]=127; + } + assertFalse(cache.put("four", fourMB)); + assertTrue(cache.put("two", twoMB)); + assertNull(cache.get("four")); + assertNotNull(cache.get("two")); + } + + public void testMaxSizeNoLimit() { + Cache<String, byte[]> cache=new Cache<>(20*1024*1024,500, -1); + byte[] fourMB = new byte[4*1024*1024]; + for (int i = 0 ; i <4*1024*1024 ; i++) { + fourMB[i]=127; + } + byte[] twoMB = new byte[2*1024*1024]; + for (int i = 0 ; i <2*1024*1024 ; i++) { + twoMB[i]=127; + } + assertTrue(cache.put("four", fourMB)); + assertTrue(cache.put("two", twoMB)); + assertNotNull(cache.get("four")); + assertNotNull(cache.get("two")); + } + + public void testGetKeysAndValuesAndClear() { + Cache<String, String> cache=new Cache<>(10*1024*1024,500, 10000); // 10 MB, .5 sec expire + assertEquals(cache.getKeys().size(), 0); + assertEquals(cache.getValues().size(), 0); + cache.put("a", "b"); + cache.put("c", "d"); + cache.put("e", "f"); + Collection<String> keys = new ArrayList<>(); + keys.add("a"); + keys.add("c"); + keys.add("e"); + Collection<String> values = new ArrayList<>(); + values.add("b"); + values.add("d"); + values.add("f"); + assertEquals(cache.getKeys().size(), 3); + assertEquals(cache.getValues().size(), 3); + assertTrue(cache.getKeys().containsAll(keys)); + assertTrue(cache.getValues().containsAll(values)); + cache.clear(); + assertEquals(cache.getKeys().size(), 0); + assertEquals(cache.getValues().size(), 0); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/cache/CalcTestCase.java b/vespajlib/src/test/java/com/yahoo/cache/CalcTestCase.java new file mode 100644 index 00000000000..fbec483debb --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/cache/CalcTestCase.java @@ -0,0 +1,176 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.cache; + +import java.util.ArrayList; +import java.util.List; + +public class CalcTestCase extends junit.framework.TestCase { + + private SizeCalculator calc; + + + public CalcTestCase (String name) { + super(name); + } + + public void setUp() { + calc = new SizeCalculator(); + } + + public void testCalc1() { + assertEquals(calc.sizeOf(new Object()), 8); + } + + public void testCalc2() { + assertEquals(calc.sizeOf(new SixtyFourBooleans()), 8+64); + } + + public void testBoolean() { + assertEquals(8+1, calc.sizeOf(new Boolean(true))); + } + + public void testArrayPrimitive() { + byte[] eightBytes = new byte[]{1,1,1,1,1,1,1,1,}; + assertEquals(16+8, calc.sizeOf(eightBytes)); + } + + public void testArrayObjects() { + SixtyFourBooleans[] bunchOfBooleans = new SixtyFourBooleans[]{new SixtyFourBooleans(), + new SixtyFourBooleans(), new SixtyFourBooleans()}; + assertEquals(16+(3*(8+64)+(3*4)), calc.sizeOf(bunchOfBooleans)); + + } + + public void testSizeOfList() { + SixtyFourBooleans sfb = new SixtyFourBooleans(); + List<Object> dupList1 = new ArrayList<>(); + dupList1.add(new Object()); + dupList1.add(sfb); + dupList1.add(sfb); + dupList1.add(sfb); + List<Object> dupList2 = new ArrayList<>(); + dupList2.addAll(dupList1); + dupList2.add(sfb); + dupList2.add(sfb); + dupList2.add(sfb); + dupList2.add(new Object()); + dupList2.add(new Object()); + assertEquals(calc.sizeOf(dupList2), calc.sizeOf(dupList1)+8+8); + } + + public void testSizeOfTuple() { + SixtyFourBooleans[] bunchOfBooleans = new SixtyFourBooleans[]{new SixtyFourBooleans(), + new SixtyFourBooleans(), new SixtyFourBooleans()}; + SixtyFourBooleans[] bunchOfBooleans2 = new SixtyFourBooleans[]{new SixtyFourBooleans(), + new SixtyFourBooleans(), new SixtyFourBooleans()}; + assertEquals(16+(3*(8+64)+(3*4)), calc.sizeOf(bunchOfBooleans)); + assertEquals(2* (16+(3*(8+64)+(3*4))), calc.sizeOf(bunchOfBooleans, bunchOfBooleans2)); + } + + /*public void testEmptyArrayList() { + assertEquals(80, calc.sizeOf(new ArrayList())); + }*/ + + /*public void testFullArrayList() { + ArrayList arrayList = new ArrayList(10000); + + for (int i = 0; i < 10000; i++) { + arrayList.add(new Object()); + } + + assertEquals(120040, calc.sizeOf(arrayList)); + }*/ + + /*public void testHashMap() { + assertEquals(120, calc.sizeOf(new HashMap())); + + Byte[] all = new Byte[256]; + for (int i = -128; i < 128; i++) { + all[i + 128] = new Byte((byte) i); + } + assertEquals(5136, calc.sizeOf(all)); + + HashMap hm = new HashMap(); + for (int i = -128; i < 128; i++) { + hm.put("" + i, new Byte((byte) i)); + } + assertEquals(30776, calc.sizeOf(hm)); + }*/ + + /*public void testThousandBooleansObjects() { + Boolean[] booleans = new Boolean[1000]; + + for (int i = 0; i < booleans.length; i++) + booleans[i] = new Boolean(true); + + assertEquals(20016, calc.sizeOf(booleans)); + }*/ + + @SuppressWarnings("unused") + private static class SixtyFourBooleans { + boolean a0; + boolean a1; + boolean a2; + boolean a3; + boolean a4; + boolean a5; + boolean a6; + boolean a7; + boolean b0; + boolean b1; + boolean b2; + boolean b3; + boolean b4; + boolean b5; + boolean b6; + boolean b7; + boolean c0; + boolean c1; + boolean c2; + boolean c3; + boolean c4; + boolean c5; + boolean c6; + boolean c7; + boolean d0; + boolean d1; + boolean d2; + boolean d3; + boolean d4; + boolean d5; + boolean d6; + boolean d7; + boolean e0; + boolean e1; + boolean e2; + boolean e3; + boolean e4; + boolean e5; + boolean e6; + boolean e7; + boolean f0; + boolean f1; + boolean f2; + boolean f3; + boolean f4; + boolean f5; + boolean f6; + boolean f7; + boolean g0; + boolean g1; + boolean g2; + boolean g3; + boolean g4; + boolean g5; + boolean g6; + boolean g7; + boolean h0; + boolean h1; + boolean h2; + boolean h3; + boolean h4; + boolean h5; + boolean h6; + boolean h7; + } +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/ArraySetTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/ArraySetTestCase.java new file mode 100644 index 00000000000..5bb15f57420 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/ArraySetTestCase.java @@ -0,0 +1,303 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import org.junit.Test; + +/** + * Check ArraySet seems to work. :) + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class ArraySetTestCase { + + @Test + public void testAdd() { + final String string = "abc"; + final String a = new String(string); + final String b = new String(string); + final ArraySet<String> t = new ArraySet<>(3); + t.add(a); + t.add(b); + assertEquals(1, t.size()); + t.add(string); + assertEquals(1, t.size()); + t.add("abd"); + t.add("abc"); + t.add("abd"); + assertEquals(2, t.size()); + + } + + @Test + public void testAddAll() { + final List<String> stuff = doubleAdd(); + final ArraySet<String> t = new ArraySet<>( + stuff.size()); + t.addAll(stuff); + assertEquals(stuff.size() / 2, t.size()); + } + + private List<String> doubleAdd() { + final List<String> stuff = new ArrayList<>(); + stuff.add("abc"); + stuff.add("abd"); + stuff.add("abe"); + stuff.add("abc"); + stuff.add("abd"); + stuff.add("abe"); + return stuff; + } + + @Test + public void testContains() { + final String string = "abc"; + final String a = new String(string); + final String b = new String(string); + final ArraySet<String> t = new ArraySet<>(2); + t.add(string); + t.add(a); + assertEquals(1, t.size()); + assertTrue(t.contains(a)); + assertTrue(t.contains(string)); + assertTrue(t.contains(b)); + } + + @Test + public void testContainsAll() { + final String string = "abc"; + final String a = new String(string); + final String b = new String(string); + final String c = "c"; + final List<String> stuff = new ArrayList<>(); + stuff.add(string); + stuff.add(a); + stuff.add(b); + final ArraySet<String> t = new ArraySet<>( + stuff.size()); + t.addAll(stuff); + assertTrue(t.containsAll(stuff)); + stuff.add(c); + assertFalse(t.containsAll(stuff)); + } + + @Test + public void testRemove() { + final String string = "abc"; + final String a = new String(string); + final String b = new String(string); + final ArraySet<String> t = new ArraySet<>(2); + t.add("abc"); + t.add("abd"); + t.add("abe"); + assertEquals(3, t.size()); + assertFalse(t.remove("ab")); + assertTrue(t.remove("abd")); + assertFalse(t.remove("abd")); + assertEquals(2, t.size()); + assertTrue(t.remove("abe")); + assertFalse(t.remove("abe")); + assertTrue(t.remove("abc")); + assertTrue(t.isEmpty()); + } + + @Test + public void testRetainAll() { + final List<String> stuff = doubleAdd(); + final ArraySet<String> t = new ArraySet<>( + stuff.size()); + t.addAll(stuff); + assertFalse(t.retainAll(stuff)); + assertEquals(stuff.size() / 2, t.size()); + t.add("nalle"); + assertEquals(stuff.size() / 2 + 1, t.size()); + assertTrue(t.retainAll(stuff)); + assertEquals(stuff.size() / 2, t.size()); + } + + @Test + public void testToArrayTArray() { + final List<String> stuff = doubleAdd(); + final ArraySet<String> t = new ArraySet<>( + stuff.size()); + t.addAll(stuff); + final String[] s = t.toArray(new String[0]); + assertEquals(t.size(), s.length); + assertEquals(stuff.size() / 2, s.length); + } + + @Test + public void testGrow() { + final ArraySet<Integer> t = new ArraySet<>(5); + final int targetSize = 100; + for (int i = 0; i < targetSize; ++i) { + t.add(i); + } + assertEquals(targetSize, t.size()); + int n = 0; + for (final Iterator<Integer> i = t.iterator(); i.hasNext();) { + assertEquals(Integer.valueOf(n++), i.next()); + } + assertEquals(targetSize, n); + } + + @Test + public void testBiggerRemoveAll() { + final int targetSize = 100; + final ArraySet<Integer> t = new ArraySet<>( + targetSize); + final Integer[] instances = new Integer[targetSize]; + final List<Integer> remove = buildSubSet(targetSize, t, instances); + t.removeAll(remove); + assertEquals(targetSize / 2, t.size()); + for (final Iterator<Integer> i = t.iterator(); i.hasNext();) { + final Integer n = i.next(); + assertTrue(n % 2 == 0); + assertFalse(remove.contains(n)); + + } + } + + @Test + public void testBiggerRetainAll() { + final int targetSize = 100; + final ArraySet<Integer> t = new ArraySet<>( + targetSize); + final Integer[] instances = new Integer[targetSize]; + final List<Integer> retain = buildSubSet(targetSize, t, instances); + t.retainAll(retain); + assertEquals(targetSize / 2, t.size()); + for (final Iterator<Integer> i = t.iterator(); i.hasNext();) { + final Integer n = i.next(); + assertTrue(n % 2 != 0); + assertTrue(retain.contains(n)); + } + } + + private List<Integer> buildSubSet(final int targetSize, + final ArraySet<Integer> t, final Integer[] instances) { + for (int i = 0; i < targetSize; ++i) { + instances[i] = Integer.valueOf(i); + t.add(instances[i]); + } + final List<Integer> subset = new ArrayList<>(50); + for (int i = 0; i < targetSize; ++i) { + if (i % 2 != 0) { + subset.add(instances[i]); + } + } + return subset; + } + + @Test + public void testMuckingAbout() { + final int targetSize = 100; + final ArraySet<Integer> t = new ArraySet<>(3); + final Integer[] instances = new Integer[targetSize]; + final List<Integer> retain = buildSubSet(targetSize, t, instances); + for (final Integer n : retain) { + t.remove(n); + assertEquals(targetSize - 1, t.size()); + t.add(n); + assertEquals(targetSize, t.size()); + } + assertEquals(targetSize, t.size()); + final Integer[] contents = t.toArray(new Integer[0]); + Arrays.sort(contents, 0, targetSize); + for (int i = 0; i < targetSize; ++i) { + assertEquals(instances[i], contents[i]); + } + } + + @Test + public void testMoreDuplicates() { + final int targetSize = 100; + final ArraySet<Integer> t = new ArraySet<>(3); + final Integer[] instances = new Integer[targetSize]; + final List<Integer> add = buildSubSet(targetSize, t, instances); + assertEquals(targetSize, t.size()); + t.addAll(add); + assertEquals(targetSize, t.size()); + } + + @Test + public void testEmptySet() { + final int targetSize = 100; + final ArraySet<Integer> t = new ArraySet<>(0); + final Integer[] instances = new Integer[targetSize]; + final List<Integer> add = buildSubSet(targetSize, t, instances); + for (Integer i : instances) { + t.remove(i); + } + assertEquals(0, t.size()); + for (Integer i : add) { + t.add(i); + } + assertEquals(targetSize / 2, t.size()); + } + + @Test + public void testSmallEmptySet() { + final ArraySet<Integer> t = new ArraySet<>(3); + Integer a = new Integer(0), b = new Integer(1), c = new Integer(2); + t.add(a); + t.add(b); + t.add(c); + assertEquals(3, t.size()); + t.remove(a); + assertEquals(2, t.size()); + t.remove(c); + assertEquals(1, t.size()); + t.remove(c); + assertEquals(1, t.size()); + t.remove(b); + assertEquals(0, t.size()); + t.add(b); + assertEquals(1, t.size()); + t.add(b); + assertEquals(1, t.size()); + t.add(a); + assertEquals(2, t.size()); + t.add(a); + assertEquals(2, t.size()); + t.add(c); + assertEquals(3, t.size()); + t.add(c); + assertEquals(3, t.size()); + } + + @Test + public void testIterator() { + final int targetSize = 100; + final ArraySet<Integer> t = new ArraySet<>(0); + final Integer[] instances = new Integer[targetSize]; + final List<Integer> remove = buildSubSet(targetSize, t, instances); + int traversed = 0; + for (Iterator<Integer> i = t.iterator(); i.hasNext();) { + Integer n = i.next(); + if (remove.contains(n)) { + i.remove(); + } + ++traversed; + } + assertEquals(targetSize, traversed); + assertEquals(targetSize / 2, t.size()); + for (int i = 0; i < instances.length; ++i) { + Integer n = instances[i]; + if (remove.contains(n)) { + assertFalse(t.contains(n)); + } else { + assertTrue(t.contains(n)); + } + } + } +} + diff --git a/vespajlib/src/test/java/com/yahoo/collections/BobHashTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/BobHashTestCase.java new file mode 100644 index 00000000000..820adffcfc5 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/BobHashTestCase.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + + +import com.yahoo.collections.BobHash; + + +/** + * Basic consistency check of BobHash implementation + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class BobHashTestCase extends junit.framework.TestCase { + + public BobHashTestCase(String name) { + super(name); + } + + public void testit() { + // Teststring: minprice + // Basic ASCII string + byte[] minprice = { 109, 105, 110, 112, 114, 105, 99, 101 }; + + assertEquals(BobHash.hash(minprice, 0), 0x90188543); + // Teststring: a\u00FFa\u00FF + // String with non-ASCII characters + byte[] ayay = { 97, -1, 97, -1 }; + + assertEquals(BobHash.hash(ayay, 0), 0x1C798331); + // lots of a's to ensure testing unsigned type emulation + byte[] aa = { + 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, + 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97, 97 }; + + assertEquals(BobHash.hash(aa, 0), 0xE09ED5E9); + // A string which caused problems during developmen of another + // feature + byte[] lastnamefirstinitial = { + 0x6c, 0x61, 0x73, 0x74, 0x6e, 0x61, 0x6d, + 0x65, 0x66, 0x69, 0x72, 0x73, 0x74, 0x69, 0x6e, 0x69, 0x74, 0x69, + 0x61, 0x6c }; + + assertEquals(BobHash.hash(lastnamefirstinitial, 0), 0xF36B4BD3); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/ByteArrayComparatorTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/ByteArrayComparatorTestCase.java new file mode 100644 index 00000000000..68b5812dd1e --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/ByteArrayComparatorTestCase.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class ByteArrayComparatorTestCase { + @Test + public void arrayLength() { + byte[] shortArr = new byte[]{(byte) 1, (byte) 2}; + byte[] longArr = new byte[]{(byte) 0, (byte) 3, (byte) 3, (byte) 3, (byte) 3, (byte) 3}; + + assertEquals(-1, ByteArrayComparator.compare(shortArr, longArr)); + } + + @Test + public void compareArrays() { + byte[] one = new byte[]{(byte) 1, (byte) 2, (byte) 3, (byte) 3, (byte) 3, (byte) 3}; + byte[] two = new byte[]{(byte) 0, (byte) 3, (byte) 3, (byte) 3, (byte) 3, (byte) 3}; + + assertEquals(1, ByteArrayComparator.compare(one, two)); + assertEquals(-1, ByteArrayComparator.compare(two, one)); + } + + @Test + public void compareEqualArrays() { + byte[] one = new byte[]{(byte) 1, (byte) 2, (byte) 3, (byte) 3, (byte) 3, (byte) 3, (byte) 9}; + byte[] two = new byte[]{(byte) 1, (byte) 2, (byte) 3, (byte) 3, (byte) 3, (byte) 3, (byte) 9}; + + assertEquals(0, ByteArrayComparator.compare(one, two)); + assertEquals(0, ByteArrayComparator.compare(two, one)); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/CollectionComparatorTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/CollectionComparatorTestCase.java new file mode 100644 index 00000000000..d77636b907f --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/CollectionComparatorTestCase.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class CollectionComparatorTestCase { + @Test + public void arrayLength() { + List<String> shortArr = Arrays.asList("x", "y"); + List<String> longArr = Arrays.asList("a", "b", "c", "d", "e"); + + assertEquals(-1, CollectionComparator.compare(shortArr, longArr)); + } + + @Test + public void compareArrays() { + List<String> one = Arrays.asList("b", "c", "d", "d", "e"); + List<String> two = Arrays.asList("a", "b", "c", "d", "e"); + + assertEquals(1, CollectionComparator.compare(one, two)); + assertEquals(-1, CollectionComparator.compare(two, one)); + } + + @Test + public void compareEqualArrays() { + List<String> one = Arrays.asList("a", "b", "c", "d", "e"); + List<String> two = Arrays.asList("a", "b", "c", "d", "e"); + + assertEquals(0, CollectionComparator.compare(one, two)); + assertEquals(0, CollectionComparator.compare(two, one)); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/CollectionUtilTest.java b/vespajlib/src/test/java/com/yahoo/collections/CollectionUtilTest.java new file mode 100644 index 00000000000..c5a20a4684c --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/CollectionUtilTest.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author tonytv + */ +public class CollectionUtilTest { + List<Integer> l1 = Arrays.asList(1, 2, 4, 5, 6, 7); + List<Integer> l2 = Arrays.asList(3, 4, 5, 6, 7); + + @Before + public void shuffle() { + Collections.shuffle(l1); + Collections.shuffle(l2); + } + + @Test + public void testMkString() { + assertEquals("1, 2, 3, 4", + CollectionUtil.mkString(Arrays.asList(1, 2, 3, 4), ", ")); + } + + @Test + public void testEqualContentsIgnoreOrder() { + List<Integer> l2Copy = new ArrayList<>(); + l2Copy.addAll(l2); + shuffle(); + assertTrue(CollectionUtil.equalContentsIgnoreOrder( + l2, l2Copy)); + assertFalse(CollectionUtil.equalContentsIgnoreOrder( + l1, l2)); + } + + @Test + public void testSymmetricDifference() { + assertTrue(CollectionUtil.equalContentsIgnoreOrder( + Arrays.asList(1, 2, 3), + CollectionUtil.symmetricDifference(l1, l2))); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/CollectionsBenchMark.java b/vespajlib/src/test/java/com/yahoo/collections/CollectionsBenchMark.java new file mode 100644 index 00000000000..51cdd11bb7d --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/CollectionsBenchMark.java @@ -0,0 +1,204 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Created by balder on 1/20/14. + */ +public class CollectionsBenchMark { + abstract static class BenchMark { + protected BenchMark(int numWarmup, int repetitions) { + this.numWarmup = numWarmup; + this.repetitions = repetitions; + } + abstract void runOnce(); + abstract String getEndComment(); + protected void run() { + System.out.println("Starting benchmark warmup '" + getClass().getName() + "'."); + for (int i=0; i < numWarmup; i++) { + runOnce(); + } + System.out.println("Starting benchmark '" + getClass().getName() + "'."); + long startTime=System.currentTimeMillis(); + for (int i=0; i < repetitions; i++) { + runOnce(); + } + long endTime=System.currentTimeMillis(); + long totalTime=(endTime-startTime); + System.out.println("Done in " + totalTime + " ms (" + ((float) totalTime * 1000 / repetitions + " microsecond per repetition.)")); // *2 because we do 2 gets + System.out.println("Final remark: " + getEndComment()); + } + + + final private int repetitions; + final private int numWarmup; + } + + static class MapFilterBenchMark extends BenchMark { + MapFilterBenchMark(Map<Integer, Integer> s, int numWarmup, int numRepetitions, int numObjects) { + super(numWarmup, numRepetitions); + this.s = s; + objects = new Integer[numObjects]; + for (int i=0; i < numObjects; i++) { + objects[i] = i; + } + } + void runOnce() { + for (Integer o : objects) { + if (s.put(o, o) == null) { + uniqueCount += o; + } + } + } + String getEndComment() { return " Unique sum is '" + uniqueCount + "'"; } + private final Map<Integer,Integer> s; + final Integer [] objects; + long uniqueCount = 0; + } + + static class SetFilterBenchMark extends BenchMark { + SetFilterBenchMark(Set<Integer> s, int numWarmup, int numRepetitions, int numObjects) { + super(numWarmup, numRepetitions); + this.s = s; + objects = new Integer[numObjects]; + for (int i=0; i < numObjects; i++) { + objects[i] = i; + } + } + void runOnce() { + for (Integer o : objects) { + if ( s.add(o) ) { + uniqueCount += o; + } + } + } + String getEndComment() { return " Unique sum is '" + uniqueCount + "'"; } + private final Set<Integer> s; + final Integer [] objects; + long uniqueCount = 0; + } + + static abstract class SmallMapsBenchMark extends BenchMark { + SmallMapsBenchMark(int numWarmup, int numRepetitions, int numObjects, int numUnique) { + super(numWarmup, numRepetitions); + objects = new Integer[numObjects]; + for (int i=0; i < numObjects; i++) { + objects[i] = i%numUnique; + } + } + void runOnce() { + Set<Integer> s = createSet(); + for (Integer o : objects) { + if ( s.add(o) ) { + uniqueCount += o; + } + } + } + abstract Set<Integer> createSet(); + String getEndComment() { return " Unique sum is '" + uniqueCount + "'"; } + final Integer [] objects; + long uniqueCount = 0; + } + + static class SmallHashSetBenchMark extends SmallMapsBenchMark + { + SmallHashSetBenchMark(int numWarmup, int numRepetitions, int numObjects, int numUnique) { + super(numWarmup, numRepetitions, numObjects, numUnique); + } + Set<Integer> createSet() { return new HashSet<Integer>();} + } + + static class SmallLazySetBenchMark extends SmallMapsBenchMark + { + SmallLazySetBenchMark(int numWarmup, int numRepetitions, int numObjects, int numUnique) { + super(numWarmup, numRepetitions, numObjects, numUnique); + } + Set<Integer> createSet() { return new LazySet<Integer>() { + @Override + protected Set<Integer> newDelegate() { + return new HashSet<Integer>(); + } + }; + } + } + + static class SmallLazyTinyBenchMark extends SmallMapsBenchMark + { + SmallLazyTinyBenchMark(int numWarmup, int numRepetitions, int numObjects, int numUnique) { + super(numWarmup, numRepetitions, numObjects, numUnique); + } + Set<Integer> createSet() { return new LazySet<Integer>() { + @Override + protected Set<Integer> newDelegate() { + return new TinyIdentitySet<Integer>(10); + } + }; + } + } + + static void benchMarkAll() { + + new MapFilterBenchMark(new HashMap<Integer, Integer>(), 100000, 10000000, 10).run(); + new MapFilterBenchMark(new IdentityHashMap<Integer, Integer>(10), 100000, 10000000, 10).run(); + new SetFilterBenchMark(new HashSet<Integer>(), 100000, 10000000, 10).run(); + new SetFilterBenchMark(new TinyIdentitySet<Integer>(10), 100000, 10000000, 10).run(); + new SetFilterBenchMark(new TinyIdentitySet<Integer>(10), 100000, 10000000, 15).run(); + new SetFilterBenchMark(new TinyIdentitySet<Integer>(10), 100000, 10000000, 20).run(); + new SetFilterBenchMark(new TinyIdentitySet<Integer>(20), 100000, 10000000, 20).run(); + new SmallHashSetBenchMark(100000, 10000000, 10, 1).run(); + new SmallLazySetBenchMark(100000, 10000000, 10, 1).run(); + new SmallHashSetBenchMark(100000, 10000000, 10, 2).run(); + new SmallLazySetBenchMark(100000, 10000000, 10, 2).run(); + new SmallLazyTinyBenchMark(100000, 10000000, 10, 2).run(); + new SmallHashSetBenchMark(100000, 10000000, 10, 10).run(); + new SmallLazySetBenchMark(100000, 10000000, 10, 10).run(); + + new SetFilterBenchMark(new HashSet<Integer>(), 100000, 10000000, 12).run(); + + new SetFilterBenchMark(new HashSet<Integer>(), 100000, 10000000, 20).run(); + new SetFilterBenchMark(new HashSet<Integer>(), 100000, 10000000, 25).run(); + new SetFilterBenchMark(new HashSet<Integer>(), 100000, 10000000, 30).run(); + new SmallHashSetBenchMark(100000, 10000000, 1, 1).run(); + + } + + static void benchMark() { + + //new MapFilterBenchMark(new HashMap<Integer, Integer>(), 100000, 10000000, 10).run(); + //new MapFilterBenchMark(new IdentityHashMap<Integer, Integer>(10), 100000, 10000000, 10).run(); + //new SetFilterBenchMark(new HashSet<Integer>(), 100000, 10000000, 10).run(); + //new SetFilterBenchMark(new TinyIdentitySet<Integer>(10), 100000, 10000000, 10).run(); + //new SmallHashSetBenchMark(100000, 10000000, 10, 1).run(); + //new SmallLazySetBenchMark(100000, 10000000, 10, 1).run(); + //new SmallHashSetBenchMark(100000, 10000000, 10, 2).run(); + //new SmallLazySetBenchMark(100000, 10000000, 10, 2).run(); + new SmallLazyTinyBenchMark(100000, 10000000, 10, 2).run(); + //new SmallHashSetBenchMark(100000, 10000000, 10, 10).run(); + //new SmallLazySetBenchMark(100000, 10000000, 10, 10).run(); + + //new SetFilterBenchMark(new HashSet<Integer>(), 100000, 10000000, 12).run(); + + //new SetFilterBenchMark(new HashSet<Integer>(), 100000, 10000000, 20).run(); + //new SetFilterBenchMark(new HashSet<Integer>(), 100000, 10000000, 25).run(); + //new SetFilterBenchMark(new HashSet<Integer>(), 100000, 10000000, 30).run(); + //new SmallHashSetBenchMark(100000, 10000000, 1, 1).run(); + + } + + + static public void main(String argv[]) { + benchMarkAll(); + ExecutorService tp = Executors.newFixedThreadPool(16); + + for (int i=0; i < 16; i++) { + tp.execute(new Runnable() { + @Override + public void run() { + benchMark(); + } + }); + } + } +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/CopyOnWriteHashMapTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/CopyOnWriteHashMapTestCase.java new file mode 100644 index 00000000000..4370a9b46b0 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/CopyOnWriteHashMapTestCase.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class CopyOnWriteHashMapTestCase { + + @Test + public void testModifySourceFirst() { + CopyOnWriteHashMap<String, String> map = new CopyOnWriteHashMap<>(); + map.put("a", "a1"); + map.put("b", "b1"); + CopyOnWriteHashMap<String,String> clone = map.clone(); + map.put("c", "c1"); + clone.remove("a"); + clone.put("b", "b2"); + clone.put("d", "d2"); + + assertEquals(3, map.size()); + assertEquals("a1", map.get("a")); + assertEquals("b1", map.get("b")); + assertEquals("c1", map.get("c")); + + assertEquals(2, clone.size()); + assertEquals("b2", clone.get("b")); + assertEquals("d2", clone.get("d")); + } + + @Test + public void testModifyTargetFirst() { + CopyOnWriteHashMap<String, String> map = new CopyOnWriteHashMap<>(); + map.put("a", "a1"); + map.put("b", "b1"); + CopyOnWriteHashMap<String,String> clone = map.clone(); + clone.remove("a"); + map.put("c", "c1"); + clone.put("b", "b2"); + clone.put("d", "d2"); + + assertEquals(3, map.size()); + assertEquals("a1", map.get("a")); + assertEquals("b1", map.get("b")); + assertEquals("c1", map.get("c")); + + assertEquals(2, clone.size()); + assertEquals("b2", clone.get("b")); + assertEquals("d2", clone.get("d")); + } + + @Test + public void testCallEntrySetThenModify() { + CopyOnWriteHashMap<String, String> map = new CopyOnWriteHashMap<>(); + map.put("a", "a1"); + map.entrySet(); + CopyOnWriteHashMap<String,String> clone = map.clone(); + clone.put("b", "b1"); + assertEquals(2, clone.size()); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/FreezableArrayListListener.java b/vespajlib/src/test/java/com/yahoo/collections/FreezableArrayListListener.java new file mode 100644 index 00000000000..762ae9d5b60 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/FreezableArrayListListener.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import org.junit.Test; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class FreezableArrayListListener { + + @Test + public void testPermitAdd() { + FreezableArrayList<String> l = new FreezableArrayList<>(true); + l.add("1"); + l.add("2"); + l.remove(1); + l.freeze(); + try { + l.remove(0); + fail("Expected exception"); + } + catch (UnsupportedOperationException expected) { + } + try { + l.set(0, "2"); + fail("Expected exception"); + } + catch (UnsupportedOperationException expected) { + } + try { + l.add(0, "2"); + fail("Expected exception"); + } + catch (UnsupportedOperationException expected) { + } + + l.add("2"); + } + + @Test + public void testDontPermitAdd() { + FreezableArrayList<String> l = new FreezableArrayList<>(); + l.add("1"); + l.add("2"); + l.remove(1); + l.freeze(); + try { + l.remove(0); + fail("Expected exception"); + } + catch (UnsupportedOperationException expected) { + } + try { + l.set(0, "2"); + fail("Expected exception"); + } + catch (UnsupportedOperationException expected) { + } + try { + l.add(0, "2"); + fail("Expected exception"); + } + catch (UnsupportedOperationException expected) { + } + try { + l.add("2"); + fail("Expected exception"); + } + catch (UnsupportedOperationException expected) { + } + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/HashletTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/HashletTestCase.java new file mode 100644 index 00000000000..1a198726994 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/HashletTestCase.java @@ -0,0 +1,186 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.*; + +public class HashletTestCase { + + @Test + public void testCopyEmptyHashlet() { + Hashlet<String, Integer> hash = new Hashlet<>(); + Hashlet<String, Integer> hash2 = new Hashlet<>(hash); + assertThat(hash.size(), is(0)); + assertThat(hash2.size(), is(0)); + hash.put("foo", 5); + hash2.put("bar", 7); + assertThat(hash.get("foo"), is(5)); + assertThat(hash.get("bar"), nullValue()); + assertThat(hash2.get("foo"), nullValue()); + assertThat(hash2.get("bar"), is(7)); + } + + private void verifyEquals(Object a, Object b) { + assertEquals(a, b); + assertEquals(b, a); + } + private void verifyNotEquals(Object a, Object b) { + assertNotEquals(a, b); + assertNotEquals(b, a); + } + + @Test + public void testThatDifferentGenericsDoesNotEqual() { + Hashlet<Long, Long> a = new Hashlet<>(); + Hashlet<String, Integer> b = new Hashlet<>(); + verifyEquals(a, b); + b.put("a", 1); + verifyNotEquals(a, b); + a.put(1L, 1L); + verifyNotEquals(a, b); + } + @Test + public void testHashCodeAndEquals() { + Hashlet<String, Integer> h1 = new Hashlet<>(); + Hashlet<String, Integer> h2 = new Hashlet<>(); + assertEquals(h1.hashCode(), h2.hashCode()); + verifyEquals(h1, h2); + + h1.put("a", 7); + assertNotEquals(h1.hashCode(), h2.hashCode()); + verifyNotEquals(h1, h2); + + h2.put("b", 8); + assertNotEquals(h1.hashCode(), h2.hashCode()); + verifyNotEquals(h1, h2); + + h2.put("a", 7); + assertNotEquals(h1.hashCode(), h2.hashCode()); + verifyNotEquals(h1, h2); + + h1.put("b", 8); + assertEquals(h1.hashCode(), h2.hashCode()); + verifyEquals(h1, h2); + + h1.put("c", null); + assertNotEquals(h1.hashCode(), h2.hashCode()); + verifyNotEquals(h1, h2); + + h2.put("d", null); + assertNotEquals(h1.hashCode(), h2.hashCode()); + verifyNotEquals(h1, h2); + + h2.put("c", null); + assertNotEquals(h1.hashCode(), h2.hashCode()); + verifyNotEquals(h1, h2); + + h1.put("d", null); + assertEquals(h1.hashCode(), h2.hashCode()); + verifyEquals(h1, h2); + } + + @Test + public void testSetValue() { + String A = "a"; + Hashlet<String, Integer> h = new Hashlet<>(); + h.put(A, 1); + int indexOfA = h.getIndexOfKey(A); + assertEquals(new Integer(1), h.value(indexOfA)); + h.setValue(indexOfA, 2); + assertEquals(new Integer(2), h.value(indexOfA)); + assertEquals(new Integer(2), h.get(A)); + } + + @Test + public void testGet() { + Hashlet<String, Integer> h = new Hashlet<>(); + h.put("a", 1); + h.put("b", null); + assertEquals(0, h.getIndexOfKey("a")); + assertEquals(h.get("a"), h.value(h.getIndexOfKey("a"))); + assertEquals(1, h.getIndexOfKey("b")); + assertEquals(h.get("b"), h.value(h.getIndexOfKey("b"))); + assertEquals(-1, h.getIndexOfKey("c")); + assertNull(h.get("c")); + } + + @Test + public void testCopyNonEmptyHashlet() { + Hashlet<String, Integer> hash = new Hashlet<>(); + hash.put("foo", 5); + hash.put("bar", 7); + Hashlet<String, Integer> hash2 = new Hashlet<>(hash); + assertThat(hash2.size(), is(2)); + assertThat(hash2.get("foo"), is(5)); + assertThat(hash2.get("bar"), is(7)); + assertThat(hash2.key(0), is("foo")); + assertThat(hash2.key(1), is("bar")); + assertThat(hash2.value(0), is(5)); + assertThat(hash2.value(1), is(7)); + assertThat(hash2.key(0), sameInstance(hash.key(0))); + assertThat(hash2.key(1), sameInstance(hash.key(1))); + assertThat(hash2.value(0), sameInstance(hash.value(0))); + assertThat(hash2.value(1), sameInstance(hash.value(1))); + } + + @Test + public void testSetValueToNull() { + Hashlet<String, Integer> hash = new Hashlet<>(); + hash.put("foo", 5); + hash.put("bar", 7); + assertThat(hash.size(), is(2)); + assertThat(hash.get("foo"), is(5)); + assertThat(hash.get("bar"), is(7)); + assertThat(hash.key(0), is("foo")); + assertThat(hash.key(1), is("bar")); + assertThat(hash.value(0), is(5)); + assertThat(hash.value(1), is(7)); + hash.put("foo", null); + assertThat(hash.size(), is(2)); + assertThat(hash.get("foo"), nullValue()); + assertThat(hash.get("bar"), is(7)); + assertThat(hash.key(0), is("foo")); + assertThat(hash.key(1), is("bar")); + assertThat(hash.value(0), nullValue()); + assertThat(hash.value(1), is(7)); + } + + @Test + public void testIterate() { + int n = 100; + Hashlet<String, Integer> hash = new Hashlet<>(); + for (int i = 0; i < n; i++) { + String str = ("" + i + "_str_" + i); + hash.put(str, i); + } + assertThat(hash.size(), is(n)); + for (int i = 0; i < n; i++) { + String str = ("" + i + "_str_" + i); + assertThat(hash.key(i), is(str)); + assertThat(hash.value(i), is(i)); + } + } + + @Test + public void testManyEntries() { + int n = 5000; + Hashlet<String, Integer> hash = new Hashlet<>(); + for (int i = 0; i < n; i++) { + String str = ("" + i + "_str_" + i); + assertThat(hash.get(str), nullValue()); + switch (i % 2) { + case 1: assertThat(hash.put(str, new Integer(i)), nullValue()); + } + } + assertThat(hash.size(), is(n / 2)); + for (int i = 0; i < n; i++) { + String str = ("" + i + "_str_" + i); + switch (i % 2) { + case 0: assertThat(hash.get(str), nullValue()); break; + case 1: assertThat(hash.get(str), is(new Integer(i))); break; + } + } + } +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/IntArrayComparatorTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/IntArrayComparatorTestCase.java new file mode 100644 index 00000000000..c4808a6605c --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/IntArrayComparatorTestCase.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class IntArrayComparatorTestCase { + @Test + public void arrayLength() { + int[] shortArr = new int[]{1, 2}; + int[] longArr = new int[]{0, 3, 3, 3, 3, 3}; + + assertEquals(-1, IntArrayComparator.compare(shortArr, longArr)); + } + + @Test + public void compareArrays() { + int[] one = new int[]{1, 2, 3, 3, 3, 3}; + int[] two = new int[]{0, 3, 3, 3, 3, 3}; + + assertEquals(1, IntArrayComparator.compare(one, two)); + assertEquals(-1, IntArrayComparator.compare(two, one)); + } + + @Test + public void compareEqualArrays() { + int[] one = new int[]{1, 2, 3, 3, 3, 3, 9}; + int[] two = new int[]{1, 2, 3, 3, 3, 3, 9}; + + assertEquals(0, IntArrayComparator.compare(one, two)); + assertEquals(0, IntArrayComparator.compare(two, one)); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/LazyMapTest.java b/vespajlib/src/test/java/com/yahoo/collections/LazyMapTest.java new file mode 100644 index 00000000000..2890d73ebaf --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/LazyMapTest.java @@ -0,0 +1,285 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NoSuchElementException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class LazyMapTest { + + @Test + public void requireThatInitialDelegateIsEmpty() { + LazyMap<String, String> map = newLazyMap(new HashMap<String, String>()); + assertEquals(LazyMap.EmptyMap.class, map.getDelegate().getClass()); + } + + @Test + public void requireThatEmptyMapPutUpgradesToSingletonMap() { + LazyMap<String, String> map = newLazyMap(new HashMap<String, String>()); + assertNull(map.put("foo", "bar")); + assertEquals(LazyMap.SingletonMap.class, map.getDelegate().getClass()); + + map = newLazyMap(new HashMap<String, String>()); + map.putAll(Collections.singletonMap("foo", "bar")); + assertEquals(LazyMap.SingletonMap.class, map.getDelegate().getClass()); + } + + @Test + public void requireThatEmptyMapPutAllEmptyMapDoesNotUpgradeToSingletonMap() { + LazyMap<String, String> map = newLazyMap(new HashMap<String, String>()); + map.putAll(Collections.<String, String>emptyMap()); + assertEquals(LazyMap.EmptyMap.class, map.getDelegate().getClass()); + } + + @Test + public void requireThatEmptyMapPutAllUpgradesToFinalMap() { + Map<String, String> delegate = new HashMap<>(); + LazyMap<String, String> map = newLazyMap(delegate); + map.putAll(new HashMapBuilder<String, String>() + .put("foo", "bar") + .put("baz", "cox").map); + assertSame(delegate, map.getDelegate()); + assertEquals(2, delegate.size()); + assertEquals("bar", delegate.get("foo")); + assertEquals("cox", delegate.get("baz")); + } + + @Test + public void requireThatSingletonMapRemoveEntryDowngradesToEmptyMap() { + LazyMap<String, String> map = newSingletonMap("foo", "bar"); + assertEquals("bar", map.remove("foo")); + assertEquals(LazyMap.EmptyMap.class, map.getDelegate().getClass()); + } + + @Test + public void requireThatSingletonMapRemoveUnknownDoesNotDowngradesToEmptyMap() { + LazyMap<String, String> map = newSingletonMap("foo", "bar"); + assertNull(map.remove("baz")); + assertEquals(LazyMap.SingletonMap.class, map.getDelegate().getClass()); + } + + @Test + public void requireThatSingletonMapValueMayBeChangedInPlace() { + LazyMap<String, String> map = newSingletonMap("foo", "bar"); + Map<String, String> delegate = map.getDelegate(); + assertEquals("bar", map.put("foo", "baz")); + assertEquals("baz", map.get("foo")); + assertSame(delegate, map.getDelegate()); + map.putAll(Collections.singletonMap("foo", "cox")); + assertSame(delegate, map.getDelegate()); + assertEquals("cox", map.get("foo")); + } + + @Test + public void requireThatSingletonMapPutAllEmptyMapDoesNotUpgradeToFinalMap() { + LazyMap<String, String> map = newSingletonMap("foo", "bar"); + map.putAll(Collections.<String, String>emptyMap()); + assertEquals(LazyMap.SingletonMap.class, map.getDelegate().getClass()); + } + + @Test + public void requireThatSingletonMapPutUpgradesToFinalMap() { + Map<String, String> delegate = new HashMap<>(); + LazyMap<String, String> map = newSingletonMap(delegate, "fooKey", "fooVal"); + map.put("barKey", "barVal"); + assertSame(delegate, map.getDelegate()); + assertEquals(2, delegate.size()); + assertEquals("fooVal", delegate.get("fooKey")); + assertEquals("barVal", delegate.get("barKey")); + } + + @Test + public void requireThatSingletonMapPutAllUpgradesToFinalMap() { + Map<String, String> delegate = new HashMap<>(); + LazyMap<String, String> map = newSingletonMap(delegate, "fooKey", "fooVal"); + map.putAll(new HashMapBuilder<String, String>() + .put("barKey", "barVal") + .put("bazKey", "bazVal").map); + assertSame(delegate, map.getDelegate()); + assertEquals(3, delegate.size()); + assertEquals("fooVal", delegate.get("fooKey")); + assertEquals("barVal", delegate.get("barKey")); + assertEquals("bazVal", delegate.get("bazKey")); + } + + @Test + public void requireThatSingletonEntryIsMutable() { + LazyMap<String, String> map = newSingletonMap("foo", "bar"); + Map.Entry<String, String> entry = map.entrySet().iterator().next(); + entry.setValue("baz"); + assertEquals("baz", map.get("foo")); + } + + @Test + public void requireThatSingletonEntryImplementsHashCode() { + assertEquals(newSingletonMap("foo", "bar").entrySet().iterator().next().hashCode(), + newSingletonMap("foo", "bar").entrySet().iterator().next().hashCode()); + } + + @Test + public void requireThatSingletonEntryImplementsEquals() { + Map.Entry<String, String> map = newSingletonMap("foo", "bar").entrySet().iterator().next(); + assertNotEquals(map, null); + assertNotEquals(map, new Object()); + assertEquals(map, map); + assertNotEquals(map, newSingletonMap("baz", "cox").entrySet().iterator().next()); + assertNotEquals(map, newSingletonMap("foo", "cox").entrySet().iterator().next()); + assertEquals(map, newSingletonMap("foo", "bar").entrySet().iterator().next()); + } + + @Test + public void requireThatSingletonEntrySetIteratorNextThrowsIfInvokedMoreThanOnce() { + LazyMap<String, String> map = newSingletonMap("foo", "bar"); + Iterator<Map.Entry<String, String>> it = map.entrySet().iterator(); + it.next(); + try { + it.next(); + fail(); + } catch (NoSuchElementException e) { + + } + try { + it.next(); + fail(); + } catch (NoSuchElementException e) { + + } + } + + @Test + public void requireThatSingletonEntrySetIteratorRemoveThrowsIfInvokedBeforeNext() { + LazyMap<String, String> map = newSingletonMap("foo", "bar"); + Iterator<Map.Entry<String, String>> it = map.entrySet().iterator(); + try { + it.remove(); + fail(); + } catch (IllegalStateException e) { + + } + } + + @SuppressWarnings("unchecked") + private static Map<String, String> makeMockMap() { + return Mockito.mock(Map.class); + } + + @Test + public void requireThatMapDelegates() { + Map<String, String> delegate = makeMockMap(); + Map<String, String> map = newLazyMap(delegate); + map.put("foo", "bar"); + map.put("baz", "cox"); // trigger the assignment of the delegate + Mockito.verify(delegate).put("foo", "bar"); + Mockito.verify(delegate).put("baz", "cox"); + + Map<String, String> arg = Collections.singletonMap("baz", "cox"); + map.putAll(arg); + Mockito.verify(delegate).putAll(arg); + + assertEquals(0, map.size()); + Mockito.verify(delegate).size(); + + assertFalse(map.isEmpty()); + Mockito.verify(delegate).isEmpty(); + + assertFalse(map.containsKey("foo")); + Mockito.verify(delegate).containsKey("foo"); + + assertFalse(map.containsValue("bar")); + Mockito.verify(delegate).containsValue("bar"); + + assertNull(map.get("foo")); + Mockito.verify(delegate).get("foo"); + + assertNull(map.remove("foo")); + Mockito.verify(delegate).remove("foo"); + + map.clear(); + Mockito.verify(delegate).clear(); + + assertTrue(map.keySet().isEmpty()); + Mockito.verify(delegate).keySet(); + + assertTrue(map.values().isEmpty()); + Mockito.verify(delegate).values(); + + assertTrue(map.entrySet().isEmpty()); + Mockito.verify(delegate).entrySet(); + } + + @Test + public void requireThatHashCodeIsImplemented() { + assertEquals(newLazyMap(null).hashCode(), + newLazyMap(null).hashCode()); + } + + @Test + public void requireThatEqualsIsImplemented() { + Map<Object, Object> lhs = newLazyMap(new HashMap<>()); + Map<Object, Object> rhs = newLazyMap(new HashMap<>()); + assertEquals(lhs, lhs); + assertEquals(lhs, rhs); + + Object key = new Object(); + Object val = new Object(); + lhs.put(key, val); + assertEquals(lhs, lhs); + assertFalse(lhs.equals(rhs)); + rhs.put(key, val); + assertEquals(lhs, rhs); + } + + @Test + public void requireThatHashMapFactoryDelegatesToAHashMap() { + LazyMap<String, String> map = LazyMap.newHashMap(); + map.put("foo", "bar"); + map.put("baz", "cox"); + assertEquals(HashMap.class, map.getDelegate().getClass()); + } + + private static <K, V> LazyMap<K, V> newSingletonMap(K key, V value) { + return newSingletonMap(new HashMap<K, V>(), key, value); + } + + private static <K, V> LazyMap<K, V> newSingletonMap(Map<K, V> delegate, K key, V value) { + LazyMap<K, V> map = newLazyMap(delegate); + map.put(key, value); + return map; + } + + private static <K, V> LazyMap<K, V> newLazyMap(final Map<K, V> delegate) { + return new LazyMap<K, V>() { + + @Override + protected Map<K, V> newDelegate() { + return delegate; + } + }; + } + + private static class HashMapBuilder<K, V> { + + final Map<K, V> map = new HashMap<>(); + + public HashMapBuilder<K, V> put(K key, V value) { + map.put(key, value); + return this; + } + } +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/LazySetTest.java b/vespajlib/src/test/java/com/yahoo/collections/LazySetTest.java new file mode 100644 index 00000000000..d71e7ca6e26 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/LazySetTest.java @@ -0,0 +1,265 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class LazySetTest { + + @Test + public void requireThatInitialDelegateIsEmpty() { + LazySet<String> set = newLazySet(new HashSet<String>()); + assertEquals(LazySet.EmptySet.class, set.getDelegate().getClass()); + } + + @Test + public void requireThatEmptySetAddUpgradesToSingletonSet() { + LazySet<String> set = newLazySet(new HashSet<String>()); + assertTrue(set.add("foo")); + assertEquals(LazySet.SingletonSet.class, set.getDelegate().getClass()); + + set = newLazySet(new HashSet<String>()); + assertTrue(set.addAll(Arrays.asList("foo"))); + assertEquals(LazySet.SingletonSet.class, set.getDelegate().getClass()); + } + + @Test + public void requireThatEmptySetAddAllEmptySetDoesNotUpgradeToSingletonSet() { + LazySet<String> set = newLazySet(new HashSet<String>()); + assertFalse(set.addAll(Collections.<String>emptySet())); + assertEquals(LazySet.EmptySet.class, set.getDelegate().getClass()); + } + + @Test + public void requireThatEmptySetAddAllUpgradesToFinalSet() { + Set<String> delegate = new HashSet<>(); + LazySet<String> set = newLazySet(delegate); + assertTrue(set.addAll(Arrays.asList("foo", "bar"))); + assertSame(delegate, set.getDelegate()); + assertEquals(2, delegate.size()); + assertTrue(delegate.contains("foo")); + assertTrue(delegate.contains("bar")); + } + + @Test + public void requireThatSingletonSetRemoveEntryDowngradesToEmptySet() { + LazySet<String> set = newSingletonSet("foo"); + assertTrue(set.remove("foo")); + assertEquals(LazySet.EmptySet.class, set.getDelegate().getClass()); + } + + @Test + public void requireThatSingletonSetRemoveUnknownDoesNotDowngradesToEmptySet() { + LazySet<String> set = newSingletonSet("foo"); + assertFalse(set.remove("bar")); + assertEquals(LazySet.SingletonSet.class, set.getDelegate().getClass()); + } + + @Test + public void requireThatSingletonSetAddAllEmptySetDoesNotUpgradeToFinalSet() { + LazySet<String> set = newSingletonSet("foo"); + assertFalse(set.addAll(Collections.<String>emptySet())); + assertEquals(LazySet.SingletonSet.class, set.getDelegate().getClass()); + } + + @Test + public void requireThatSingletonSetAddKnownDoesNotUpgradeToFinalSet() { + LazySet<String> set = newSingletonSet("foo"); + assertFalse(set.add("foo")); + assertEquals(LazySet.SingletonSet.class, set.getDelegate().getClass()); + } + + @Test + public void requireThatSingletonSetAddUpgradesToFinalSet() { + Set<String> delegate = new HashSet<>(); + LazySet<String> set = newSingletonSet(delegate, "foo"); + assertTrue(set.add("bar")); + assertSame(delegate, set.getDelegate()); + assertEquals(2, delegate.size()); + assertTrue(delegate.contains("foo")); + assertTrue(delegate.contains("bar")); + } + + @Test + public void requireThatSingletonSetAddAllUpgradesToFinalSet() { + Set<String> delegate = new HashSet<>(); + LazySet<String> set = newSingletonSet(delegate, "foo"); + assertTrue(set.addAll(Arrays.asList("bar"))); + assertSame(delegate, set.getDelegate()); + assertEquals(2, delegate.size()); + assertTrue(delegate.contains("foo")); + assertTrue(delegate.contains("bar")); + + delegate = new HashSet<>(); + set = newSingletonSet(delegate, "foo"); + assertTrue(set.addAll(Arrays.asList("bar", "baz"))); + assertSame(delegate, set.getDelegate()); + assertEquals(3, delegate.size()); + assertTrue(delegate.contains("foo")); + assertTrue(delegate.contains("bar")); + assertTrue(delegate.contains("baz")); + } + + @Test + public void requireThatSingletonIteratorNextThrowsIfInvokedMoreThanOnce() { + LazySet<String> set = newSingletonSet("foo"); + Iterator<String> it = set.iterator(); + it.next(); + try { + it.next(); + fail(); + } catch (NoSuchElementException e) { + + } + try { + it.next(); + fail(); + } catch (NoSuchElementException e) { + + } + } + + @Test + public void requireThatSingletonIteratorRemoveDowngradesToEmptySet() { + LazySet<String> set = newSingletonSet("foo"); + Iterator<String> it = set.iterator(); + it.next(); + it.remove(); + assertEquals(LazySet.EmptySet.class, set.getDelegate().getClass()); + } + + @Test + public void requireThatSingletonIteratorRemoveThrowsIfInvokedBeforeNext() { + LazySet<String> set = newSingletonSet("foo"); + Iterator<String> it = set.iterator(); + try { + it.remove(); + fail(); + } catch (IllegalStateException e) { + + } + } + + @SuppressWarnings("unchecked") + private static Set<String> makeMockSet() { + return Mockito.mock(Set.class); + } + + @Test + public void requireThatSetDelegates() { + Set<String> delegate = makeMockSet(); + Set<String> set = newLazySet(delegate); + set.add("foo"); + set.add("bar"); // trigger the assignment of the delegate + Mockito.verify(delegate).add("foo"); + Mockito.verify(delegate).add("bar"); + + Set<String> addAllArg = Collections.singleton("foo"); + set.addAll(addAllArg); + Mockito.verify(delegate).addAll(addAllArg); + + assertEquals(0, set.size()); + Mockito.verify(delegate).size(); + + assertFalse(set.isEmpty()); + Mockito.verify(delegate).isEmpty(); + + assertFalse(set.contains("foo")); + Mockito.verify(delegate).contains("foo"); + + assertNull(set.iterator()); + Mockito.verify(delegate).iterator(); + + assertNull(set.toArray()); + Mockito.verify(delegate).toArray(); + + String[] toArrayArg = new String[69]; + assertNull(set.toArray(toArrayArg)); + Mockito.verify(delegate).toArray(toArrayArg); + + assertFalse(set.remove("foo")); + Mockito.verify(delegate).remove("foo"); + + Collection<String> containsAllArg = Collections.singletonList("foo"); + assertFalse(set.containsAll(containsAllArg)); + Mockito.verify(delegate).containsAll(containsAllArg); + + Collection<String> retainAllArg = Collections.singletonList("foo"); + assertFalse(set.retainAll(retainAllArg)); + Mockito.verify(delegate).retainAll(retainAllArg); + + Collection<String> removeAllArg = Collections.singletonList("foo"); + assertFalse(set.removeAll(removeAllArg)); + Mockito.verify(delegate).removeAll(removeAllArg); + + set.clear(); + Mockito.verify(delegate).clear(); + } + + @Test + public void requireThatHashCodeIsImplemented() { + assertEquals(newLazySet(null).hashCode(), + newLazySet(null).hashCode()); + } + + @Test + public void requireThatEqualsIsImplemented() { + Set<Object> lhs = newLazySet(new HashSet<>()); + Set<Object> rhs = newLazySet(new HashSet<>()); + assertEquals(lhs, lhs); + assertEquals(lhs, rhs); + + Object obj = new Object(); + lhs.add(obj); + assertEquals(lhs, lhs); + assertFalse(lhs.equals(rhs)); + rhs.add(obj); + assertEquals(lhs, rhs); + } + + @Test + public void requireThatHashSetFactoryDelegatesToAHashSet() { + LazySet<Integer> set = LazySet.newHashSet(); + set.add(6); + set.add(9); + assertEquals(HashSet.class, set.getDelegate().getClass()); + } + + private static <E> LazySet<E> newSingletonSet(E element) { + return newSingletonSet(new HashSet<E>(), element); + } + + private static <E> LazySet<E> newSingletonSet(Set<E> delegate, E element) { + LazySet<E> set = newLazySet(delegate); + set.add(element); + return set; + } + + private static <E> LazySet<E> newLazySet(final Set<E> delegate) { + return new LazySet<E>() { + + @Override + protected Set<E> newDelegate() { + return delegate; + } + }; + } +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/ListMapTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/ListMapTestCase.java new file mode 100644 index 00000000000..29668676222 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/ListMapTestCase.java @@ -0,0 +1,153 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import org.junit.Test; + +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class ListMapTestCase { + + @Test + public void testSimple() { + ListMap<String, String> stringMap = new ListMap<>(); + stringMap.put("foo", "bar"); + stringMap.put("foo", "far"); + stringMap.put("bar", "rab"); + + List<String> fooValues = stringMap.get("foo"); + assertEquals(2, fooValues.size()); + assertEquals("bar", fooValues.get(0)); + assertEquals("far", fooValues.get(1)); + + List<String> barValues = stringMap.get("bar"); + assertEquals(1, barValues.size()); + assertEquals("rab", barValues.get(0)); + } + + @Test + public void testAnotherImplementation() { + ListMap<String, String> stringMap = new ListMap<>(IdentityHashMap.class); + String foo = "foo"; + String bar = "bar"; + String far = "far"; + String rab = "rab"; + + stringMap.put(foo, bar); + stringMap.put(foo, far); + stringMap.put(bar, rab); + + List<String> fooValues = stringMap.get(new String("foo")); + assertEquals(0, fooValues.size()); + fooValues = stringMap.get(foo); + assertEquals(2, fooValues.size()); + assertEquals("bar", fooValues.get(0)); + assertEquals("far", fooValues.get(1)); + + + List<String> barValues = stringMap.get(new String("bar")); + assertEquals(0, barValues.size()); + barValues = stringMap.get(bar); + assertEquals(1, barValues.size()); + assertEquals("rab", barValues.get(0)); + } + + @SuppressWarnings("serial") + private static class BoomMap extends HashMap<String, String> { + @SuppressWarnings("unused") + BoomMap() { + throw new RuntimeException(); + } + } + + @Test + public void testExplodingImplementation() { + boolean illegalArgument = false; + try { + new ListMap<String, String>(BoomMap.class); + } catch (IllegalArgumentException e) { + assertTrue(e.getCause().getClass() == RuntimeException.class); + illegalArgument = true; + } + assertTrue(illegalArgument); + } + + private static final String A = "A"; + private static final String B = "B"; + private static final String B0 = "b0"; + + private ListMap<String, String> initSimpleMap() { + ListMap<String, String> lm = new ListMap<>(); + lm.put(A, "a0"); + lm.put(A, "a1"); + lm.put(B, B0); + lm.put(B, "b1"); + lm.put("C", "c"); + lm.put("D", "d"); + return lm; + } + + @Test + public void testRemoval() { + ListMap<String, String> lm = initSimpleMap(); + assertEquals(2, lm.getList(A).size()); + assertEquals(4, lm.entrySet().size()); + lm.removeAll(A); + assertEquals(3, lm.entrySet().size()); + assertEquals(0, lm.getList(A).size()); + assertEquals(2, lm.getList(B).size()); + assertTrue(lm.removeValue(B, B0)); + assertFalse(lm.removeValue(B, B0)); + assertEquals(1, lm.getList(B).size()); + assertEquals(3, lm.entrySet().size()); + } + + @Test + public void testGetSet() { + ListMap<String, String> lm = initSimpleMap(); + lm.removeAll(B); + Set<Map.Entry<String, List<String>>> l = lm.entrySet(); + assertEquals(3, l.size()); + boolean hasA = false; + boolean hasB = false; + for (Map.Entry<String, List<String>> e : l) { + if (e.getKey().equals(A)) { + hasA = true; + } else if (e.getKey().equals(B)) { + hasB = true; + } + } + assertTrue(hasA); + assertFalse(hasB); + } + + @Test + public void testFreeze() { + ListMap<String, String> map = initSimpleMap(); + map.freeze(); + try { + map.put("key", "value"); + fail("Expected exception"); + } + catch (Exception expected) { + } + try { + map.entrySet().iterator().next().getValue().add("foo"); + fail("Expected exception"); + } + catch (Exception expected) { + } + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/ListenableArrayListTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/ListenableArrayListTestCase.java new file mode 100644 index 00000000000..e3fb48c7a0e --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/ListenableArrayListTestCase.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import org.junit.Test; +import static org.junit.Assert.*; + +import java.util.Arrays; +import java.util.ListIterator; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ListenableArrayListTestCase { + + @Test + public void testIt() { + ListenableArrayList<String> list = new ListenableArrayList<>(); + ArrayListListener listener = new ArrayListListener(); + list.addListener(listener); + assertEquals(0,listener.invoked); + list.add("a"); + assertEquals(1,listener.invoked); + list.add(0,"b"); + assertEquals(2,listener.invoked); + list.addAll(Arrays.asList(new String[]{"c", "d"})); + assertEquals(3,listener.invoked); + list.addAll(1,Arrays.asList(new String[]{"e", "f"})); + assertEquals(4,listener.invoked); + list.set(0,"g"); + assertEquals(5,listener.invoked); + ListIterator<String> i = list.listIterator(); + i.add("h"); + assertEquals(6,listener.invoked); + i.next(); + i.set("i"); + assertEquals(7,listener.invoked); + } + + private static class ArrayListListener implements Runnable { + + int invoked; + + @Override + public void run() { + invoked++; + } + + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/MD5TestCase.java b/vespajlib/src/test/java/com/yahoo/collections/MD5TestCase.java new file mode 100644 index 00000000000..a107b21abb1 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/MD5TestCase.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class MD5TestCase extends junit.framework.TestCase { + public void testMD5() { + MD5 md5 = new MD5(); + int a = md5.hash("foobar"); + int b = md5.hash("foobar"); + + assertEquals(a, b); + + int c = md5.hash("foo"); + + assertTrue(a != c); + assertTrue(b != c); + + //rudimentary check; see that all four bytes contain something: + + assertTrue((a & 0xFF000000) != 0); + assertTrue((a & 0x00FF0000) != 0); + assertTrue((a & 0x0000FF00) != 0); + assertTrue((a & 0x000000FF) != 0); + + + assertTrue((c & 0xFF000000) != 0); + assertTrue((c & 0x00FF0000) != 0); + assertTrue((c & 0x0000FF00) != 0); + assertTrue((c & 0x000000FF) != 0); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/PredicateSplitTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/PredicateSplitTestCase.java new file mode 100644 index 00000000000..d1c040809de --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/PredicateSplitTestCase.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import org.junit.Test; + +import java.util.List; +import java.util.ArrayList; + +import static org.junit.Assert.assertEquals; + +public class PredicateSplitTestCase { + @Test + public void requireThatSplitWorks() { + List<Integer> l = new ArrayList<Integer>(); + l.add(1); + l.add(6); + l.add(2); + l.add(4); + l.add(5); + PredicateSplit<Integer> result = PredicateSplit.partition(l, x -> (x % 2 == 0)); + assertEquals((long) result.falseValues.size(), 2L); + assertEquals((long) result.falseValues.get(0), 1L); + assertEquals((long) result.falseValues.get(1), 5L); + + assertEquals((long) result.trueValues.size(), 3L); + assertEquals((long) result.trueValues.get(0), 6L); + assertEquals((long) result.trueValues.get(1), 2L); + assertEquals((long) result.trueValues.get(2), 4L); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/TinyIdentitySetTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/TinyIdentitySetTestCase.java new file mode 100644 index 00000000000..2ba7262530b --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/TinyIdentitySetTestCase.java @@ -0,0 +1,302 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import org.junit.Test; + +/** + * Check TinyIdentitySet seems to work. :) + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class TinyIdentitySetTestCase { + + @Test + public void testAdd() { + final String string = "abc"; + final String a = new String(string); + final String b = new String(string); + final TinyIdentitySet<String> t = new TinyIdentitySet<>(3); + t.add(a); + t.add(b); + assertEquals(2, t.size()); + t.add(string); + assertEquals(3, t.size()); + t.add(string); + t.add(a); + t.add(b); + assertEquals(3, t.size()); + + } + + @Test + public void testAddAll() { + final List<String> stuff = doubleAdd(); + final TinyIdentitySet<String> t = new TinyIdentitySet<>( + stuff.size()); + t.addAll(stuff); + assertEquals(stuff.size() / 2, t.size()); + } + + private List<String> doubleAdd() { + final String string = "abc"; + final String a = new String(string); + final String b = new String(string); + final String c = "c"; + final List<String> stuff = new ArrayList<>(); + stuff.add(string); + stuff.add(a); + stuff.add(b); + stuff.add(c); + stuff.add(string); + stuff.add(a); + stuff.add(b); + stuff.add(c); + return stuff; + } + + @Test + public void testContains() { + final String string = "abc"; + final String a = new String(string); + final String b = new String(string); + final TinyIdentitySet<String> t = new TinyIdentitySet<>(2); + t.add(string); + t.add(a); + assertTrue(t.contains(a)); + assertTrue(t.contains(string)); + assertFalse(t.contains(b)); + } + + @Test + public void testContainsAll() { + final String string = "abc"; + final String a = new String(string); + final String b = new String(string); + final String c = "c"; + final List<String> stuff = new ArrayList<>(); + stuff.add(string); + stuff.add(a); + stuff.add(b); + final TinyIdentitySet<String> t = new TinyIdentitySet<>( + stuff.size()); + t.addAll(stuff); + assertTrue(t.containsAll(stuff)); + stuff.add(c); + assertFalse(t.containsAll(stuff)); + } + + @Test + public void testRemove() { + final String string = "abc"; + final String a = new String(string); + final String b = new String(string); + final TinyIdentitySet<String> t = new TinyIdentitySet<>(2); + t.add(string); + t.add(a); + assertFalse(t.remove(b)); + assertTrue(t.remove(a)); + assertFalse(t.remove(a)); + assertTrue(t.remove(string)); + assertFalse(t.remove(b)); + } + + @Test + public void testRetainAll() { + final List<String> stuff = doubleAdd(); + final TinyIdentitySet<String> t = new TinyIdentitySet<>( + stuff.size()); + t.addAll(stuff); + assertFalse(t.retainAll(stuff)); + assertEquals(stuff.size() / 2, t.size()); + t.add("nalle"); + assertEquals(stuff.size() / 2 + 1, t.size()); + assertTrue(t.retainAll(stuff)); + assertEquals(stuff.size() / 2, t.size()); + } + + @Test + public void testToArrayTArray() { + final List<String> stuff = doubleAdd(); + final TinyIdentitySet<String> t = new TinyIdentitySet<>( + stuff.size()); + t.addAll(stuff); + final String[] s = t.toArray(new String[0]); + assertEquals(t.size(), s.length); + assertEquals(stuff.size() / 2, s.length); + } + + @Test + public void testGrow() { + final TinyIdentitySet<Integer> t = new TinyIdentitySet<>(5); + final int targetSize = 100; + for (int i = 0; i < targetSize; ++i) { + t.add(i); + } + assertEquals(targetSize, t.size()); + int n = 0; + for (final Iterator<Integer> i = t.iterator(); i.hasNext();) { + assertEquals(Integer.valueOf(n++), i.next()); + } + assertEquals(targetSize, n); + } + + @Test + public void testBiggerRemoveAll() { + final int targetSize = 100; + final TinyIdentitySet<Integer> t = new TinyIdentitySet<>( + targetSize); + final Integer[] instances = new Integer[targetSize]; + final List<Integer> remove = buildSubSet(targetSize, t, instances); + t.removeAll(remove); + assertEquals(targetSize / 2, t.size()); + for (final Iterator<Integer> i = t.iterator(); i.hasNext();) { + final Integer n = i.next(); + assertTrue(n % 2 == 0); + assertFalse(remove.contains(n)); + + } + } + + @Test + public void testBiggerRetainAll() { + final int targetSize = 100; + final TinyIdentitySet<Integer> t = new TinyIdentitySet<>( + targetSize); + final Integer[] instances = new Integer[targetSize]; + final List<Integer> retain = buildSubSet(targetSize, t, instances); + t.retainAll(retain); + assertEquals(targetSize / 2, t.size()); + for (final Iterator<Integer> i = t.iterator(); i.hasNext();) { + final Integer n = i.next(); + assertTrue(n % 2 != 0); + assertTrue(retain.contains(n)); + } + } + + private List<Integer> buildSubSet(final int targetSize, + final TinyIdentitySet<Integer> t, final Integer[] instances) { + for (int i = 0; i < targetSize; ++i) { + instances[i] = Integer.valueOf(i); + t.add(instances[i]); + } + final List<Integer> subset = new ArrayList<>(50); + for (int i = 0; i < targetSize; ++i) { + if (i % 2 != 0) { + subset.add(instances[i]); + } + } + return subset; + } + + @Test + public void testMuckingAbout() { + final int targetSize = 100; + final TinyIdentitySet<Integer> t = new TinyIdentitySet<>(3); + final Integer[] instances = new Integer[targetSize]; + final List<Integer> retain = buildSubSet(targetSize, t, instances); + for (final Integer n : retain) { + t.remove(n); + assertEquals(targetSize - 1, t.size()); + t.add(n); + assertEquals(targetSize, t.size()); + } + assertEquals(targetSize, t.size()); + final Integer[] contents = t.toArray(new Integer[0]); + Arrays.sort(contents, 0, targetSize); + for (int i = 0; i < targetSize; ++i) { + assertEquals(instances[i], contents[i]); + } + } + + @Test + public void testMoreDuplicates() { + final int targetSize = 100; + final TinyIdentitySet<Integer> t = new TinyIdentitySet<>(3); + final Integer[] instances = new Integer[targetSize]; + final List<Integer> add = buildSubSet(targetSize, t, instances); + assertEquals(targetSize, t.size()); + t.addAll(add); + assertEquals(targetSize, t.size()); + } + + @Test + public void testEmptySet() { + final int targetSize = 100; + final TinyIdentitySet<Integer> t = new TinyIdentitySet<>(0); + final Integer[] instances = new Integer[targetSize]; + final List<Integer> add = buildSubSet(targetSize, t, instances); + for (Integer i : instances) { + t.remove(i); + } + assertEquals(0, t.size()); + for (Integer i : add) { + t.add(i); + } + assertEquals(targetSize / 2, t.size()); + } + + @Test + public void testSmallEmptySet() { + final TinyIdentitySet<Integer> t = new TinyIdentitySet<>(3); + Integer a = new Integer(0), b = new Integer(1), c = new Integer(2); + t.add(a); + t.add(b); + t.add(c); + assertEquals(3, t.size()); + t.remove(a); + assertEquals(2, t.size()); + t.remove(c); + assertEquals(1, t.size()); + t.remove(c); + assertEquals(1, t.size()); + t.remove(b); + assertEquals(0, t.size()); + t.add(b); + assertEquals(1, t.size()); + t.add(b); + assertEquals(1, t.size()); + t.add(a); + assertEquals(2, t.size()); + t.add(a); + assertEquals(2, t.size()); + t.add(c); + assertEquals(3, t.size()); + t.add(c); + assertEquals(3, t.size()); + } + + @Test + public void testIterator() { + final int targetSize = 100; + final TinyIdentitySet<Integer> t = new TinyIdentitySet<>(0); + final Integer[] instances = new Integer[targetSize]; + final List<Integer> remove = buildSubSet(targetSize, t, instances); + int traversed = 0; + for (Iterator<Integer> i = t.iterator(); i.hasNext();) { + Integer n = i.next(); + if (remove.contains(n)) { + i.remove(); + } + ++traversed; + } + assertEquals(targetSize, traversed); + assertEquals(targetSize / 2, t.size()); + for (int i = 0; i < instances.length; ++i) { + Integer n = instances[i]; + if (remove.contains(n)) { + assertFalse(t.contains(n)); + } else { + assertTrue(t.contains(n)); + } + } + } +} diff --git a/vespajlib/src/test/java/com/yahoo/collections/TupleTestCase.java b/vespajlib/src/test/java/com/yahoo/collections/TupleTestCase.java new file mode 100644 index 00000000000..8c7d25431a2 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/collections/TupleTestCase.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.collections; + +import static org.junit.Assert.*; + +import org.junit.Test; + +/** + * Test case used for testing and experimenting with the tuple APIs. It seems + * Tuple4 is just as horrible as I first assumed, but using quick-fix funtions + * in the IDE made writing the code less painful than I guessed.. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class TupleTestCase { + + private static final String _12 = "12"; + private static final Integer _11 = Integer.valueOf(11); + + Tuple2<Integer, String> instance = new Tuple2<>(_11, _12); + + + @Test + public final void objectStuff() { + boolean hashException = false; + boolean equalsException = false; + assertEquals("Tuple2(11, 12)", instance.toString()); + try { + instance.hashCode(); + } catch (UnsupportedOperationException e) { + hashException = true; + } + assertTrue(hashException); + try { + instance.equals(null); + } catch (UnsupportedOperationException e) { + equalsException = true; + } + assertTrue(equalsException); + } + + @Test + public final void basicUse() { + assertSame(_11, instance.first); + assertSame(_12, instance.second); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/compress/IntegerCompressorTest.java b/vespajlib/src/test/java/com/yahoo/compress/IntegerCompressorTest.java new file mode 100644 index 00000000000..46a70a4c956 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/compress/IntegerCompressorTest.java @@ -0,0 +1,105 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.compress; + +import org.junit.Test; + +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * TODO: balder + */ +public class IntegerCompressorTest { + private void verifyPositiveNumber(int n, byte [] expected) { + ByteBuffer buf = ByteBuffer.allocate(expected.length); + IntegerCompressor.putCompressedPositiveNumber(n, buf); + assertArrayEquals(expected, buf.array()); + } + private void verifyNumber(int n, byte [] expected) { + ByteBuffer buf = ByteBuffer.allocate(expected.length); + IntegerCompressor.putCompressedNumber(n, buf); + assertArrayEquals(expected, buf.array()); + } + + @Test + public void requireThatPositiveNumberCompressCorrectly() { + byte [] zero = {0}; + verifyPositiveNumber(0, zero); + byte [] one = {0x01}; + verifyPositiveNumber(1, one); + byte [] x3f = {0x3f}; + verifyPositiveNumber(0x3f, x3f); + byte [] x40 = {(byte)0x80,0x40}; + verifyPositiveNumber(0x40, x40); + byte [] x3fff = {(byte)0xbf, (byte)0xff}; + verifyPositiveNumber(0x3fff, x3fff); + byte [] x4000 = {(byte)0xc0, 0x00, 0x40, 0x00}; + verifyPositiveNumber(0x4000, x4000); + byte [] x3fffffff = {(byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff}; + verifyPositiveNumber(0x3fffffff, x3fffffff); + byte [] x40000000 = {0,0,0,0}; + try { + verifyPositiveNumber(0x40000000, x40000000); + assertTrue(false); + } catch (IllegalArgumentException e) { + assertEquals("Number '1073741824' too big, must extend encoding", e.getMessage()); + } + try { + verifyPositiveNumber(-1, x40000000); + assertTrue(false); + } catch (IllegalArgumentException e) { + assertEquals("Number '-1' must be positive", e.getMessage()); + } + } + + @Test + public void requireThatNumberCompressCorrectly() { + byte [] zero = {0}; + verifyNumber(0, zero); + byte [] one = {0x01}; + verifyNumber(1, one); + byte [] x1f = {0x1f}; + verifyNumber(0x1f, x1f); + byte [] x20 = {0x40,0x20}; + verifyNumber(0x20, x20); + byte [] x1fff = {0x5f, (byte)0xff}; + verifyNumber(0x1fff, x1fff); + byte [] x2000 = {0x60, 0x00, 0x20, 0x00}; + verifyNumber(0x2000, x2000); + byte [] x1fffffff = {0x7f, (byte)0xff, (byte)0xff, (byte)0xff}; + verifyNumber(0x1fffffff, x1fffffff); + byte [] x20000000 = {0,0,0,0}; + try { + verifyNumber(0x20000000, x20000000); + assertTrue(false); + } catch (IllegalArgumentException e) { + assertEquals("Number '536870912' too big, must extend encoding", e.getMessage()); + } + byte [] mzero = {(byte)0x81}; + verifyNumber(-1, mzero); + byte [] mone = {(byte)0x82}; + verifyNumber(-2, mone); + byte [] mx1f = {(byte)0x9f}; + verifyNumber(-0x1f, mx1f); + byte [] mx20 = {(byte)0xc0,0x20}; + verifyNumber(-0x20, mx20); + byte [] mx1fff = {(byte)0xdf, (byte)0xff}; + verifyNumber(-0x1fff, mx1fff); + byte [] mx2000 = {(byte)0xe0, 0x00, 0x20, 0x00}; + verifyNumber(-0x2000, mx2000); + byte [] mx1fffffff = {(byte)0xff, (byte)0xff, (byte)0xff, (byte)0xff}; + verifyNumber(-0x1fffffff, mx1fffffff); + byte [] mx20000000 = {0,0,0,0}; + try { + verifyNumber(-0x20000000, mx20000000); + assertTrue(false); + } catch (IllegalArgumentException e) { + assertEquals("Number '-536870912' too big, must extend encoding", e.getMessage()); + } + + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/CopyOnWriteHashMapTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/CopyOnWriteHashMapTest.java new file mode 100644 index 00000000000..22619e3865e --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/concurrent/CopyOnWriteHashMapTest.java @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import org.junit.Test; + +import java.util.Iterator; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ +public class CopyOnWriteHashMapTest { + + @Test + public void requireThatAccessorsWork() { + Map<String, String> map = new CopyOnWriteHashMap<>(); + assertEquals(0, map.size()); + assertEquals(true, map.isEmpty()); + assertEquals(false, map.containsKey("fooKey")); + assertEquals(false, map.containsValue("fooVal")); + assertNull(map.get("fooKey")); + assertNull(map.remove("fooKey")); + assertEquals(0, map.keySet().size()); + assertEquals(0, map.entrySet().size()); + assertEquals(0, map.values().size()); + + map.put("fooKey", "fooVal"); + assertEquals(1, map.size()); + assertEquals(false, map.isEmpty()); + assertEquals(true, map.containsKey("fooKey")); + assertEquals(true, map.containsValue("fooVal")); + assertEquals("fooVal", map.get("fooKey")); + assertEquals(1, map.keySet().size()); + assertEquals(1, map.entrySet().size()); + assertEquals(1, map.values().size()); + + map.put("barKey", "barVal"); + assertEquals(2, map.size()); + assertEquals(false, map.isEmpty()); + assertEquals(true, map.containsKey("fooKey")); + assertEquals(true, map.containsKey("barKey")); + assertEquals(true, map.containsValue("fooVal")); + assertEquals(true, map.containsValue("barVal")); + assertEquals("fooVal", map.get("fooKey")); + assertEquals("barVal", map.get("barKey")); + assertEquals(2, map.keySet().size()); + assertEquals(2, map.entrySet().size()); + assertEquals(2, map.values().size()); + + assertEquals("fooVal", map.remove("fooKey")); + assertEquals(1, map.size()); + assertEquals(false, map.isEmpty()); + assertEquals(false, map.containsKey("fooKey")); + assertEquals(true, map.containsKey("barKey")); + assertEquals(false, map.containsValue("fooVal")); + assertEquals(true, map.containsValue("barVal")); + assertNull(map.get("fooKey")); + assertEquals("barVal", map.get("barKey")); + assertEquals(1, map.keySet().size()); + assertEquals(1, map.entrySet().size()); + assertEquals(1, map.values().size()); + } + + @Test + public void requireThatEntrySetDoesNotReflectConcurrentModifications() { + Map<String, String> map = new CopyOnWriteHashMap<>(); + map.put("fooKey", "fooVal"); + + Iterator<Map.Entry<String, String>> it = map.entrySet().iterator(); + assertEquals("fooVal", map.remove("fooKey")); + + assertTrue(it.hasNext()); + Map.Entry<String, String> entry = it.next(); + assertEquals("fooKey", entry.getKey()); + assertEquals("fooVal", entry.getValue()); + } + + @Test + public void requireThatKeySetDoesNotReflectConcurrentModifications() { + Map<String, String> map = new CopyOnWriteHashMap<>(); + map.put("fooKey", "fooVal"); + + Iterator<String> it = map.keySet().iterator(); + assertEquals("fooVal", map.remove("fooKey")); + + assertTrue(it.hasNext()); + assertEquals("fooKey", it.next()); + } + + @Test + public void requireThatValuesDoNotReflectConcurrentModifications() { + Map<String, String> map = new CopyOnWriteHashMap<>(); + map.put("fooKey", "fooVal"); + + Iterator<String> it = map.values().iterator(); + assertEquals("fooVal", map.remove("fooKey")); + + assertTrue(it.hasNext()); + assertEquals("fooVal", it.next()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/EventBarrierTestCase.java b/vespajlib/src/test/java/com/yahoo/concurrent/EventBarrierTestCase.java new file mode 100644 index 00000000000..eae792effd4 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/concurrent/EventBarrierTestCase.java @@ -0,0 +1,168 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent;
+
+import junit.framework.TestCase;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class EventBarrierTestCase extends TestCase {
+
+ public void testEmpty() {
+ // waiting for an empty set of events
+ Barrier b = new Barrier();
+ EventBarrier eb = new EventBarrier();
+
+ assertTrue(!eb.startBarrier(b));
+ assertTrue(!b.done);
+ assertEquals(eb.getNumEvents(), 0);
+ assertEquals(eb.getNumBarriers(), 0);
+
+ int token = eb.startEvent();
+ eb.completeEvent(token);
+
+ assertTrue(!eb.startBarrier(b));
+ assertTrue(!b.done);
+ assertEquals(eb.getNumEvents(), 0);
+ assertEquals(eb.getNumBarriers(), 0);
+ }
+
+ public void testSimple() {
+ // a single barrier waiting for a single event
+ Barrier b = new Barrier();
+ EventBarrier eb = new EventBarrier();
+ assertEquals(eb.getNumEvents(), 0);
+ assertEquals(eb.getNumBarriers(), 0);
+
+ int token = eb.startEvent();
+ assertEquals(eb.getNumEvents(), 1);
+ assertEquals(eb.getNumBarriers(), 0);
+
+ assertTrue(eb.startBarrier(b));
+ assertTrue(!b.done);
+ assertEquals(eb.getNumEvents(), 1);
+ assertEquals(eb.getNumBarriers(), 1);
+
+ eb.completeEvent(token);
+ assertTrue(b.done);
+ assertEquals(eb.getNumEvents(), 0);
+ assertEquals(eb.getNumBarriers(), 0);
+ }
+
+ public void testBarrierChain() {
+ // more than one barrier waiting for the same set of events
+ Barrier b1 = new Barrier();
+ Barrier b2 = new Barrier();
+ Barrier b3 = new Barrier();
+ EventBarrier eb = new EventBarrier();
+ assertEquals(eb.getNumEvents(), 0);
+ assertEquals(eb.getNumBarriers(), 0);
+
+ int token = eb.startEvent();
+ assertEquals(eb.getNumEvents(), 1);
+ assertEquals(eb.getNumBarriers(), 0);
+
+ assertTrue(eb.startBarrier(b1));
+ assertTrue(eb.startBarrier(b2));
+ assertTrue(eb.startBarrier(b3));
+ assertTrue(!b1.done);
+ assertTrue(!b2.done);
+ assertTrue(!b3.done);
+
+ assertEquals(eb.getNumEvents(), 1);
+ assertEquals(eb.getNumBarriers(), 3);
+
+ eb.completeEvent(token);
+ assertTrue(b1.done);
+ assertTrue(b2.done);
+ assertTrue(b3.done);
+ assertEquals(eb.getNumEvents(), 0);
+ assertEquals(eb.getNumBarriers(), 0);
+ }
+
+ public void testEventAfter() {
+ // new events starting after the start of a barrier
+ Barrier b = new Barrier();
+ EventBarrier eb = new EventBarrier();
+ assertEquals(eb.getNumEvents(), 0);
+ assertEquals(eb.getNumBarriers(), 0);
+
+ int token = eb.startEvent();
+ assertEquals(eb.getNumEvents(), 1);
+ assertEquals(eb.getNumBarriers(), 0);
+
+ assertTrue(eb.startBarrier(b));
+ assertTrue(!b.done);
+ assertEquals(eb.getNumEvents(), 1);
+ assertEquals(eb.getNumBarriers(), 1);
+
+ int t2 = eb.startEvent();
+ assertTrue(!b.done);
+ assertEquals(eb.getNumEvents(), 2);
+ assertEquals(eb.getNumBarriers(), 1);
+
+ eb.completeEvent(token);
+ assertTrue(b.done);
+ assertEquals(eb.getNumEvents(), 1);
+ assertEquals(eb.getNumBarriers(), 0);
+
+ eb.completeEvent(t2);
+ assertEquals(eb.getNumEvents(), 0);
+ assertEquals(eb.getNumBarriers(), 0);
+ }
+
+ public void testReorder() {
+ // events completing in a different order than they started
+ Barrier b1 = new Barrier();
+ Barrier b2 = new Barrier();
+ Barrier b3 = new Barrier();
+ EventBarrier eb = new EventBarrier();
+
+ int t1 = eb.startEvent();
+ eb.startBarrier(b1);
+ int t2 = eb.startEvent();
+ eb.startBarrier(b2);
+ int t3 = eb.startEvent();
+ eb.startBarrier(b3);
+ int t4 = eb.startEvent();
+
+ assertEquals(eb.getNumEvents(), 4);
+ assertEquals(eb.getNumBarriers(), 3);
+
+ assertTrue(!b1.done);
+ assertTrue(!b2.done);
+ assertTrue(!b3.done);
+
+ eb.completeEvent(t4);
+ assertTrue(!b1.done);
+ assertTrue(!b2.done);
+ assertTrue(!b3.done);
+
+ eb.completeEvent(t3);
+ assertTrue(!b1.done);
+ assertTrue(!b2.done);
+ assertTrue(!b3.done);
+
+ eb.completeEvent(t1);
+ assertTrue(b1.done);
+ assertTrue(!b2.done);
+ assertTrue(!b3.done);
+
+ eb.completeEvent(t2);
+ assertTrue(b1.done);
+ assertTrue(b2.done);
+ assertTrue(b3.done);
+
+ assertEquals(eb.getNumEvents(), 0);
+ assertEquals(eb.getNumBarriers(), 0);
+ }
+
+ private static class Barrier implements EventBarrier.BarrierWaiter {
+ boolean done = false;
+
+ @Override
+ public void completeBarrier() {
+ done = true;
+ }
+ }
+}
diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/ExecutorsTestCase.java b/vespajlib/src/test/java/com/yahoo/concurrent/ExecutorsTestCase.java new file mode 100644 index 00000000000..b8f2b0e5c58 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/concurrent/ExecutorsTestCase.java @@ -0,0 +1,139 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Ignore; +import org.junit.Test; + +import java.util.LinkedList; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +public class ExecutorsTestCase { + static private class Runner implements Runnable { + static private AtomicInteger threadCount = new AtomicInteger(0); + static private class ThreadId extends ThreadLocal<Integer> { + @Override + protected Integer initialValue() { + return new Integer(threadCount.getAndIncrement()); + } + } + static private ThreadId threadId = new ThreadId(); + private volatile int runBy = -1; + @Override + public void run() { + runBy = threadId.get(); + } + int getRunBy() { return runBy; } + } + + private static class Producer implements Runnable { + private volatile int maxThreadId = 0; + private final long timeOutMS; + private final ExecutorService consumer; + Producer(ExecutorService consumer, long timeOutMS) { + this.timeOutMS = timeOutMS; + this.consumer = consumer; + } + @Override + public void run() { + long now = System.currentTimeMillis(); + Runner r = new Runner(); + try { + while (now + timeOutMS > System.currentTimeMillis()) { + Future<?> f = consumer.submit(r); + f.get(); + maxThreadId = Math.max(maxThreadId, r.getRunBy()); + Thread.sleep(1); + + } + } catch (InterruptedException e) { + assertTrue(false); + } catch (ExecutionException e) { + assertTrue(false); + } + + } + } + + private void assertThreadId(ExecutorService s, int id) throws InterruptedException, ExecutionException { + Runner r = new Runner(); + Future<?> f = s.submit(r); + assertNull(f.get()); + assertEquals(id, r.getRunBy()); + } + private void assertRoundRobinOrder(ExecutorService s) throws InterruptedException, ExecutionException { + assertThreadId(s, 0); + assertThreadId(s, 1); + assertThreadId(s, 2); + assertThreadId(s, 0); + assertThreadId(s, 1); + assertThreadId(s, 2); + assertThreadId(s, 0); + assertThreadId(s, 1); + } + private int measureMaxNumThreadsUsage(ThreadPoolExecutor s, long durationMS, int maxProducers) throws InterruptedException, ExecutionException { + s.prestartAllCoreThreads(); + ExecutorService consumers = Executors.newCachedThreadPool(); + LinkedList<Future<Producer>> futures = new LinkedList<>(); + for (int i = 0; i < maxProducers; i++) { + Producer p = new Producer(s, durationMS); + futures.add(consumers.submit(p, p)); + } + int maxThreadId = 0; + try { + while (! futures.isEmpty()) { + Producer p = futures.remove().get(); + maxThreadId = Math.max(maxThreadId, p.maxThreadId); + } + } catch (InterruptedException e) { + assertTrue(false); + } catch (ExecutionException e) { + assertTrue(false); + } + return maxThreadId; + } + private void assertStackOrder(ThreadPoolExecutor s) throws InterruptedException, ExecutionException { + s.prestartAllCoreThreads(); + Thread.sleep(10); //Sleep to allow last executing thread to get back on the stack + assertThreadId(s, 0); + Thread.sleep(10); + assertThreadId(s, 0); + Thread.sleep(10); + assertThreadId(s, 0); + Thread.sleep(10); + assertThreadId(s, 0); + Thread.sleep(10); + assertThreadId(s, 0); + Thread.sleep(10); + assertThreadId(s, 0); + Thread.sleep(10); + assertThreadId(s, 0); + } + + @Ignore // Ignored as it is not deterministic, and probably hard to make deterministic to. + @Test + public void requireThatExecutionOrderIsPredictable() throws InterruptedException, ExecutionException { + Runner.threadCount.set(0); + assertRoundRobinOrder(new ThreadPoolExecutor(3, 3, 0L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>())); + Runner.threadCount.set(0); + assertRoundRobinOrder(new ThreadPoolExecutor(3, 3, 0L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(true))); + Runner.threadCount.set(0); + assertStackOrder(new ThreadPoolExecutor(3, 3, 0L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(false))); + } + + @Ignore // Ignored as it might not be deterministic + public void requireThatExecutionOrderIsPredictableUnderLoad() throws InterruptedException, ExecutionException { + Runner.threadCount.set(0); + assertEquals(99, measureMaxNumThreadsUsage(new ThreadPoolExecutor(100, 100, 0L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()), 3000, 10)); + Runner.threadCount.set(0); + assertEquals(99, measureMaxNumThreadsUsage(new ThreadPoolExecutor(100, 100, 0L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(true)), 3000, 10)); + Runner.threadCount.set(0); + //Max 9 concurrent tasks. Might not be deterministic + assertEquals(9, measureMaxNumThreadsUsage(new ThreadPoolExecutor(100, 100, 0L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(false)), 3000, 10)); + Runner.threadCount.set(0); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/ReceiverTestCase.java b/vespajlib/src/test/java/com/yahoo/concurrent/ReceiverTestCase.java new file mode 100644 index 00000000000..88d5283f46a --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/concurrent/ReceiverTestCase.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import com.yahoo.collections.Tuple2; + +/** + * Check for com.yahoo.concurrent.Receiver. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ReceiverTestCase { + + private static class Worker implements Runnable { + private static final String HELLO_WORLD = "Hello, World!"; + private final Receiver<String> receiver; + private final long timeToWait; + + Worker(Receiver<String> receiver, long timeToWait) { + this.receiver = receiver; + this.timeToWait = timeToWait; + } + + @Override + public void run() { + try { + Thread.sleep(timeToWait); + } catch (InterruptedException e) { + fail("Test was interrupted."); + } + receiver.put(HELLO_WORLD); + } + } + + @Test + public void testPut() throws InterruptedException { + Receiver<String> receiver = new Receiver<>(); + Worker runnable = new Worker(receiver, 0); + Thread worker = new Thread(runnable); + worker.start(); + Tuple2<Receiver.MessageState, String> answer = receiver.get(1000L * 1000L * 1000L); + assertEquals(Receiver.MessageState.VALID, answer.first); + assertEquals(answer.second, Worker.HELLO_WORLD); + } + + @Test + public void testTimeOut() throws InterruptedException { + Receiver<String> receiver = new Receiver<>(); + Worker runnable = new Worker(receiver, 1000L * 1000L * 1000L); + Thread worker = new Thread(runnable); + worker.start(); + Tuple2<Receiver.MessageState, String> answer = receiver.get(500L); + assertEquals(Receiver.MessageState.TIMEOUT, answer.first); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/ThreadFactoryFactoryTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/ThreadFactoryFactoryTest.java new file mode 100644 index 00000000000..7fc6a9cc390 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/concurrent/ThreadFactoryFactoryTest.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import org.junit.Test; + + +import java.util.concurrent.ThreadFactory; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 26.04.13 + * Time: 12:01 + * To change this template use File | Settings | File Templates. + */ +public class ThreadFactoryFactoryTest { + + static class Runner implements Runnable { + @Override + public void run() { + + } + } + + @Test + public void requireThatFactoryCreatesCorrectlyNamedThreads() { + Thread thread = ThreadFactoryFactory.getThreadFactory("a").newThread(new Runner()); + assertEquals("a-1-thread-1", thread.getName()); + thread = ThreadFactoryFactory.getThreadFactory("a").newThread(new Runner()); + assertEquals("a-2-thread-1", thread.getName()); + thread = ThreadFactoryFactory.getThreadFactory("b").newThread(new Runner()); + assertEquals("b-1-thread-1", thread.getName()); + ThreadFactory factory = ThreadFactoryFactory.getThreadFactory("a"); + thread = factory.newThread(new Runner()); + assertEquals("a-3-thread-1", thread.getName()); + thread = factory.newThread(new Runner()); + assertEquals("a-3-thread-2", thread.getName()); + thread = factory.newThread(new Runner()); + assertEquals("a-3-thread-3", thread.getName()); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/ThreadLocalDirectoryTestCase.java b/vespajlib/src/test/java/com/yahoo/concurrent/ThreadLocalDirectoryTestCase.java new file mode 100644 index 00000000000..d813ae1e18d --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/concurrent/ThreadLocalDirectoryTestCase.java @@ -0,0 +1,125 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import static org.junit.Assert.*; + +import java.util.List; + +import org.junit.Test; + +/** + * Smoke test for multi producer data structure. + * + * <p> + * TODO sorely needs nastier cases + * </p> + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ThreadLocalDirectoryTestCase { + private static class SumUpdater implements ThreadLocalDirectory.Updater<Integer, Integer> { + + @Override + public Integer update(Integer current, Integer x) { + return Integer.valueOf(current.intValue() + x.intValue()); + } + + @Override + public Integer createGenerationInstance(Integer previous) { + return Integer.valueOf(0); + } + } + + private static class ObservableSumUpdater extends SumUpdater implements ThreadLocalDirectory.ObservableUpdater<Integer, Integer> { + + @Override + public Integer copy(Integer current) { + return current; + } + } + + + private static class Counter implements Runnable { + ThreadLocalDirectory<Integer, Integer> r; + + Counter(ThreadLocalDirectory<Integer, Integer> r) { + this.r = r; + } + + @Override + public void run() { + LocalInstance<Integer, Integer> s = r.getLocalInstance(); + for (int i = 0; i < 500; ++i) { + put(s, i); + } + } + + void put(LocalInstance<Integer, Integer> s, int i) { + r.update(Integer.valueOf(i), s); + } + } + + private static class CounterAndViewer extends Counter { + CounterAndViewer(ThreadLocalDirectory<Integer, Integer> r) { + super(r); + } + + @Override + void put(LocalInstance<Integer, Integer> s, int i) { + super.put(s, i); + if (i % 10 == 0) { + r.view(); + } + } + } + + @Test + public void sumFromMultipleThreads() { + SumUpdater updater = new SumUpdater(); + ThreadLocalDirectory<Integer, Integer> s = new ThreadLocalDirectory<>(updater); + Thread[] threads = new Thread[500]; + for (int i = 0; i < 500; ++i) { + Counter c = new Counter(s); + threads[i] = new Thread(c); + } + runAll(threads); + List<Integer> measurements = s.fetch(); + int sum = 0; + for (Integer i : measurements) { + sum += i.intValue(); + } + assertTrue("Data lost.", 62375000 == sum); + } + + @Test + public void sumAndViewFromMultipleThreads() { + ObservableSumUpdater updater = new ObservableSumUpdater(); + ThreadLocalDirectory<Integer, Integer> s = new ThreadLocalDirectory<>(updater); + Thread[] threads = new Thread[500]; + for (int i = 0; i < 500; ++i) { + CounterAndViewer c = new CounterAndViewer(s); + threads[i] = new Thread(c); + } + runAll(threads); + List<Integer> measurements = s.fetch(); + int sum = 0; + for (Integer i : measurements) { + sum += i.intValue(); + } + assertTrue("Data lost.", 62375000 == sum); + } + + + private void runAll(Thread[] threads) { + for (Thread t : threads) { + t.start(); + } + for (Thread t : threads) { + try { + t.join(); + } catch (InterruptedException e) { + // nop + } + } + } +} diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/ThreadRobustListTestCase.java b/vespajlib/src/test/java/com/yahoo/concurrent/ThreadRobustListTestCase.java new file mode 100644 index 00000000000..88c7a962c95 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/concurrent/ThreadRobustListTestCase.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +import java.util.Iterator; + +import org.junit.Test; + +/** + * Check we keep the consistent view when reading and writing in parallell. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ThreadRobustListTestCase { + + private static class Writer implements Runnable { + private final ThreadRobustList<String> l; + private final Receiver<Boolean> sharedLock; + + public Writer(final ThreadRobustList<String> l, + final Receiver<Boolean> sharedLock) { + this.sharedLock = sharedLock; + this.l = l; + } + + @Override + public void run() { + for (int i = 0; i < 5; ++i) { + l.add(String.valueOf(i)); + } + sharedLock.put(Boolean.TRUE); + for (int i = 5; i < 100 * 1000; ++i) { + l.add(String.valueOf(i)); + } + } + + } + + private static class Reader implements Runnable { + private final ThreadRobustList<String> l; + private final Receiver<Boolean> sharedLock; + + public Reader(final ThreadRobustList<String> l, + final Receiver<Boolean> sharedLock) { + this.sharedLock = sharedLock; + this.l = l; + } + + @Override + public void run() { + int n; + int previous; + + try { + sharedLock.get(5 * 60 * 1000); + } catch (final InterruptedException e) { + fail("Test interrupted."); + } + n = countElements(); + assertFalse(n < 5); + previous = n; + for (int i = 0; i < 1000; ++i) { + int reverse = reverseCountElements(); + n = countElements(); + assertFalse(n < reverse); + assertFalse(n < previous); + previous = n; + } + } + + private int reverseCountElements() { + int n = 0; + for (final Iterator<String> j = l.reverseIterator(); j.hasNext(); j.next()) { + ++n; + } + return n; + } + + private int countElements() { + int n = 0; + for (final Iterator<String> j = l.iterator(); j.hasNext(); j.next()) { + ++n; + } + return n; + } + } + + @Test + public final void test() throws InterruptedException { + final ThreadRobustList<String> container = new ThreadRobustList<>(); + final Receiver<Boolean> lock = new Receiver<>(); + final Reader r = new Reader(container, lock); + final Writer w = new Writer(container, lock); + final Thread wt = new Thread(w); + wt.start(); + r.run(); + wt.join(); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/data/access/InspectorConformanceTestBase.java b/vespajlib/src/test/java/com/yahoo/data/access/InspectorConformanceTestBase.java new file mode 100644 index 00000000000..4cf449fbf4f --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/data/access/InspectorConformanceTestBase.java @@ -0,0 +1,365 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access; + +import org.junit.Test; +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.util.ArrayList; +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.ArrayTraverser; +import com.yahoo.data.access.ObjectTraverser; +import com.yahoo.data.access.Type; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + +abstract public class InspectorConformanceTestBase { + + public abstract static class Try { + abstract void f(); + public Exception call() { + try { + f(); + } catch (Exception e) { + return e; + } + return null; + } + } + + public static class Entries implements ArrayTraverser { + List<Inspector> entries = new ArrayList<>(); + public void entry(int idx, Inspector inspector) { + entries.add(inspector); + } + public Entries traverse(Inspector value) { + value.traverse(this); + return this; + } + public Entries iterate(Inspector value) { + for (Inspector itr: value.entries()) { + entries.add(itr); + } + return this; + } + public Entries add(Inspector value) { + entries.add(value); + return this; + } + } + + public static class Fields implements ObjectTraverser { + Map<String,Inspector> fields = new HashMap<>(); + public void field(String name, Inspector inspector) { + fields.put(name, inspector); + } + public Fields traverse(Inspector value) { + value.traverse(this); + return this; + } + public Fields iterate(Inspector value) { + for (Map.Entry<String,Inspector> itr: value.fields()) { + fields.put(itr.getKey(), itr.getValue()); + } + return this; + } + public Fields add(String name, Inspector value) { + fields.put(name, value); + return this; + } + } + + // This method must be implemented by all tests of concrete + // implementations to return an inspector to a structured object + // on the following form (for an example, take a look at + // com.yahoo.data.access.simple.InspectorConformanceTestCase): + // + // ARRAY { + // [0]: EMPTY + // [1]: BOOL: true + // [2]: LONG: 10 + // [3]: DOUBLE: 5.75 + // [4]: OBJECT { + // "foo": STRING: "foo_value" + // "bar": DATA: 0x04 0x02 + // "nested": ARRAY { + // [0]: OBJECT { + // "hidden": STRING: "treasure" + // } + // } + // } + // } + public abstract Inspector getData(); + + @Test + public void testSelfInspectableInspector() throws Exception { + final Inspector value = getData(); + final Inspector self = value.inspect(); + assertThat(self, is(value)); + } + + @Test + public void testInvalidValue() throws Exception { + final Inspector value = getData().entry(10).field("bogus").entry(0); + assertThat(value.valid(), is(false)); + assertThat(value.type(), is(Type.EMPTY)); + assertThat(value.entryCount(), is(0)); + assertThat(value.fieldCount(), is(0)); + assertThat(new Try(){void f() { value.asBool(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asLong(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asDouble(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asString(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asUtf8(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asData(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(value.asBool(true), is(true)); + assertThat(value.asLong(50), is(50L)); + assertThat(value.asDouble(20.25), is(20.25)); + assertThat(value.asString("default"), is("default")); + assertThat(value.asUtf8("utf8".getBytes("UTF-8")), is("utf8".getBytes("UTF-8"))); + assertThat(value.asData("data".getBytes("UTF-8")), is("data".getBytes("UTF-8"))); + assertThat(new Entries().traverse(value).entries.size(), is(0)); + assertThat(new Fields().traverse(value).fields.size(), is(0)); + assertThat(value.entry(0).valid(), is(false)); + assertThat(value.field("foo").valid(), is(false)); + assertThat(new Entries().iterate(value).entries.size(), is(0)); + assertThat(new Fields().iterate(value).fields.size(), is(0)); + } + + @Test + public void testEmptyValue() throws Exception { + final Inspector value = getData().entry(0); + assertThat(value.valid(), is(true)); + assertThat(value.type(), is(Type.EMPTY)); + assertThat(value.entryCount(), is(0)); + assertThat(value.fieldCount(), is(0)); + assertThat(value.asBool(), is(false)); + assertThat(value.asLong(), is(0L)); + assertThat(value.asDouble(), is(0.0)); + assertThat(value.asString(), is("")); + assertThat(value.asUtf8(), is(new byte[0])); + assertThat(value.asData(), is(new byte[0])); + assertThat(value.asBool(true), is(true)); + assertThat(value.asLong(50), is(50L)); + assertThat(value.asDouble(20.25), is(20.25)); + assertThat(value.asString("default"), is("default")); + assertThat(value.asUtf8("utf8".getBytes("UTF-8")), is("utf8".getBytes("UTF-8"))); + assertThat(value.asData("data".getBytes("UTF-8")), is("data".getBytes("UTF-8"))); + assertThat(new Entries().traverse(value).entries.size(), is(0)); + assertThat(new Fields().traverse(value).fields.size(), is(0)); + assertThat(value.entry(0).valid(), is(false)); + assertThat(value.field("foo").valid(), is(false)); + assertThat(new Entries().iterate(value).entries.size(), is(0)); + assertThat(new Fields().iterate(value).fields.size(), is(0)); + } + + @Test + public void testBoolValue() throws Exception { + final Inspector value = getData().entry(1); + assertThat(value.valid(), is(true)); + assertThat(value.type(), is(Type.BOOL)); + assertThat(value.entryCount(), is(0)); + assertThat(value.fieldCount(), is(0)); + assertThat(value.asBool(), is(true)); + assertThat(new Try(){void f() { value.asLong(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asDouble(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asString(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asUtf8(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asData(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(value.asBool(false), is(true)); + assertThat(value.asLong(50), is(50L)); + assertThat(value.asDouble(20.25), is(20.25)); + assertThat(value.asString("default"), is("default")); + assertThat(value.asUtf8("utf8".getBytes("UTF-8")), is("utf8".getBytes("UTF-8"))); + assertThat(value.asData("data".getBytes("UTF-8")), is("data".getBytes("UTF-8"))); + assertThat(new Entries().traverse(value).entries.size(), is(0)); + assertThat(new Fields().traverse(value).fields.size(), is(0)); + assertThat(value.entry(0).valid(), is(false)); + assertThat(value.field("foo").valid(), is(false)); + assertThat(new Entries().iterate(value).entries.size(), is(0)); + assertThat(new Fields().iterate(value).fields.size(), is(0)); + } + + @Test + public void testLongValue() throws Exception { + final Inspector value = getData().entry(2); + assertThat(value.valid(), is(true)); + assertThat(value.type(), is(Type.LONG)); + assertThat(value.entryCount(), is(0)); + assertThat(value.fieldCount(), is(0)); + assertThat(new Try(){void f() { value.asBool(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(value.asLong(), is(10L)); + assertThat(value.asDouble(), is(10.0)); + assertThat(new Try(){void f() { value.asString(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asUtf8(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asData(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(value.asBool(true), is(true)); + assertThat(value.asLong(50), is(10L)); + assertThat(value.asDouble(20.25), is(10.0)); + assertThat(value.asString("default"), is("default")); + assertThat(value.asUtf8("utf8".getBytes("UTF-8")), is("utf8".getBytes("UTF-8"))); + assertThat(value.asData("data".getBytes("UTF-8")), is("data".getBytes("UTF-8"))); + assertThat(new Entries().traverse(value).entries.size(), is(0)); + assertThat(new Fields().traverse(value).fields.size(), is(0)); + assertThat(value.entry(0).valid(), is(false)); + assertThat(value.field("foo").valid(), is(false)); + assertThat(new Entries().iterate(value).entries.size(), is(0)); + assertThat(new Fields().iterate(value).fields.size(), is(0)); + } + + @Test + public void testDoubleValue() throws Exception { + final Inspector value = getData().entry(3); + assertThat(value.valid(), is(true)); + assertThat(value.type(), is(Type.DOUBLE)); + assertThat(value.entryCount(), is(0)); + assertThat(value.fieldCount(), is(0)); + assertThat(new Try(){void f() { value.asBool(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(value.asLong(), is(5L)); + assertThat(value.asDouble(), is(5.75)); + assertThat(new Try(){void f() { value.asString(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asUtf8(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asData(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(value.asBool(true), is(true)); + assertThat(value.asLong(50), is(5L)); + assertThat(value.asDouble(20.25), is(5.75)); + assertThat(value.asString("default"), is("default")); + assertThat(value.asUtf8("utf8".getBytes("UTF-8")), is("utf8".getBytes("UTF-8"))); + assertThat(value.asData("data".getBytes("UTF-8")), is("data".getBytes("UTF-8"))); + assertThat(new Entries().traverse(value).entries.size(), is(0)); + assertThat(new Fields().traverse(value).fields.size(), is(0)); + assertThat(value.entry(0).valid(), is(false)); + assertThat(value.field("foo").valid(), is(false)); + assertThat(new Entries().iterate(value).entries.size(), is(0)); + assertThat(new Fields().iterate(value).fields.size(), is(0)); + } + + @Test + public void testStringValue() throws Exception { + final Inspector value = getData().entry(4).field("foo"); + assertThat(value.valid(), is(true)); + assertThat(value.type(), is(Type.STRING)); + assertThat(value.entryCount(), is(0)); + assertThat(value.fieldCount(), is(0)); + assertThat(new Try(){void f() { value.asBool(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asLong(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asDouble(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(value.asString(), is("foo_value")); + assertThat(value.asUtf8(), is("foo_value".getBytes("UTF-8"))); + assertThat(new Try(){void f() { value.asData(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(value.asBool(true), is(true)); + assertThat(value.asLong(50), is(50L)); + assertThat(value.asDouble(20.25), is(20.25)); + assertThat(value.asString("default"), is("foo_value")); + assertThat(value.asUtf8("utf8".getBytes("UTF-8")), is("foo_value".getBytes("UTF-8"))); + assertThat(value.asData("data".getBytes("UTF-8")), is("data".getBytes("UTF-8"))); + assertThat(new Entries().traverse(value).entries.size(), is(0)); + assertThat(new Fields().traverse(value).fields.size(), is(0)); + assertThat(value.entry(0).valid(), is(false)); + assertThat(value.field("foo").valid(), is(false)); + assertThat(new Entries().iterate(value).entries.size(), is(0)); + assertThat(new Fields().iterate(value).fields.size(), is(0)); + } + + @Test + public void testDataValue() throws Exception { + final Inspector value = getData().entry(4).field("bar"); + assertThat(value.valid(), is(true)); + assertThat(value.type(), is(Type.DATA)); + assertThat(value.entryCount(), is(0)); + assertThat(value.fieldCount(), is(0)); + assertThat(new Try(){void f() { value.asBool(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asLong(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asDouble(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asString(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asUtf8(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(value.asData(), is(new byte[] { (byte)4, (byte)2 })); + assertThat(value.asBool(true), is(true)); + assertThat(value.asLong(50), is(50L)); + assertThat(value.asDouble(20.25), is(20.25)); + assertThat(value.asString("default"), is("default")); + assertThat(value.asUtf8("utf8".getBytes("UTF-8")), is("utf8".getBytes("UTF-8"))); + assertThat(value.asData("data".getBytes("UTF-8")), is(new byte[] { (byte)4, (byte)2 })); + assertThat(new Entries().traverse(value).entries.size(), is(0)); + assertThat(new Fields().traverse(value).fields.size(), is(0)); + assertThat(value.entry(0).valid(), is(false)); + assertThat(value.field("foo").valid(), is(false)); + assertThat(new Entries().iterate(value).entries.size(), is(0)); + assertThat(new Fields().iterate(value).fields.size(), is(0)); + } + + @Test + public void testArrayValue() throws Exception { + final Inspector value = getData(); + List<Inspector> expected_entries = new Entries() + .add(value.entry(0)) + .add(value.entry(1)) + .add(value.entry(2)) + .add(value.entry(3)) + .add(value.entry(4)).entries; + assertThat(value.valid(), is(true)); + assertThat(value.type(), is(Type.ARRAY)); + assertThat(value.entryCount(), is(expected_entries.size())); + assertThat(value.fieldCount(), is(0)); + assertThat(new Try(){void f() { value.asBool(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asLong(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asDouble(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asString(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asUtf8(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asData(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(value.asBool(true), is(true)); + assertThat(value.asLong(50), is(50L)); + assertThat(value.asDouble(20.25), is(20.25)); + assertThat(value.asString("default"), is("default")); + assertThat(value.asUtf8("utf8".getBytes("UTF-8")), is("utf8".getBytes("UTF-8"))); + assertThat(value.asData("data".getBytes("UTF-8")), is("data".getBytes("UTF-8"))); + assertThat(new Entries().traverse(value).entries, is(expected_entries)); + assertThat(new Fields().traverse(value).fields.size(), is(0)); + assertThat(value.entry(10).valid(), is(false)); + assertThat(value.field("foo").valid(), is(false)); + assertThat(new Entries().iterate(value).entries, is(expected_entries)); + assertThat(new Fields().iterate(value).fields.size(), is(0)); + } + + @Test + public void testObjectValue() throws Exception { + final Inspector value = getData().entry(4); + Map<String,Inspector> expected_fields = new Fields() + .add("foo", value.field("foo")) + .add("bar", value.field("bar")) + .add("nested", value.field("nested")).fields; + assertThat(value.valid(), is(true)); + assertThat(value.type(), is(Type.OBJECT)); + assertThat(value.entryCount(), is(0)); + assertThat(value.fieldCount(), is(expected_fields.size())); + assertThat(new Try(){void f() { value.asBool(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asLong(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asDouble(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asString(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asUtf8(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(new Try(){void f() { value.asData(); }}.call(), instanceOf(IllegalStateException.class)); + assertThat(value.asBool(true), is(true)); + assertThat(value.asLong(50), is(50L)); + assertThat(value.asDouble(20.25), is(20.25)); + assertThat(value.asString("default"), is("default")); + assertThat(value.asUtf8("utf8".getBytes("UTF-8")), is("utf8".getBytes("UTF-8"))); + assertThat(value.asData("data".getBytes("UTF-8")), is("data".getBytes("UTF-8"))); + assertThat(new Entries().traverse(value).entries.size(), is(0)); + assertThat(new Fields().traverse(value).fields, is(expected_fields)); + assertThat(value.entry(0).valid(), is(false)); + assertThat(value.field("bogus").valid(), is(false)); + assertThat(new Entries().iterate(value).entries.size(), is(0)); + assertThat(new Fields().iterate(value).fields, is(expected_fields)); + } + + @Test + public void testNesting() throws Exception { + Inspector value1 = getData().entry(4).field("nested"); + assertThat(value1.type(), is(Type.ARRAY)); + Inspector value2 = value1.entry(0); + assertThat(value2.type(), is(Type.OBJECT)); + Inspector value3 = value2.field("hidden"); + assertThat(value3.type(), is(Type.STRING)); + assertThat(value3.asString(), is("treasure")); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/data/access/simple/SimpleConformanceTestCase.java b/vespajlib/src/test/java/com/yahoo/data/access/simple/SimpleConformanceTestCase.java new file mode 100644 index 00000000000..4db5d0afde7 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/data/access/simple/SimpleConformanceTestCase.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access.simple; + + +import org.junit.Test; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + + +public class SimpleConformanceTestCase extends com.yahoo.data.access.InspectorConformanceTestBase { + + // ARRAY { + // [0]: EMPTY + // [1]: BOOL: true + // [2]: LONG: 10 + // [3]: DOUBLE: 5.75 + // [4]: OBJECT { + // "foo": STRING: "foo_value" + // "bar": DATA: 0x04 0x02 + // "nested": ARRAY { + // [0]: OBJECT { + // "hidden": STRING: "treasure" + // } + // } + // } + // } + public com.yahoo.data.access.Inspector getData() { + return new Value.ArrayValue() + .add(new Value.EmptyValue()) + .add(new Value.BoolValue(true)) + .add(new Value.LongValue(10L)) + .add(new Value.DoubleValue(5.75)) + .add(new Value.ObjectValue() + .put("foo", new Value.StringValue("foo_value")) + .put("bar", new Value.DataValue(new byte[] { (byte)4, (byte)2 })) + .put("nested", new Value.ArrayValue() + .add(new Value.ObjectValue() + .put("hidden", new Value.StringValue("treasure"))))); + } + + @Test + public void testSingletons() { + assertThat(Value.empty().valid(), is(true)); + assertThat(Value.empty().type(), is(com.yahoo.data.access.Type.EMPTY)); + assertThat(Value.invalid().valid(), is(false)); + assertThat(Value.invalid().type(), is(com.yahoo.data.access.Type.EMPTY)); + } + + @Test + public void testToString() { + String json = getData().toString(); + String correct = "[null,true,10,5.75,{\"foo\":\"foo_value\",\"bar\":\"0x0402\",\"nested\":[{\"hidden\":\"treasure\"}]}]"; + assertThat(json, is(correct)); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/data/access/slime/SlimeConformanceTestCase.java b/vespajlib/src/test/java/com/yahoo/data/access/slime/SlimeConformanceTestCase.java new file mode 100644 index 00000000000..ff6e98cfa37 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/data/access/slime/SlimeConformanceTestCase.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.data.access.slime; + + +public class SlimeConformanceTestCase extends com.yahoo.data.access.InspectorConformanceTestBase { + + // ARRAY { + // [0]: EMPTY + // [1]: BOOL: true + // [2]: LONG: 10 + // [3]: DOUBLE: 5.75 + // [4]: OBJECT { + // "foo": STRING: "foo_value" + // "bar": DATA: 0x04 0x02 + // "nested": ARRAY { + // [0]: OBJECT { + // "hidden": STRING: "treasure" + // } + // } + // } + // } + public com.yahoo.data.access.Inspector getData() { + com.yahoo.slime.Slime slime = new com.yahoo.slime.Slime(); + { + com.yahoo.slime.Cursor arr = slime.setArray(); + arr.addNix(); + arr.addBool(true); + arr.addLong(10); + arr.addDouble(5.75); + { + com.yahoo.slime.Cursor obj = arr.addObject(); + obj.setString("foo", "foo_value"); + obj.setData("bar", new byte[] { (byte)4, (byte)2 }); + { + com.yahoo.slime.Cursor nested_array = obj.setArray("nested"); + { + com.yahoo.slime.Cursor nested_object = nested_array.addObject(); + nested_object.setString("hidden", "treasure"); + } + } + } + } + return new SlimeAdapter(slime.get()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/data/inspect/slime/.gitignore b/vespajlib/src/test/java/com/yahoo/data/inspect/slime/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/data/inspect/slime/.gitignore diff --git a/vespajlib/src/test/java/com/yahoo/geo/BoundingBoxParserTestCase.java b/vespajlib/src/test/java/com/yahoo/geo/BoundingBoxParserTestCase.java new file mode 100644 index 00000000000..47a8ade2235 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/geo/BoundingBoxParserTestCase.java @@ -0,0 +1,162 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.geo; + +/** + * Tests for the BoundingBoxParser class. + * + * @author Arne J + */ +public class BoundingBoxParserTestCase extends junit.framework.TestCase { + + private BoundingBoxParser parser; + + public BoundingBoxParserTestCase(String name) { + super(name); + } + + private void allZero(BoundingBoxParser data) { + assertEquals(0d, data.n); + assertEquals(0d, data.s); + assertEquals(0d, data.e); + assertEquals(0d, data.w); + } + + private void all1234(BoundingBoxParser data) { + assertEquals(1d, data.n); + assertEquals(2d, data.s); + assertEquals(3d, data.e); + assertEquals(4d, data.w); + } + + /** + * Tests different inputs that should all produce 0 + */ + public void testZero() { + parser = new BoundingBoxParser("n=0,s=0,e=0,w=0"); + allZero(parser); + parser = new BoundingBoxParser("N=0,S=0,E=0,W=0"); + allZero(parser); + parser = new BoundingBoxParser("NORTH=0,SOUTH=0,EAST=0,WEST=0"); + allZero(parser); + parser = new BoundingBoxParser("north=0,south=0,east=0,west=0"); + allZero(parser); + parser = new BoundingBoxParser("n=0.0,s=0.0e-17,e=0.0e0,w=0.0e100"); + allZero(parser); + parser = new BoundingBoxParser("s:0.0,w:0.0,n:0.0,e:0.0"); + allZero(parser); + parser = new BoundingBoxParser("s:0.0,w:0.0,n:0.0,e:0.0"); + allZero(parser); + } + + public void testOneTwoThreeFour() { + parser = new BoundingBoxParser("n=1,s=2,e=3,w=4"); + all1234(parser); + parser = new BoundingBoxParser("n=1.0,s=2.0,e=3.0,w=4.0"); + all1234(parser); + parser = new BoundingBoxParser("s=2,w=4,n=1,e=3"); + all1234(parser); + parser = new BoundingBoxParser("N=1,S=2,E=3,W=4"); + all1234(parser); + parser = new BoundingBoxParser("S=2,W=4,N=1,E=3"); + all1234(parser); + parser = new BoundingBoxParser("north=1.0,south=2.0,east=3.0,west=4.0"); + all1234(parser); + parser = new BoundingBoxParser("South=2.0 West=4.0 North=1.0 East=3.0"); + all1234(parser); + } + + /** + * Tests various legal inputs and print the output + */ + public void testPrint() { + String here = "n=63.418417 E=10.433033 S=37.7 W=-122.02"; + parser = new BoundingBoxParser(here); + System.out.println(here+" -> "+parser); + } + + public void testGeoPlanetExample() { + /* example XML: + <boundingBox> + <southWest> + <latitude>40.183868</latitude> + <longitude>-74.819519</longitude> + </southWest> + <northEast> + <latitude>40.248291</latitude> + <longitude>-74.728798</longitude> + </northEast> + </boundingBox> + + can be input as: + + s=40.183868,w=-74.819519,n=40.248291,e=-74.728798 + */ + parser = new BoundingBoxParser("south=40.183868,west=-74.819519,north=40.248291,east=-74.728798"); + assertEquals(40.183868d, parser.s, 0.0000001); + assertEquals(-74.819519d, parser.w, 0.0000001); + assertEquals(40.248291d, parser.n, 0.0000001); + assertEquals(-74.728798d, parser.e, 0.0000001); + } + + public void testGwsExample() { + /* example XML: + <boundingbox> + <north>37.44899</north><south>37.3323</south><east>-121.98241</east><west>-122.06566</west> + </boundingbox> + can be input as: north:37.44899 south:37.3323, east:-121.98241 west:-122.06566 + */ + parser = new BoundingBoxParser(" north:37.44899 south:37.3323, east:-121.98241 west:-122.06566 "); + assertEquals(37.44899d, parser.n, 0.000001); + assertEquals(37.33230d, parser.s, 0.000001); + assertEquals(-121.98241d, parser.e, 0.000001); + assertEquals(-122.06566d, parser.w, 0.000001); + } + + /** + * Tests various inputs that contain syntax errors. + */ + public void testInputErrors() { + String message = ""; + try { + parser = new BoundingBoxParser("n=10.11,e=2.02"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("Missing bounding box limits, n=true s=false e=true w=false", message); + + try { + parser = new BoundingBoxParser("n=11.01,s=10.11,e=xyzzy,w=-122.2"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("Could not parse e limit 'xyzzy' as a number", message); + + try { + parser = new BoundingBoxParser("n=11.01,n=10.11,e=-122.0,w=-122.2"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("multiple limits for 'n' boundary", message); + + try { + parser = new BoundingBoxParser("s=11.01,s=10.11,e=-122.0,w=-122.2"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("multiple limits for 's' boundary", message); + + try { + parser = new BoundingBoxParser("n=11.01,s=10.11,e=-122.0,e=-122.2"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("multiple limits for 'e' boundary", message); + + try { + parser = new BoundingBoxParser("n=11.01,s=10.11,w=-122.0,w=-122.2"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("multiple limits for 'w' boundary", message); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/geo/DegreesParserTestCase.java b/vespajlib/src/test/java/com/yahoo/geo/DegreesParserTestCase.java new file mode 100644 index 00000000000..ed6fed5cbc7 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/geo/DegreesParserTestCase.java @@ -0,0 +1,282 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.geo; + +/** + * Tests for the DegreesParser class. + * + * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + */ +public class DegreesParserTestCase extends junit.framework.TestCase { + + private DegreesParser parser; + + public DegreesParserTestCase(String name) { + super(name); + } + + /** + * Tests different inputs that should all produce 0 or -0. + */ + public void testZero() { + parser = new DegreesParser("N0;E0"); + assertEquals(0d, parser.latitude); + assertEquals(0d, parser.longitude); + parser = new DegreesParser("S0;W0"); + assertEquals(-0d, parser.latitude); + assertEquals(-0d, parser.longitude); + parser = new DegreesParser("N0.0;E0.0"); + assertEquals(0d, parser.latitude); + assertEquals(0d, parser.longitude); + parser = new DegreesParser("S0.0;W0.0"); + assertEquals(-0d, parser.latitude); + assertEquals(-0d, parser.longitude); + parser = new DegreesParser("N0\u00B00'0;E0\u00B00'0"); + assertEquals(0d, parser.latitude); + assertEquals(0d, parser.longitude); + parser = new DegreesParser("S0\u00B00'0;W0\u00B00'0"); + assertEquals(-0d, parser.latitude); + assertEquals(-0d, parser.longitude); + parser = new DegreesParser("S0o0'0;W0o0'0"); + assertEquals(-0d, parser.latitude); + assertEquals(-0d, parser.longitude); + } + + /** + * Tests various legal inputs and print the output + */ + public void testPrint() { + String here = "63N025.105;010E25.982"; + parser = new DegreesParser(here); + System.out.println(here+" -> "+parser.latitude+"/"+parser.longitude+" (lat/long)"); + + here = "N63.418417 E10.433033"; + parser = new DegreesParser(here); + System.out.println(here+" -> "+parser.latitude+"/"+parser.longitude+" (lat/long)"); + + here = "N63o025.105;E010o25.982"; + parser = new DegreesParser(here); + System.out.println(here+" -> "+parser.latitude+"/"+parser.longitude+" (lat/long)"); + + here = "N63.418417;E10.433033"; + parser = new DegreesParser(here); + System.out.println(here+" -> "+parser.latitude+"/"+parser.longitude+" (lat/long)"); + + here = "63.418417N;10.433033E"; + parser = new DegreesParser(here); + System.out.println(here+" -> "+parser.latitude+"/"+parser.longitude+" (lat/long)"); + + here = "N37.417075;W122.025358"; + parser = new DegreesParser(here); + System.out.println(here+" -> "+parser.latitude+"/"+parser.longitude+" (lat/long)"); + + here = "N37\u00B024.983;W122\u00B001.481"; + parser = new DegreesParser(here); + System.out.println(here+" -> "+parser.latitude+"/"+parser.longitude+" (lat/long)"); + } + + /** + * Tests inputs that are close to 0. + */ + public void testNearZero() { + parser = new DegreesParser("N0.0001;E0.0001"); + assertEquals(0.0001, parser.latitude); + assertEquals(0.0001, parser.longitude); + parser = new DegreesParser("S0.0001;W0.0001"); + assertEquals(-0.0001, parser.latitude); + assertEquals(-0.0001, parser.longitude); + + parser = new DegreesParser("N0.000001;E0.000001"); + assertEquals(0.000001, parser.latitude); + assertEquals(0.000001, parser.longitude); + parser = new DegreesParser("S0.000001;W0.000001"); + assertEquals(-0.000001, parser.latitude); + assertEquals(-0.000001, parser.longitude); + + parser = new DegreesParser("N0\u00B00'1;E0\u00B00'1"); + assertEquals(1/3600d, parser.latitude); + assertEquals(1/3600d, parser.longitude); + parser = new DegreesParser("S0\u00B00'1;W0\u00B00'1"); + assertEquals(-1/3600d, parser.latitude); + assertEquals(-1/3600d, parser.longitude); + } + + /** + * Tests inputs that are close to latitude 90/-90 degrees and longitude 180/-180 degrees. + */ + public void testNearBoundary() { + + parser = new DegreesParser("N89.9999;E179.9999"); + assertEquals(89.9999, parser.latitude); + assertEquals(179.9999, parser.longitude); + parser = new DegreesParser("S89.9999;W179.9999"); + assertEquals(-89.9999, parser.latitude); + assertEquals(-179.9999, parser.longitude); + + parser = new DegreesParser("N89.999999;E179.999999"); + assertEquals(89.999999, parser.latitude); + assertEquals(179.999999, parser.longitude); + parser = new DegreesParser("S89.999999;W179.999999"); + assertEquals(-89.999999, parser.latitude); + assertEquals(-179.999999, parser.longitude); + + parser = new DegreesParser("N89\u00B059'59;E179\u00B059'59"); + assertEquals(89+59/60d+59/3600d, parser.latitude); + assertEquals(179+59/60d+59/3600d, parser.longitude); + parser = new DegreesParser("S89\u00B059'59;W179\u00B059'59"); + assertEquals(-(89+59/60d+59/3600d), parser.latitude); + assertEquals(-(179+59/60d+59/3600d), parser.longitude); + } + + /** + * Tests inputs that are on latitude 90/-90 degrees and longitude 180/-180 degrees. + */ + public void testOnBoundary() { + parser = new DegreesParser("N90;E180"); + assertEquals(90d, parser.latitude); + assertEquals(180d, parser.longitude); + parser = new DegreesParser("S90;W180"); + assertEquals(-90d, parser.latitude); + assertEquals(-180d, parser.longitude); + + parser = new DegreesParser("N90\u00B00'0;E180\u00B00'0"); + assertEquals(90d, parser.latitude); + assertEquals(180d, parser.longitude); + parser = new DegreesParser("S90\u00B00'0;W180\u00B00'0"); + assertEquals(-90d, parser.latitude); + assertEquals(-180d, parser.longitude); + } + + /** + * Tests inputs that are above latitude 90/-90 degrees and longitude 180/-180 degrees. + */ + public void testAboveBoundary() { + String message = ""; + try { + parser = new DegreesParser("N90.0001;E0"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("out of range [-90,+90]: 90.0001", message); + try { + parser = new DegreesParser("S90.0001;E0"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("out of range [-90,+90]: -90.0001", message); + try { + parser = new DegreesParser("N0;E180.0001"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("out of range [-180,+180]: 180.0001", message); + try { + parser = new DegreesParser("N0;W180.0001"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("out of range [-180,+180]: -180.0001", message); + try { + parser = new DegreesParser("N90.000001;E0"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("out of range [-90,+90]: 90.000001", message); + try { + parser = new DegreesParser("S90.000001;E0"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("out of range [-90,+90]: -90.000001", message); + try { + parser = new DegreesParser("N0;E180.000001"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("out of range [-180,+180]: 180.000001", message); + try { + parser = new DegreesParser("N0;W180.000001"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("out of range [-180,+180]: -180.000001", message); + } + + /** + * Tests various inputs that contain syntax errors. + */ + public void testInputErrors() { + String message = ""; + try { + parser = new DegreesParser("N90;S90"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("found latitude (N or S) twice", message); + try { + parser = new DegreesParser("E120;W120"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("found longitude (E or W) twice", message); + try { + parser = new DegreesParser("N90;90"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("end of field without any compass direction seen", message); + try { + parser = new DegreesParser("N90;E"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("end of field without any number seen", message); + try { + parser = new DegreesParser(";"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("end of field without any compass direction seen", message); + try { + parser = new DegreesParser("25;60"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("end of field without any compass direction seen", message); + try { + parser = new DegreesParser("NW25;SW60"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("already set direction once, cannot add direction: W", message); + try { + parser = new DegreesParser("N16.25\u00B0;W60"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("cannot have fractional degrees before degrees sign", message); + try { + parser = new DegreesParser("N16\u00B022.40';W60"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("cannot have fractional minutes before minutes sign", message); + try { + parser = new DegreesParser(""); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("end of field without any compass direction seen", message); + try { + parser = new DegreesParser("Yahoo!"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("invalid character: Y", message); + try { + parser = new DegreesParser("N63O025.105;E010O25.982"); + } catch (IllegalArgumentException e) { + message = e.getMessage(); + } + assertEquals("invalid character: O", message); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/geo/ZCurveTestCase.java b/vespajlib/src/test/java/com/yahoo/geo/ZCurveTestCase.java new file mode 100644 index 00000000000..b5536fca510 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/geo/ZCurveTestCase.java @@ -0,0 +1,204 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.geo; + +/** + * Tests for the ZCurve class. + * + * @author gjoranv + */ +public class ZCurveTestCase extends junit.framework.TestCase { + + public ZCurveTestCase(String name) { + super(name); + } + + /** + * Verify that encoded values return the expected bit pattern + */ + public void testEncoding() { + int x = 0; + int y = 0; + long z = ZCurve.encode(x, y); + assertEquals(0, z); + + x = Integer.MAX_VALUE; + y = Integer.MAX_VALUE; + z = ZCurve.encode(x, y); + assertEquals(0x3fffffffffffffffL, z); + + x = Integer.MIN_VALUE; + y = Integer.MIN_VALUE; + z = ZCurve.encode(x, y); + assertEquals(0xc000000000000000L, z); + + x = Integer.MIN_VALUE; + y = Integer.MAX_VALUE; + z = ZCurve.encode(x, y); + assertEquals(0x6aaaaaaaaaaaaaaaL, z); + + x = -1; + y = -1; + z = ZCurve.encode(x, y); + assertEquals(0xffffffffffffffffL, z); + + x = Integer.MAX_VALUE / 2; + y = Integer.MIN_VALUE / 2; + z = ZCurve.encode(x, y); + assertEquals(0xa555555555555555L, z); + } + + /** + * Verify that decoded values are equal to inputs in different cases + */ + public void testDecoding() { + int x = 0; + int y = 0; + long z = ZCurve.encode(x, y); + int[] xy = ZCurve.decode(z); + assertEquals(x, xy[0]); + assertEquals(y, xy[1]); + + x = Integer.MAX_VALUE; + y = Integer.MAX_VALUE; + z = ZCurve.encode(x, y); + xy = ZCurve.decode(z); + assertEquals(x, xy[0]); + assertEquals(y, xy[1]); + + x = Integer.MIN_VALUE; + y = Integer.MIN_VALUE; + z = ZCurve.encode(x, y); + xy = ZCurve.decode(z); + assertEquals(x, xy[0]); + assertEquals(y, xy[1]); + + x = Integer.MIN_VALUE; + y = Integer.MAX_VALUE; + z = ZCurve.encode(x, y); + xy = ZCurve.decode(z); + assertEquals(x, xy[0]); + assertEquals(y, xy[1]); + + x = -18; + y = 1333; + z = ZCurve.encode(x, y); + xy = ZCurve.decode(z); + assertEquals(x, xy[0]); + assertEquals(y, xy[1]); + + x = -1333; + y = 18; + z = ZCurve.encode(x, y); + xy = ZCurve.decode(z); + assertEquals(x, xy[0]); + assertEquals(y, xy[1]); + } + + + + /** + * Verify that encoded values return the expected bit pattern + */ + public void testEncoding_slow() { + int x = 0; + int y = 0; + long z = ZCurve.encode_slow(x, y); + assertEquals(0, z); + + x = Integer.MIN_VALUE; + y = Integer.MIN_VALUE; + z = ZCurve.encode_slow(x, y); + assertEquals(0xc000000000000000L, z); + + x = Integer.MIN_VALUE; + y = Integer.MAX_VALUE; + z = ZCurve.encode_slow(x, y); + assertEquals(0x6aaaaaaaaaaaaaaaL, z); + + x = Integer.MAX_VALUE; + y = Integer.MAX_VALUE; + z = ZCurve.encode_slow(x, y); + assertEquals(0x3fffffffffffffffL, z); + + x = -1; + y = -1; + z = ZCurve.encode_slow(x, y); + assertEquals(0xffffffffffffffffL, z); + + x = Integer.MAX_VALUE / 2; + y = Integer.MIN_VALUE / 2; + z = ZCurve.encode_slow(x, y); + assertEquals(0xa555555555555555L, z); + } + + /** + * Verify that decoded values are equal to inputs in different cases + */ + public void testDecoding_slow() { + int x = 0; + int y = 0; + long z = ZCurve.encode_slow(x, y); + int[] xy = ZCurve.decode_slow(z); + assertEquals(xy[0], x); + assertEquals(xy[1], y); + + x = Integer.MAX_VALUE; + y = Integer.MAX_VALUE; + z = ZCurve.encode_slow(x, y); + xy = ZCurve.decode_slow(z); + assertEquals(xy[0], x); + assertEquals(xy[1], y); + + x = Integer.MIN_VALUE; + y = Integer.MIN_VALUE; + z = ZCurve.encode_slow(x, y); + xy = ZCurve.decode_slow(z); + assertEquals(xy[0], x); + assertEquals(xy[1], y); + + x = Integer.MIN_VALUE; + y = Integer.MAX_VALUE; + z = ZCurve.encode_slow(x, y); + xy = ZCurve.decode_slow(z); + assertEquals(xy[0], x); + assertEquals(xy[1], y); + + x = -18; + y = 1333; + z = ZCurve.encode_slow(x, y); + xy = ZCurve.decode_slow(z); + assertEquals(xy[0], x); + assertEquals(xy[1], y); + + x = -1333; + y = 18; + z = ZCurve.encode_slow(x, y); + xy = ZCurve.decode_slow(z); + assertEquals(xy[0], x); + assertEquals(xy[1], y); + } + + public void testBenchmarkEncoding() { + int limit = 2000000; + + long z1 = 0L; + long start = System.currentTimeMillis(); + for (int i=0; i<limit; i++) { + z1 += ZCurve.encode(i,-i); + } + long elapsed = System.currentTimeMillis() - start; + System.out.println("Fast method: elapsed time: " + elapsed + " ms"); + System.out.println("Per encoding: " + elapsed/(1.0*limit) * 1000000 + " ns"); + + long z2 = 0L; + start = System.currentTimeMillis(); + for (int i=0; i<limit; i++) { + z2 += ZCurve.encode_slow(i,-i); + } + elapsed = System.currentTimeMillis() - start; + System.out.println("Slow method: elapsed time: " + elapsed + " ms"); + System.out.println("Per encoding: " + elapsed/(1.0*limit) * 1000000 + " ns"); + assertEquals(z1, z2); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/io/BlobTestCase.java b/vespajlib/src/test/java/com/yahoo/io/BlobTestCase.java new file mode 100644 index 00000000000..9643d70ba5e --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/io/BlobTestCase.java @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.ByteBuffer; + +public class BlobTestCase extends junit.framework.TestCase { + + public void testEmpty() { + Blob empty = new Blob(); + assertTrue(empty.get() != null); + assertEquals(0, empty.get().length); + } + + public void testCopyArray() { + byte[] d = { 1, 2, 3 }; + Blob b = new Blob(d); + d[0] = 7; + d[1] = 8; + d[2] = 9; + assertEquals(3, b.get().length); + assertEquals(1, b.get()[0]); + assertEquals(2, b.get()[1]); + assertEquals(3, b.get()[2]); + } + + public void testCopyArraySubset() { + byte[] d = { 1, 2, 3 }; + Blob b = new Blob(d, 1, 1); + d[0] = 7; + d[1] = 8; + d[2] = 9; + assertEquals(1, b.get().length); + assertEquals(2, b.get()[0]); + } + + public void testCopyBlob() { + byte[] d = { 1, 2, 3 }; + Blob b = new Blob(d); + Blob x = new Blob(b); + b.get()[1] = 4; + assertEquals(3, x.get().length); + assertEquals(1, x.get()[0]); + assertEquals(4, b.get()[1]); + assertEquals(2, x.get()[1]); + assertEquals(3, x.get()[2]); + } + + public void testReadBuffer() { + ByteBuffer buf = ByteBuffer.allocate(100); + buf.put((byte)1); + buf.put((byte)2); + buf.put((byte)3); + buf.flip(); + assertEquals(3, buf.remaining()); + Blob b = new Blob(buf); + assertEquals(0, buf.remaining()); + assertEquals(3, b.get().length); + assertEquals(1, b.get()[0]); + assertEquals(2, b.get()[1]); + assertEquals(3, b.get()[2]); + } + + public void testReadPartialBuffer() { + ByteBuffer buf = ByteBuffer.allocate(100); + buf.put((byte)1); + buf.put((byte)2); + buf.put((byte)3); + buf.put((byte)4); + buf.put((byte)5); + buf.flip(); + assertEquals(5, buf.remaining()); + Blob b = new Blob(buf, 3); + assertEquals(2, buf.remaining()); + assertEquals(3, b.get().length); + assertEquals(1, b.get()[0]); + assertEquals(2, b.get()[1]); + assertEquals(3, b.get()[2]); + assertEquals(4, buf.get()); + assertEquals(5, buf.get()); + assertEquals(0, buf.remaining()); + } + + public void testWriteBuffer() { + byte[] d = { 1, 2, 3 }; + Blob b = new Blob(d); + ByteBuffer buf = ByteBuffer.allocate(100); + b.write(buf); + buf.flip(); + assertEquals(3, buf.remaining()); + assertEquals(1, buf.get()); + assertEquals(2, buf.get()); + assertEquals(3, buf.get()); + assertEquals(0, buf.remaining()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/io/ByteWriterTestCase.java b/vespajlib/src/test/java/com/yahoo/io/ByteWriterTestCase.java new file mode 100644 index 00000000000..d5025dd03aa --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/io/ByteWriterTestCase.java @@ -0,0 +1,444 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import static org.junit.Assert.assertArrayEquals; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; +import java.util.Arrays; +import java.util.IdentityHashMap; + +import com.yahoo.text.Utf8; +import com.yahoo.text.Utf8Array; + +/** + * Test the ByteWriter class. ByteWriter is also implicitly tested in + * com.yahoo.prelude.templates.test.TemplateTestCase. + * + * @author <a href="mailt:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ByteWriterTestCase extends junit.framework.TestCase { + private CharsetEncoder encoder; + private ByteArrayOutputStream stream; + + // TODO split BufferChain tests from ByteWriter tests + public ByteWriterTestCase (String name) { + super(name); + Charset cs = Charset.forName("UTF-8"); + encoder = cs.newEncoder(); + stream = new ByteArrayOutputStream(); + + } + + /** + * A stream which does nothing, but complains if it is called and asked to + * do nothing. + */ + private static class CurmudgeonlyStream extends OutputStream { + + static final String ZERO_LENGTH_WRITE = "Was asked to do zero length write."; + + @Override + public void write(int b) throws IOException { + // NOP + + } + + @Override + public void close() throws IOException { + // NOP + } + + @Override + public void flush() throws IOException { + // NOP + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + if (len == 0) { + throw new IOException(ZERO_LENGTH_WRITE); + } + } + + @Override + public void write(byte[] b) throws IOException { + if (b.length == 0) { + throw new IOException(ZERO_LENGTH_WRITE); + } + } + + } + + public void testMuchData() throws java.io.IOException { + final int SINGLE_BUFFER = 500; + final int APPENDS = 500; + assertTrue("Code has been changed making constants in this test meaningless, review test.", + BufferChain.BUFFERSIZE * BufferChain.MAXBUFFERS < SINGLE_BUFFER * APPENDS); + assertTrue("Code has been changed making constants in this test meaningless, review test.", + BufferChain.WATERMARK > SINGLE_BUFFER); + stream.reset(); + byte[] c = new byte[SINGLE_BUFFER]; + Arrays.fill(c, (byte) 'a'); + ByteWriter bw = new ByteWriter(stream, encoder); + for (int i = APPENDS; i > 0; --i) { + bw.append(c); + } + bw.close(); + byte[] res = stream.toByteArray(); + assertEquals("BufferChain has duplicated or lost a buffer.", SINGLE_BUFFER * APPENDS, res.length); + byte[] completeData = new byte[SINGLE_BUFFER * APPENDS]; + Arrays.fill(completeData, (byte) 'a'); + assertTrue("ByteWriter seems to have introduced data errors.", Arrays.equals(completeData, res)); + } + + public void testLongString() throws IOException { + final int length = BufferChain.BUFFERSIZE * BufferChain.MAXBUFFERS * 3; + StringBuilder b = new StringBuilder(length); + String s; + for (int i = length; i > 0; --i) { + b.append("\u00E5"); + } + s = b.toString(); + stream.reset(); + ByteWriter bw = new ByteWriter(stream, encoder); + bw.write(s); + bw.close(); + String res = stream.toString("UTF-8"); + assertEquals(s, res); + } + + public void testNoSpuriousWrite() throws IOException { + OutputStream grumpy = new CurmudgeonlyStream(); + ByteWriter bw = new ByteWriter(grumpy, encoder); + final int SINGLE_BUFFER = 500; + final int APPENDS = 500; + assertTrue("Code has been changed making constants in this test meaningless, review test.", + BufferChain.BUFFERSIZE * BufferChain.MAXBUFFERS < SINGLE_BUFFER * APPENDS); + assertTrue("Code has been changed making constants in this test meaningless, review test.", + BufferChain.WATERMARK > SINGLE_BUFFER); + stream.reset(); + byte[] c = new byte[SINGLE_BUFFER]; + for (int i = APPENDS; i > 0; --i) { + try { + bw.append(c); + } catch (IOException e) { + if (e.getMessage() == CurmudgeonlyStream.ZERO_LENGTH_WRITE) { + fail(CurmudgeonlyStream.ZERO_LENGTH_WRITE); + } else { + throw e; + } + } + } + try { + bw.close(); + } catch (IOException e) { + if (e.getMessage() == CurmudgeonlyStream.ZERO_LENGTH_WRITE) { + fail(CurmudgeonlyStream.ZERO_LENGTH_WRITE); + } else { + throw e; + } + } + } + + public void testDoubleFlush() throws IOException { + stream.reset(); + byte[] c = new byte[] { 97, 98, 99 }; + ByteWriter bw = new ByteWriter(stream, encoder); + bw.append(c); + bw.flush(); + bw.flush(); + bw.close(); + byte[] res = stream.toByteArray(); + assertTrue(Arrays.equals(new byte[] { 97, 98, 99 }, res)); + } + + public void testCharArrays() throws java.io.IOException { + stream.reset(); + char[] c = new char[] { 'a', 'b', 'c', '\u00F8' }; + ByteWriter bw = new ByteWriter(stream, encoder); + bw.write(c); + bw.close(); + byte[] res = stream.toByteArray(); + assertTrue(Arrays.equals(new byte[] { 97, 98, 99, (byte) 0xc3, (byte) 0xb8 }, res)); + } + + public void testByteBuffers() throws java.io.IOException { + stream.reset(); + ByteBuffer b = ByteBuffer.allocate(16); + b.put((byte) 97); + b.put((byte) 98); + b.put((byte) 99); + ByteWriter bw = new ByteWriter(stream, encoder); + b.flip(); + bw.append(b); + bw.close(); + byte[] res = stream.toByteArray(); + assertTrue(Arrays.equals(new byte[] { 97, 98, 99 }, res)); + } + + public void testByteArrays() throws java.io.IOException { + stream.reset(); + byte[] c = new byte[] { 97, 98, 99 }; + ByteWriter bw = new ByteWriter(stream, encoder); + bw.append(c); + bw.close(); + byte[] res = stream.toByteArray(); + assertTrue(Arrays.equals(new byte[] { 97, 98, 99 }, res)); + } + + public void testByteArrayWithOffset() throws java.io.IOException { + final int length = BufferChain.BUFFERSIZE * 3 / 2; + final int offset = 1; + final byte invalid = 3; + final byte valid = 2; + stream.reset(); + byte[] c = new byte[length]; + c[0] = invalid; + for (int i = offset; i < length; ++i) { + c[i] = valid; + } + ByteWriter bw = new ByteWriter(stream, encoder); + bw.append(c, offset, length - offset); + bw.close(); + byte[] res = stream.toByteArray(); + assertEquals(length - offset, res.length); + assertEquals(valid, res[0]); + } + + public void testStrings() throws java.io.IOException { + stream.reset(); + String c = "abc\u00F8"; + ByteWriter bw = new ByteWriter(stream, encoder); + bw.write(c); + bw.close(); + byte[] res = stream.toByteArray(); + assertTrue(Arrays.equals(new byte[] { 97, 98, 99, (byte) 0xc3, (byte) 0xb8 }, res)); + } + + public void testStringsAndByteArrays() throws java.io.IOException { + stream.reset(); + String c = "abc\u00F8"; + byte[] b = new byte[] { 97, 98, 99 }; + ByteWriter bw = new ByteWriter(stream, encoder); + bw.write(c); + bw.append(b); + bw.close(); + byte[] res = stream.toByteArray(); + assertTrue(Arrays.equals(new byte[] { 97, 98, 99, (byte) 0xc3, (byte) 0xb8, 97, 98, 99 }, res)); + } + + public void testByteBuffersAndByteArrays() throws java.io.IOException { + stream.reset(); + ByteBuffer b = ByteBuffer.allocate(16); + b.put((byte) 97); + b.put((byte) 98); + b.put((byte) 99); + b.flip(); + byte[] c = new byte[] { 100, 101, 102 }; + ByteWriter bw = new ByteWriter(stream, encoder); + bw.append(b); + bw.append(c); + bw.close(); + byte[] res = stream.toByteArray(); + assertTrue(Arrays.equals(new byte[] { 97, 98, 99, 100, 101, 102 }, res)); + } + + public void testOverFlow() throws java.io.IOException { + stream.reset(); + byte[] b = new byte[] { 97, 98, 99 }; + ByteWriter bw = new ByteWriter(stream, encoder); + int i = 0; + while (i < 5000) { + bw.append(b); + ++i; + } + bw.close(); + byte[] res = stream.toByteArray(); + assertEquals(15000, res.length); + i = 0; + int base = 0; + while (i < 5000) { + byte[] sub = new byte[3]; + System.arraycopy(res, base, sub, 0, 3); + assertTrue(Arrays.equals(new byte[] { 97, 98, 99 }, sub)); + base += 3; + ++i; + } + } + + public void testUnMappableCharacter() throws java.io.IOException { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + ByteWriter writer = new ByteWriter(stream, Charset.forName("ascii").newEncoder()); + writer.write("yahoo\u9999bahoo"); + writer.close(); + assertTrue(stream.toString("ascii").contains("yahoo")); + assertTrue(stream.toString("ascii").contains("bahoo")); + } + + public void testNoRecycling() throws IOException { + final int SINGLE_BUFFER = 500; + final int APPENDS = 500; + assertTrue( + "Code has been changed making constants in this test meaningless, review test.", + BufferChain.BUFFERSIZE * BufferChain.MAXBUFFERS < SINGLE_BUFFER + * APPENDS); + assertTrue( + "Code has been changed making constants in this test meaningless, review test.", + BufferChain.WATERMARK > SINGLE_BUFFER); + byte[] c = new byte[SINGLE_BUFFER]; + Arrays.fill(c, (byte) 'a'); + OnlyUniqueBuffers b = new OnlyUniqueBuffers(); + try { + for (int i = APPENDS; i > 0; --i) { + b.insert(ByteBuffer.wrap(c)); + } + b.flush(); + } catch (IOException e) { + if (e.getMessage() == OnlyUniqueBuffers.RECYCLED_BYTE_BUFFER) { + fail(OnlyUniqueBuffers.RECYCLED_BYTE_BUFFER); + } else { + throw e; + } + } + } + + public void testGetEncoding() throws java.io.IOException { + stream.reset(); + ByteWriter b = new ByteWriter(stream, encoder); + assertEquals(Utf8.getCharset(), b.getEncoding()); + b.close(); + } + + public void testWriteLong() throws java.io.IOException { + stream.reset(); + ByteWriter b = new ByteWriter(stream, encoder); + b.write(1000L * 1000L * 1000L * 1000L); + b.close(); + assertArrayEquals(Utf8.toBytes("1000000000000"), stream.toByteArray()); + } + + public void testWriteInt() throws java.io.IOException { + stream.reset(); + ByteWriter b = new ByteWriter(stream, encoder); + b.write((int) 'z'); + b.close(); + assertArrayEquals(Utf8.toBytes("z"), stream.toByteArray()); + } + + public void testSurrogatePairs() throws java.io.IOException { + stream.reset(); + ByteWriter b = new ByteWriter(stream, encoder); + b.write(0xD800); + b.write(0xDFD0); + b.close(); + assertArrayEquals(Utf8.toBytes("\uD800\uDFD0"), stream.toByteArray()); + } + + public void testSurrogatePairsMixedWithSingleCharacters() throws java.io.IOException { + stream.reset(); + ByteWriter b = new ByteWriter(stream, encoder); + b.write(0x00F8); + b.write(0xD800); + b.write(0xDFD0); + b.write(0x00F8); + b.write(0xD800); + b.write((int) 'a'); + b.write(0xDFD0); + b.write((int) 'b'); + b.close(); + assertArrayEquals(Utf8.toBytes("\u00F8\uD800\uDFD0\u00F8ab"), stream.toByteArray()); + } + + + public void testWriteDouble() throws java.io.IOException { + stream.reset(); + ByteWriter b = new ByteWriter(stream, encoder); + b.write(12.0d); + b.close(); + assertArrayEquals(Utf8.toBytes("12.0"), stream.toByteArray()); + } + + public void testWriteFloat() throws java.io.IOException { + stream.reset(); + ByteWriter b = new ByteWriter(stream, encoder); + b.write(12.0f); + b.close(); + assertArrayEquals(Utf8.toBytes("12.0"), stream.toByteArray()); + } + + public void testWriteShort() throws java.io.IOException { + stream.reset(); + ByteWriter b = new ByteWriter(stream, encoder); + b.write((short) 12); + b.close(); + assertArrayEquals(Utf8.toBytes("12"), stream.toByteArray()); + } + + public void testWriteBoolean() throws java.io.IOException { + stream.reset(); + ByteWriter b = new ByteWriter(stream, encoder); + b.write(true); + b.close(); + assertArrayEquals(Utf8.toBytes("true"), stream.toByteArray()); + } + + public void testAppendSingleByte() throws java.io.IOException { + stream.reset(); + ByteWriter b = new ByteWriter(stream, encoder); + b.append((byte) 'a'); + b.close(); + assertArrayEquals(new byte[] { (byte) 'a' }, stream.toByteArray()); + } + + public void testAppended() throws IOException { + stream.reset(); + ByteWriter b = new ByteWriter(stream, encoder); + final String s = "nalle"; + b.write(s); + b.close(); + final byte[] bytes = Utf8.toBytes(s); + assertArrayEquals(bytes, stream.toByteArray()); + assertEquals(bytes.length, b.appended()); + } + + public void testWriteUtf8Array() throws IOException { + stream.reset(); + ByteWriter b = new ByteWriter(stream, encoder); + final byte[] bytes = Utf8.toBytes("nalle"); + b.write(new Utf8Array(bytes)); + b.close(); + assertArrayEquals(bytes, stream.toByteArray()); + } + + private static class OnlyUniqueBuffers implements WritableByteTransmitter { + static final String RECYCLED_BYTE_BUFFER = "Got a ByteBuffer instance twice."; + private final IdentityHashMap<ByteBuffer, ?> buffers = new IdentityHashMap<ByteBuffer, Object>(); + private final BufferChain datastore; + + public OnlyUniqueBuffers() { + datastore = new BufferChain(this); + } + + public void insert(ByteBuffer b) throws IOException { + datastore.append(b); + } + + @Override + public void send(ByteBuffer src) throws IOException { + if (buffers.containsKey(src)) { + throw new IOException(RECYCLED_BYTE_BUFFER); + } else { + buffers.put(src, null); + } + } + + public void flush() throws IOException { + datastore.flush(); + } + } +} diff --git a/vespajlib/src/test/java/com/yahoo/io/FatalErrorHandlerTestCase.java b/vespajlib/src/test/java/com/yahoo/io/FatalErrorHandlerTestCase.java new file mode 100644 index 00000000000..d4889c1fa96 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/io/FatalErrorHandlerTestCase.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import static org.junit.Assert.*; + +import java.security.Permission; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Just to remove noise from the coverage report. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class FatalErrorHandlerTestCase { + private static final class AvoidExiting extends SecurityManager { + + @Override + public void checkPermission(Permission perm) { + } + + @Override + public void checkExit(int status) { + throw new SecurityException(); + } + + } + + private FatalErrorHandler h; + + @Before + public void setUp() throws Exception { + h = new FatalErrorHandler(); + System.setSecurityManager(new AvoidExiting()); + } + + @After + public void tearDown() throws Exception { + System.setSecurityManager(null); + } + + @Test + public final void testHandle() { + boolean caught = false; + try { + h.handle(new Throwable(), "abc"); + } catch (SecurityException e) { + caught = true; + } + assertTrue(caught); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/io/FileReadTestCase.java b/vespajlib/src/test/java/com/yahoo/io/FileReadTestCase.java new file mode 100644 index 00000000000..5ebdeeb797e --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/io/FileReadTestCase.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.nio.charset.Charset; + + +public class FileReadTestCase extends junit.framework.TestCase { + + @Test + public void testReadByteArray() throws IOException { + byte[] thisFile = IOUtils.readFileBytes(new File("src/test/java/com/yahoo/io/FileReadTestCase.java")); + String str = new String(thisFile, Charset.forName("US-ASCII")); + assertTrue(str.startsWith("// Copyright 2016 Yahoo Inc.")); + assertTrue(str.endsWith("// Yeppers\n")); + } + + @Test + public void testReadString() throws IOException { + String str = IOUtils.readFile(new File("src/test/java/com/yahoo/io/FileReadTestCase.java")); + assertTrue(str.startsWith("// Copyright 2016 Yahoo Inc.")); + assertTrue(str.endsWith("// Yeppers\n")); + } + + @Test + public void testReadAllFromReader() throws IOException { + assertEquals(IOUtils.readAll(new StringReader("")), ""); + assertEquals(IOUtils.readAll(new StringReader("hei")), "hei"); + assertEquals(IOUtils.readAll(new StringReader("hei\nhaa")), "hei\nhaa"); + assertEquals(IOUtils.readAll(new StringReader("hei\nhaa\n")), "hei\nhaa\n"); + } + +} + +// Yeppers diff --git a/vespajlib/src/test/java/com/yahoo/io/GrowableBufferOutputStreamTestCase.java b/vespajlib/src/test/java/com/yahoo/io/GrowableBufferOutputStreamTestCase.java new file mode 100644 index 00000000000..71568a3cfbc --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/io/GrowableBufferOutputStreamTestCase.java @@ -0,0 +1,126 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.channels.WritableByteChannel; +import java.nio.ByteBuffer; +import java.io.IOException; +import com.yahoo.io.GrowableBufferOutputStream; + + +/** + * Tests the GrowableBufferOutputStream + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class GrowableBufferOutputStreamTestCase extends junit.framework.TestCase { + private byte[] testData; + + static class DummyWritableByteChannel implements WritableByteChannel { + private ByteBuffer buffer; + + public DummyWritableByteChannel(ByteBuffer buffer) { + this.buffer = buffer; + } + + public int write(ByteBuffer src) throws IOException { + int written = Math.min(src.remaining(), buffer.remaining()); + + if (buffer.remaining() < src.remaining()) { + ByteBuffer tmp = src.slice(); + + tmp.limit(written); + src.position(src.position() + written); + } else { + buffer.put(src); + } + return written; + } + + public boolean isOpen() { + return true; + } + + public void close() throws IOException {} + } + + public GrowableBufferOutputStreamTestCase(String name) { + super(name); + } + + public void setUp() { + testData = new byte[100]; + for (int i = 0; i < 100; ++i) { + testData[i] = (byte) i; + } + } + + public void testSimple() throws IOException { + GrowableBufferOutputStream g = new GrowableBufferOutputStream(10, 5); + + g.write(testData, 0, 100); + g.flush(); + assertEquals(10, g.numWritableBuffers()); + assertEquals(100, g.writableSize()); + + ByteBuffer sink = ByteBuffer.allocate(60); + DummyWritableByteChannel channel = new DummyWritableByteChannel(sink); + int written = g.channelWrite(channel); + + assertEquals(60, written); + assertEquals(60, sink.position()); + assertEquals(40, g.writableSize()); + + // there should be 4 buffers left now + assertEquals(4, g.numWritableBuffers()); + + // ensure that we got what we expected + for (int i = 0; i < 60; ++i) { + if (((int) sink.get(i)) != i) { + fail(); + } + } + + // then we write more data + g.write(testData, 0, 100); + g.flush(); + assertEquals(140, g.writableSize()); + + // ...which implies that we should now have 14 writable buffers + assertEquals(14, g.numWritableBuffers()); + + // reset the sink so it can consume more data + sink.clear(); + + // then write more to the DummyWritableByteChannel + written = g.channelWrite(channel); + assertEquals(60, written); + assertEquals(60, sink.position()); + assertEquals(80, g.writableSize()); + + // now there should be 8 buffers + assertEquals(8, g.numWritableBuffers()); + + // ensure that we got what we expected + for (int i = 0; i < 60; ++i) { + int val = (int) sink.get(i); + int expected = (i + 60) % 100; + + if (val != expected) { + fail("Value was " + val + " and not " + i); + } + } + + // when we clear there should be no buffers + g.clear(); + assertEquals(0, g.numWritableBuffers()); + assertEquals(0, g.writableSize()); + + // ditto after flush after clear + g.flush(); + assertEquals(0, g.numWritableBuffers()); + + // flush the cache too + g.clearAll(); + assertEquals(0, g.numWritableBuffers()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/io/GrowableByteBufferTestCase.java b/vespajlib/src/test/java/com/yahoo/io/GrowableByteBufferTestCase.java new file mode 100644 index 00000000000..ffc3d6dfe64 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/io/GrowableByteBufferTestCase.java @@ -0,0 +1,756 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.InvalidMarkException; +import java.nio.ReadOnlyBufferException; +import java.util.Arrays; + +import static org.junit.Assert.assertArrayEquals;; + +/** + * Tests GrowableByteBuffer. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class GrowableByteBufferTestCase extends junit.framework.TestCase { + public void testBuffer() { + GrowableByteBuffer buf = new GrowableByteBuffer(20, 1.5f); + + buf.putChar((char) 5); + assertEquals(2, buf.position()); + + buf.putDouble(983.982d); + assertEquals(10, buf.position()); + + buf.putFloat(94.322f); + assertEquals(14, buf.position()); + + buf.putInt(98); + assertEquals(18, buf.position()); + + assertEquals(20, buf.capacity()); + + buf.putLong(983L); + assertEquals(26, buf.position()); + + // Adding fudge factors and other fun to the growth rate, + // makes capacity() suboptimal to test, so this should perhaps + // be removed + // TODO: Better test of growth rate + assertEquals(130, buf.capacity()); + + buf.putShort((short) 4); + assertEquals(28, buf.position()); + + + buf.position(0); + assertEquals((char) 5, buf.getChar()); + assertEquals(2, buf.position()); + + assertEquals((int) (983.982d * 1000d), (int) (buf.getDouble() * 1000d)); + assertEquals(10, buf.position()); + + assertEquals((int) (94.322f * 1000f), (int) (buf.getFloat() * 1000f)); + assertEquals(14, buf.position()); + + assertEquals(98, buf.getInt()); + assertEquals(18, buf.position()); + + assertEquals(983L, buf.getLong()); + assertEquals(26, buf.position()); + + assertEquals((short) 4, buf.getShort()); + assertEquals(28, buf.position()); + + + byte[] twoBytes = new byte[2]; + buf.put(twoBytes); + assertEquals(30, buf.position()); + assertEquals(130, buf.capacity()); + + buf.put((byte) 1); + assertEquals(31, buf.position()); + assertEquals(130, buf.capacity()); + + ByteBuffer tmpBuf = ByteBuffer.allocate(15); + tmpBuf.putInt(56); + tmpBuf.position(0); + buf.put(tmpBuf); + assertEquals(46, buf.position()); + assertEquals(130, buf.capacity()); + } + + public void testGrowth() { + GrowableByteBuffer buf = new GrowableByteBuffer(256, 2.0f); + + //add bytes almost to the boundary + for (int i = 0; i < 255; i++) { + buf.put((byte) 0); + } + + //We are just before the boundary now. + assertEquals(255, buf.position()); + assertEquals(256, buf.capacity()); + assertEquals(256, buf.limit()); + + //Test adding one more byte. + buf.put((byte) 0); + //The buffer is full. + assertEquals(256, buf.position()); + assertEquals(256, buf.capacity()); + assertEquals(256, buf.limit()); + + //Adding one more byte should make it grow. + buf.put((byte) 0); + assertEquals(257, buf.position()); + assertEquals(612, buf.capacity()); + assertEquals(612, buf.limit()); + + //add a buffer exactly to the boundary + byte[] bytes = new byte[355]; + buf.put(bytes); + assertEquals(612, buf.position()); + assertEquals(612, buf.capacity()); + assertEquals(612, buf.limit()); + + //adding a one-byte buffer should make it grow again + byte[] oneByteBuf = new byte[1]; + buf.put(oneByteBuf); + assertEquals(613, buf.position()); + assertEquals(1324, buf.capacity()); + assertEquals(1324, buf.limit()); + + //add a large buffer that goes waaay past the boundary and makes it grow yet again, + //but that is not enough + byte[] largeBuf = new byte[3000]; + buf.put(largeBuf); + //the buffer should be doubled twice now + assertEquals(3613, buf.position()); + assertEquals(5596, buf.capacity()); + assertEquals(5596, buf.limit()); + + //let's try that again, and make the buffer double three times + byte[] veryLargeBuf = new byte[20000]; + buf.put(veryLargeBuf); + //the buffer should be doubled three times now + assertEquals(23613, buf.position()); + assertEquals(45468, buf.capacity()); + assertEquals(45468, buf.limit()); + } + + public void testBadGrowthFactors() { + try { + new GrowableByteBuffer(100, 1.0f); + assertTrue(false); + } catch (IllegalArgumentException iae) { + //we're OK + } + GrowableByteBuffer buf = new GrowableByteBuffer(16, 1.0000001f); + buf.putInt(1); + assertEquals(16, buf.capacity()); + buf.putInt(1); + assertEquals(16, buf.capacity()); + buf.putInt(1); + assertEquals(16, buf.capacity()); + buf.putInt(1); + assertEquals(16, buf.capacity()); + + buf.putInt(1); + assertEquals(116, buf.capacity()); + + } + + public void testPropertiesNonDirect() { + GrowableByteBuffer buf = new GrowableByteBuffer(10, 1.5f); + buf.order(ByteOrder.LITTLE_ENDIAN); + + assertEquals(0, buf.position()); + // GrowableByteBuffer never makes a buffer smaller than 16 bytes + assertEquals(16, buf.capacity()); + assertEquals(16, buf.limit()); + assertEquals(false, buf.isReadOnly()); + assertEquals(ByteOrder.LITTLE_ENDIAN, buf.order()); + assertEquals(false, buf.isDirect()); + + buf.put(new byte[17]); + + assertEquals(17, buf.position()); + assertEquals(124, buf.capacity()); + assertEquals(124, buf.limit()); + assertEquals(false, buf.isReadOnly()); + assertEquals(ByteOrder.LITTLE_ENDIAN, buf.order()); + assertEquals(false, buf.isDirect()); + } + + public void testPropertiesDirect() { + // allocate* are simply encapsulated, so don't add logic to them, + // therefore minimum size becomes what it says + GrowableByteBuffer buf = GrowableByteBuffer.allocateDirect(10, 1.5f); + buf.order(ByteOrder.LITTLE_ENDIAN); + + assertEquals(0, buf.position()); + assertEquals(10, buf.capacity()); + assertEquals(10, buf.limit()); + assertEquals(false, buf.isReadOnly()); + assertEquals(ByteOrder.LITTLE_ENDIAN, buf.order()); + assertEquals(true, buf.isDirect()); + + buf.put(new byte[11]); + + assertEquals(11, buf.position()); + assertEquals(115, buf.capacity()); + assertEquals(115, buf.limit()); + assertEquals(false, buf.isReadOnly()); + assertEquals(ByteOrder.LITTLE_ENDIAN, buf.order()); + assertEquals(true, buf.isDirect()); + } + + public void testNumberEncodings() { + GrowableByteBuffer buf = new GrowableByteBuffer(); + buf.putInt1_2_4Bytes(124); + buf.putInt2_4_8Bytes(124); + buf.putInt1_4Bytes(124); + + buf.putInt1_2_4Bytes(127); + buf.putInt2_4_8Bytes(127); + buf.putInt1_4Bytes(127); + + buf.putInt1_2_4Bytes(128); + buf.putInt2_4_8Bytes(128); + buf.putInt1_4Bytes(128); + + buf.putInt1_2_4Bytes(255); + buf.putInt2_4_8Bytes(255); + buf.putInt1_4Bytes(255); + + buf.putInt1_2_4Bytes(256); + buf.putInt2_4_8Bytes(256); + buf.putInt1_4Bytes(256); + + buf.putInt1_2_4Bytes(0); + buf.putInt2_4_8Bytes(0); + buf.putInt1_4Bytes(0); + + buf.putInt1_2_4Bytes(1); + buf.putInt2_4_8Bytes(1); + buf.putInt1_4Bytes(1); + + try { + buf.putInt1_2_4Bytes(Integer.MAX_VALUE); + fail("Should have gotten exception here..."); + } catch (Exception e) { } + buf.putInt2_4_8Bytes(Integer.MAX_VALUE); + buf.putInt1_4Bytes(Integer.MAX_VALUE); + + try { + buf.putInt2_4_8Bytes(Long.MAX_VALUE); + fail("Should have gotten exception here..."); + } catch (Exception e) { } + + buf.putInt1_2_4Bytes(Short.MAX_VALUE); + buf.putInt2_4_8Bytes(Short.MAX_VALUE); + buf.putInt1_4Bytes(Short.MAX_VALUE); + + buf.putInt1_2_4Bytes(Byte.MAX_VALUE); + buf.putInt2_4_8Bytes(Byte.MAX_VALUE); + buf.putInt1_4Bytes(Byte.MAX_VALUE); + + try { + buf.putInt1_2_4Bytes(-1); + fail("Should have gotten exception here..."); + } catch (Exception e) { } + try { + buf.putInt2_4_8Bytes(-1); + fail("Should have gotten exception here..."); + } catch (Exception e) { } + try { + buf.putInt1_4Bytes(-1); + fail("Should have gotten exception here..."); + } catch (Exception e) { } + + try { + buf.putInt1_2_4Bytes(Integer.MIN_VALUE); + fail("Should have gotten exception here..."); + } catch (Exception e) { } + try { + buf.putInt2_4_8Bytes(Integer.MIN_VALUE); + fail("Should have gotten exception here..."); + } catch (Exception e) { } + try { + buf.putInt1_4Bytes(Integer.MIN_VALUE); + fail("Should have gotten exception here..."); + } catch (Exception e) { } + + try { + buf.putInt2_4_8Bytes(Long.MIN_VALUE); + fail("Should have gotten exception here..."); + } catch (Exception e) { } + + int endWritePos = buf.position(); + buf.position(0); + + assertEquals(124, buf.getInt1_2_4Bytes()); + assertEquals(124, buf.getInt2_4_8Bytes()); + assertEquals(124, buf.getInt1_4Bytes()); + + assertEquals(127, buf.getInt1_2_4Bytes()); + assertEquals(127, buf.getInt2_4_8Bytes()); + assertEquals(127, buf.getInt1_4Bytes()); + + assertEquals(128, buf.getInt1_2_4Bytes()); + assertEquals(128, buf.getInt2_4_8Bytes()); + assertEquals(128, buf.getInt1_4Bytes()); + + assertEquals(255, buf.getInt1_2_4Bytes()); + assertEquals(255, buf.getInt2_4_8Bytes()); + assertEquals(255, buf.getInt1_4Bytes()); + + assertEquals(256, buf.getInt1_2_4Bytes()); + assertEquals(256, buf.getInt2_4_8Bytes()); + assertEquals(256, buf.getInt1_4Bytes()); + + assertEquals(0, buf.getInt1_2_4Bytes()); + assertEquals(0, buf.getInt2_4_8Bytes()); + assertEquals(0, buf.getInt1_4Bytes()); + + assertEquals(1, buf.getInt1_2_4Bytes()); + assertEquals(1, buf.getInt2_4_8Bytes()); + assertEquals(1, buf.getInt1_4Bytes()); + + assertEquals(Integer.MAX_VALUE, buf.getInt2_4_8Bytes()); + assertEquals(Integer.MAX_VALUE, buf.getInt1_4Bytes()); + + assertEquals(Short.MAX_VALUE, buf.getInt1_2_4Bytes()); + assertEquals(Short.MAX_VALUE, buf.getInt2_4_8Bytes()); + assertEquals(Short.MAX_VALUE, buf.getInt1_4Bytes()); + + assertEquals(Byte.MAX_VALUE, buf.getInt1_2_4Bytes()); + assertEquals(Byte.MAX_VALUE, buf.getInt2_4_8Bytes()); + assertEquals(Byte.MAX_VALUE, buf.getInt1_4Bytes()); + + int endReadPos = buf.position(); + + assertEquals(endWritePos, endReadPos); + } + public void testNumberLengths() { + assertEquals(1, GrowableByteBuffer.getSerializedSize1_4Bytes(0)); + assertEquals(1, GrowableByteBuffer.getSerializedSize1_4Bytes(1)); + assertEquals(1, GrowableByteBuffer.getSerializedSize1_4Bytes(4)); + assertEquals(1, GrowableByteBuffer.getSerializedSize1_4Bytes(31)); + assertEquals(1, GrowableByteBuffer.getSerializedSize1_4Bytes(126)); + assertEquals(1, GrowableByteBuffer.getSerializedSize1_4Bytes(127)); + assertEquals(4, GrowableByteBuffer.getSerializedSize1_4Bytes(128)); + assertEquals(4, GrowableByteBuffer.getSerializedSize1_4Bytes(129)); + assertEquals(4, GrowableByteBuffer.getSerializedSize1_4Bytes(255)); + assertEquals(4, GrowableByteBuffer.getSerializedSize1_4Bytes(256)); + assertEquals(4, GrowableByteBuffer.getSerializedSize1_4Bytes(0x7FFFFFFF)); + + assertEquals(2, GrowableByteBuffer.getSerializedSize2_4_8Bytes(0)); + assertEquals(2, GrowableByteBuffer.getSerializedSize2_4_8Bytes(1)); + assertEquals(2, GrowableByteBuffer.getSerializedSize2_4_8Bytes(4)); + assertEquals(2, GrowableByteBuffer.getSerializedSize2_4_8Bytes(31)); + assertEquals(2, GrowableByteBuffer.getSerializedSize2_4_8Bytes(126)); + assertEquals(2, GrowableByteBuffer.getSerializedSize2_4_8Bytes(127)); + assertEquals(2, GrowableByteBuffer.getSerializedSize2_4_8Bytes(128)); + assertEquals(2, GrowableByteBuffer.getSerializedSize2_4_8Bytes(32767)); + assertEquals(4, GrowableByteBuffer.getSerializedSize2_4_8Bytes(32768)); + assertEquals(4, GrowableByteBuffer.getSerializedSize2_4_8Bytes(32769)); + assertEquals(4, GrowableByteBuffer.getSerializedSize2_4_8Bytes(1030493)); + assertEquals(4, GrowableByteBuffer.getSerializedSize2_4_8Bytes(0x3FFFFFFF)); + assertEquals(8, GrowableByteBuffer.getSerializedSize2_4_8Bytes(0x40000000)); + assertEquals(8, GrowableByteBuffer.getSerializedSize2_4_8Bytes(0x40000001)); + + assertEquals(1, GrowableByteBuffer.getSerializedSize1_2_4Bytes(0)); + assertEquals(1, GrowableByteBuffer.getSerializedSize1_2_4Bytes(1)); + assertEquals(1, GrowableByteBuffer.getSerializedSize1_2_4Bytes(4)); + assertEquals(1, GrowableByteBuffer.getSerializedSize1_2_4Bytes(31)); + assertEquals(1, GrowableByteBuffer.getSerializedSize1_2_4Bytes(126)); + assertEquals(1, GrowableByteBuffer.getSerializedSize1_2_4Bytes(127)); + assertEquals(2, GrowableByteBuffer.getSerializedSize1_2_4Bytes(128)); + assertEquals(2, GrowableByteBuffer.getSerializedSize1_2_4Bytes(16383)); + assertEquals(4, GrowableByteBuffer.getSerializedSize1_2_4Bytes(16384)); + assertEquals(4, GrowableByteBuffer.getSerializedSize1_2_4Bytes(16385)); + } + + public void testSize0() { + GrowableByteBuffer buf = new GrowableByteBuffer(0, 2.0f); + buf.put((byte) 1); + buf.put((byte) 1); + } + + public void testExceptionSafety() { + GrowableByteBuffer g = new GrowableByteBuffer(32); + ByteBuffer b = ByteBuffer.allocate(232); + for (int i = 0; i < 232; ++i) { + b.put((byte) 32); + } + b.flip(); + g.put(b); + b.flip(); + g.put(b); + assertEquals(464, g.position()); + g.flip(); + for (int i = 0; i < 464; ++i) { + assertEquals(32, (int) g.get()); + } + } + + public void testGrowthFactorAccessor() { + GrowableByteBuffer g = new GrowableByteBuffer(32); + assertEquals(GrowableByteBuffer.DEFAULT_GROW_FACTOR, g.getGrowFactor()); + } + + public void testGrowthWithNonZeroMark() { + GrowableByteBuffer g = new GrowableByteBuffer(32); + final int mark = 16; + byte[] stuff = new byte[mark]; + Arrays.fill(stuff, (byte) 37); + g.put(stuff); + g.mark(); + stuff = new byte[637]; + Arrays.fill(stuff, (byte) 38); + g.put(stuff); + assertEquals(mark, g.getByteBuffer().reset().position()); + } + + public void testPutInt2_4_8BytesMore() { + GrowableByteBuffer g = new GrowableByteBuffer(32); + g.putInt2_4_8Bytes(0x9000); + assertEquals(4, g.position()); + } + + public void testPutInt2_4_8BytesAs4() { + GrowableByteBuffer g = new GrowableByteBuffer(32); + boolean caught = false; + try { + g.putInt2_4_8BytesAs4(-1); + } catch (IllegalArgumentException e) { + caught = true; + } + assertTrue(caught); + caught = false; + try { + g.putInt2_4_8BytesAs4(1L << 37); + } catch (IllegalArgumentException e) { + caught = true; + } + assertTrue(caught); + g.putInt2_4_8BytesAs4(37); + assertEquals(4, g.position()); + } + + public void testGetInt2_4_8Bytes() { + GrowableByteBuffer g = new GrowableByteBuffer(32); + final long expected3 = 37L; + g.putInt2_4_8Bytes(expected3); + final long expected2 = 0x9000L; + g.putInt2_4_8Bytes(expected2); + final long expected = 1L << 56; + g.putInt2_4_8Bytes(expected); + g.flip(); + assertEquals(expected3, g.getInt2_4_8Bytes()); + assertEquals(expected2, g.getInt2_4_8Bytes()); + assertEquals(expected, g.getInt2_4_8Bytes()); + } + + public void testSerializedSize2_4_8BytesIllegalValues() { + boolean caught = false; + try { + GrowableByteBuffer.getSerializedSize2_4_8Bytes(-1); + } catch (IllegalArgumentException e) { + caught = true; + } + assertTrue(caught); + caught = false; + try { + GrowableByteBuffer.getSerializedSize2_4_8Bytes((1L << 62) + 1L); + } catch (IllegalArgumentException e) { + caught = true; + } + assertTrue(caught); + } + + public void testPutInt1_2_4BytesAs4IllegalValues() { + GrowableByteBuffer g = new GrowableByteBuffer(32); + boolean caught = false; + try { + g.putInt1_2_4BytesAs4(-1); + } catch (IllegalArgumentException e) { + caught = true; + } + assertTrue(caught); + caught = false; + try { + g.putInt1_2_4BytesAs4((1 << 30) + 1); + } catch (IllegalArgumentException e) { + caught = true; + } + assertTrue(caught); + } + + public void testSerializedSize1_2_4BytesIllegalValues() { + boolean caught = false; + try { + GrowableByteBuffer.getSerializedSize1_2_4Bytes(-1); + } catch (IllegalArgumentException e) { + caught = true; + } + assertTrue(caught); + caught = false; + try { + GrowableByteBuffer.getSerializedSize1_2_4Bytes((1 << 30) + 1); + } catch (IllegalArgumentException e) { + caught = true; + } + assertTrue(caught); + } + + public void testPutInt1_4BytesAs4() { + GrowableByteBuffer g = new GrowableByteBuffer(32); + boolean caught = false; + try { + g.putInt1_4BytesAs4(-1); + } catch (IllegalArgumentException e) { + caught = true; + } + assertTrue(caught); + g.putInt1_4BytesAs4(37); + assertEquals(4, g.position()); + } + + public void testSerializedSize1_4BytesIllegalValues() { + boolean caught = false; + try { + GrowableByteBuffer.getSerializedSize1_4Bytes(-1); + } catch (IllegalArgumentException e) { + caught = true; + } + assertTrue(caught); + } + + public void testBuilders() { + GrowableByteBuffer g = GrowableByteBuffer.allocate(1063); + assertEquals(1063, g.capacity()); + g = GrowableByteBuffer.allocate(1063, 37.0f); + assertEquals(1063, g.capacity()); + assertEquals(37.0f, g.getGrowFactor()); + g = GrowableByteBuffer.allocateDirect(1063); + assertTrue(g.isDirect()); + } + + public void testForwarding() { + GrowableByteBuffer g = new GrowableByteBuffer(1063); + int first = g.arrayOffset(); + g.put(0, (byte) 37); + assertTrue(g.hasArray()); + assertEquals((byte) 37, g.array()[first]); + g.putChar(0, 'a'); + assertEquals('a', g.getChar(0)); + assertEquals('a', g.asCharBuffer().get(0)); + g.putDouble(0, 10.0d); + assertEquals(10.0d, g.getDouble(0)); + assertEquals(10.0d, g.asDoubleBuffer().get(0)); + g.putFloat(0, 10.0f); + assertEquals(10.0f, g.getFloat(0)); + assertEquals(10.0f, g.asFloatBuffer().get(0)); + g.putInt(0, 10); + assertEquals(10, g.getInt(0)); + assertEquals(10, g.asIntBuffer().get(0)); + g.putLong(0, 10L); + assertEquals(10L, g.getLong(0)); + assertEquals(10L, g.asLongBuffer().get(0)); + boolean caught = false; + try { + g.asReadOnlyBuffer().put((byte) 10); + } catch (ReadOnlyBufferException e) { + caught = true; + } + assertTrue(caught); + g.putShort(0, (short) 10); + assertEquals((short) 10, g.getShort(0)); + assertEquals((short) 10, g.asShortBuffer().get(0)); + g.position(0); + g.put((byte) 0); + g.put((byte) 10); + g.limit(2); + g.position(1); + g.compact(); + assertEquals((byte) 10, g.get(0)); + } + + public void testComparison() { + GrowableByteBuffer g0 = new GrowableByteBuffer(32); + GrowableByteBuffer g1 = new GrowableByteBuffer(32); + assertEquals(g0.hashCode(), g1.hashCode()); + assertFalse(g0.equals(Integer.valueOf(12))); + assertFalse(g0.hashCode() == new GrowableByteBuffer(1063).hashCode()); + assertTrue(g0.equals(g1)); + assertEquals(0, g0.compareTo(g1)); + g0.put((byte) 9); + assertFalse(g0.equals(g1)); + assertEquals(-1, g0.compareTo(g1)); + } + + public void testDuplicate() { + GrowableByteBuffer g0 = new GrowableByteBuffer(32); + GrowableByteBuffer g1 = g0.duplicate(); + g0.put((byte) 12); + assertEquals(12, g1.get()); + } + + public void testGetByteArrayOffsetLen() { + GrowableByteBuffer g = new GrowableByteBuffer(32); + byte[] expected = new byte[] { (byte) 1, (byte) 2, (byte) 3 }; + for (int i = 0; i < expected.length; ++i) { + g.put(expected[i]); + } + byte[] got = new byte[3]; + g.flip(); + g.get(got, 0, got.length); + assertArrayEquals(expected, got); + } + + public void testPutByteArrayOffsetLen() { + GrowableByteBuffer g = new GrowableByteBuffer(32); + byte[] expected = new byte[] { (byte) 1, (byte) 2, (byte) 3 }; + g.put(expected, 0, expected.length); + byte[] got = new byte[3]; + g.flip(); + g.get(got, 0, got.length); + assertArrayEquals(expected, got); + } + + public void testPutGrowableBuffer() { + GrowableByteBuffer g0 = new GrowableByteBuffer(32); + byte[] expected = new byte[] { (byte) 1, (byte) 2, (byte) 3 }; + GrowableByteBuffer g1 = new GrowableByteBuffer(32); + g0.put(expected, 0, expected.length); + byte[] got = new byte[3]; + g0.flip(); + g1.put(g0); + g1.flip(); + g1.get(got, 0, got.length); + assertArrayEquals(expected, got); + } + + private GrowableByteBuffer fullBuffer() { + GrowableByteBuffer g = new GrowableByteBuffer(32); + byte[] stuffer = new byte[g.remaining()]; + Arrays.fill(stuffer, (byte) 'a'); + g.put(stuffer); + return g; + } + + public void testPutWithGrow() { + GrowableByteBuffer g = fullBuffer(); + final int capacity = g.capacity(); + byte[] b = new byte[] { (byte) 'b' }; + g.put(b, 0, b.length); + assertTrue(capacity < g.capacity()); + + g = fullBuffer(); + GrowableByteBuffer toPut = fullBuffer(); + toPut.flip(); + g.put(toPut); + + assertTrue(capacity < g.capacity()); + g = fullBuffer(); + g.put(g.position(), (byte) 'b'); + assertTrue(capacity < g.capacity()); + + g = fullBuffer(); + g.putChar('b'); + assertTrue(capacity < g.capacity()); + g = fullBuffer(); + g.putChar(g.position(), 'b'); + assertTrue(capacity < g.capacity()); + + g = fullBuffer(); + g.putDouble(1.0d); + assertTrue(capacity < g.capacity()); + g = fullBuffer(); + g.putDouble(g.position(), 1.0d); + assertTrue(capacity < g.capacity()); + + g = fullBuffer(); + g.putFloat(1.0f); + assertTrue(capacity < g.capacity()); + g = fullBuffer(); + g.putFloat(g.position(), 1.0f); + assertTrue(capacity < g.capacity()); + + g = fullBuffer(); + g.putInt(g.position(), 1); + assertTrue(capacity < g.capacity()); + + g = fullBuffer(); + g.putLong(g.position(), 1L); + assertTrue(capacity < g.capacity()); + + g = fullBuffer(); + g.putShort((short) 1); + assertTrue(capacity < g.capacity()); + g = fullBuffer(); + g.putShort(g.position(), (short) 1); + assertTrue(capacity < g.capacity()); + } + + public void testSlice() { + GrowableByteBuffer g0 = new GrowableByteBuffer(32); + GrowableByteBuffer g1 = g0.slice(); + final int expected = 37; + g0.putInt(expected); + assertEquals(expected, g1.getInt()); + } + + public void testToString() { + assertEquals("GrowableByteBuffer[pos=32 lim=32 cap=32 grow=2.0]", + fullBuffer().toString()); + } + + public void testWrappers() { + final byte expected = (byte) 2; + byte[] data = new byte[] { (byte) 1, expected, (byte) 3 }; + final float grow = 9e5f; + GrowableByteBuffer g = GrowableByteBuffer.wrap(data, grow); + assertEquals(expected, g.get(1)); + assertEquals(grow, g.getGrowFactor()); + g = GrowableByteBuffer.wrap(data, 1, 1); + assertEquals(expected, g.get()); + assertEquals(2, g.limit()); + g = GrowableByteBuffer.wrap(data, 1, 1, grow); + assertEquals(expected, g.get()); + assertEquals(2, g.limit()); + assertEquals(grow, g.getGrowFactor()); + } + + public void testByteBufferMethods() { + GrowableByteBuffer g = fullBuffer(); + assertFalse(g.hasRemaining()); + g.clear(); + assertTrue(g.hasRemaining()); + g = fullBuffer(); + g.mark(); + g.limit(16); + boolean caught = false; + try { + g.reset(); + } catch (InvalidMarkException e) { + caught = true; + } + assertTrue(caught); + caught = false; + g = fullBuffer(); + g.mark(); + g.position(16); + try { + g.reset(); + } catch (InvalidMarkException e) { + caught = true; + } + assertTrue(caught); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/io/HexDumpTestCase.java b/vespajlib/src/test/java/com/yahoo/io/HexDumpTestCase.java new file mode 100644 index 00000000000..940f0150540 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/io/HexDumpTestCase.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import org.junit.Test; + +import com.yahoo.text.Utf8; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class HexDumpTestCase { + + private static final Charset UTF8 = Charset.forName("UTF-8"); + private static final Charset UTF16 = Charset.forName("UTF-16"); + + @Test + public void requireThatToHexStringAcceptsNull() { + assertNull(HexDump.toHexString(null)); + } + + @Test + public void requireThatToHexStringIsUnformatted() { + assertEquals("6162636465666768696A6B6C6D6E6F707172737475767778797A", + HexDump.toHexString("abcdefghijklmnopqrstuvwxyz".getBytes(UTF8))); + assertEquals("FEFF006100620063006400650066006700680069006A006B006C00" + + "6D006E006F0070007100720073007400750076007700780079007A", + HexDump.toHexString("abcdefghijklmnopqrstuvwxyz".getBytes(UTF16))); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/io/IOUtilsTestCase.java b/vespajlib/src/test/java/com/yahoo/io/IOUtilsTestCase.java new file mode 100644 index 00000000000..7f89cccc6c8 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/io/IOUtilsTestCase.java @@ -0,0 +1,149 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import java.io.*; +import java.util.Arrays; +import java.util.List; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class IOUtilsTestCase extends junit.framework.TestCase { + + public void testCloseNUllDoesNotFail() { + IOUtils.closeWriter(null); + IOUtils.closeReader(null); + IOUtils.closeInputStream(null); + IOUtils.closeOutputStream(null); + } + + public void testFileWriter() throws IOException { + IOUtils.writeFile("temp1.txt", "hello",false); + assertEquals("hello", IOUtils.readFile(new File("temp1.txt"))); + new File("temp1.txt").delete(); + } + + public void testFileWriterWithoutEncoding() throws IOException { + BufferedWriter writer=null; + try { + writer=IOUtils.createWriter(new File("temp2.txt"),false); + writer.write("hello"); + } + finally { + IOUtils.closeWriter(writer); + } + assertEquals("hello", IOUtils.readFile(new File("temp2.txt"))); + new File("temp2.txt").delete(); + } + + public void testFileWriterWithoutEncodingFromFileName() throws IOException { + BufferedWriter writer=null; + try { + writer=IOUtils.createWriter("temp3.txt",false); + writer.write("hello"); + } + finally { + IOUtils.closeWriter(writer); + } + assertEquals("hello",IOUtils.readFile(new File("temp3.txt"))); + new File("temp3.txt").delete(); + } + + public void testFileCounting() throws IOException { + IOUtils.writeFile("temp4.txt","hello\nworld",false); + assertEquals(2,IOUtils.countLines("temp4.txt")); + new File("temp4.txt").delete(); + } + + public void testFileCopy() throws IOException { + IOUtils.writeFile("temp5.txt","hello",false); + IOUtils.copy(new File("temp5.txt"), new File("temp5copy.txt")); + assertEquals("hello", IOUtils.readFile(new File("temp5copy.txt"))); + new File("temp5.txt").delete(); + new File("temp5copy.txt").delete(); + } + + public void testFileCopyWithLineCap() throws IOException { + IOUtils.writeFile("temp6.txt","hello\nyou\nworld",false); + IOUtils.copy("temp6.txt","temp6copy.txt",2); + assertEquals("hello\nyou\n", IOUtils.readFile(new File("temp6copy.txt"))); + new File("temp6.txt").delete(); + new File("temp6copy.txt").delete(); + } + + public void testGetLines() throws IOException { + IOUtils.writeFile("temp7.txt","hello\nworld",false); + List<String> lines=IOUtils.getLines("temp7.txt"); + assertEquals(2,lines.size()); + assertEquals("hello",lines.get(0)); + assertEquals("world",lines.get(1)); + new File("temp7.txt").delete(); + } + + public void testFileWriterAppend() throws IOException { + boolean append=true; + IOUtils.writeFile("temp8.txt", "hello",!append); + BufferedWriter writer=null; + try { + writer=IOUtils.createWriter(new File("temp8.txt"),append); + writer.write("\nworld"); + } + finally { + IOUtils.closeWriter(writer); + } + assertEquals("hello\nworld", IOUtils.readFile(new File("temp8.txt"))); + new File("temp8.txt").delete(); + } + + public void testCloseAllReaders() throws IOException { + StringReader reader1=new StringReader("hello"); + StringReader reader2=new StringReader("world"); + IOUtils.closeAll(Arrays.<Reader>asList(reader1, reader2)); + try { + reader1.ready(); + fail("Expected exception due to reader closed"); + } + catch (IOException e) { + // Expected + } + try { + reader2.ready(); + fail("Expected exception due to reader closed"); + } + catch (IOException e) { + // Expected + } + } + + public void testDirCopying() throws IOException { + IOUtils.writeFile("temp1/temp1.txt","hello",false); + IOUtils.writeFile("temp1/temp2.txt","world",false); + IOUtils.copyDirectory(new File("temp1"), new File("temp2")); + assertEquals("hello", IOUtils.readFile(new File("temp2/temp1.txt"))); + assertEquals("world", IOUtils.readFile(new File("temp2/temp2.txt"))); + IOUtils.recursiveDeleteDir(new File("temp1")); + IOUtils.recursiveDeleteDir(new File("temp2")); + assertTrue(!new File("temp1").exists()); + assertTrue(!new File("temp2").exists()); + } + + public void testDirCopyingWithFilter() throws IOException { + IOUtils.writeFile("temp1/temp1.txt","hello",false); + IOUtils.writeFile("temp1/temp2.txt","world",false); + IOUtils.writeFile("temp1/temp3.json", "world", false); + IOUtils.copyDirectory(new File("temp1"), new File("temp2"), -1, new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".json"); + } + }); + assertEquals("world", IOUtils.readFile(new File("temp2/temp3.json"))); + assertFalse(new File("temp2/temp1.txt").exists()); + assertFalse(new File("temp2/temp2.txt").exists()); + IOUtils.recursiveDeleteDir(new File("temp1")); + IOUtils.recursiveDeleteDir(new File("temp2")); + assertTrue(!new File("temp1").exists()); + assertTrue(!new File("temp2").exists()); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/io/ListenerTestCase.java b/vespajlib/src/test/java/com/yahoo/io/ListenerTestCase.java new file mode 100644 index 00000000000..e2049017076 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/io/ListenerTestCase.java @@ -0,0 +1,108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.SocketChannel; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.collections.Tuple2; +import com.yahoo.concurrent.Receiver; +import com.yahoo.concurrent.Receiver.MessageState; + +/** + * Test a NIO based Reactor pattern implementation, com.yahoo.io.Listener. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ListenerTestCase { + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + Receiver<Byte> r = new Receiver<>(); + + private final class MockConnection implements Connection { + + private SocketChannel channel; + + MockConnection(SocketChannel channel, Listener listener) { + this.channel = channel; + } + + @Override + public void write() throws IOException { + } + + @Override + public void read() throws IOException { + ByteBuffer b = ByteBuffer.allocate(1); + channel.read(b); + b.flip(); + r.put(b.get()); + } + + @Override + public void close() throws IOException { + channel.close(); + + } + + @Override + public void connect() throws IOException { + } + + @Override + public int selectOps() { + return SelectionKey.OP_READ; + } + + @Override + public SocketChannel socketChannel() { + return channel; + } + } + + private final class GetConnection implements ConnectionFactory { + + @Override + public Connection newConnection(SocketChannel channel, Listener listener) { + return new MockConnection(channel, listener); + } + } + + @Test + public final void testRun() throws IOException, InterruptedException { + Listener l = new Listener("ListenerTestCase"); + l.listen(new GetConnection(), 0); + l.start(); + int port = ((InetSocketAddress) l.acceptors.get(0).socket.getLocalAddress()).getPort(); + Socket s = new Socket("127.0.0.1", port); + final byte expected = 42; + s.getOutputStream().write(expected); + s.getOutputStream().flush(); + s.close(); + Tuple2<MessageState, Byte> received = r.get(60 * 1000); + l.acceptors.get(0).interrupt(); + l.acceptors.get(0).socket.close(); + l.acceptors.get(0).join(); + l.interrupt(); + l.join(); + assertTrue("Test timed out.", received.first == MessageState.VALID); + assertEquals(expected, received.second.byteValue()); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/io/SlowInflateTestCase.java b/vespajlib/src/test/java/com/yahoo/io/SlowInflateTestCase.java new file mode 100644 index 00000000000..48d8e8cdffe --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/io/SlowInflateTestCase.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io; + +import static org.junit.Assert.*; + +import java.util.Arrays; +import java.util.zip.Deflater; + +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.text.Utf8; + +/** + * Check decompressor used among other things for packed summary fields. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class SlowInflateTestCase { + + private String value; + private byte[] raw; + private byte[] output; + private byte[] compressed; + private int compressedDataLength; + + @Before + public void setUp() throws Exception { + value = "000000000000000000000000000000000000000000000000000000000000000"; + raw = Utf8.toBytesStd(value); + output = new byte[raw.length * 2]; + Deflater compresser = new Deflater(); + compresser.setInput(raw); + compresser.finish(); + compressedDataLength = compresser.deflate(output); + compresser.end(); + compressed = Arrays.copyOf(output, compressedDataLength); + } + + @Test + public final void test() { + byte[] unpacked = new SlowInflate().unpack(compressed, raw.length); + assertArrayEquals(raw, unpacked); + } + + @Test + public final void testCorruptData() { + compressed[0] = (byte) (compressed[0] ^ compressed[1]); + compressed[1] = (byte) (compressed[1] ^ compressed[2]); + compressed[2] = (byte) (compressed[2] ^ compressed[3]); + compressed[3] = (byte) (compressed[3] ^ compressed[4]); + boolean caught = false; + try { + new SlowInflate().unpack(compressed, raw.length); + } catch (RuntimeException e) { + caught = true; + } + assertTrue(caught); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/io/reader/NamedReaderTestCase.java b/vespajlib/src/test/java/com/yahoo/io/reader/NamedReaderTestCase.java new file mode 100644 index 00000000000..280d0782bd2 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/io/reader/NamedReaderTestCase.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.io.reader; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.nio.CharBuffer; +import java.util.Collections; + +import com.yahoo.io.reader.NamedReader; +import com.yahoo.protect.ClassValidator; + +/** + * Tests all method of NamedReader. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class NamedReaderTestCase extends junit.framework.TestCase { + + public void testIt() { + StringReader stringReader=new StringReader("hello world"); + NamedReader r=new NamedReader("test1",stringReader); + assertEquals("test1",r.getName()); + assertEquals("test1",r.toString()); + assertEquals(stringReader,r.getReader()); + NamedReader.closeAll(Collections.singletonList(r)); + NamedReader.closeAll(null); // noop, nor exception + } + + public void testMethodMasking() { + assertEquals(0, + ClassValidator.unmaskedMethodsFromSuperclass(NamedReader.class).size()); + } + + private static class MarkerReader extends Reader { + static final String READ_CHAR_BUFFER = "com.yahoo.io.reader.NamedReaderTestCase.MarkerReader.read(CharBuffer)"; + static final String READ = "com.yahoo.io.reader.NamedReaderTestCase.MarkerReader.read()"; + static final String READ_CHAR = "com.yahoo.io.reader.NamedReaderTestCase.MarkerReader.read(char[])"; + static final String READ_CHAR_INT_INT = "com.yahoo.io.reader.NamedReaderTestCase.MarkerReader.read(char[], int, int)"; + static final String SKIP_LONG = "com.yahoo.io.reader.NamedReaderTestCase.MarkerReader.skip(long)"; + static final String READY = "com.yahoo.io.reader.NamedReaderTestCase.MarkerReader.ready()"; + static final String MARK_SUPPORTED = "com.yahoo.io.reader.NamedReaderTestCase.MarkerReader.markSupported()"; + static final String MARK_INT = "com.yahoo.io.reader.NamedReaderTestCase.MarkerReader.mark(int)"; + static final String RESET = "com.yahoo.io.reader.NamedReaderTestCase.MarkerReader.reset()"; + static final String CLOSE = "com.yahoo.io.reader.NamedReaderTestCase.MarkerReader.close()"; + String lastMethodHit = null; + + @Override + public int read(CharBuffer target) throws IOException { + lastMethodHit = READ_CHAR_BUFFER; + return 0; + } + + @Override + public int read() throws IOException { + lastMethodHit = READ; + return -1; + } + + @Override + public int read(char[] cbuf) throws IOException { + lastMethodHit = READ_CHAR; + return 0; + } + + @Override + public int read(char[] cbuf, int off, int len) throws IOException { + lastMethodHit = READ_CHAR_INT_INT; + return 0; + } + + @Override + public long skip(long n) throws IOException { + lastMethodHit = SKIP_LONG; + return 0; + } + + @Override + public boolean ready() throws IOException { + lastMethodHit = READY; + return false; + } + + @Override + public boolean markSupported() { + lastMethodHit = MARK_SUPPORTED; + return false; + } + + @Override + public void mark(int readAheadLimit) throws IOException { + lastMethodHit = MARK_INT; + } + + @Override + public void reset() throws IOException { + lastMethodHit = RESET; + } + + @Override + public void close() throws IOException { + lastMethodHit = CLOSE; + } + } + + public void testAllDelegators() throws IOException { + MarkerReader m = new MarkerReader(); + NamedReader r = new NamedReader("nalle", m); + r.read(CharBuffer.allocate(5000)); + assertEquals(MarkerReader.READ_CHAR_BUFFER, m.lastMethodHit); + r.read(); + assertEquals(MarkerReader.READ, m.lastMethodHit); + r.read(new char[5]); + assertEquals(MarkerReader.READ_CHAR, m.lastMethodHit); + r.read(new char[5], 0, 5); + assertEquals(MarkerReader.READ_CHAR_INT_INT, m.lastMethodHit); + r.skip(5L); + assertEquals(MarkerReader.SKIP_LONG, m.lastMethodHit); + r.ready(); + assertEquals(MarkerReader.READY, m.lastMethodHit); + r.markSupported(); + assertEquals(MarkerReader.MARK_SUPPORTED, m.lastMethodHit); + r.mark(5); + assertEquals(MarkerReader.MARK_INT, m.lastMethodHit); + r.reset(); + assertEquals(MarkerReader.RESET, m.lastMethodHit); + r.close(); + assertEquals(MarkerReader.CLOSE, m.lastMethodHit); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/java7compat/UtilTest.java b/vespajlib/src/test/java/com/yahoo/java7compat/UtilTest.java new file mode 100644 index 00000000000..1f919978b7a --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/java7compat/UtilTest.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.java7compat; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @since 5.2 + */ + +public class UtilTest { + + @Test + public void requireJava7CompatibleDoublePrinting() { + if (Util.isJava7Compatible()) { + assertEquals("0.004", String.valueOf(0.0040)); + }else { + assertEquals("0.0040", String.valueOf(0.0040)); + } + assertEquals("0.004", Util.toJava7String(0.0040) ); + } + + @Test + public void nonCompatible() { + assertEquals(Util.nonJava7CompatibleString("0.0040"), "0.004"); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/javacc/FastCharStreamTestCase.java b/vespajlib/src/test/java/com/yahoo/javacc/FastCharStreamTestCase.java new file mode 100644 index 00000000000..a73fffc6c5c --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/javacc/FastCharStreamTestCase.java @@ -0,0 +1,181 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.javacc; + +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class FastCharStreamTestCase { + + @Test + public void requireThatInputCanBeRead() throws IOException { + FastCharStream input = new FastCharStream("foo"); + assertEquals('f', input.readChar()); + assertEquals('o', input.readChar()); + assertEquals('o', input.readChar()); + try { + input.readChar(); + fail(); + } catch (IOException e) { + + } + } + + @Test + public void requireThatColumnIsTracked() throws IOException { + FastCharStream input = new FastCharStream("foo"); + assertEquals(1, input.getColumn()); + input.readChar(); + assertEquals(2, input.getColumn()); + input.readChar(); + assertEquals(3, input.getColumn()); + input.readChar(); + assertEquals(4, input.getColumn()); + } + + @Test + public void requireThatLineIsNotTracked() throws IOException { + FastCharStream input = new FastCharStream("f\no"); + assertEquals(-1, input.getLine()); + input.readChar(); + assertEquals(-1, input.getLine()); + input.readChar(); + assertEquals(-1, input.getLine()); + input.readChar(); + assertEquals(-1, input.getLine()); + } + + + @Test + public void requireThatBackupIsSupported() throws IOException { + FastCharStream input = new FastCharStream("foo"); + assertEquals('f', input.readChar()); + input.backup(1); + assertEquals('f', input.readChar()); + assertEquals('o', input.readChar()); + input.backup(2); + assertEquals('f', input.readChar()); + assertEquals('o', input.readChar()); + assertEquals('o', input.readChar()); + input.backup(3); + assertEquals('f', input.readChar()); + assertEquals('o', input.readChar()); + assertEquals('o', input.readChar()); + input.backup(2); + assertEquals('o', input.readChar()); + assertEquals('o', input.readChar()); + input.backup(1); + assertEquals('o', input.readChar()); + } + + @Test + public void requireThatSuffixIsNotSupported() { + try { + new FastCharStream("foo").GetSuffix(0); + fail(); + } catch (UnsupportedOperationException e) { + + } + } + + @Test + public void requireThatDoneDoesNotThrowException() { + FastCharStream input = new FastCharStream("foo"); + input.Done(); + } + + @Test + public void requireThatTokensCanBeRetrieved() throws IOException { + FastCharStream input = new FastCharStream("foo bar baz"); + input.readChar(); + input.readChar(); + input.readChar(); + input.readChar(); + assertEquals('b', input.BeginToken()); + assertEquals(5, input.getBeginColumn()); + assertEquals(-1, input.getBeginLine()); + assertEquals(6, input.getEndColumn()); + assertEquals(-1, input.getEndLine()); + assertEquals('a', input.readChar()); + assertEquals('r', input.readChar()); + assertEquals(8, input.getEndColumn()); + assertEquals(-1, input.getEndLine()); + assertEquals("bar", input.GetImage()); + } + + @Test + public void requireThatExceptionsDetectLineNumber() { + FastCharStream input = new FastCharStream("foo\nbar"); + assertEquals("line 2, column 1\n" + + "At position:\n" + + "bar\n" + + "^", + input.formatException("line -1, column 5")); + assertEquals("line 2, column 2\n" + + "At position:\n" + + "bar\n" + + " ^", + input.formatException("line -1, column 6")); + assertEquals("line 2, column 3\n" + + "At position:\n" + + "bar\n" + + " ^", + input.formatException("line -1, column 7")); + assertEquals("line 2, column 4\n" + + "At position:\n" + + "bar\n" + + " ^", + input.formatException("line -1, column 8")); + assertEquals("foo line 2, column 2\n" + + "At position:\n" + + "bar\n" + + " ^", + input.formatException("foo line -1, column 6")); + assertEquals("foo line 2, column 2 bar\n" + + "At position:\n" + + "bar\n" + + " ^", + input.formatException("foo line -1, column 6 bar")); + assertEquals("line 2, column 2 bar\n" + + "At position:\n" + + "bar\n" + + " ^", + input.formatException("line -1, column 6 bar")); + } + + @Test + public void requireErrorMsgExceptionAtEOF() { + FastCharStream input = new FastCharStream("\n"); + assertEquals("line 1, column 1\n" + + "At position:\n" + + "EOF\n" + + "^", + input.formatException("line -1, column 1")); + } + + @Test + public void requireThatUnknownExceptionFormatIsIgnored() { + FastCharStream input = new FastCharStream("foo\nbar"); + assertEquals("", + input.formatException("")); + assertEquals("foo", + input.formatException("foo")); + assertEquals("foo line -1, column ", + input.formatException("foo line -1, column ")); + assertEquals("foo line -1, column bar", + input.formatException("foo line -1, column bar")); + } + + @Test + public void requireThatIllegalExceptionColumnIsIgnored() { + FastCharStream input = new FastCharStream("foo\nbar"); + assertEquals("line -1, column 9", + input.formatException("line -1, column 9")); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/javacc/UnicodeUtilitiesTestCase.java b/vespajlib/src/test/java/com/yahoo/javacc/UnicodeUtilitiesTestCase.java new file mode 100644 index 00000000000..81329e06308 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/javacc/UnicodeUtilitiesTestCase.java @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.javacc; + +import org.junit.Test; + +import static com.yahoo.javacc.UnicodeUtilities.quote; +import static com.yahoo.javacc.UnicodeUtilities.unquote; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class UnicodeUtilitiesTestCase { + + @Test + public void testQuote() { + assertEquals("'\\f'", quote("\f", '\'')); + assertEquals("'\\n'", quote("\n", '\'')); + assertEquals("'\\r'", quote("\r", '\'')); + assertEquals("'\\t'", quote("\t", '\'')); + + for (char c = 'a'; c <= 'z'; ++c) { + assertEquals("'" + c + "'", quote(String.valueOf(c), '\'')); + } + + assertEquals("'\\u4f73'", quote("\u4f73", '\'')); + assertEquals("'\\u80fd'", quote("\u80fd", '\'')); + assertEquals("'\\u7d22'", quote("\u7d22", '\'')); + assertEquals("'\\u5c3c'", quote("\u5c3c", '\'')); + assertEquals("'\\u60e0'", quote("\u60e0", '\'')); + assertEquals("'\\u666e'", quote("\u666e", '\'')); + + assertEquals("\"foo\"", quote("foo", '"')); + assertEquals("\"'foo\"", quote("'foo", '"')); + assertEquals("\"foo'\"", quote("foo'", '"')); + assertEquals("\"'foo'\"", quote("'foo'", '"')); + assertEquals("\"\\\"foo\"", quote("\"foo", '"')); + assertEquals("\"foo\\\"\"", quote("foo\"", '"')); + assertEquals("\"\\\"foo\\\"\"", quote("\"foo\"", '"')); + assertEquals("\"\\\"'foo'\\\"\"", quote("\"'foo'\"", '"')); + assertEquals("\"'\\\"foo\\\"'\"", quote("'\"foo\"'", '"')); + assertEquals("\"'f\\\\'o\\\"o\\\\\\\\'\"", quote("'f\\'o\"o\\\\'", '"')); + + assertEquals("\"\\female \\nude fa\\rt fe\\tish\"", quote("\female \nude fa\rt fe\tish", '"')); + assertEquals("\"\\u666e\"", quote("\u666e", '"')); + } + + @Test + public void testQuoteUnquote() { + assertEquals("\"foo\"", quote(unquote("'foo'"), '"')); + assertEquals("\"\\foo\"", quote(unquote(quote("\foo", '"')), '"')); + assertEquals("\u666e", unquote(quote("\u666e", '"'))); + } + + @Test + public void testUnquote() { + assertEquals("foo", unquote("foo")); + assertEquals("'foo", unquote("'foo")); + assertEquals("foo'", unquote("foo'")); + assertEquals("foo", unquote("'foo'")); + assertEquals("\"foo", unquote("\"foo")); + assertEquals("foo\"", unquote("foo\"")); + assertEquals("foo", unquote("\"foo\"")); + assertEquals("'foo'", unquote("\"'foo'\"")); + assertEquals("\"foo\"", unquote("'\"foo\"'")); + assertEquals("f'o\"o\\", unquote("'f\\'o\"o\\\\'")); + + assertEquals("\female \nude fa\rt fe\tish", unquote("'\\female \\nude fa\\rt fe\\tish'")); + assertEquals("\u666e", unquote("\"\\u666e\"")); + + try { + unquote("\"\\uSiM0N\""); + fail(); + } catch (IllegalArgumentException e) { + + } + assertEquals("simo\n", unquote("'\\s\\i\\m\\o\\n'")); + try { + unquote("\"foo\"bar\""); + fail(); + } catch (IllegalArgumentException e) { + + } + try { + unquote("'foo'bar'"); + fail(); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void requireThatTokenIncludesOnlyAcceptedChars() { + assertEquals("\"\\u0000\",\"\\u7777\",\"\\uffff\",", + UnicodeUtilities.generateToken(new UnicodeUtilities.Predicate() { + + @Override + public boolean accepts(char c) { + return c == 0x0000 || c == 0x7777 || c == 0xffff; + } + })); + assertEquals("\"\\u0006\",\"\\u0009\",\"\\u0060\"-\"\\u0069\",", + UnicodeUtilities.generateToken(new UnicodeUtilities.Predicate() { + + @Override + public boolean accepts(char c) { + return c == 0x6 || c == 0x9 || (c >= 0x60 && c <= 0x69); + } + })); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/net/HostNameTestCase.java b/vespajlib/src/test/java/com/yahoo/net/HostNameTestCase.java new file mode 100644 index 00000000000..98be9f0ef6f --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/net/HostNameTestCase.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import org.junit.Test; + +import static org.junit.Assert.assertFalse; + +/** + * @author lulf + */ +public class HostNameTestCase { + @Test + public void testHostnameIsFound() { + assertFalse(HostName.getLocalhost().isEmpty()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/net/LinuxInetAddressTestCase.java b/vespajlib/src/test/java/com/yahoo/net/LinuxInetAddressTestCase.java new file mode 100755 index 00000000000..fc1945d2d39 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/net/LinuxInetAddressTestCase.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.net;
+
+import java.net.UnknownHostException;
+import java.net.InetAddress;
+import java.net.Inet4Address;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LinuxInetAddressTestCase extends junit.framework.TestCase {
+
+ public void testPreferIPv4() throws UnknownHostException {
+ try {
+ // This test only works if there is at least one inet address returned.
+ InetAddress[] arr = LinuxInetAddress.getAllLocal();
+ if (arr.length > 0) {
+ // System.out.println("Got " + arr.length + " addresses.");
+
+ // And it can only make sure it is preferred if there is at least one ip v4 address.
+ boolean ipv4 = false;
+ for (int i = 0; i < arr.length; ++i) {
+ // System.out.println("Address " + i + " is an instance of " + arr[i].getClass() + ".");
+ if (arr[i] instanceof Inet4Address) {
+ ipv4 = true;
+ }
+ }
+
+ // And the only thing we test is that an ip v4 address is preferred.
+ if (ipv4) {
+ InetAddress addr = LinuxInetAddress.getLocalHost();
+ assertNotNull("IPv4 is prefered", addr instanceof Inet4Address);
+ }
+ }
+ }
+ catch (java.net.UnknownHostException e) {
+ // We're on vpn or have no network
+ }
+ }
+
+}
diff --git a/vespajlib/src/test/java/com/yahoo/net/URITestCase.java b/vespajlib/src/test/java/com/yahoo/net/URITestCase.java new file mode 100644 index 00000000000..7bb2303d7bb --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/net/URITestCase.java @@ -0,0 +1,512 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + + +import java.util.List; + + +/** + * Tests the URI class + * + * @author <a href="mailto:bratseth@fast.no">Jon S Bratseth</a> + */ +public class URITestCase extends junit.framework.TestCase { + + public URITestCase(String name) { + super(name); + } + + public void testEquality() { + URI one = new URI("http://www.nils.arne.com"); + URI two = new URI("http://www.nils.arne.com"); + + assertEquals(one, two); + assertEquals(one.hashCode(), two.hashCode()); + assertEquals("http://www.nils.arne.com/", one.toString()); + + assertEqualURIs( + "http://info.t.fast.no/art.php?sid=29&mode=thread&order=0", + "http://info.t.fast.no/art.php?sid=29&mode=thread&order=0"); + assertEqualURIs("http://a/g/", "http://a/g/"); + assertEquals("http://a/g;x?y#s", + new URI("http://a/g;x?y#s", true).stringValue()); + assertEquals("http://a/g?y#s", + new URI("http://a/g?y#s", true).stringValue()); + assertEqualURIs("http://a/b/c/.g", "http://a/b/c/.g"); + assertEqualURIs("http://a/b/c/..g", "http://a/b/c/..g"); + assertEqualURIs("http://a/b/c/g;x=1/y", "http://a/b/c/g;x=1/y"); + assertEquals("http://a/b/c/g#s/../x", + new URI("http://a/b/c/g#s/../x", true).stringValue()); + assertEquals("http://www.strange_host.com/b", + new URI("http://www.strange_host.com/b", true).stringValue()); + } + + public void testOpaque() { + URI uri = new URI("mailto:knut"); + + assertEquals("mailto:knut", uri.toString()); + assertTrue(uri.isOpaque()); + } + + public void testValid() { + assertTrue( + new URI("http://www.one.com/isValid?even=if&theres=args").isValid()); + assertTrue( + !new URI("http://www.one.com/isValid?even=if&theres=args").isOpaque()); + + assertTrue(!(new URI("not\\uri?", false, true).isValid())); + + assertTrue(new URI("http://www.strange_host.com/b").isValid()); + assertTrue(!new URI("http://www.strange_host.com/b").isOpaque()); + } + + public void testSorting() { + URI first = new URI("http://aisfirst.kanoo.com"); + URI second = new URI("www.thentheresw.com"); + + assertTrue(first.compareTo(second) < 0); + assertTrue(second.compareTo(second) == 0); + assertTrue(second.compareTo(first) > 1); + } + + public void testHost() { + assertEquals("a.b.c", new URI("http://A.B.C:567").getHost()); + assertEquals("www.kanoo.com", + new URI("www.kanoo.com/foo", false, true).getHost()); + assertEquals("a.b.c", new URI("http://a.b.C/foo").getHost()); + assertEquals("a.b.c", new URI("http://a.b.C").getHost()); + assertEquals("a", new URI("http://A").getHost()); + assertEquals("a", new URI("http://A:80").getHost()); + } + + public void testUnfragmenting() { + assertEquals("http://www.sng.no/a/b/dee?kanoos&at=nught#chapter3", + new URI("http://www.sng.no/a/b/cee/../dee?kanoos&at=nught#chapter3", true).stringValue()); + assertEquals("http://www.sng.no/a/b/dee?kanoos&at=nught", + new URI("http://www.sng.no/a/b/cee/../dee?kanoos&at=nught#chapter3", false).stringValue()); + } + + public void testNormalizing() { + // Abbreviation resolving heuristics + assertEquals("http://www.a.b/c", + new URI("www.a.b/c", false, true).toString()); + assertEquals("file://x:\\a", new URI("x:\\a", false, true).toString()); + assertEquals("file://c:/a", new URI("c:/a", false, true).toString()); + + // RFC 2396 normalizing + assertEqualURIs("http://a/c/d", "http://a/b/../c/d"); + assertEqualURIs("http://a/b", "http://a/./b"); + + // FAST normalizing + assertEqualURIs("http://a/", " http://a "); + assertEqualURIs("http://a/%e6;m%e5;ha%f8;l", "http://a/\u00E6m\u00E5ha\u00F8l"); + assertEqualURIs("http://a/&b", "http://a/&b"); + assertEqualURIs("http://a/", "http://A"); + assertEqualURIs("http://a/", "http://a:80"); + assertEqualURIs("https://a/", "https://a:443"); + assertEqualURIs("http://a/", "http://a."); + assertEqualURIs("http://a/b", "http://a//b"); + assertEqualURIs("http://a/b/", "http://A/b/"); + assertEqualURIs("http://a/b/", "http://a./b/"); + assertEqualURIs("http://a/", "http://a/b/../"); + assertEqualURIs("http://a/../", "http://a/b/../a/../../"); + assertEqualURIs("http://a/", "http://a/b/../"); + assertEqualURIs("http://a/b/c/d", "http://a/b/c/d"); + assertEqualURIs("http://a/b/c", "http://a/b/c#kanoo"); + + // Everything combined + assertEquals("http://www.a.b/m%e5;l/&/%f8;l&&/", + new URI(" WWW.a.B:80//m\u00E5l/.//&/./\u00F8l&&/foo/../upp/./..", true, true).toString()); + } + + public void testParemeterAdding() { + assertEquals("http://a/?knug=zagg", + new URI("http://a/").addParameter("knug", "zagg").stringValue()); + assertEquals("http://a/b?knug=zagg&fjukk=barra", + new URI("http://a/b?knug=zagg").addParameter("fjukk", "barra").stringValue()); + } + + private void assertEqualURIs(String fasit, String test) { + assertEquals(fasit, new URI(test).toString()); + } + + public void testDepth() { + assertEquals(0, new URI("test:hit").getDepth()); + assertEquals(0, new URI("test://hit").getDepth()); + assertEquals(0, new URI("test://hit/").getDepth()); + assertEquals(1, new URI("test://hit.test/hello ").getDepth()); + assertEquals(1, new URI("test://hit.test/hello/").getDepth()); + assertEquals(0, new URI("test:// ").getDepth()); + assertEquals(0, new URI("test:///").getDepth()); + assertEquals(1, new URI("test:////").getDepth()); + assertEquals(2, new URI("test://hit.test/hello/test2/").getDepth()); + } + + public void testURLEmpty() { + URI uri = new URI("", true); + assertTrue(uri.isValid()); + assertNull(uri.getScheme()); + assertNull(uri.getHost()); + assertNull(uri.getDomain()); + assertNull(uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertNull(uri.getPath()); + assertNull(uri.getFilename()); + assertNull(uri.getExtension()); + assertNull(uri.getParams()); + assertNull(uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLDot() { + URI uri = new URI(".", true); + assertTrue(uri.isValid()); + assertNull(uri.getScheme()); + assertNull(uri.getHost()); + assertNull(uri.getDomain()); + assertNull(uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertNull(uri.getPath()); //differs from FastS_URL, "." + assertNull(uri.getFilename()); //differs from FastS_URL, "." + assertNull(uri.getExtension()); + assertNull(uri.getParams()); + assertNull(uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLDotDot() { + URI uri = new URI("..", true); + assertTrue(uri.isValid()); + assertNull(uri.getScheme()); + assertNull(uri.getHost()); + assertNull(uri.getDomain()); + assertNull(uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertNull(uri.getPath()); //differs from FastS_URL, ".." + assertNull(uri.getFilename()); //differs from FastS_URL, ".." + assertNull(uri.getExtension()); + assertNull(uri.getParams()); + assertNull(uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLUninett() { + URI uri = new URI("http://180.uninett.no/servlet/online.Bransje", true); + assertTrue(uri.isValid()); + assertEquals("http", uri.getScheme()); + assertEquals("180.uninett.no", uri.getHost()); + assertEquals("uninett.no", uri.getDomain()); + assertEquals("no", uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertEquals("/servlet/online.Bransje", uri.getPath()); + assertEquals("online.Bransje", uri.getFilename()); + assertEquals("Bransje", uri.getExtension()); + assertNull(uri.getParams()); + assertNull(uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLUnderdusken() { + URI uri = new URI("http://www.underdusken.no", true); + assertTrue(uri.isValid()); + assertEquals("http", uri.getScheme()); + assertEquals("www.underdusken.no", uri.getHost()); + assertEquals("underdusken.no", uri.getDomain()); + assertEquals("no", uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertEquals("", uri.getPath()); + assertEquals("", uri.getFilename()); + assertNull(uri.getExtension()); + assertNull(uri.getParams()); + assertNull(uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLUnderduskenUholdbar() { + URI uri = + new URI("http://www.underdusken.no/?page=dusker/html/0008/Uholdbar.html", true); + assertTrue(uri.isValid()); + assertEquals("http", uri.getScheme()); + assertEquals("www.underdusken.no", uri.getHost()); + assertEquals("underdusken.no", uri.getDomain()); + assertEquals("no", uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertEquals("/", uri.getPath()); + assertEquals("", uri.getFilename()); + assertNull(uri.getExtension()); + assertNull(uri.getParams()); + assertEquals("page=dusker/html/0008/Uholdbar.html", uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLUniKarlsruhe() { + URI uri = new URI("http://www.uni-karlsruhe.de/~ig25/ssh-faq/", true); + assertTrue(uri.isValid()); + assertEquals("http", uri.getScheme()); + assertEquals("www.uni-karlsruhe.de", uri.getHost()); + assertEquals("uni-karlsruhe.de", uri.getDomain()); + assertEquals("de", uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertEquals("/~ig25/ssh-faq/", uri.getPath()); + assertEquals("", uri.getFilename()); + assertNull(uri.getExtension()); + assertNull(uri.getParams()); + assertNull(uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLDetteErEn() { + URI uri = new URI("https://dette.er.en:2020/~janie/index.htm?param1=q¶m2=r", true); + assertTrue(uri.isValid()); + assertEquals("https", uri.getScheme()); + assertEquals("dette.er.en", uri.getHost()); + assertEquals("er.en", uri.getDomain()); + assertEquals("en", uri.getMainTld()); + assertEquals(2020, uri.getPort()); + assertEquals("/~janie/index.htm", uri.getPath()); + assertEquals("index.htm", uri.getFilename()); + assertEquals("htm", uri.getExtension()); + assertNull(uri.getParams()); + assertEquals("param1=q¶m2=r", uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLSonyCoUk() { + URI uri = new URI("http://www.sony.co.uk/", true); + assertTrue(uri.isValid()); + assertEquals("http", uri.getScheme()); + assertEquals("www.sony.co.uk", uri.getHost()); + assertEquals("sony.co.uk", uri.getDomain()); + assertEquals("uk", uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertEquals("/", uri.getPath()); + assertEquals("", uri.getFilename()); + assertNull(uri.getExtension()); + assertNull(uri.getParams()); + assertNull(uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLSonyCoUk2() { + URI uri = new URI("http://sony.co.uk/", true); + assertTrue(uri.isValid()); + assertEquals("http", uri.getScheme()); + assertEquals("sony.co.uk", uri.getHost()); + //TODO: Fix when tldlist is implemented: + //assertEquals("sony.co.uk", uri.getDomain()); + assertEquals("co.uk", uri.getDomain()); + assertEquals("uk", uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertEquals("/", uri.getPath()); + assertEquals("", uri.getFilename()); + assertNull(uri.getExtension()); + assertNull(uri.getParams()); + assertNull(uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLSomehostSomedomain() { + URI uri = new URI("http://somehost.somedomain/this!is!it/boom", true); + assertTrue(uri.isValid()); + assertEquals("http", uri.getScheme()); + assertEquals("somehost.somedomain", uri.getHost()); + assertEquals("somehost.somedomain", uri.getDomain()); + assertEquals("somedomain", uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertEquals("/this!is!it/boom", uri.getPath()); + assertEquals("boom", uri.getFilename()); + assertNull(uri.getExtension()); + assertNull(uri.getParams()); + assertNull(uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLTestCom() { + URI uri = new URI("http://test.com/index.htm?p1=q%20test&p2=r%10d", true); + assertTrue(uri.isValid()); + assertEquals("http", uri.getScheme()); + assertEquals("test.com", uri.getHost()); + assertEquals("test.com", uri.getDomain()); + assertEquals("com", uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertEquals("/index.htm", uri.getPath()); + assertEquals("index.htm", uri.getFilename()); + assertEquals("htm", uri.getExtension()); + assertNull(uri.getParams()); + assertEquals("p1=q%20test&p2=r%10d", uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLArthur() { + URI uri = new URI("http://arthur/qm/images/qm1.gif", true); + assertTrue(uri.isValid()); + assertEquals("http", uri.getScheme()); + assertEquals("arthur", uri.getHost()); + assertEquals("arthur", uri.getDomain()); + assertNull(uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertEquals("/qm/images/qm1.gif", uri.getPath()); + assertEquals("qm1.gif", uri.getFilename()); + assertEquals("gif", uri.getExtension()); + assertNull(uri.getParams()); + assertNull(uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLFooCom() { + URI uri = new URI("http://foo.com/ui;.gif", true); + assertTrue(uri.isValid()); + assertEquals("http", uri.getScheme()); + assertEquals("foo.com", uri.getHost()); + assertEquals("foo.com", uri.getDomain()); + assertEquals("com", uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertEquals("/ui;.gif", uri.getPath()); + assertEquals("ui", uri.getFilename()); + assertNull(uri.getExtension()); + assertEquals(".gif", uri.getParams()); + assertNull(uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLFooCom2() { + URI uri = new URI("http://foo.com/ui;par1=1/par2=2", true); + assertTrue(uri.isValid()); + assertEquals("http", uri.getScheme()); + assertEquals("foo.com", uri.getHost()); + assertEquals("foo.com", uri.getDomain()); + assertEquals("com", uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertEquals("/ui;par1=1/par2=2", uri.getPath()); + assertEquals("ui", uri.getFilename()); + assertNull(uri.getExtension()); + assertEquals("par1=1/par2=2", uri.getParams()); + assertNull(uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testURLFooNo() { + URI uri = new URI( + "http://www.foo.no:8080/path/filename.ext;par1=hello/par2=world?query=test#fragment", true); + assertTrue(uri.isValid()); + assertEquals("http", uri.getScheme()); + assertEquals("www.foo.no", uri.getHost()); + assertEquals("foo.no", uri.getDomain()); + assertEquals("no", uri.getMainTld()); + assertEquals(8080, uri.getPort()); + assertEquals("/path/filename.ext;par1=hello/par2=world", uri.getPath()); + assertEquals("filename.ext", uri.getFilename()); + assertEquals("ext", uri.getExtension()); + assertEquals("par1=hello/par2=world", uri.getParams()); + assertEquals("query=test", uri.getQuery()); + assertEquals("fragment", uri.getFragment()); + } + + public void testURLAmpersand() { + URI uri = new URI("http://canonsarang.com/zboard/data/gallery04/HU&BANG.jpg", true); + assertTrue(uri.isValid()); + assertEquals("http", uri.getScheme()); + assertEquals("canonsarang.com", uri.getHost()); + assertEquals("canonsarang.com", uri.getDomain()); + assertEquals("com", uri.getMainTld()); + assertEquals(-1, uri.getPort()); + assertEquals("/zboard/data/gallery04/HU&BANG.jpg", uri.getPath()); + assertEquals("HU&BANG.jpg", uri.getFilename()); + assertEquals("jpg", uri.getExtension()); + assertNull(uri.getParams()); + assertNull(uri.getQuery()); + assertNull(uri.getFragment()); + } + + public void testQMark() { + URI uri = new URI("http://foobar/?"); + assertTrue(uri.isValid()); + assertEquals("http", uri.getScheme()); + assertEquals("foobar", uri.getHost()); + assertEquals("", uri.getQuery()); + } + + public void testTokenization() { + URI uri = new URI("http://this.i_s:5000/wo_ho;ba-lo?gobo#banana", true); + List<URI.Token> tokens = uri.tokenize(); + URI.Token token; + + token = tokens.get(0); + assertEquals("http", token.getToken()); + assertEquals(URI.URLContext.URL_SCHEME, token.getContext()); + + token = tokens.get(1); + assertEquals("this", token.getToken()); + assertEquals(URI.URLContext.URL_HOST, token.getContext()); + + token = tokens.get(2); + assertEquals("i_s", token.getToken()); + assertEquals(URI.URLContext.URL_HOST, token.getContext()); + + token = tokens.get(3); + assertEquals("5000", token.getToken()); + assertEquals(URI.URLContext.URL_PORT, token.getContext()); + + token = tokens.get(4); + assertEquals("wo_ho", token.getToken()); + assertEquals(URI.URLContext.URL_PATH, token.getContext()); + + token = tokens.get(5); + assertEquals("ba-lo", token.getToken()); + assertEquals(URI.URLContext.URL_PATH, token.getContext()); + + token = tokens.get(6); + assertEquals("gobo", token.getToken()); + assertEquals(URI.URLContext.URL_QUERY, token.getContext()); + + token = tokens.get(7); + assertEquals("banana", token.getToken()); + assertEquals(URI.URLContext.URL_FRAGMENT, token.getContext()); + + try { + tokens.get(8); + fail(); + } catch (IndexOutOfBoundsException ioobe) { + } + } + + // Error reported int bug #2466528 + public void testFileURIEmptyHost() { + URI uri = new URI("file:///C:/Inetpub/wwwroot/DW_SHORTCUTS.htm"); + List<URI.Token> tokens = uri.tokenize(); + URI.Token token; + token = tokens.get(0); + assertEquals("file", token.getToken()); + assertEquals(URI.URLContext.URL_SCHEME, token.getContext()); + + token = tokens.get(1); + assertEquals("localhost", token.getToken()); + assertEquals(URI.URLContext.URL_HOST, token.getContext()); + + token = tokens.get(2); + assertEquals("C", token.getToken()); + assertEquals(URI.URLContext.URL_PATH, token.getContext()); + + token = tokens.get(3); + assertEquals("Inetpub", token.getToken()); + assertEquals(URI.URLContext.URL_PATH, token.getContext()); + + token = tokens.get(4); + assertEquals("wwwroot", token.getToken()); + assertEquals(URI.URLContext.URL_PATH, token.getContext()); + + token = tokens.get(5); + assertEquals("DW_SHORTCUTS", token.getToken()); + assertEquals(URI.URLContext.URL_PATH, token.getContext()); + + token = tokens.get(6); + assertEquals("htm", token.getToken()); + assertEquals(URI.URLContext.URL_PATH, token.getContext()); + + try { + tokens.get(7); + fail(); + } catch (IndexOutOfBoundsException ioobe) { + } + } +} diff --git a/vespajlib/src/test/java/com/yahoo/net/UriToolsTestCase.java b/vespajlib/src/test/java/com/yahoo/net/UriToolsTestCase.java new file mode 100644 index 00000000000..9a17fa341be --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/net/UriToolsTestCase.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import static org.junit.Assert.*; + +import java.net.URISyntaxException; + +import org.junit.Test; + +/** + * Check validity of the URI helper methods. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class UriToolsTestCase { + + private static final String SEARCH_QUERY = "/search/?query=sddocname:music#trick"; + + @Test + public final void testRawRequest() throws URISyntaxException { + java.net.URI uri = new java.net.URI("http://localhost:" + 8080 + SEARCH_QUERY); + assertEquals(SEARCH_QUERY, UriTools.rawRequest(uri)); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/net/UrlTestCase.java b/vespajlib/src/test/java/com/yahoo/net/UrlTestCase.java new file mode 100644 index 00000000000..7cf99cf2c5a --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/net/UrlTestCase.java @@ -0,0 +1,192 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class UrlTestCase { + + @Test + public void requireThatAccessorsWork() { + Url url = Url.fromString("scheme://user:pass@host:69/path?query#fragment"); + assertEquals("scheme://user:pass@host:69/path?query#fragment", url.toString()); + assertEquals("scheme", url.getScheme()); + assertEquals(0, url.getSchemeBegin()); + assertEquals(6, url.getSchemeEnd()); + assertEquals("user", url.getUserInfo()); + assertEquals(9, url.getUserInfoBegin()); + assertEquals(13, url.getUserInfoEnd()); + assertEquals("pass", url.getPassword()); + assertEquals(14, url.getPasswordBegin()); + assertEquals(18, url.getPasswordEnd()); + assertEquals("host", url.getHost()); + assertEquals(19, url.getHostBegin()); + assertEquals(23, url.getHostEnd()); + assertEquals("69", url.getPortString()); + assertEquals(24, url.getPortBegin()); + assertEquals(26, url.getPortEnd()); + assertEquals(Integer.valueOf(69), url.getPort()); + assertEquals("/path", url.getPath()); + assertEquals(26, url.getPathBegin()); + assertEquals(31, url.getPathEnd()); + assertEquals("query", url.getQuery()); + assertEquals(32, url.getQueryBegin()); + assertEquals(37, url.getQueryEnd()); + assertEquals("fragment", url.getFragment()); + assertEquals(38, url.getFragmentBegin()); + assertEquals(46, url.getFragmentEnd()); + } + + @Test + public void requireThatOffsetsAreNeverOutOfBounds() { + Url url = Url.fromString("http:"); + assertEquals(0, url.getSchemeBegin()); + assertEquals(4, url.getSchemeEnd()); + assertEquals(5, url.getUserInfoBegin()); + assertEquals(5, url.getUserInfoEnd()); + assertEquals(5, url.getPasswordBegin()); + assertEquals(5, url.getPasswordEnd()); + assertEquals(5, url.getHostBegin()); + assertEquals(5, url.getHostEnd()); + assertEquals(5, url.getPortBegin()); + assertEquals(5, url.getPortEnd()); + assertEquals(5, url.getPathBegin()); + assertEquals(5, url.getPathEnd()); + assertEquals(5, url.getQueryBegin()); + assertEquals(5, url.getQueryEnd()); + assertEquals(5, url.getFragmentBegin()); + assertEquals(5, url.getFragmentEnd()); + + url = Url.fromString("//host"); + assertEquals(0, url.getSchemeBegin()); + assertEquals(0, url.getSchemeEnd()); + assertEquals(2, url.getUserInfoBegin()); + assertEquals(2, url.getUserInfoEnd()); + assertEquals(2, url.getPasswordBegin()); + assertEquals(2, url.getPasswordEnd()); + assertEquals(2, url.getHostBegin()); + assertEquals(6, url.getHostEnd()); + assertEquals(6, url.getPortBegin()); + assertEquals(6, url.getPortEnd()); + assertEquals(6, url.getPathBegin()); + assertEquals(6, url.getPathEnd()); + assertEquals(6, url.getQueryBegin()); + assertEquals(6, url.getQueryEnd()); + assertEquals(6, url.getFragmentBegin()); + assertEquals(6, url.getFragmentEnd()); + } + + @Test + public void requireThatCommonSchemesCanBeParsed() { + assertParse("ftp://ftp.is.co.za/rfc/rfc1808.txt", + "ftp", null, null, "ftp.is.co.za", null, "/rfc/rfc1808.txt", null, null); + assertParse("http://www.ietf.org/rfc/rfc 2396.txt", + "http", null, null, "www.ietf.org", null, "/rfc/rfc 2396.txt", null, null); + assertParse("ldap://[2001:db8::7]/c=GB?objectClass?one", + "ldap", null, null, "2001:db8::7", null, "/c=GB", "objectClass?one", null); + assertParse("mailto:John.Doe@example.com", + "mailto", null, null, null, null, "John.Doe@example.com", null, null); + assertParse("news:comp.infosystems.www.servers.unix", + "news", null, null, null, null, "comp.infosystems.www.servers.unix", null, null); + assertParse("tel:+1-816-555-1212", + "tel", null, null, null, null, "+1-816-555-1212", null, null); + assertParse("telnet://192.0.2.16:80/", + "telnet", null, null, "192.0.2.16", 80, "/", null, null); + assertParse("urn:oasis:names:specification:docbook:dtd:xml:4.1.2", + "urn", null, null, null, null, "oasis:names:specification:docbook:dtd:xml:4.1.2", null, null); + } + + @Test + public void requireThatAllComponentsCanBeParsed() { + assertParse("scheme:", + "scheme", null, null, null, null, null, null, null); + assertParse("scheme://", + "scheme", null, null, null, null, "//", null, null); + assertParse("scheme://host", + "scheme", null, null, "host", null, null, null, null); + try { + assertParse("scheme://host:foo", + null, null, null, null, null, null, null, null); + fail(); + } catch (NumberFormatException e) { + // expected + } + assertParse("scheme://host:69", + "scheme", null, null, "host", 69, null, null, null); + assertParse("scheme://user@host:69", + "scheme", "user", null, "host", 69, null, null, null); + assertParse("scheme://user:pass@host:69", + "scheme", "user", "pass", "host", 69, null, null, null); + assertParse("scheme://user:pass@host:69", + "scheme", "user", "pass", "host", 69, null, null, null); + assertParse("scheme://user:pass@host:69/", + "scheme", "user", "pass", "host", 69, "/", null, null); + assertParse("scheme://user:pass@host:69/path", + "scheme", "user", "pass", "host", 69, "/path", null, null); + assertParse("scheme://user:pass@host:69/path?query", + "scheme", "user", "pass", "host", 69, "/path", "query", null); + assertParse("scheme://user:pass@host:69/path?query#fragment", + "scheme", "user", "pass", "host", 69, "/path", "query", "fragment"); + assertParse("scheme://user@host:69/path?query#fragment", + "scheme", "user", null, "host", 69, "/path", "query", "fragment"); + assertParse("scheme://host:69/path?query#", + "scheme", null, null, "host", 69, "/path", "query", null); + assertParse("scheme://host:69/path?query#fragment", + "scheme", null, null, "host", 69, "/path", "query", "fragment"); + assertParse("scheme://host/path?query#fragment", + "scheme", null, null, "host", null, "/path", "query", "fragment"); + assertParse("scheme:///path?query#fragment", + "scheme", null, null, null, null, "///path", "query", "fragment"); + assertParse("scheme://?query#fragment", + "scheme", null, null, null, null, "//", "query", "fragment"); + assertParse("scheme://#fragment", + "scheme", null, null, null, null, "//", null, "fragment"); + } + + @Test + public void requireThatIPv6CanBeParsed() { + assertParse("http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", + "http", null, null, "2001:0db8:85a3:0000:0000:8a2e:0370:7334", null, null, null, null); + assertParse("http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]/path", + "http", null, null, "2001:0db8:85a3:0000:0000:8a2e:0370:7334", null, "/path", null, null); + + assertParse("http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:80", + "http", null, null, "2001:0db8:85a3:0000:0000:8a2e:0370:7334", 80, null, null, null); + assertParse("http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:80/path", + "http", null, null, "2001:0db8:85a3:0000:0000:8a2e:0370:7334", 80, "/path", null, null); + } + + private static void assertParse(String input, String scheme, String userInfo, String password, String host, + Integer port, String path, String query, String fragment) + { + Url urlA = Url.fromString(input); + assertEquals("Image", input, urlA.toString()); + assertUrl(urlA, scheme, userInfo, password, host, port, path, query, fragment); + + Url urlB = new Url(urlA.getScheme(), urlA.getUserInfo(), urlA.getPassword(), urlA.getHost(), urlA.getPort(), + urlA.getPath(), urlA.getQuery(), urlA.getFragment()); + assertUrl(urlB, scheme, userInfo, password, host, port, path, query, fragment); + + Url urlC = Url.fromString(urlB.toString()); + assertEquals(urlB, urlC); + assertUrl(urlC, scheme, userInfo, password, host, port, path, query, fragment); + } + + private static void assertUrl(Url url, String scheme, String userInfo, String password, String host, Integer port, + String path, String query, String fragment) + { + assertEquals("Scheme", scheme, url.getScheme()); + assertEquals("User", userInfo, url.getUserInfo()); + assertEquals("Password", password, url.getPassword()); + assertEquals("Host", host, url.getHost()); + assertEquals("Port", port, url.getPort()); + assertEquals("Path", path, url.getPath()); + assertEquals("Query", query, url.getQuery()); + assertEquals("Fragment", fragment, url.getFragment()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/net/UrlTokenTestCase.java b/vespajlib/src/test/java/com/yahoo/net/UrlTokenTestCase.java new file mode 100644 index 00000000000..ca42a701655 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/net/UrlTokenTestCase.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class UrlTokenTestCase { + + @Test + public void requireThatAccessorsWork() { + UrlToken token = new UrlToken(UrlToken.Type.FRAGMENT, 69, "foo", "bar"); + assertEquals(UrlToken.Type.FRAGMENT, token.getType()); + assertEquals(69, token.getOffset()); + assertEquals(3, token.getLength()); + assertEquals("foo", token.getOrig()); + assertEquals("bar", token.getTerm()); + } + + @Test + public void requireThatTypeCanNotBeNull() { + try { + new UrlToken(null, 0, "foo", "bar"); + fail(); + } catch (NullPointerException e) { + + } + } + + @Test + public void requireThatOrigAndTermCanBeNull() { + UrlToken token = new UrlToken(UrlToken.Type.SCHEME, 0, null, "foo"); + assertNull(token.getOrig()); + assertEquals("foo", token.getTerm()); + + token = new UrlToken(UrlToken.Type.SCHEME, 0, "foo", null); + assertEquals("foo", token.getOrig()); + assertNull(token.getTerm()); + + token = new UrlToken(UrlToken.Type.SCHEME, 0, null, null); + assertNull(token.getOrig()); + assertNull(token.getTerm()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/net/UrlTokenizerTestCase.java b/vespajlib/src/test/java/com/yahoo/net/UrlTokenizerTestCase.java new file mode 100644 index 00000000000..d192c0901a6 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/net/UrlTokenizerTestCase.java @@ -0,0 +1,385 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.net; + +import org.junit.Test; + +import java.util.*; + +import static org.junit.Assert.*; + +import static com.yahoo.text.Lowercase.toLowerCase; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class UrlTokenizerTestCase { + + @Test + public void requireThatAllTokenCharactersAreAccepted() { + assertTerms("a", "a"); + assertTerms("aa", "aa"); + assertTerms("aaa", "aaa"); + for (int c = Character.MIN_VALUE; c < Character.MAX_VALUE; ++c) { + if (c == '%') { + continue; // escape + } + String img = String.format("a%ca", c); + if ((c >= '0' && c <= '9') || + (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c == '-' || c == '_')) + { + assertTerms(img, toLowerCase(img)); + } else { + assertTerms(img, "a", "a"); + } + } + } + + @Test + public void requireThatUrlCanBeTokenized() { + assertTokenize("", + new UrlToken(UrlToken.Type.SCHEME, 0, null, "http"), + new UrlToken(UrlToken.Type.PORT, 0, null, "80")); + assertTokenize("scheme:", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme")); + assertTokenize("scheme://host", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.HOST, 9, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 9, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 13, null, UrlTokenizer.TERM_ENDHOST)); + assertTokenize("scheme://user@host", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"), + new UrlToken(UrlToken.Type.HOST, 14, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 14, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 18, null, UrlTokenizer.TERM_ENDHOST)); + assertTokenize("scheme://user:pass@host", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"), + new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"), + new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 19, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST)); + assertTokenize("scheme://user:pass@host:69", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"), + new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"), + new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 19, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 24, "69", "69")); + assertTokenize("scheme://user:pass@host:69/path", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"), + new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"), + new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 19, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 24, "69", "69"), + new UrlToken(UrlToken.Type.PATH, 27, "path", "path")); + assertTokenize("scheme://user:pass@host:69/path?query", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"), + new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"), + new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 19, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 24, "69", "69"), + new UrlToken(UrlToken.Type.PATH, 27, "path", "path"), + new UrlToken(UrlToken.Type.QUERY, 32, "query", "query")); + assertTokenize("scheme://user:pass@host:69/path?query#fragment", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"), + new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"), + new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 19, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 24, "69", "69"), + new UrlToken(UrlToken.Type.PATH, 27, "path", "path"), + new UrlToken(UrlToken.Type.QUERY, 32, "query", "query"), + new UrlToken(UrlToken.Type.FRAGMENT, 38, "fragment", "fragment")); + } + + @Test + public void requireThatComponentsCanHaveMultipleTokens() { + assertTokenize("sch+eme://us+er:pa+ss@ho+st:69/pa/th?que+ry#frag+ment", + new UrlToken(UrlToken.Type.SCHEME, 0, "sch", "sch"), + new UrlToken(UrlToken.Type.SCHEME, 4, "eme", "eme"), + new UrlToken(UrlToken.Type.USERINFO, 10, "us", "us"), + new UrlToken(UrlToken.Type.USERINFO, 13, "er", "er"), + new UrlToken(UrlToken.Type.PASSWORD, 16, "pa", "pa"), + new UrlToken(UrlToken.Type.PASSWORD, 19, "ss", "ss"), + new UrlToken(UrlToken.Type.HOST, 22, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 22, "ho", "ho"), + new UrlToken(UrlToken.Type.HOST, 25, "st", "st"), + new UrlToken(UrlToken.Type.HOST, 27, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 28, "69", "69"), + new UrlToken(UrlToken.Type.PATH, 31, "pa", "pa"), + new UrlToken(UrlToken.Type.PATH, 34, "th", "th"), + new UrlToken(UrlToken.Type.QUERY, 37, "que", "que"), + new UrlToken(UrlToken.Type.QUERY, 41, "ry", "ry"), + new UrlToken(UrlToken.Type.FRAGMENT, 44, "frag", "frag"), + new UrlToken(UrlToken.Type.FRAGMENT, 49, "ment", "ment")); + } + @Test + public void requireThatSequencesOfDelimitersAreCollapsed() { + assertTokenize("sch++eme://us++er:pa++ss@ho++st:69/pa/th?que++ry#frag++ment", + new UrlToken(UrlToken.Type.SCHEME, 0, "sch", "sch"), + new UrlToken(UrlToken.Type.SCHEME, 5, "eme", "eme"), + new UrlToken(UrlToken.Type.USERINFO, 11, "us", "us"), + new UrlToken(UrlToken.Type.USERINFO, 15, "er", "er"), + new UrlToken(UrlToken.Type.PASSWORD, 18, "pa", "pa"), + new UrlToken(UrlToken.Type.PASSWORD, 22, "ss", "ss"), + new UrlToken(UrlToken.Type.HOST, 25, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 25, "ho", "ho"), + new UrlToken(UrlToken.Type.HOST, 29, "st", "st"), + new UrlToken(UrlToken.Type.HOST, 31, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 32, "69", "69"), + new UrlToken(UrlToken.Type.PATH, 35, "pa", "pa"), + new UrlToken(UrlToken.Type.PATH, 38, "th", "th"), + new UrlToken(UrlToken.Type.QUERY, 41, "que", "que"), + new UrlToken(UrlToken.Type.QUERY, 46, "ry", "ry"), + new UrlToken(UrlToken.Type.FRAGMENT, 49, "frag", "frag"), + new UrlToken(UrlToken.Type.FRAGMENT, 55, "ment", "ment")); + } + + @Test + public void requireThatIPv6CanBeTokenized() { + assertTokenize("scheme://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.HOST, 10, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 10, "2001", "2001"), + new UrlToken(UrlToken.Type.HOST, 15, "0db8", "0db8"), + new UrlToken(UrlToken.Type.HOST, 20, "85a3", "85a3"), + new UrlToken(UrlToken.Type.HOST, 25, "0000", "0000"), + new UrlToken(UrlToken.Type.HOST, 30, "0000", "0000"), + new UrlToken(UrlToken.Type.HOST, 35, "8a2e", "8a2e"), + new UrlToken(UrlToken.Type.HOST, 40, "0370", "0370"), + new UrlToken(UrlToken.Type.HOST, 45, "7334", "7334"), + new UrlToken(UrlToken.Type.HOST, 49, null, UrlTokenizer.TERM_ENDHOST)); + } + + @Test + public void requireThatTermsAreLowerCased() { + assertTokenize("SCHEME://USER:PASS@HOST:69/PATH?QUERY#FRAGMENT", + new UrlToken(UrlToken.Type.SCHEME, 0, "SCHEME", "scheme"), + new UrlToken(UrlToken.Type.USERINFO, 9, "USER", "user"), + new UrlToken(UrlToken.Type.PASSWORD, 14, "PASS", "pass"), + new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 19, "HOST", "host"), + new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 24, "69", "69"), + new UrlToken(UrlToken.Type.PATH, 27, "PATH", "path"), + new UrlToken(UrlToken.Type.QUERY, 32, "QUERY", "query"), + new UrlToken(UrlToken.Type.FRAGMENT, 38, "FRAGMENT", "fragment")); + } + + @Test + public void requireThatEscapedCharsAreDecoded() { + assertTokenize("sch%65me://%75ser:p%61ss@h%6fst:69/p%61th?q%75ery#fr%61gment", + new UrlToken(UrlToken.Type.SCHEME, 0, "sch%65me", "scheme"), + new UrlToken(UrlToken.Type.USERINFO, 11, "%75ser", "user"), + new UrlToken(UrlToken.Type.PASSWORD, 18, "p%61ss", "pass"), + new UrlToken(UrlToken.Type.HOST, 25, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 25, "h%6fst", "host"), + new UrlToken(UrlToken.Type.HOST, 31, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 32, "69", "69"), + new UrlToken(UrlToken.Type.PATH, 35, "p%61th", "path"), + new UrlToken(UrlToken.Type.QUERY, 42, "q%75ery", "query"), + new UrlToken(UrlToken.Type.FRAGMENT, 50, "fr%61gment", "fragment")); + } + + @Test + public void requireThatDecodedCharsAreLowerCased() { + assertTokenize("sch%45me://%55ser:p%41ss@h%4fst:69/p%41th?q%55ery#fr%41gment", + new UrlToken(UrlToken.Type.SCHEME, 0, "sch%45me", "scheme"), + new UrlToken(UrlToken.Type.USERINFO, 11, "%55ser", "user"), + new UrlToken(UrlToken.Type.PASSWORD, 18, "p%41ss", "pass"), + new UrlToken(UrlToken.Type.HOST, 25, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 25, "h%4fst", "host"), + new UrlToken(UrlToken.Type.HOST, 31, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 32, "69", "69"), + new UrlToken(UrlToken.Type.PATH, 35, "p%41th", "path"), + new UrlToken(UrlToken.Type.QUERY, 42, "q%55ery", "query"), + new UrlToken(UrlToken.Type.FRAGMENT, 50, "fr%41gment", "fragment")); + } + + @Test + public void requireThatDecodedCharsCanSplitTokens() { + assertTokenize("sch%2beme://us%2ber:pa%2bss@ho%2bst:69/pa/th?que%2bry#frag%2bment", + new UrlToken(UrlToken.Type.SCHEME, 0, "sch", "sch"), + new UrlToken(UrlToken.Type.SCHEME, 6, "eme", "eme"), + new UrlToken(UrlToken.Type.USERINFO, 12, "us", "us"), + new UrlToken(UrlToken.Type.USERINFO, 17, "er", "er"), + new UrlToken(UrlToken.Type.PASSWORD, 20, "pa", "pa"), + new UrlToken(UrlToken.Type.PASSWORD, 25, "ss", "ss"), + new UrlToken(UrlToken.Type.HOST, 28, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 28, "ho", "ho"), + new UrlToken(UrlToken.Type.HOST, 33, "st", "st"), + new UrlToken(UrlToken.Type.HOST, 35, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 36, "69", "69"), + new UrlToken(UrlToken.Type.PATH, 39, "pa", "pa"), + new UrlToken(UrlToken.Type.PATH, 42, "th", "th"), + new UrlToken(UrlToken.Type.QUERY, 45, "que", "que"), + new UrlToken(UrlToken.Type.QUERY, 51, "ry", "ry"), + new UrlToken(UrlToken.Type.FRAGMENT, 54, "frag", "frag"), + new UrlToken(UrlToken.Type.FRAGMENT, 61, "ment", "ment")); + } + + @Test + public void requireThatSchemeCanBeGuessed() { + assertTokenize("//host:80", + new UrlToken(UrlToken.Type.SCHEME, 0, null, "http"), + new UrlToken(UrlToken.Type.HOST, 2, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 2, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 6, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 7, "80", "80")); + } + + @Test + public void requireThatHostCanBeGuessed() { + assertTokenize("file:/path", + new UrlToken(UrlToken.Type.SCHEME, 0, "file", "file"), + new UrlToken(UrlToken.Type.HOST, 4, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 4, null, "localhost"), + new UrlToken(UrlToken.Type.HOST, 4, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PATH, 6, "path", "path")); + } + + @Test + public void requireThatPortCanBeGuessed() { + assertTokenize("http://host", + new UrlToken(UrlToken.Type.SCHEME, 0, "http", "http"), + new UrlToken(UrlToken.Type.HOST, 7, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 7, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 11, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 11, null, "80")); + } + + @Test + public void requireThatComponentsAreOptional() { + assertTokenize("scheme", "user", "pass", "host", 99, "/path", "query", "fragment", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"), + new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"), + new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 19, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 24, "99", "99"), + new UrlToken(UrlToken.Type.PATH, 27, "path", "path"), + new UrlToken(UrlToken.Type.QUERY, 32, "query", "query"), + new UrlToken(UrlToken.Type.FRAGMENT, 38, "fragment", "fragment")); + assertTokenize(null, "user", "pass", "host", 99, "/path", "query", "fragment", + new UrlToken(UrlToken.Type.SCHEME, 0, null, "http"), + new UrlToken(UrlToken.Type.USERINFO, 2, "user", "user"), + new UrlToken(UrlToken.Type.PASSWORD, 7, "pass", "pass"), + new UrlToken(UrlToken.Type.HOST, 12, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 12, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 16, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 17, "99", "99"), + new UrlToken(UrlToken.Type.PATH, 20, "path", "path"), + new UrlToken(UrlToken.Type.QUERY, 25, "query", "query"), + new UrlToken(UrlToken.Type.FRAGMENT, 31, "fragment", "fragment")); + assertTokenize("scheme", null, "pass", "host", 99, "/path", "query", "fragment", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.PASSWORD, 10, "pass", "pass"), + new UrlToken(UrlToken.Type.HOST, 15, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 15, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 20, "99", "99"), + new UrlToken(UrlToken.Type.PATH, 23, "path", "path"), + new UrlToken(UrlToken.Type.QUERY, 28, "query", "query"), + new UrlToken(UrlToken.Type.FRAGMENT, 34, "fragment", "fragment")); + assertTokenize("scheme", null, null, "host", 99, "/path", "query", "fragment", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.HOST, 9, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 9, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 13, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 14, "99", "99"), + new UrlToken(UrlToken.Type.PATH, 17, "path", "path"), + new UrlToken(UrlToken.Type.QUERY, 22, "query", "query"), + new UrlToken(UrlToken.Type.FRAGMENT, 28, "fragment", "fragment")); + assertTokenize("scheme", null, null, null, 99, "/path", "query", "fragment", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.PORT, 8, "99", "99"), + new UrlToken(UrlToken.Type.PATH, 11, "path", "path"), + new UrlToken(UrlToken.Type.QUERY, 16, "query", "query"), + new UrlToken(UrlToken.Type.FRAGMENT, 22, "fragment", "fragment")); + assertTokenize("scheme", "user", "pass", "host", null, "/path", "query", "fragment", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"), + new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"), + new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 19, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PATH, 24, "path", "path"), + new UrlToken(UrlToken.Type.QUERY, 29, "query", "query"), + new UrlToken(UrlToken.Type.FRAGMENT, 35, "fragment", "fragment")); + assertTokenize("scheme", "user", "pass", "host", 99, null, "query", "fragment", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"), + new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"), + new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 19, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 24, "99", "99"), + new UrlToken(UrlToken.Type.QUERY, 27, "query", "query"), + new UrlToken(UrlToken.Type.FRAGMENT, 33, "fragment", "fragment")); + assertTokenize("scheme", "user", "pass", "host", 99, "/path", null, "fragment", + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"), + new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"), + new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 19, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 24, "99", "99"), + new UrlToken(UrlToken.Type.PATH, 27, "path", "path"), + new UrlToken(UrlToken.Type.FRAGMENT, 32, "fragment", "fragment")); + assertTokenize("scheme", "user", "pass", "host", 99, "/path", "query", null, + new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"), + new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"), + new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"), + new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST), + new UrlToken(UrlToken.Type.HOST, 19, "host", "host"), + new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST), + new UrlToken(UrlToken.Type.PORT, 24, "99", "99"), + new UrlToken(UrlToken.Type.PATH, 27, "path", "path"), + new UrlToken(UrlToken.Type.QUERY, 32, "query", "query")); + } + + private static void assertTokenize(String scheme, String userInfo, String password, String host, Integer port, + String path, String query, String fragment, UrlToken... expected) + { + assertTokenize(new Url(scheme, userInfo, password, host, port, path, query, fragment), expected); + } + + private static void assertTokenize(String url, UrlToken... expected) { + assertTokenize(Url.fromString(url), expected); + } + + private static void assertTokenize(Url url, UrlToken... expected) { + Iterator<UrlToken> expectedIt = Arrays.asList(expected).iterator(); + Iterator<UrlToken> actualIt = new UrlTokenizer(url).tokenize().iterator(); + while (expectedIt.hasNext()) { + assertTrue(actualIt.hasNext()); + assertEquals(expectedIt.next(), actualIt.next()); + } + assertFalse(expectedIt.hasNext()); + assertFalse(actualIt.hasNext()); + } + + private static void assertTerms(String img, String... expected) { + List<UrlToken> actual = new LinkedList<>(); + UrlTokenizer.addTokens(actual, UrlToken.Type.PATH, 0, img, true); + + Iterator<String> expectedIt = Arrays.asList(expected).iterator(); + Iterator<UrlToken> actualIt = actual.iterator(); + while (expectedIt.hasNext()) { + assertTrue(actualIt.hasNext()); + assertEquals(expectedIt.next(), actualIt.next().getTerm()); + } + assertFalse(expectedIt.hasNext()); + assertFalse(actualIt.hasNext()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/path/PathTest.java b/vespajlib/src/test/java/com/yahoo/path/PathTest.java new file mode 100644 index 00000000000..4d78df0f4e9 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/path/PathTest.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.path; + +import org.junit.Test; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class PathTest { + @Test + public void testGetName() { + assertThat(getAbsolutePath().getName(), is("baz")); + assertThat(getRelativePath().getName(), is("baz")); + assertThat(getWithSlashes().getName(), is("baz")); + assertThat(getAppended().getName(), is("baz")); + assertThat(getOne().getName(), is("foo")); + } + + @Test + public void testEquals() { + assertTrue(getAbsolutePath().equals(getAbsolutePath())); + assertTrue(getAbsolutePath().equals(getRelativePath())); + assertTrue(getAbsolutePath().equals(getWithSlashes())); + assertTrue(getAbsolutePath().equals(getAppended())); + assertFalse(getAbsolutePath().equals(getOne())); + + assertTrue(getRelativePath().equals(getAbsolutePath())); + assertTrue(getRelativePath().equals(getRelativePath())); + assertTrue(getRelativePath().equals(getWithSlashes())); + assertTrue(getRelativePath().equals(getAppended())); + assertFalse(getRelativePath().equals(getOne())); + + assertTrue(getWithSlashes().equals(getAbsolutePath())); + assertTrue(getWithSlashes().equals(getRelativePath())); + assertTrue(getWithSlashes().equals(getWithSlashes())); + assertTrue(getWithSlashes().equals(getAppended())); + assertFalse(getWithSlashes().equals(getOne())); + + assertTrue(getAppended().equals(getAbsolutePath())); + assertTrue(getAppended().equals(getRelativePath())); + assertTrue(getAppended().equals(getWithSlashes())); + assertTrue(getAppended().equals(getAppended())); + assertFalse(getAppended().equals(getOne())); + + assertFalse(getOne().equals(getAbsolutePath())); + assertFalse(getOne().equals(getRelativePath())); + assertFalse(getOne().equals(getWithSlashes())); + assertFalse(getOne().equals(getAppended())); + assertTrue(getOne().equals(getOne())); + } + + @Test + public void testGetPath() { + assertThat(getAbsolutePath().getRelative(), is("foo/bar/baz")); + assertThat(getRelativePath().getRelative(), is("foo/bar/baz")); + assertThat(getWithSlashes().getRelative(), is("foo/bar/baz")); + assertThat(getAppended().getRelative(), is("foo/bar/baz")); + assertThat(getOne().getRelative(), is("foo")); + } + + @Test + public void testGetParentPath() { + assertThat(getAbsolutePath().getParentPath().getRelative(), is("foo/bar")); + assertThat(getRelativePath().getParentPath().getRelative(), is("foo/bar")); + assertThat(getWithSlashes().getParentPath().getRelative(), is("foo/bar")); + assertThat(getAppended().getParentPath().getRelative(), is("foo/bar")); + assertThat(getOne().getParentPath().getRelative(), is("")); + } + + @Test + public void testGetAbsolutePath() { + assertThat(getAbsolutePath().getAbsolute(), is("/foo/bar/baz")); + assertThat(getAbsolutePath().getParentPath().getAbsolute(), is("/foo/bar")); + } + + @Test + public void testEmptyPath() { + assertThat(Path.createRoot().getName(), is("")); + assertThat(Path.createRoot().getRelative(), is("")); + assertThat(Path.createRoot().getParentPath().getRelative(), is("")); + assertTrue(Path.createRoot().isRoot()); + } + + @Test + public void testDelimiters() { + assertThat(Path.fromString("foo/bar", ",").getName(), is("foo/bar")); + assertThat(Path.fromString("foo/bar", "/").getName(), is("bar")); + assertThat(Path.fromString("foo,bar", "/").getName(), is("foo,bar")); + assertThat(Path.fromString("foo,bar", ",").getName(), is("bar")); + assertThat(Path.createRoot(",").append("foo").append("bar").getRelative(), is("foo,bar")); + } + + @Test + public void testAppendPath() { + Path p1 = getAbsolutePath(); + Path p2 = getAbsolutePath(); + Path p3 = p1.append(p2); + assertThat(p1.getAbsolute(), is("/foo/bar/baz")); + assertThat(p2.getAbsolute(), is("/foo/bar/baz")); + assertThat(p3.getAbsolute(), is("/foo/bar/baz/foo/bar/baz")); + } + + private Path getRelativePath() { + return Path.fromString("foo/bar/baz"); + } + + private Path getAbsolutePath() { + return Path.fromString("/foo/bar/baz"); + } + + private Path getWithSlashes() { + return Path.fromString("/foo//bar///baz/"); + } + + private Path getAppended() { + return Path.createRoot().append("foo").append("bar").append("baz"); + } + + private Path getOne() { + return Path.fromString("foo"); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/protect/TestErrorMessage.java b/vespajlib/src/test/java/com/yahoo/protect/TestErrorMessage.java new file mode 100644 index 00000000000..fa611d8fd71 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/protect/TestErrorMessage.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.protect; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class TestErrorMessage extends junit.framework.TestCase { + + public void testErrorMessages() { + ErrorMessage m1=new ErrorMessage(17,"Message"); + ErrorMessage m2=new ErrorMessage(17,"Message","Detail"); + ErrorMessage m3=new ErrorMessage(17,"Message","Detail",new Exception("Throwable message")); + assertEquals(17,m1.getCode()); + assertEquals("Message",m1.getMessage()); + assertEquals("Detail",m2.getDetailedMessage()); + assertEquals("Throwable message",m3.getCause().getMessage()); + assertEquals("error : Message (Detail: Throwable message)",m3.toString()); + } + + public void testErrorMessageEquality() { + assertEquals(new ErrorMessage(17,"Message"),new ErrorMessage(17,"Message")); + assertFalse(new ErrorMessage(16,"Message").equals(new ErrorMessage(17,"Message"))); + assertFalse(new ErrorMessage(17,"Message").equals(new ErrorMessage(17,"Other message"))); + assertFalse(new ErrorMessage(17,"Message").equals(new ErrorMessage(17,"Message","Detail"))); + assertFalse(new ErrorMessage(17,"Message","Detail").equals(new ErrorMessage(17,"Message"))); + assertEquals(new ErrorMessage(17,"Message","Detail"),new ErrorMessage(17,"Message","Detail",new Exception())); + assertTrue(new ErrorMessage(17,"Message","Detail").equals(new ErrorMessage(17,"Message","Detail"))); + assertFalse(new ErrorMessage(17,"Message","Detail").equals(new ErrorMessage(17,"Message","Other detail"))); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/protect/ValidatorTestCase.java b/vespajlib/src/test/java/com/yahoo/protect/ValidatorTestCase.java new file mode 100644 index 00000000000..6acbda729e5 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/protect/ValidatorTestCase.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.protect; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ValidatorTestCase extends junit.framework.TestCase { + + public void testEnsureNotNull() { + try { + Validator.ensureNotNull("Description",null); + fail("No exception"); + } + catch (Exception e) { + // success + } + } + + public void testEnsureNotInitialized() { + try { + Validator.ensureNotInitialized("Description","Field-owner","Initialized-field-value"); + fail("No exception"); + } + catch (Exception e) { + // success + } + } + + public void testEnsureInRange() { + try { + Validator.ensureInRange("Description",2,4,5); + fail("No exception"); + } + catch (Exception e) { + // success + } + } + + public void testSmallerInts() { + try { + Validator.ensureSmaller("Small-description",3,"Large-description",2); + fail("No exception"); + } + catch (Exception e) { + // success + } + } + + public void testSmallerComparables() { + try { + Validator.ensureSmaller("Small-description","b","Large-description","a"); + fail("No exception"); + } + catch (Exception e) { + // success + } + } + + public void testEnsure() { + try { + Validator.ensure("Description",false); + fail("No exception"); + } + catch (Exception e) { + // success + } + } + + public void testEnsureInstanceOf() { + try { + Validator.ensureInstanceOf("Description","item",Integer.class); + fail("No exception"); + } + catch (Exception e) { + // success + } + } + + public void testVarArgsEnsure() { + Validator.ensure(true, "ignored"); + try { + Validator.ensure(false, "a", "b", "c"); + } catch (Exception e) { + assertEquals("abc", e.getMessage()); + } + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/reflection/CastingTest.java b/vespajlib/src/test/java/com/yahoo/reflection/CastingTest.java new file mode 100644 index 00000000000..32b506b206f --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/reflection/CastingTest.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.reflection; + +import org.junit.Test; + +import java.util.Optional; + +import static com.yahoo.text.StringUtilities.quote; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; +import static com.yahoo.reflection.Casting.cast; + +public class CastingTest { + @Test + public void valid_cast_gives_present_optional() { + Object objectToCast = 12; + Optional<Integer> value = cast(Integer.class, objectToCast); + assertTrue("Value is not present", value.isPresent()); + assertThat(value.get(), is(objectToCast)); + } + + @Test + public void invalid_cast_gives_empty_optional() { + Object objectToCast = "string"; + Optional<Integer> value = cast(Integer.class, objectToCast); + assertTrue("Value is present", !value.isPresent()); + } + + @Test(expected = IllegalArgumentException.class) + public void cast_sample_usage() { + Object objectToCast = "illegal"; + int result = cast(Integer.class, objectToCast). + filter(i -> !i.equals(0)). + orElseThrow(() -> new IllegalArgumentException("Expected non-zero integer, got " + quote(objectToCast))); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/rmi/.gitignore b/vespajlib/src/test/java/com/yahoo/rmi/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/rmi/.gitignore diff --git a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java new file mode 100644 index 00000000000..7b42d4e6bda --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java @@ -0,0 +1,567 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.hamcrest.CoreMatchers.*; + +import static com.yahoo.slime.BinaryFormat.*; + +public class BinaryFormatTestCase { + + static final int TYPE_LIMIT = 8; + static final int META_LIMIT = 32; + static final int MAX_CMPR_SIZE = 10; + static final int MAX_NUM_SIZE = 8; + + static final byte enc_t_and_sz(Type t, int size) { + assert size <= 30; + return encode_type_and_meta(t.ID, size + 1); + } + static final byte enc_t_and_m(Type t, int meta) { + assert meta <= 31; + return encode_type_and_meta(t.ID, meta); + } + + void verify_cmpr_long(long value, byte[] expect) { + BinaryEncoder bof = new BinaryEncoder(); + bof.encode_cmpr_long(value); + byte[] actual = bof.out.toArray(); + assertThat(actual, is(expect)); + + BinaryDecoder bif = new BinaryDecoder(); + bif.in = new BufferedInput(expect); + long got = bif.read_cmpr_long(); + assertThat(got, is(value)); + } + + // was verifyBasic + void verifyEncoding(Slime slime, byte[] expect) { + assertThat(BinaryFormat.encode(slime), is(expect)); + verifyMultiEncode(expect); + } + + void verifyMultiEncode(byte[] expect) { + byte[][] buffers = new byte[6][]; + buffers[0] = expect; + + for (int i = 0; i < 5; ++i) { + Slime slime = BinaryFormat.decode(buffers[i]); + buffers[i+1] = BinaryFormat.encode(slime); + assertThat(buffers[i+1], is(expect)); + } + } + + @Test + public void testZigZagConversion() { + System.out.println("test zigzag conversion"); + assertThat(encode_zigzag(0), is((long)0)); + assertThat(decode_zigzag(encode_zigzag(0)), is(0L)); + + assertThat(encode_zigzag(-1), is(1L)); + assertThat(decode_zigzag(encode_zigzag(-1)), is(-1L)); + + assertThat(encode_zigzag(1), is(2L)); + assertThat(decode_zigzag(encode_zigzag(1)), is(1L)); + + assertThat(encode_zigzag(-2), is(3L)); + assertThat(decode_zigzag(encode_zigzag(-2)), is(-2L)); + + assertThat(encode_zigzag(2), is(4L)); + assertThat(decode_zigzag(encode_zigzag(2)), is(2L)); + + assertThat(encode_zigzag(-1000), is(1999L)); + assertThat(decode_zigzag(encode_zigzag(-1000)), is(-1000L)); + + assertThat(encode_zigzag(1000), is(2000L)); + assertThat(decode_zigzag(encode_zigzag(1000)), is(1000L)); + + assertThat(encode_zigzag(-0x8000000000000000L), is(-1L)); + assertThat(decode_zigzag(encode_zigzag(-0x8000000000000000L)), is(-0x8000000000000000L)); + + assertThat(encode_zigzag(0x7fffffffffffffffL), is(-2L)); + assertThat(decode_zigzag(encode_zigzag(0x7fffffffffffffffL)), is(0x7fffffffffffffffL)); + } + + @Test + public void testDoubleConversion() { + System.out.println("test double conversion"); + assertThat(encode_double(0.0), is(0L)); + assertThat(decode_double(encode_double(0.0)), is(0.0)); + + assertThat(encode_double(1.0), is(0x3ff0000000000000L)); + assertThat(decode_double(encode_double(1.0)), is(1.0)); + + assertThat(encode_double(-1.0), is(0xbff0000000000000L)); + assertThat(decode_double(encode_double(-1.0)), is(-1.0)); + + assertThat(encode_double(2.0), is(0x4000000000000000L)); + assertThat(decode_double(encode_double(2.0)), is(2.0)); + + assertThat(encode_double(-2.0), is(0xc000000000000000L)); + assertThat(decode_double(encode_double(-2.0)), is(-2.0)); + + assertThat(encode_double(-0.0), is(0x8000000000000000L)); + assertThat(decode_double(encode_double(-0.0)), is(-0.0)); + + assertThat(encode_double(3.5), is(0x400c000000000000L)); + assertThat(decode_double(encode_double(3.5)), is(3.5)); + + assertThat(encode_double(65535.875), is(0x40EFFFFC00000000L)); + assertThat(decode_double(encode_double(65535.875)), is(65535.875)); + } + + @Test + public void testTypeAndMetaMangling() { + System.out.println("test type and meta mangling"); + for (byte type = 0; type < TYPE_LIMIT; ++type) { + for (int meta = 0; meta < META_LIMIT; ++meta) { + byte mangled = encode_type_and_meta(type, meta); + assertThat(decode_type(mangled).ID, is(type)); + assertThat(decode_meta(mangled), is(meta)); + } + } + } + + // was testCmprUlong + @Test + public void testCmprLong() { + System.out.println("test compressed long"); + { + long value = 0; + byte[] wanted = { 0 }; + verify_cmpr_long(value, wanted); + }{ + long value = 127; + byte[] wanted = { 127 }; + verify_cmpr_long(value, wanted); + }{ + long value = 128; + byte[] wanted = { -128, 1 }; + verify_cmpr_long(value, wanted); + }{ + long value = 16383; + byte[] wanted = { -1, 127 }; + verify_cmpr_long(value, wanted); + }{ + long value = 16384; + byte[] wanted = { -128, -128, 1 }; + verify_cmpr_long(value, wanted); + }{ + long value = 2097151; + byte[] wanted = { -1, -1, 127 }; + verify_cmpr_long(value, wanted); + }{ + long value = 2097152; + byte[] wanted = { -128, -128, -128, 1 }; + verify_cmpr_long(value, wanted); + }{ + long value = 268435455; + byte[] wanted = { -1, -1, -1, 127 }; + verify_cmpr_long(value, wanted); + }{ + long value = 268435456; + byte[] wanted = { -128, -128, -128, -128, 1 }; + verify_cmpr_long(value, wanted); + }{ + long value = 34359738367L; + byte[] wanted = { -1, -1, -1, -1, 127 }; + verify_cmpr_long(value, wanted); + }{ + long value = 34359738368L; + byte[] wanted = { -128, -128, -128, -128, -128, 1 }; + verify_cmpr_long(value, wanted); + }{ + long value = 4398046511103L; + byte[] wanted = { -1, -1, -1, -1, -1, 127 }; + verify_cmpr_long(value, wanted); + }{ + long value = 4398046511104L; + byte[] wanted = { -128, -128, -128, -128, -128, -128, 1 }; + verify_cmpr_long(value, wanted); + }{ + long value = 562949953421311L; + byte[] wanted = { -1, -1, -1, -1, -1, -1, 127 }; + verify_cmpr_long(value, wanted); + }{ + long value = 562949953421312L; + byte[] wanted = { -128, -128, -128, -128, -128, -128, -128, 1 }; + verify_cmpr_long(value, wanted); + }{ + long value = 72057594037927935L; + byte[] wanted = { -1, -1, -1, -1, -1, -1, -1, 127 }; + verify_cmpr_long(value, wanted); + }{ + long value = 72057594037927936L; + byte[] wanted = { -128, -128, -128, -128, -128, -128, -128, -128, 1 }; + verify_cmpr_long(value, wanted); + }{ + long value = 9223372036854775807L; + byte[] wanted = { -1, -1, -1, -1, -1, -1, -1, -1, 127 }; + verify_cmpr_long(value, wanted); + }{ + long value = -9223372036854775808L; + byte[] wanted = { -128, -128, -128, -128, -128, -128, -128, -128, -128, 1 }; + verify_cmpr_long(value, wanted); + }{ + long value = -1; + byte[] wanted = { -1, -1, -1, -1, -1, -1, -1, -1, -1, 1 }; + verify_cmpr_long(value, wanted); + } + } + + // testWriteByte -> buffered IO test + // testWriteBytes -> buffered IO test + // testReadByte -> buffered IO test + // testReadBytes -> buffered IO test + + @Test + public void testTypeAndSize() { + System.out.println("test type and size conversion"); + + for (byte type = 0; type < TYPE_LIMIT; ++type) { + for (long size = 0; size < 500; ++size) { + BufferedOutput expect = new BufferedOutput(); + BufferedOutput actual = new BufferedOutput(); + + if ((size + 1) < META_LIMIT) { + expect.put(encode_type_and_meta((int)type, (int)(size +1))); + } else { + expect.put(type); + BinaryEncoder encoder = new BinaryEncoder(); + encoder.out = expect; + encoder.encode_cmpr_long(size); + } + { + BinaryEncoder encoder = new BinaryEncoder(); + encoder.out = actual; + encoder.write_type_and_size(type, size); + } + assertThat(actual.toArray(), is(expect.toArray())); + + byte[] got = expect.toArray(); + BinaryDecoder bif = new BinaryDecoder(); + bif.in = new BufferedInput(got); + byte b = bif.in.getByte(); + Type decodedType = decode_type(b); + long decodedSize = bif.read_size(decode_meta(b)); + assertThat(decodedType.ID, is(type)); + assertThat(decodedSize, is(size)); + assertThat(bif.in.getConsumedSize(), is(got.length)); + assertThat(bif.in.failed(), is(false)); + } + } + + } + + static long build_bits(int type, int n, int pre, boolean hi, BufferedOutput expect) { + long value = 0; + expect.put(encode_type_and_meta(type, n)); + for (int i = 0; i < n; ++i) { + byte b = (i < pre) ? 0x00 : (byte)(0x11 * (i - pre + 1)); + expect.put(b); + int shift = hi ? ((7 - i) * 8) : (i * 8); + long bits = b & 0xff; + value |= bits << shift; + } + return value; + } + + @Test + public void testTypeAndBytes() { + System.out.println("test encoding and decoding of type and bytes"); + for (byte type = 0; type < TYPE_LIMIT; ++type) { + for (int n = 0; n < MAX_NUM_SIZE; ++n) { + for (int pre = 0; (pre == 0) || (pre < n); ++pre) { + for (int hi = 0; hi < 2; ++hi) { + BufferedOutput expbuf = new BufferedOutput(); + long bits = build_bits(type, n, pre, (hi != 0), expbuf); + byte[] expect = expbuf.toArray(); + + // test output: + BinaryEncoder bof = new BinaryEncoder(); + bof.out = new BufferedOutput(); + if (hi != 0) { + bof.write_type_and_bytes_be(type, bits); + } else { + bof.write_type_and_bytes_le(type, bits); + } + byte[] actual = bof.out.toArray(); + assertThat(actual, is(expect)); + + // test input: + BinaryDecoder bif = new BinaryDecoder(); + bif.in = new BufferedInput(expect); + int size = decode_meta(bif.in.getByte()); + long decodedBits = (hi != 0) ? bif.read_bytes_be(size) : bif.read_bytes_le(size); + assertThat(decodedBits, is(bits)); + assertThat(bif.in.getConsumedSize(), is(expect.length)); + assertThat(bif.in.failed(), is(false)); + } + } + } + } + } + + @Test + public void testEmpty() { + System.out.println("test encoding empty slime"); + + Slime slime = new Slime(); + BufferedOutput expect = new BufferedOutput(); + expect.put((byte)0); // num symbols + expect.put((byte)0); // nix + byte[] actual = BinaryFormat.encode(slime); + + assertThat(actual, is(expect.toArray())); + verifyMultiEncode(expect.toArray()); + } + + @Test + public void testBasic() { + System.out.println("test encoding slime holding a single basic value"); + { + Slime slime = new Slime(); + slime.setBool(false); + byte[] expect = { 0, Type.BOOL.ID }; + verifyEncoding(slime, expect); + } + { + Slime slime = new Slime(); + slime.setBool(true); + byte[] expect = { 0, enc_t_and_m(Type.BOOL, 1) }; + verifyEncoding(slime, expect); + } + { + Slime slime = new Slime(); + slime.setLong(0); + byte[] expect = { 0, Type.LONG.ID }; + verifyEncoding(slime, expect); + } + { + Slime slime = new Slime(); + slime.setLong(13); + byte[] expect = { 0, enc_t_and_m(Type.LONG, 1), 13*2 }; + verifyEncoding(slime, expect); + } + { + Slime slime = new Slime(); + slime.setLong(-123456789); + final long ev = (2 * 123456789) - 1; + byte b1 = (byte)(ev); + byte b2 = (byte)(ev >> 8); + byte b3 = (byte)(ev >> 16); + byte b4 = (byte)(ev >> 24); + + byte[] expect = { 0, enc_t_and_m(Type.LONG, 4), b1, b2, b3, b4 }; + verifyEncoding(slime, expect); + } + { + Slime slime = new Slime(); + slime.setDouble(0.0); + byte[] expect = { 0, Type.DOUBLE.ID }; + verifyEncoding(slime, expect); + } + { + Slime slime = new Slime(); + slime.setDouble(1.0); + byte[] expect = { 0, enc_t_and_m(Type.DOUBLE, 2), (byte)0x3f, (byte)0xf0 }; + verifyEncoding(slime, expect); + } + { + Slime slime = new Slime(); + slime.setString(""); + byte[] expect = { 0, enc_t_and_sz(Type.STRING, 0) }; + verifyEncoding(slime, expect); + } + { + Slime slime = new Slime(); + slime.setString("fo"); + byte[] expect = { 0, enc_t_and_sz(Type.STRING, 2), 'f', 'o' }; + verifyEncoding(slime, expect); + } + { + Slime slime = new Slime(); + slime.setString("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"); + byte[] expect = { 0, Type.STRING.ID, 26*2, + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', + 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', + 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', + 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', + 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', + 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z' + }; + verifyEncoding(slime, expect); + } + { + Slime slime = new Slime(); + slime.setData(new byte[0]); + byte[] expect = { 0, enc_t_and_sz(Type.DATA, 0) }; + verifyEncoding(slime, expect); + } + { + Slime slime = new Slime(); + byte[] data = { 42, -123 }; + slime.setData(data); + byte[] expect = { 0, enc_t_and_sz(Type.DATA, 2), 42, -123 }; + verifyEncoding(slime, expect); + } + } + + @Test + public void testBufferedInputWithOffset() { + Slime slime = new Slime(); + byte[] data = { 42, -123 }; + slime.setData(data); + byte[] expect = { 0, enc_t_and_sz(Type.DATA, 2), 42, -123 }; + verifyEncoding(slime, expect); + byte [] overlappingBuffer = new byte [expect.length + 2]; + System.arraycopy(expect, 0, overlappingBuffer, 1, expect.length); + overlappingBuffer[overlappingBuffer.length - 1] = 0; + Slime copy = BinaryFormat.decode(overlappingBuffer, 1, expect.length); + assertThat(BinaryFormat.encode(slime), is(BinaryFormat.encode(copy))); + } + + @Test + public void testArray() { + System.out.println("test encoding slime holding an array of various basic values"); + Slime slime = new Slime(); + Cursor c = slime.setArray(); + byte[] data = { 'd', 'a', 't', 'a' }; + c.addNix(); + c.addBool(true); + c.addLong(42); + c.addDouble(3.5); + c.addString("string"); + c.addData(data); + byte[] expect = { + 0, // num symbols + enc_t_and_sz(Type.ARRAY, 6), // value type and size + 0, // nix + enc_t_and_m(Type.BOOL, 1), + enc_t_and_m(Type.LONG, 1), 42*2, + enc_t_and_m(Type.DOUBLE, 2), 0x40, 0x0c, // 3.5 + enc_t_and_sz(Type.STRING, 6), 's', 't', 'r', 'i', 'n', 'g', + enc_t_and_sz(Type.DATA, 4), 'd', 'a', 't', 'a' + }; + verifyEncoding(slime, expect); + } + + @Test + public void testObject() { + System.out.println("test encoding slime holding an object of various basic values"); + Slime slime = new Slime(); + Cursor c = slime.setObject(); + byte[] data = { 'd', 'a', 't', 'a' }; + c.setNix("a"); + c.setBool("b", true); + c.setLong("c", 42); + c.setDouble("d", 3.5); + c.setString("e", "string"); + c.setData("f", data); + byte[] expect = { + 6, // num symbols + 1, 'a', 1, 'b', 1, 'c', 1, 'd', 1, 'e', 1, 'f', // symbol table + enc_t_and_sz(Type.OBJECT, 6), // value type and size + 0, 0, // nix + 1, enc_t_and_m(Type.BOOL, 1), + 2, enc_t_and_m(Type.LONG, 1), 42*2, + 3, enc_t_and_m(Type.DOUBLE, 2), 0x40, 0x0c, // 3.5 + 4, enc_t_and_sz(Type.STRING, 6), 's', 't', 'r', 'i', 'n', 'g', + 5, enc_t_and_sz(Type.DATA, 4), 'd', 'a', 't', 'a' + }; + verifyEncoding(slime, expect); + } + + @Test + public void testNesting() { + System.out.println("test encoding slime holding a more complex structure"); + Slime slime = new Slime(); + Cursor c1 = slime.setObject(); + c1.setLong("bar", 10); + Cursor c2 = c1.setArray("foo"); + c2.addLong(20); + Cursor c3 = c2.addObject(); + c3.setLong("answer", 42); + byte[] expect = { + 3, // num symbols + 3, 'b', 'a', 'r', + 3, 'f', 'o', 'o', + 6, 'a', 'n', 's', 'w', 'e', 'r', + enc_t_and_sz(Type.OBJECT, 2), // value type and size + 0, enc_t_and_m(Type.LONG, 1), 10*2, + 1, enc_t_and_sz(Type.ARRAY, 2), // nested value type and size + enc_t_and_m(Type.LONG, 1), 20*2, + enc_t_and_sz(Type.OBJECT, 1), // doubly nested value + 2, enc_t_and_m(Type.LONG, 1), 42*2 + }; + verifyEncoding(slime, expect); + } + + @Test + public void testSymbolReuse() { + System.out.println("test encoding slime reusing symbols"); + Slime slime = new Slime(); + Cursor c1 = slime.setArray(); + { + Cursor c2 = c1.addObject(); + c2.setLong("foo", 10); + c2.setLong("bar", 20); + } + { + Cursor c2 = c1.addObject(); + c2.setLong("foo", 100); + c2.setLong("bar", 200); + } + byte[] expect = { + 2, // num symbols + 3, 'f', 'o', 'o', + 3, 'b', 'a', 'r', + enc_t_and_sz(Type.ARRAY, 2), // value type and size + enc_t_and_sz(Type.OBJECT, 2), // nested value + 0, enc_t_and_m(Type.LONG, 1), 10*2, // foo + 1, enc_t_and_m(Type.LONG, 1), 20*2, // bar + enc_t_and_sz(Type.OBJECT, 2), // nested value + 0, enc_t_and_m(Type.LONG, 1), (byte)(100*2), // foo + 1, enc_t_and_m(Type.LONG, 2), (byte)144, 1 // bar: 2*200 = 400 = 256 + 144 + }; + verifyEncoding(slime, expect); + } + + @Test + public void testOptionalDecodeOrder() { + System.out.println("test decoding slime with different symbol order"); + byte[] data = { + 5, // num symbols + 1, 'd', 1, 'e', 1, 'f', 1, 'b', 1, 'c', // symbol table + enc_t_and_sz(Type.OBJECT, 5), // value type and size + 3, enc_t_and_m(Type.BOOL, 1), // b + 1, enc_t_and_sz(Type.STRING, 6), // e + 's', 't', 'r', 'i', 'n', 'g', + 0, enc_t_and_m(Type.DOUBLE, 2), 0x40, 0x0c, // d + 4, enc_t_and_m(Type.LONG, 1), 5*2, // c + 2, enc_t_and_sz(Type.DATA, 4), // f + 'd', 'a', 't', 'a' + }; + Slime slime = new Slime(); + BinaryDecoder decoder = new BinaryDecoder(); + slime = decoder.decode(data); + int consumed = decoder.in.getConsumedSize(); + assertThat(consumed, is(data.length)); + Cursor c = slime.get(); + assertThat(c.valid(), is(true)); + assertThat(c.type(), is(Type.OBJECT)); + assertThat(c.children(), is(5)); + assertThat(c.field("b").asBool(), is(true)); + assertThat(c.field("c").asLong(), is(5L)); + assertThat(c.field("d").asDouble(), is(3.5)); + assertThat(c.field("e").asString(), is("string")); + byte[] expd = { 'd', 'a', 't', 'a' }; + assertThat(c.field("f").asData(), is(expd)); + assertThat(c.entry(5).valid(), is(false)); // not ARRAY + } +} diff --git a/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java b/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java new file mode 100644 index 00000000000..8ee1a91c970 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/slime/JsonBenchmark.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.Integer; + +/** + * Created by balder on 2/26/14. + */ +public class JsonBenchmark { + private static byte [] createJson(int numElements) { + Slime slime = new Slime(); + Cursor a = slime.setArray(); + for (int i=0; i < numElements; i++) { + Cursor e = a.addObject(); + e.setString("key", "i"); + e.setLong("weight", i); + } + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + JsonFormat json = new JsonFormat(false); + try { + json.encode(bs, slime); + } catch (IOException e) { + throw new RuntimeException(e); + } + return bs.toByteArray(); + } + private static long benchmarkJacksonStreaming(byte [] json, int numIterations) { + long count = 0; + JsonFactory jsonFactory = new JsonFactory(); + + try { + for (int i=0; i < numIterations; i++) { + JsonParser jsonParser = jsonFactory.createParser(json); + JsonToken array = jsonParser.nextToken(); + for (JsonToken token = jsonParser.nextToken(); ! JsonToken.END_ARRAY.equals(token); token = jsonParser.nextToken()) { + if (JsonToken.FIELD_NAME.equals(token) && "weight".equals(jsonParser.getCurrentName())) { + token = jsonParser.nextToken(); + count += jsonParser.getLongValue(); + } + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return count; + } + private static long benchmarkJacksonTree(byte [] json, int numIterations) { + long count = 0; + ObjectMapper mapper = new ObjectMapper(); + // use the ObjectMapper to read the json string and create a tree + try { + for (int i=0; i < numIterations; i++) { + JsonNode node = mapper.readTree(json); + for(JsonNode item : node) { + count += item.get("weight").asLong(); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return count; + } + private static long benchmarkSlime(byte [] json, int numIterations) { + long count = 0; + for (int i=0; i < numIterations; i++) { + JsonDecoder decoder = new JsonDecoder(); + Slime slime = decoder.decode(new Slime(), json); + + Cursor array = slime.get(); + int weightSymbol = slime.lookup("weight"); + for (int j=0, m=slime.get().children(); j < m; j++) { + count += array.entry(j).field(weightSymbol).asLong(); + } + } + return count; + } + private static void warmup(byte [] json) { + System.out.println(System.currentTimeMillis() + " Warming up"); + benchmarkSlime(json, 5000); + System.out.println(System.currentTimeMillis() + " Done Warming up"); + } + + /** + * jacksons 1000 40000 = 5.6 seconds + * jacksont 1000 40000 = 11.0 seconds + * slime 1000 40000 = 17.5 seconds + * @param argv type, num elements in weigted set, num iterations + */ + static public void main(String argv[]) { + String type = argv[0]; + byte [] json = createJson(Integer.valueOf(argv[1])); + warmup(json); + int count = Integer.valueOf(argv[2]); + System.out.println(System.currentTimeMillis() + " Start"); + long start = System.currentTimeMillis(); + long numValues; + if ("jacksons".equals(type)) { + numValues = benchmarkJacksonStreaming(json, count); + } else if ("jacksont".equals(type)) { + numValues = benchmarkJacksonTree(json, count); + } else{ + numValues = benchmarkSlime(json, count); + } + System.out.println(System.currentTimeMillis() + " End with " + numValues + " values in " + (System.currentTimeMillis() - start) + " milliseconds."); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java new file mode 100644 index 00000000000..e48a717f150 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java @@ -0,0 +1,273 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import com.yahoo.text.Utf8; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class JsonFormatTestCase { + + @Test + public void testBasic() { + System.out.println("test encoding slime holding a single basic value"); + { + Slime slime = new Slime(); + slime.setBool(false); + verifyEncoding(slime, "false"); + } + + { + Slime slime = new Slime(); + slime.setBool(true); + verifyEncoding(slime, "true"); + } + + { + Slime slime = new Slime(); + slime.setLong(0); + verifyEncoding(slime, "0"); + } + { + Slime slime = new Slime(); + slime.setLong(13); + verifyEncoding(slime, "13"); + } + { + Slime slime = new Slime(); + slime.setLong(-123456789); + verifyEncoding(slime, "-123456789"); + } + { + Slime slime = new Slime(); + slime.setDouble(0.0); + verifyEncoding(slime, "0.0"); + } + { + Slime slime = new Slime(); + slime.setDouble(1.5); + verifyEncoding(slime, "1.5"); + } + { + Slime slime = new Slime(); + slime.setString(""); + verifyEncoding(slime, "\"\""); + } + { + Slime slime = new Slime(); + slime.setString("fo"); + verifyEncoding(slime, "\"fo\""); + } + { + Slime slime = new Slime(); + slime.setString("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"); + verifyEncoding(slime, "\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\""); + } + { + Slime slime = new Slime(); + slime.setData(new byte[0]); + verifyEncoding(slime, "\"0x\""); + } + + { + Slime slime = new Slime(); + byte[] data = { 42, -123 }; + slime.setData(data); + String expect = "\"0x2A85\""; + verifyEncoding(slime, expect); + } + + { + Slime slime = new Slime(); + String expected = "\"my\\nencoded\\rsting\\\\is\\bthe\\fnicest\\t\\\"string\\\"\\u0005\""; + slime.setString("my\nencoded\rsting\\is\bthe\fnicest\t\"string\"" + Character.toString((char) 5)); + verifyEncoding(slime, expected); + } + + { + Slime slime = new Slime(); + slime.setDouble(Double.NaN); + verifyEncoding(slime, "null"); + slime.setDouble(Double.NEGATIVE_INFINITY); + verifyEncoding(slime, "null"); + slime.setDouble(Double.POSITIVE_INFINITY); + verifyEncoding(slime, "null"); + } + } + + @Test + public void testArray() { + System.out.println("test encoding slime holding an array of various basic values"); + Slime slime = new Slime(); + Cursor c = slime.setArray(); + byte[] data = { 'd', 'a', 't', 'a' }; + c.addNix(); + c.addBool(true); + c.addLong(42); + c.addDouble(3.5); + c.addString("string"); + c.addData(data); + + verifyEncoding(slime, "[null,true,42,3.5,\"string\",\"0x64617461\"]"); + } + + @Test + public void testObject() { + System.out.println("test encoding slime holding an object of various basic values"); + Slime slime = new Slime(); + Cursor c = slime.setObject(); + byte[] data = { 'd', 'a', 't', 'a' }; + c.setNix("a"); + c.setBool("b", true); + c.setLong("c", 42); + c.setDouble("d", 3.5); + c.setString("e", "string"); + c.setData("f", data); + verifyEncoding(slime, "{\"a\":null,\"b\":true,\"c\":42,\"d\":3.5,\"e\":\"string\",\"f\":\"0x64617461\"}"); + String expected = "{\n" + + " \"a\": null,\n" + + " \"b\": true,\n" + + " \"c\": 42,\n" + + " \"d\": 3.5,\n" + + " \"e\": \"string\",\n" + + " \"f\": \"0x64617461\"\n" + + "}\n"; + verifyEncoding(slime, expected, false); + } + + @Test + public void testNesting() { + System.out.println("test encoding slime holding a more complex structure"); + Slime slime = new Slime(); + Cursor c1 = slime.setObject(); + c1.setLong("bar", 10); + Cursor c2 = c1.setArray("foo"); + c2.addLong(20); + Cursor c3 = c2.addObject(); + c3.setLong("answer", 42); + verifyEncoding(slime, "{\"bar\":10,\"foo\":[20,{\"answer\":42}]}"); + } + + @Test + public void testDecodeEncode() { + System.out.println("test decoding and encoding a json string yields the same string"); + verifyEncodeDecode("{\"bar\":10,\"foo\":[20,{\"answer\":42}]}", true); + String expected = "{\n" + + " \"a\": null,\n" + + " \"b\": true,\n" + + " \"c\": 42,\n" + + " \"d\": 3.5,\n" + + " \"e\": \"string\",\n" + + " \"f\": \"0x64617461\"\n" + + "}\n"; + verifyEncodeDecode(expected, false); + } + + @Test + public void testDecodeEncodeUtf8() { + final String json = "{\n" + + " \"rules\": \"# Use unicode equivalents in java source:\\n" + + " #\\n" + + " # ä½³:\u4f73\"\n" + + "}\n"; + verifyEncodeDecode(json, false); + } + + @Test + public void testDecodeUtf8() { + final String str = "\u4f73:\u4f73"; + final String json = " {\n" + + " \"rules\": \"" + str + "\"\n" + + " }\n"; + + Slime slime = new Slime(); + slime = new JsonDecoder().decode(slime, Utf8.toBytesStd(json)); + Cursor a = slime.get().field("rules"); + assertThat(a.asString(), is(str)); + } + + @Test(expected = UnsupportedOperationException.class) + public void testThatDecodeIsNotImplemented() throws IOException { + new JsonFormat(true).decode(null, null); + } + + private void verifyEncoding(Slime slime, String expected) { + verifyEncoding(slime, expected, true); + } + + @Test + public void testEncodingUTF8() throws IOException { + Slime slime = new Slime(); + slime.setString("M\u00E6L"); + ByteArrayOutputStream a = new ByteArrayOutputStream(); + new JsonFormat(true).encode(a, slime); + String val = new String(a.toByteArray(), "UTF-8"); + assertEquals("\"M\u00E6L\"", val); + + // TODO Some issues with newline + /* + slime = new Slime(); + final String str = "# Use unicode equivalents in java source:\n" + + " #\n" + + " #\n" + + " # ä½³:\u4f73\n"; + slime.setString(str); + a = new ByteArrayOutputStream(); + new JsonFormat(true).encode(a, slime); + val = new String(a.toByteArray(), "UTF-8"); + assertEquals(str, val); + */ + } + + private void verifyEncoding(Slime slime, String expected, boolean compact) { + try { + ByteArrayOutputStream a = new ByteArrayOutputStream(); + new JsonFormat(compact).encode(a, slime); + assertEquals(expected, new String(a.toByteArray(), StandardCharsets.UTF_8)); + } catch (Exception e) { + fail("Exception thrown when encoding slime: " + e.getMessage()); + } + } + + private void verifyEncodeDecode(String json, boolean compact) { + try { + Slime slime = new Slime(); + new JsonDecoder().decode(slime, Utf8.toBytesStd(json)); + ByteArrayOutputStream a = new ByteArrayOutputStream(); + new JsonFormat(compact).encode(a, slime); + assertEquals(json, Utf8.toString(a.toByteArray())); + } catch (Exception e) { + fail("Exception thrown when encoding slime: " + e.getMessage()); + } + } + + private String formatDecimal(double value) { + try { + Slime slime = new Slime(); + slime.setDouble(value); + ByteArrayOutputStream a = new ByteArrayOutputStream(); + new JsonFormat(true).encode(a, slime); + return new String(a.toByteArray(), StandardCharsets.UTF_8); + } catch (Exception e) { + return ""; + } + } + + @Test + public void testDecimalFormat() { + assertEquals("0.0", formatDecimal(0.0)); + assertEquals("1.0", formatDecimal(1.0)); + assertEquals("2.0", formatDecimal(2.0)); + assertEquals("1.2", formatDecimal(1.2)); + assertEquals("3.333333", formatDecimal(3.333333)); + assertEquals("1.0E20", formatDecimal(1e20)); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java new file mode 100644 index 00000000000..371668d3821 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java @@ -0,0 +1,330 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import org.junit.Test; + +import static org.junit.Assert.assertThat; +import static org.hamcrest.CoreMatchers.*; + +public class SlimeTestCase { + + @Test + public void testTypeIds() { + System.out.println("testing type identifiers..."); + + assertThat(Type.NIX.ID, is((byte)0)); + assertThat(Type.BOOL.ID, is((byte)1)); + assertThat(Type.LONG.ID, is((byte)2)); + assertThat(Type.DOUBLE.ID, is((byte)3)); + assertThat(Type.STRING.ID, is((byte)4)); + assertThat(Type.DATA.ID, is((byte)5)); + assertThat(Type.ARRAY.ID, is((byte)6)); + assertThat(Type.OBJECT.ID, is((byte)7)); + + assertThat(Type.values().length, is(8)); + + assertThat(Type.values()[0], sameInstance(Type.NIX)); + assertThat(Type.values()[1], sameInstance(Type.BOOL)); + assertThat(Type.values()[2], sameInstance(Type.LONG)); + assertThat(Type.values()[3], sameInstance(Type.DOUBLE)); + assertThat(Type.values()[4], sameInstance(Type.STRING)); + assertThat(Type.values()[5], sameInstance(Type.DATA)); + assertThat(Type.values()[6], sameInstance(Type.ARRAY)); + assertThat(Type.values()[7], sameInstance(Type.OBJECT)); + } + + @Test + public void testEmpty() { + System.out.println("testing empty slime..."); + Slime slime = new Slime(); + Cursor cur; + for (int i = 0; i < 2; i++) { + if (i == 0) { + cur = slime.get(); + assertThat(cur.valid(), is(true)); + } else { + cur = NixValue.invalid(); + assertThat(cur.valid(), is(false)); + } + assertThat(cur.type(), is(Type.NIX)); + assertThat(cur.children(), is(0)); + assertThat(cur.asBool(), is(false)); + assertThat(cur.asLong(), is((long)0)); + assertThat(cur.asDouble(), is(0.0)); + assertThat(cur.asString(), is("")); + assertThat(cur.asData(), is(new byte[0])); + assertThat(cur.entry(0).valid(), is(false)); + assertThat(cur.field(0).valid(), is(false)); + assertThat(cur.field("foo").valid(), is(false)); + } + Inspector insp; + for (int i = 0; i < 2; i++) { + if (i == 0) { + insp = slime.get(); + assertThat(insp.valid(), is(true)); + } else { + insp = NixValue.invalid(); + assertThat(insp.valid(), is(false)); + } + assertThat(insp.type(), is(Type.NIX)); + assertThat(insp.children(), is(0)); + assertThat(insp.asBool(), is(false)); + assertThat(insp.asLong(), is((long)0)); + assertThat(insp.asDouble(), is(0.0)); + assertThat(insp.asString(), is("")); + assertThat(insp.asData(), is(new byte[0])); + assertThat(insp.entry(0).valid(), is(false)); + assertThat(insp.field(0).valid(), is(false)); + assertThat(insp.field("foo").valid(), is(false)); + } + } + + @Test + public void testBasic() { + System.out.println("testing basic values..."); + Slime slime = new Slime(); + + System.out.println("testing boolean value"); + slime.setBool(true); + Inspector insp = slime.get(); + assertThat(insp.valid(), is(true)); + assertThat(insp.type(), sameInstance(Type.BOOL)); + assertThat(insp.asBool(), is(true)); + Cursor cur = slime.get(); + assertThat(cur.valid(), is(true)); + assertThat(cur.type(), sameInstance(Type.BOOL)); + assertThat(cur.asBool(), is(true)); + + System.out.println("testing long value"); + slime.setLong(42); + cur = slime.get(); + insp = slime.get(); + assertThat(cur.valid(), is(true)); + assertThat(insp.valid(), is(true)); + assertThat(cur.type(), sameInstance(Type.LONG)); + assertThat(insp.type(), sameInstance(Type.LONG)); + assertThat(cur.asLong(), is((long)42)); + assertThat(insp.asLong(), is((long)42)); + + System.out.println("testing double value"); + slime.setDouble(4.2); + cur = slime.get(); + insp = slime.get(); + assertThat(cur.valid(), is(true)); + assertThat(insp.valid(), is(true)); + assertThat(cur.type(), sameInstance(Type.DOUBLE)); + assertThat(insp.type(), sameInstance(Type.DOUBLE)); + assertThat(cur.asDouble(), is(4.2)); + assertThat(insp.asDouble(), is(4.2)); + + System.out.println("testing string value"); + slime.setString("fortytwo"); + cur = slime.get(); + insp = slime.get(); + assertThat(cur.valid(), is(true)); + assertThat(insp.valid(), is(true)); + assertThat(cur.type(), sameInstance(Type.STRING)); + assertThat(insp.type(), sameInstance(Type.STRING)); + assertThat(cur.asString(), is("fortytwo")); + assertThat(insp.asString(), is("fortytwo")); + + System.out.println("testing data value"); + byte[] data = { (byte)4, (byte)2 }; + slime.setData(data); + cur = slime.get(); + insp = slime.get(); + assertThat(cur.valid(), is(true)); + assertThat(insp.valid(), is(true)); + assertThat(cur.type(), sameInstance(Type.DATA)); + assertThat(insp.type(), sameInstance(Type.DATA)); + assertThat(cur.asData(), is(data)); + assertThat(insp.asData(), is(data)); + data[0] = 10; + data[1] = 20; + byte[] data2 = { 10, 20 }; + assertThat(cur.asData(), is(data2)); + assertThat(insp.asData(), is(data2)); + } + + @Test + public void testArray() { + System.out.println("testing array values..."); + Slime slime = new Slime(); + Cursor c = slime.setArray(); + assertThat(c.valid(), is(true)); + assertThat(c.type(), is(Type.ARRAY)); + assertThat(c.children(), is(0)); + Inspector i = slime.get(); + assertThat(i.valid(), is(true)); + assertThat(i.type(), is(Type.ARRAY)); + assertThat(i.children(), is(0)); + c.addNix(); + c.addBool(true); + c.addLong(5); + c.addDouble(3.5); + c.addString("string"); + byte[] data = { (byte)'d', (byte)'a', (byte)'t', (byte)'a' }; + c.addData(data); + assertThat(c.children(), is(6)); + assertThat(c.entry(0).valid(), is(true)); + assertThat(c.entry(1).asBool(), is(true)); + assertThat(c.entry(2).asLong(), is((long)5)); + assertThat(c.entry(3).asDouble(), is(3.5)); + assertThat(c.entry(4).asString(), is("string")); + assertThat(c.entry(5).asData(), is(data)); + assertThat(c.field(5).valid(), is(false)); // not OBJECT + + assertThat(i.children(), is(6)); + assertThat(i.entry(0).valid(), is(true)); + assertThat(i.entry(1).asBool(), is(true)); + assertThat(i.entry(2).asLong(), is((long)5)); + assertThat(i.entry(3).asDouble(), is(3.5)); + assertThat(i.entry(4).asString(), is("string")); + assertThat(i.entry(5).asData(), is(data)); + assertThat(i.field(5).valid(), is(false)); // not OBJECT + } + + @Test + public void testObject() { + System.out.println("testing object values..."); + Slime slime = new Slime(); + Cursor c = slime.setObject(); + + assertThat(c.valid(), is(true)); + assertThat(c.type(), is(Type.OBJECT)); + assertThat(c.children(), is(0)); + Inspector i = slime.get(); + assertThat(i.valid(), is(true)); + assertThat(i.type(), is(Type.OBJECT)); + assertThat(i.children(), is(0)); + + c.setNix("a"); + c.setBool("b", true); + c.setLong("c", 5); + c.setDouble("d", 3.5); + c.setString("e", "string"); + byte[] data = { (byte)'d', (byte)'a', (byte)'t', (byte)'a' }; + c.setData("f", data); + + assertThat(c.children(), is(6)); + assertThat(c.field("a").valid(), is(true)); + assertThat(c.field("b").asBool(), is(true)); + assertThat(c.field("c").asLong(), is((long)5)); + assertThat(c.field("d").asDouble(), is(3.5)); + assertThat(c.field("e").asString(), is("string")); + assertThat(c.field("f").asData(), is(data)); + assertThat(c.entry(4).valid(), is(false)); // not ARRAY + + assertThat(i.children(), is(6)); + assertThat(i.field("a").valid(), is(true)); + assertThat(i.field("b").asBool(), is(true)); + assertThat(i.field("c").asLong(), is((long)5)); + assertThat(i.field("d").asDouble(), is(3.5)); + assertThat(i.field("e").asString(), is("string")); + assertThat(i.field("f").asData(), is(data)); + assertThat(i.entry(4).valid(), is(false)); // not ARRAY + } + + @Test + public void testChaining() { + System.out.println("testing cursor chaining..."); + { + Slime slime = new Slime(); + Cursor c = slime.setArray(); + assertThat(c.addLong(5).asLong(), is((long)5)); + } + { + Slime slime = new Slime(); + Cursor c = slime.setObject(); + assertThat(c.setLong("a", 5).asLong(), is((long)5)); + } + } + + @Test + public void testCursorToInspector() { + System.out.println("testing proxy conversion..."); + + Slime slime = new Slime(); + Cursor c = slime.setLong(10); + Inspector i1 = c; + assertThat(i1.asLong(), is((long)10)); + + Inspector i2 = slime.get(); + assertThat(i2.asLong(), is((long)10)); + } + + @Test + public void testNesting() { + System.out.println("testing data nesting..."); + Slime slime = new Slime(); + { + Cursor c1 = slime.setObject(); + c1.setLong("bar", 10); + Cursor c2 = c1.setArray("foo"); + c2.addLong(20); + Cursor c3 = c2.addObject(); + c3.setLong("answer", 42); + } + Inspector i = slime.get(); + assertThat(i.field("bar").asLong(), is((long)10)); + assertThat(i.field("foo").entry(0).asLong(), is((long)20)); + assertThat(i.field("foo").entry(1).field("answer").asLong(), is((long)42)); + + Cursor c = slime.get(); + assertThat(c.field("bar").asLong(), is((long)10)); + assertThat(c.field("foo").entry(0).asLong(), is((long)20)); + assertThat(c.field("foo").entry(1).field("answer").asLong(), is((long)42)); + } + + @Test + public void testLotsOfSymbolsAndFields() { + // put pressure on symbol table and object fields + int n = 1000; + Slime slime = new Slime(); + Cursor c = slime.setObject(); + for (int i = 0; i < n; i++) { + String str = ("" + i + "_str_" + i); + assertThat(slime.lookup(str), is(SymbolTable.INVALID)); + assertThat(c.field(str).type(), sameInstance(Type.NIX)); + switch (i % 2) { + case 0: assertThat((int)c.setLong(str, i).asLong(), is(i)); break; + case 1: assertThat(slime.insert(str), is(i)); break; + } + } + for (int i = 0; i < n; i++) { + String str = ("" + i + "_str_" + i); + assertThat(slime.lookup(str), is(i)); + switch (i % 2) { + case 0: assertThat((int)c.field(str).asLong(), is(i)); break; + case 1: assertThat((int)c.field(str).asLong(), is(0)); break; + } + } + } + + @Test + public void testLotsOfEntries() { + // put pressure on array entries + int n = 1000; + Slime slime = new Slime(); + Cursor c = slime.setArray(); + for (int i = 0; i < n; i++) { + assertThat((int)c.addLong(i).asLong(), is(i)); + } + for (int i = 0; i < n; i++) { + assertThat((int)c.entry(i).asLong(), is(i)); + } + assertThat((int)c.entry(n).asLong(), is(0)); + } + + @Test + public void testToString() { + Slime slime = new Slime(); + Cursor c1 = slime.setArray(); + c1.addLong(20); + Cursor c2 = c1.addObject(); + c2.setLong("answer", 42); + assertThat(slime.get().toString(), is("[20,{\"answer\":42}]")); + c1.addString("\u2008"); + assertThat(slime.get().toString(), is("[20,{\"answer\":42},\"\u2008\"]")); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java new file mode 100644 index 00000000000..949cb4eecf2 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import org.junit.Test; +import org.mockito.Mockito; + +import static org.mockito.Matchers.argThat; +import static org.hamcrest.CoreMatchers.sameInstance; + +public class VisitorTestCase { + + @Test + public void testVisitInvalid() { + Visitor visitor = Mockito.mock(Visitor.class); + Inspector inspector = new Slime().get().field("invalid"); + inspector.accept(visitor); + Mockito.verify(visitor).visitInvalid(); + Mockito.verifyNoMoreInteractions(visitor); + } + + @Test + public void testVisitNix() { + Visitor visitor = Mockito.mock(Visitor.class); + Inspector inspector = new Slime().get(); + inspector.accept(visitor); + Mockito.verify(visitor).visitNix(); + Mockito.verifyNoMoreInteractions(visitor); + } + + @Test + public void testVisitBool() { + Visitor visitor = Mockito.mock(Visitor.class); + Inspector inspector = new Slime().setBool(true); + inspector.accept(visitor); + Mockito.verify(visitor).visitBool(true); + Mockito.verifyNoMoreInteractions(visitor); + } + + @Test + public void testVisitLong() { + Visitor visitor = Mockito.mock(Visitor.class); + Inspector inspector = new Slime().setLong(123); + inspector.accept(visitor); + Mockito.verify(visitor).visitLong(123); + Mockito.verifyNoMoreInteractions(visitor); + } + + @Test + public void testVisitDouble() { + Visitor visitor = Mockito.mock(Visitor.class); + Inspector inspector = new Slime().setDouble(123.0); + inspector.accept(visitor); + Mockito.verify(visitor).visitDouble(123.0); + Mockito.verifyNoMoreInteractions(visitor); + } + + @Test + public void testVisitStringUtf16() { + Visitor visitor = Mockito.mock(Visitor.class); + Inspector inspector = new Slime().setString("abc"); + inspector.accept(visitor); + Mockito.verify(visitor).visitString("abc"); + Mockito.verifyNoMoreInteractions(visitor); + } + + @Test + public void testVisitStringUtf8() { + Visitor visitor = Mockito.mock(Visitor.class); + Inspector inspector = new Slime().setString(new byte[] {65,66,67}); + inspector.accept(visitor); + Mockito.verify(visitor).visitString(new byte[] {65,66,67}); + Mockito.verifyNoMoreInteractions(visitor); + } + + @Test + public void testVisitData() { + Visitor visitor = Mockito.mock(Visitor.class); + Inspector inspector = new Slime().setData(new byte[] {1,2,3}); + inspector.accept(visitor); + Mockito.verify(visitor).visitData(new byte[] {1,2,3}); + Mockito.verifyNoMoreInteractions(visitor); + } + + @Test + public void testVisitArray() { + Visitor visitor = Mockito.mock(Visitor.class); + Inspector inspector = new Slime().setArray(); + inspector.accept(visitor); + Mockito.verify(visitor).visitArray(argThat(sameInstance(inspector))); + Mockito.verifyNoMoreInteractions(visitor); + } + + @Test + public void testVisitObject() { + Visitor visitor = Mockito.mock(Visitor.class); + Inspector inspector = new Slime().setObject(); + inspector.accept(visitor); + Mockito.verify(visitor).visitObject(argThat(sameInstance(inspector))); + Mockito.verifyNoMoreInteractions(visitor); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/system/Bar.java b/vespajlib/src/test/java/com/yahoo/system/Bar.java new file mode 100644 index 00000000000..747b19edaee --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/system/Bar.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +/** + * Dummy class to be used to test force loading + **/ +public class Bar {} diff --git a/vespajlib/src/test/java/com/yahoo/system/CatchSigTermTestCase.java b/vespajlib/src/test/java/com/yahoo/system/CatchSigTermTestCase.java new file mode 100644 index 00000000000..dfe508eb2d6 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/system/CatchSigTermTestCase.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +import com.yahoo.system.CatchSigTerm; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author arnej27959 + */ +public class CatchSigTermTestCase extends junit.framework.TestCase { + + public CatchSigTermTestCase(String name) { + super(name); + } + + public void testThatSetupCompiles() { + CatchSigTerm.setup(new AtomicBoolean(false)); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/system/CommandLineParserTestCase.java b/vespajlib/src/test/java/com/yahoo/system/CommandLineParserTestCase.java new file mode 100644 index 00000000000..a2a086e4c65 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/system/CommandLineParserTestCase.java @@ -0,0 +1,125 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +import junit.framework.TestCase; + +public class CommandLineParserTestCase extends TestCase { + + public void testParse1() { + String[] args = new String[] {"-d", "-f", "hello.txt"}; + CommandLineParser parser = new CommandLineParser(args); + parser.addLegalBinarySwitch("-f"); + parser.addLegalUnarySwitch("-d"); + parser.parse(); + assertNull(parser.getBinarySwitches().get("-g")); + assertFalse(parser.getUnarySwitches().contains("-e")); + assertEquals(parser.getBinarySwitches().get("-f"), "hello.txt"); + assertTrue(parser.getUnarySwitches().contains("-d")); + assertFalse(parser.getArguments().contains("-d")); + assertFalse(parser.getArguments().contains("-f")); + assertFalse(parser.getArguments().contains("-hello.txt")); + assertEquals(parser.getArguments().size(), 0); + } + + public void testParse2() { + String[] args = new String[] {"-d", "-f", "hello.txt", "-XX", "myName", "-o", "output file", "myLastField"}; + CommandLineParser parser = new CommandLineParser("progname", args); + parser.setArgumentExplanation("Bla bla1"); + parser.setExtendedHelpText("Bla bla blaaaaaaa bla2"); + parser.addLegalBinarySwitch("-f"); + parser.addLegalBinarySwitch("-o"); + parser.addLegalUnarySwitch("-d"); + parser.addLegalUnarySwitch("-XX"); + parser.parse(); + assertNull(parser.getBinarySwitches().get("-g")); + assertFalse(parser.getUnarySwitches().contains("-e")); + assertEquals(parser.getBinarySwitches().get("-f"), "hello.txt"); + assertTrue(parser.getUnarySwitches().contains("-d")); + assertTrue(parser.getUnarySwitches().contains("-XX")); + assertEquals(parser.getBinarySwitches().get("-o"), "output file"); + assertTrue(parser.getArguments().contains("myName")); + assertTrue(parser.getArguments().contains("myLastField")); + assertEquals(parser.getUnarySwitches().size(), 2); + assertEquals(parser.getBinarySwitches().size(), 2); + assertEquals(parser.getArguments().size(), 2); + assertEquals(parser.getArguments().get(0), "myName"); + assertEquals(parser.getArguments().get(1), "myLastField"); + assertEquals(parser.getUnarySwitches().get(0), "-d"); + assertEquals(parser.getUnarySwitches().get(1), "-XX"); + + try { + parser.usageAndThrow(); + fail("usageAndThrow didn't throw"); + } catch (Exception e) { + assertTrue(e.getMessage().replaceAll("\n", "").matches(".*bla1.*")); + assertTrue(e.getMessage().replaceAll("\n", "").matches(".*bla2.*")); + } + } + + public void testIllegal() { + String[] args = new String[] {"-d", "-f", "hello.txt", "-XX", "myName", "-o", "output file", "myLastField"}; + CommandLineParser parser = new CommandLineParser(args); + parser.addLegalBinarySwitch("-f"); + parser.addLegalBinarySwitch("-o"); + parser.addLegalUnarySwitch("-d"); + try { + parser.parse(); + fail("Parse of cmd line with illegal arg worked"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().startsWith("\nusage")); + } + + args = new String[] {"-d", "-f", "hello.txt", "-XX", "myName", "-o", "output file", "myLastField"}; + parser = new CommandLineParser(args); + parser.addLegalBinarySwitch("-f"); + parser.addLegalUnarySwitch("-d"); + parser.addLegalUnarySwitch("-XX"); + try { + parser.parse(); + fail("Parse of cmd line with illegal arg worked"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().startsWith("\nusage")); + } + } + + public void testRequired() { + String[] args1 = new String[] {"-d", "-f", "hello.txt", "-XX", "myName", "-o", "output file", "myLastField"}; + String[] args2 = new String[] {"-XX", "myName", "-o", "output file", "myLastField"}; + CommandLineParser parser = new CommandLineParser(args1); + parser.addLegalBinarySwitch("-f", "test1"); + parser.addRequiredBinarySwitch("-o", "test2"); + parser.addLegalUnarySwitch("-d", "test3"); + parser.addLegalUnarySwitch("-XX", "test4"); + parser.parse(); + + parser = new CommandLineParser(args2); + parser.addRequiredBinarySwitch("-o", "test2"); + parser.addLegalUnarySwitch("-XX", "test4"); + parser.parse(); + assertEquals(parser.getBinarySwitches().size(),1); + assertEquals(parser.getUnarySwitches().size(),1); + + parser = new CommandLineParser(args2); + parser.addLegalUnarySwitch("-XX", "test4"); + parser.addRequiredBinarySwitch("-f", "test5"); + parser.addRequiredBinarySwitch("-o", "test6"); + try { + parser.parse(); + fail("Illegal cmd line parsed"); + } catch (Exception e) { + assertTrue(e.getMessage().startsWith("\nusage")); + } + + args1 = new String[] {"-d"}; + parser = new CommandLineParser(args1); + parser.addRequiredUnarySwitch("-d", "(required, there are so many bugs)"); + try { + parser.addLegalBinarySwitch("-d"); + fail("Switch clobber didn't throw"); + } catch (Exception e) { + assertTrue(e.getMessage().matches(".*already.*")); + } + parser.parse(); + assertEquals(parser.getUnarySwitches().get(0), "-d"); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/system/Foo.java b/vespajlib/src/test/java/com/yahoo/system/Foo.java new file mode 100644 index 00000000000..ea51a80caaa --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/system/Foo.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +/** + * Dummy class to be used to test force loading + **/ +public class Foo {} diff --git a/vespajlib/src/test/java/com/yahoo/system/ForceLoadTestCase.java b/vespajlib/src/test/java/com/yahoo/system/ForceLoadTestCase.java new file mode 100644 index 00000000000..ec4f716247e --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/system/ForceLoadTestCase.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +public class ForceLoadTestCase extends junit.framework.TestCase { + + public ForceLoadTestCase(String name) { + super(name); + } + + public void testLoadClasses() { + try { + ForceLoad.forceLoad(getClass().getPackage().getName(), new String[] { "Foo", "Bar" }); + } catch (ForceLoadError e) { + e.printStackTrace(); + assertTrue(false); + } + } + + public void testLoadBogusClass() { + try { + ForceLoad.forceLoad(getClass().getPackage().getName(), new String[] { "Foo", "Bar", "Baz" }); + } catch (ForceLoadError e) { + return; + } + assertTrue(false); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/system/ProcessExecuterTestCase.java b/vespajlib/src/test/java/com/yahoo/system/ProcessExecuterTestCase.java new file mode 100644 index 00000000000..2bf1d8c094a --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/system/ProcessExecuterTestCase.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.system; + +import com.yahoo.collections.Pair; +import com.yahoo.io.IOUtils; + +import java.io.File; +import java.io.IOException; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ProcessExecuterTestCase extends junit.framework.TestCase { + + public void testIt() throws IOException { + IOUtils.writeFile("tmp123.txt","hello\nworld",false); + ProcessExecuter exec=new ProcessExecuter(); + assertEquals(new Pair<>(0, "hello\nworld"), exec.exec("cat tmp123.txt")); + assertEquals(new Pair<>(0, "hello\nworld"), exec.exec(new String[]{"cat", "tmp123.txt"})); + new File("tmp123.txt").delete(); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/tensor/MapTensorBuilderTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/MapTensorBuilderTestCase.java new file mode 100644 index 00000000000..92f0e71c7f5 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/tensor/MapTensorBuilderTestCase.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import com.google.common.collect.Sets; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class MapTensorBuilderTestCase { + + @Test + public void requireThatEmptyTensorCanBeBuilt() { + Tensor tensor = new MapTensorBuilder().build(); + assertEquals(0, tensor.dimensions().size()); + assertEquals("{}", tensor.toString()); + } + + @Test + public void requireThatOneDimensionalTensorCanBeBuilt() { + Tensor tensor = new MapTensorBuilder(). + cell().label("x", "0").value(1). + cell().label("x", "1").value(2).build(); + assertEquals(Sets.newHashSet("x"), tensor.dimensions()); + assertEquals("{{x:0}:1.0,{x:1}:2.0}", tensor.toString()); + } + + @Test + public void requireThatTwoDimensionalTensorCanBeBuilt() { + Tensor tensor = new MapTensorBuilder(). + cell().label("x", "0").label("y", "0").value(1). + cell().label("x", "1").label("y", "0").value(2).build(); + assertEquals(Sets.newHashSet("x", "y"), tensor.dimensions()); + assertEquals("{{x:1,y:0}:2.0,{x:0,y:0}:1.0}", tensor.toString()); + } + + @Test + public void requireThatExtraDimensionsCanBeSpecified() { + Tensor tensor = new MapTensorBuilder().dimension("y").dimension("z"). + cell().label("x", "0").value(1).build(); + assertEquals(Sets.newHashSet("x", "y", "z"), tensor.dimensions()); + assertEquals("( {{y:-,z:-}:1.0} * {{x:0}:1.0} )", tensor.toString()); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/tensor/MapTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/MapTensorTestCase.java new file mode 100644 index 00000000000..13ea0e95dc8 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/tensor/MapTensorTestCase.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import org.junit.Test; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Basic tensor tests. Tensor operations are tested in EvaluationTestCase + * + * @author bratseth + */ +public class MapTensorTestCase { + + @Test + public void testStringForm() { + assertEquals("{}", MapTensor.from("{}").toString()); + assertEquals("{{d1:l1}:5.0,{d1:l1,d2:l2}:6.0}", MapTensor.from("{ {d1:l1}:5, {d2:l2, d1:l1}:6.0} ").toString()); + assertEquals("{{d1:l1}:-5.3,{d1:l1,d2:l2}:0.0}", MapTensor.from("{ {d1:l1}:-5.3, {d2:l2, d1:l1}:0}").toString()); + } + + @Test + public void testParseError() { + try { + MapTensor.from("--"); + fail("Expected parse error"); + } + catch (IllegalArgumentException expected) { + assertEquals("Excepted a string starting by { or (, got '--'", expected.getMessage()); + } + } + + @Test + public void testConstruction() { + assertEquals("{}", new MapTensor(Collections.emptyMap()).toString()); + assertEquals("{{}:5.0}", new MapTensor(Collections.singletonMap(TensorAddress.empty, 5.0)).toString()); + + Map<TensorAddress, Double> cells = new LinkedHashMap<>(); + cells.put(TensorAddress.fromSorted(Collections.singletonList(new TensorAddress.Element("d1","l1"))), 5.0); + cells.put(TensorAddress.fromSorted(Collections.singletonList(new TensorAddress.Element("d2","l1"))), 6.0); + cells.put(TensorAddress.empty, 7.0); + assertEquals("{{}:7.0,{d1:l1}:5.0,{d2:l1}:6.0}", new MapTensor(cells).toString()); + } + + @Test + public void testDimensions() { + Set<String> dimensions1 = MapTensor.from("{} ").dimensions(); + assertEquals(0, dimensions1.size()); + + Set<String> dimensions2 = MapTensor.from("{ {d1:l1}:5, {d2:l2, d1:l1}:6.0} ").dimensions(); + assertEquals(2, dimensions2.size()); + assertTrue(dimensions2.contains("d1")); + assertTrue(dimensions2.contains("d2")); + + Set<String> dimensions3 = MapTensor.from("{ {d1:l1, d2:l1}:5, {d2:l2, d3:l1}:6.0} ").dimensions(); + assertEquals(3, dimensions3.size()); + assertTrue(dimensions3.contains("d1")); + assertTrue(dimensions3.contains("d2")); + assertTrue(dimensions3.contains("d3")); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java new file mode 100644 index 00000000000..59d77f6569a --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTypeTestCase.java @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor; + +import org.junit.Test; + +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class TensorTypeTestCase { + + @Test + public void requireThatAnEmptyTensorTypeCanBeSpecified() { + assertTensorType("tensor()"); + } + + @Test + public void requireThatBoundIndexedDimensionsCanBeSpecified() { + assertTensorType("tensor(x[5])"); + assertTensorType("tensor(x[5],y[10],z[100])"); + assertTensorType("tensor(x[5],y[10],z[100])", "tensor( x[5] , y[10] , z[100] )"); + assertTensorType("tensor(baR_09[10])"); + } + + @Test + public void requireThatUnboundIndexedDimensionsCanBeSpecified() { + assertTensorType("tensor(x[])"); + assertTensorType("tensor(x[],y[],z[])"); + assertTensorType("tensor(x[],y[],z[])", "tensor( x[] , y[] , z[] )"); + assertTensorType("tensor(baR_09[])"); + } + + @Test + public void requireThatMappedDimensionsCanBeSpecified() { + assertTensorType("tensor(x{})"); + assertTensorType("tensor(x{},y{},z{})"); + assertTensorType("tensor(x{},y{},z{})", "tensor( x{} , y{} , z{} )"); + assertTensorType("tensor(baR_09{})"); + } + + @Test + public void requireThatIndexedBoundDimensionMustHaveNonZeroSize() { + assertIllegalTensorType("tensor(x[0])", "Size of bound dimension 'x' must be at least 1"); + } + + @Test + public void requireThatDimensionsMustHaveUniqueNames() { + assertIllegalTensorType("tensor(x[10],y[20],x[30])", "'x[10]' and 'x[30]' have the same name"); + assertIllegalTensorType("tensor(x{},y{},x{})", "'x{}' and 'x{}' have the same name"); + } + + @Test + public void requireThatDimensionsAreOfSameType() { + assertIllegalTensorType("tensor(x[10],y[])", "'x[10]' does not have the same type as 'y[]'"); + assertIllegalTensorType("tensor(x[10],y{})", "'x[10]' does not have the same type as 'y{}'"); + assertIllegalTensorType("tensor(x[10],y[20],z{})", "'y[20]' does not have the same type as 'z{}'"); + assertIllegalTensorType("tensor(x[],y{})", "'x[]' does not have the same type as 'y{}'"); + } + + @Test + public void requireThatIllegalSyntaxInSpecThrowsException() { + assertIllegalTensorType("foo(x[10])", "Tensor type spec must start with 'tensor(' and end with ')', but was 'foo(x[10])'"); + assertIllegalTensorType("tensor(x_@[10])", "Failed parsing element 'x_@[10]' in type spec 'tensor(x_@[10])'"); + assertIllegalTensorType("tensor(x[10a])", "Failed parsing element 'x[10a]' in type spec 'tensor(x[10a])'"); + assertIllegalTensorType("tensor(x{10})", "Failed parsing element 'x{10}' in type spec 'tensor(x{10})'"); + } + + private static void assertTensorType(String typeSpec) { + assertTensorType(typeSpec, typeSpec); + } + + private static void assertTensorType(String expected, String typeSpec) { + assertEquals(expected, TensorType.fromSpec(typeSpec).toString()); + } + + private static void assertIllegalTensorType(String typeSpec, String messageSubstring) { + try { + TensorType.fromSpec(typeSpec); + fail("Exception exception to be thrown with message: '" + messageSubstring + "'"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString(messageSubstring)); + } + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/CompactBinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/CompactBinaryFormatTestCase.java new file mode 100644 index 00000000000..23589577c0c --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/CompactBinaryFormatTestCase.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.tensor.serialization; + +import com.google.common.collect.Sets; +import com.yahoo.tensor.MapTensor; +import com.yahoo.tensor.Tensor; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +/** + * Tests for the compact binary format. + * + * TODO: When new formats are added we should refactor this test to test all formats + * with the same set of tensor inputs (if feasible). + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class CompactBinaryFormatTestCase { + + private static void assertSerialization(String tensorString) { + assertSerialization(MapTensor.from(tensorString)); + } + + private static void assertSerialization(String tensorString, Set<String> dimensions) { + Tensor tensor = MapTensor.from(tensorString); + assertEquals(dimensions, tensor.dimensions()); + assertSerialization(tensor); + } + + private static void assertSerialization(Tensor tensor) { + byte[] encodedTensor = TypedBinaryFormat.encode(tensor); + Tensor decodedTensor = TypedBinaryFormat.decode(encodedTensor); + assertEquals(tensor, decodedTensor); + } + + @Test + public void testSerializationOfTensorsWithDenseTensorAddresses() { + assertSerialization("{}"); + assertSerialization("{{x:0}:2.0}"); + assertSerialization("{{x:0}:2.0,{x:1}:3.0}"); + assertSerialization("{{x:0,y:0}:2.0}"); + assertSerialization("{{x:0,y:0}:2.0,{x:0,y:1}:3.0}"); + assertSerialization("{{y:0,x:0}:2.0}"); + assertSerialization("{{y:0,x:0}:2.0,{y:1,x:0}:3.0}"); + assertSerialization("{{dimX:labelA,dimY:labelB}:2.0,{dimY:labelC,dimX:labelD}:3.0}"); + } + + @Test + public void testSerializationOfTensorsWithSparseTensorAddresses() { + assertSerialization("{{x:0}:2.0, {}:3.0}", Sets.newHashSet("x")); + assertSerialization("({{y:-}:1} * {{x:0}:2.0})", Sets.newHashSet("x", "y")); + assertSerialization("({{y:-}:1} * {{x:0}:2.0, {}:3.0})", Sets.newHashSet("x", "y")); + assertSerialization("({{y:-}:1} * {{x:0}:2.0,{x:1}:3.0})", Sets.newHashSet("x", "y")); + assertSerialization("({{z:-}:1} * {{x:0,y:0}:2.0})", Sets.newHashSet("x", "y", "z")); + assertSerialization("({{z:-}:1} * {{x:0,y:0}:2.0,{x:0,y:1}:3.0})", Sets.newHashSet("x", "y", "z")); + assertSerialization("({{z:-}:1} * {{y:0,x:0}:2.0})", Sets.newHashSet("x", "y", "z")); + assertSerialization("({{z:-}:1} * {{y:0,x:0}:2.0,{y:1,x:0}:3.0})", Sets.newHashSet("x", "y", "z")); + assertSerialization("({{z:-}:1} * {{}:2.0,{x:0}:3.0,{x:0,y:0}:5.0})", Sets.newHashSet("x", "y", "z")); + } + + @Test + public void requireThatCompactSerializationFormatDoNotChange() { + byte[] encodedTensor = new byte[] {1, // binary format type + 2, // num dimensions + 2, (byte)'x', (byte)'y', 1, (byte)'z', // dimensions + 2, // num cells, + 2, (byte)'a', (byte)'b', 0, 64, 0, 0, 0, 0, 0, 0, 0, // cell 0 + 2, (byte)'c', (byte)'d', 1, (byte)'e', 64, 8, 0, 0, 0, 0, 0, 0}; // cell 1 + assertEquals(Arrays.toString(encodedTensor), + Arrays.toString(TypedBinaryFormat.encode(MapTensor.from("{{xy:ab}:2.0,{xy:cd,z:e}:3.0}")))); + } + +} + diff --git a/vespajlib/src/test/java/com/yahoo/text/AsciiTest.java b/vespajlib/src/test/java/com/yahoo/text/AsciiTest.java new file mode 100644 index 00000000000..4d598e7b1bb --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/AsciiTest.java @@ -0,0 +1,187 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class AsciiTest { + + @Test + public void requireThatAdditionalCodePointsCanBeEscaped() { + assertEquals("\\x66\\x6F\\x6F \\x62ar \\x62az", + Ascii.newEncoder(StandardCharsets.UTF_8, 'f', 'o', 'b').encode("foo bar baz")); + } + + @Test + public void requireThatReadableCharactersAreNotEscaped() { + StringBuilder str = new StringBuilder(); + for (int i = 0x20; i < 0x7F; ++i) { + if (i != '\\') { + str.appendCodePoint(i); + } + } + assertEncodeUtf8(str.toString(), str.toString()); + } + + @Test + public void requireThatNonReadableCharactersAreEscapedAsUtf8() { + for (int i = Character.MIN_CODE_POINT; i < 0x20; ++i) { + String expected; + switch (i) { + case '\f': + expected = "\\f"; + break; + case '\n': + expected = "\\n"; + break; + case '\r': + expected = "\\r"; + break; + case '\t': + expected = "\\t"; + break; + default: + expected = String.format("\\x%02X", i); + break; + } + assertEncodeUtf8(expected, new StringBuilder().appendCodePoint(i).toString()); + } + for (int i = 0x80; i < 0xC0; ++i) { + String expected = String.format("\\xC2\\x%02X", i); + assertEncodeUtf8(expected, new StringBuilder().appendCodePoint(i).toString()); + } + for (int i = 0xC0; i < 0x0100; ++i) { + String expected = String.format("\\xC3\\x%02X", i - 0x40); + assertEncodeUtf8(expected, new StringBuilder().appendCodePoint(i).toString()); + } + for (int i = 0x0100; i < 0x0140; ++i) { + String expected = String.format("\\xC4\\x%02X", i - 0x80); + assertEncodeUtf8(expected, new StringBuilder().appendCodePoint(i).toString()); + } + } + + @Test + public void requireThatBackslashIsEscaped() { + assertEncodeUtf8("\\\\", "\\"); + } + + @Test + public void requireThatQuoteIsEscaped() { + assertEncodeUtf8("\\x62az", "baz", 'b'); + assertEncodeUtf8("b\\x61z", "baz", 'a'); + assertEncodeUtf8("ba\\x7A", "baz", 'z'); + } + + @Test + public void requireThatAnyEscapedCharacterCanBeUnescaped() { + assertDecodeUtf8("baz", "\\baz"); + assertDecodeUtf8("baz", "b\\az"); + assertDecodeUtf8("baz", "ba\\z"); + } + + @Test + public void requireThatUtf8SequencesAreUnescaped() { + for (int i = 0x80; i < 0xC0; ++i) { + String str = String.format("\\xC2\\x%02X", i); + assertDecodeUtf8(new StringBuilder().appendCodePoint(i).toString(), str); + } + for (int i = 0xC0; i < 0x0100; ++i) { + String str = String.format("\\xC3\\x%02X", i - 0x40); + assertDecodeUtf8(new StringBuilder().appendCodePoint(i).toString(), str); + } + for (int i = 0x0100; i < 0x0140; ++i) { + String str = String.format("\\xC4\\x%02X", i - 0x80); + assertDecodeUtf8(new StringBuilder().appendCodePoint(i).toString(), str); + } + } + + @Test + public void requireThatUtf8CanBeEncoded() { + // First possible sequence of a certain length + assertEncodeUtf8("\\x00", "\u0000"); + assertEncodeUtf8("\\xC2\\x80", "\u0080"); + assertEncodeUtf8("\\xE0\\xA0\\x80", "\u0800"); + assertEncodeUtf8("\\x01\\x00", "\u0001\u0000"); + assertEncodeUtf8("\\x20\\x00", "\u0020\u0000", ' '); + assertEncodeUtf8("\\xD0\\x80\\x00", "\u0400\u0000"); + + // Last possible sequence of a certain length + assertEncodeUtf8("\\x7F", "\u007F"); + assertEncodeUtf8("\\xDF\\xBF", "\u07FF"); + assertEncodeUtf8("\\xEF\\xBF\\xBF", "\uFFFF"); + assertEncodeUtf8("\\x1F\\xEF\\xBF\\xBF", "\u001F\uFFFF"); + assertEncodeUtf8("\\xCF\\xBF\\xEF\\xBF\\xBF", "\u03FF\uFFFF"); + assertEncodeUtf8("\\xE7\\xBF\\xBF\\xEF\\xBF\\xBF", "\u7FFF\uFFFF"); + + // Other boundary conditions + assertEncodeUtf8("\\xED\\x9F\\xBF", "\uD7FF"); + assertEncodeUtf8("\\xEE\\x80\\x80", "\uE000"); + assertEncodeUtf8("\\xEF\\xBF\\xBD", "\uFFFD"); + assertEncodeUtf8("\\x10\\xEF\\xBF\\xBF", "\u0010\uFFFF"); + assertEncodeUtf8("\\x11\\x00", "\u0011\u0000"); + } + + @Test + public void requireThatUTf8CanBeDecoded() { + // First possible sequence of a certain length + assertDecodeUtf8("\u0000", "\\x00"); + assertDecodeUtf8("\u0080", "\\xC2\\x80"); + assertDecodeUtf8("\u0800", "\\xE0\\xA0\\x80"); + assertDecodeUtf8("\u0001\u0000", "\\x01\\x00"); + assertDecodeUtf8("\u0020\u0000", "\\x20\\x00"); + assertDecodeUtf8("\u0400\u0000", "\\xD0\\x80\\x00"); + + // Last possible sequence of a certain length + assertDecodeUtf8("\u007F", "\\x7F"); + assertDecodeUtf8("\u07FF", "\\xDF\\xBF"); + assertDecodeUtf8("\uFFFF", "\\xEF\\xBF\\xBF"); + assertDecodeUtf8("\u001F\uFFFF", "\\x1F\\xEF\\xBF\\xBF"); + assertDecodeUtf8("\u03FF\uFFFF", "\\xCF\\xBF\\xEF\\xBF\\xBF"); + assertDecodeUtf8("\u7FFF\uFFFF", "\\xE7\\xBF\\xBF\\xEF\\xBF\\xBF"); + + // Other boundary conditions + assertDecodeUtf8("\uD7FF", "\\xED\\x9F\\xBF"); + assertDecodeUtf8("\uE000", "\\xEE\\x80\\x80"); + assertDecodeUtf8("\uFFFD", "\\xEF\\xBF\\xBD"); + assertDecodeUtf8("\u0010\uFFFF", "\\x10\\xEF\\xBF\\xBF"); + assertDecodeUtf8("\u0011\u0000", "\\x11\\x00"); + } + + @Test + public void requireThatUnicodeCanBeEncoded() { + assertEncodeUtf8("\\xE4\\xB8\\x9C\\xE8\\xA5\\xBF\\xE8\\x87\\xAA\\xE8\\xA1\\x8C\\xE8\\xBD\\xA6", + "\u4E1C\u897F\u81EA\u884C\u8F66"); + } + + @Test + public void requireThatUnicodeCanBeDecoded() { + assertDecodeUtf8("\u4E1C\u897F\u81EA\u884C\u8F66", + "\\xE4\\xB8\\x9C\\xE8\\xA5\\xBF\\xE8\\x87\\xAA\\xE8\\xA1\\x8C\\xE8\\xBD\\xA6"); + } + + @Test + public void requireThatUnicodeIsAllowedInInputString() { + assertDecodeUtf8("\u4E1C\u897F\u81EA\u884C\u8F66", + "\u4E1C\u897F\u81EA\u884C\u8F66"); + } + + private static void assertEncodeUtf8(String expected, String str, int... requiresEscape) { + String actual = Ascii.encode(str, StandardCharsets.UTF_8, requiresEscape); + for (int i = 0; i < actual.length(); i += actual.offsetByCodePoints(i, 1)) { + int c = actual.codePointAt(i); + assertTrue(Integer.toHexString(c), c >= 0x20 && c <= 0x7F); + } + assertEquals(expected, actual); + } + + private static void assertDecodeUtf8(String expected, String str) { + assertEquals(expected, Ascii.decode(str, StandardCharsets.UTF_8)); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/text/Benchmark.java b/vespajlib/src/test/java/com/yahoo/text/Benchmark.java new file mode 100644 index 00000000000..caa4b57c099 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/Benchmark.java @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +// import com.google.common.base.Preconditions; +// import com.google.inject.Provider; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +class Benchmark { + + public static interface Task { + public long run(CyclicBarrier barrier, int numIterations) throws Exception; + } + + + public static class TaskProvider { + final Class<? extends Task> taskClass; + public Task get() { + try { + return taskClass.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + public TaskProvider(final Class<? extends Task> taskClass) { + this.taskClass = taskClass; + } + } + + private final TaskProvider taskProvider; + private final int numIterationsPerThread; + private final int numThreads; + + private Benchmark(Builder builder) { + Objects.requireNonNull(builder.taskProvider, "taskProvider"); +/* + Preconditions.checkArgument(builder.numIterationsPerThread > 0, "numIterationsPerThread; %s", + builder.numIterationsPerThread); + Preconditions.checkArgument(builder.numThreads > 0, "numThreads; %s", + builder.numThreads); +*/ + taskProvider = builder.taskProvider; + numIterationsPerThread = builder.numIterationsPerThread; + numThreads = builder.numThreads; + } + + public long run() throws Exception { + final CyclicBarrier barrier = new CyclicBarrier(numThreads); + List<Callable<Long>> clients = new ArrayList<>(numThreads); + for (int i = 0; i < numThreads; ++i) { + final Task task = taskProvider.get(); + clients.add(new Callable<Long>() { + + @Override + public Long call() throws Exception { + return task.run(barrier, numIterationsPerThread); + } + }); + } + long maxNanosPerClient = 0; + for (Future<Long> result : Executors.newFixedThreadPool(numThreads).invokeAll(clients)) { + maxNanosPerClient = Math.max(maxNanosPerClient, result.get()); + } + return TimeUnit.SECONDS.toNanos(1) * numThreads * numIterationsPerThread / maxNanosPerClient; + } + + public static class Builder { + + private TaskProvider taskProvider; + private int numIterationsPerThread = 1000; + private int numThreads = 1; + + public Builder setNumThreads(int numThreads) { + this.numThreads = numThreads; + return this; + } + + public Builder setNumIterationsPerThread(int numIterationsPerThread) { + this.numIterationsPerThread = numIterationsPerThread; + return this; + } + + public Builder setTaskClass(final Class<? extends Task> taskClass) { + return setTaskProvider(new TaskProvider(taskClass)); + } + + public Builder setTaskProvider(TaskProvider taskProvider) { + this.taskProvider = taskProvider; + return this; + } + + public Benchmark build() { + return new Benchmark(this); + } + } +} diff --git a/vespajlib/src/test/java/com/yahoo/text/BooleanParserTestCase.java b/vespajlib/src/test/java/com/yahoo/text/BooleanParserTestCase.java new file mode 100644 index 00000000000..756d0cd23ff --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/BooleanParserTestCase.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * Control argument checking in BooleanParser. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class BooleanParserTestCase { + + @Test + public final void testParseBoolean() { + boolean gotException = false; + try { + BooleanParser.parseBoolean(null); + } catch (final NullPointerException e) { + gotException = true; + } + assertTrue(gotException); + gotException = false; + try { + BooleanParser.parseBoolean("nalle"); + } catch (final IllegalArgumentException e) { + gotException = true; + } + assertTrue(BooleanParser.parseBoolean("true")); + assertFalse(BooleanParser.parseBoolean("false")); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/CaseInsensitiveIdentifierTestCase.java b/vespajlib/src/test/java/com/yahoo/text/CaseInsensitiveIdentifierTestCase.java new file mode 100644 index 00000000000..6c6b5b62506 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/CaseInsensitiveIdentifierTestCase.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + + +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 11.11.12 + * Time: 11:37 + * To change this template use File | Settings | File Templates. + */ +public class CaseInsensitiveIdentifierTestCase { + @Test + public void testCaseInsentivitity() { + assertEquals(new CaseInsensitiveIdentifier("").toString(), ""); + assertEquals(new CaseInsensitiveIdentifier("a").toString(), "a"); + assertEquals(new CaseInsensitiveIdentifier("z").toString(), "z"); + assertEquals(new CaseInsensitiveIdentifier("B").toString(), "B"); + assertEquals(new CaseInsensitiveIdentifier("Z").toString(), "Z"); + assertEquals(new CaseInsensitiveIdentifier("_").toString(), "_"); + try { + assertEquals(new CaseInsensitiveIdentifier("0").toString(), "0"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Illegal starting character '0' of identifier '0'."); + } + try { + assertEquals(new CaseInsensitiveIdentifier("-").toString(), "-"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Illegal starting character '-' of identifier '-'."); + } + assertEquals(new CaseInsensitiveIdentifier("a0_9").toString(), "a0_9"); + assertEquals(new Identifier("a9Z_").toString(), "a9Z_"); + try { + assertEquals(new CaseInsensitiveIdentifier("a-b").toString(), "a-b"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Illegal character '-' of identifier 'a-b'."); + } + assertEquals(new CaseInsensitiveIdentifier("AbC"), new CaseInsensitiveIdentifier("ABC")); + assertEquals(new CaseInsensitiveIdentifier("AbC").hashCode(), new CaseInsensitiveIdentifier("ABC").hashCode()); + + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/DataTypeIdentifierTestCase.java b/vespajlib/src/test/java/com/yahoo/text/DataTypeIdentifierTestCase.java new file mode 100644 index 00000000000..b79f65d9eb2 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/DataTypeIdentifierTestCase.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 12.11.12 + * Time: 08:10 + * To change this template use File | Settings | File Templates. + */ +public class DataTypeIdentifierTestCase { + @Test + public void testDataTypeIdentifier() { + assertEquals("", new DataTypeIdentifier("").toString()); + assertEquals("a", new DataTypeIdentifier("a").toString()); + assertEquals("_", new DataTypeIdentifier("_").toString()); + try { + assertEquals("aB", new DataTypeIdentifier("aB").toString()); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Illegal character 'B' of identifier 'aB'."); + } + try { + assertEquals("1", new DataTypeIdentifier("1").toString()); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Illegal starting character '1' of identifier '1'."); + } + assertEquals("a1", new DataTypeIdentifier("a1").toString()); + assertEquals("array<b>", DataTypeIdentifier.createArrayDataTypeIdentifier(new DataTypeIdentifier("b")).toString()); + assertEquals("weightedset<b>", DataTypeIdentifier.createWeightedSetTypeIdentifier(new DataTypeIdentifier("b"), false, false).toString()); + assertEquals("weightedset<b>;add", DataTypeIdentifier.createWeightedSetTypeIdentifier(new DataTypeIdentifier("b"), true, false).toString()); + assertEquals("weightedset<b>;remove", DataTypeIdentifier.createWeightedSetTypeIdentifier(new DataTypeIdentifier("b"), false, true).toString()); + assertEquals("weightedset<b>;add;remove", DataTypeIdentifier.createWeightedSetTypeIdentifier(new DataTypeIdentifier("b"), true, true).toString()); + assertEquals("annotationreference<b>", DataTypeIdentifier.createAnnotationReferenceDataTypeIdentifier(new DataTypeIdentifier("b")).toString()); + assertEquals("map<k,v>", DataTypeIdentifier.createMapDataTypeIdentifier(new DataTypeIdentifier("k"), new DataTypeIdentifier("v")).toString()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/text/DoubleFormatterTestCase.java b/vespajlib/src/test/java/com/yahoo/text/DoubleFormatterTestCase.java new file mode 100644 index 00000000000..21be58e5fd2 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/DoubleFormatterTestCase.java @@ -0,0 +1,204 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +/** + * @author arnej27959 + */ +public class DoubleFormatterTestCase { + + @Test + public void testZero() { + String zero = DoubleFormatter.stringValue(0.0); + //assertEquals("0.0", zero); + } + + @Test + public void testOne() { + String one = DoubleFormatter.stringValue(1.0); + assertEquals("1.0", one); + } + + @Test + public void testMinusOne() { + String one = DoubleFormatter.stringValue(-1.0); + assertEquals("-1.0", one); + } + + @Test + public void testNanInf() { + String plusInf = DoubleFormatter.stringValue(Double.POSITIVE_INFINITY); + assertEquals("Infinity", plusInf); + + String notAnum = DoubleFormatter.stringValue(Double.NaN); + assertEquals("NaN", notAnum); + + String negInf = DoubleFormatter.stringValue(Double.NEGATIVE_INFINITY); + assertEquals("-Infinity", negInf); + } + + @Test + public void testSeven() { + String seven = DoubleFormatter.stringValue(7.0); + assertEquals("7.0", seven); + + seven = DoubleFormatter.stringValue(77.0); + assertEquals("77.0", seven); + + seven = DoubleFormatter.stringValue(7777.0); + assertEquals("7777.0", seven); + + seven = DoubleFormatter.stringValue(7777007777.0); + assertEquals("7.777007777E9", seven); + } + + + @Test + public void testSomeChosenNumbers() { + String s = DoubleFormatter.stringValue(4097.0); + assertEquals("4097.0", s); + + s = DoubleFormatter.stringValue(4097.5); + assertEquals("4097.5", s); + + s = DoubleFormatter.stringValue(1073741823.0); + assertEquals("1.073741823E9", s); + + s = DoubleFormatter.stringValue(1073741823.5); + assertEquals("1.0737418235E9", s); + + s = DoubleFormatter.stringValue(1073741825.5); + assertEquals("1.0737418255E9", s); + + s = DoubleFormatter.stringValue(1.23456789012345669); + assertEquals("1.234567890123457", s); + s = DoubleFormatter.stringValue(12.3456789012345673); + assertEquals("12.34567890123457", s); + s = DoubleFormatter.stringValue(123.456789012345666); + assertEquals("123.4567890123457", s); + s = DoubleFormatter.stringValue(1234.56789012345666); + assertEquals("1234.567890123457", s); + s = DoubleFormatter.stringValue(12345.6789012345670); + assertEquals("12345.67890123457", s); + s = DoubleFormatter.stringValue(123456.789012345674); + assertEquals("123456.7890123457", s); + s = DoubleFormatter.stringValue(1234567.89012345671); + assertEquals("1234567.890123457", s); + + s = DoubleFormatter.stringValue(0.99); + // assertEquals("0.99", s); + + s = DoubleFormatter.stringValue(0.5); + assertEquals("0.5", s); + + s = DoubleFormatter.stringValue(0.1); + // assertEquals("0.1", s); + + s = DoubleFormatter.stringValue(0.00123456789); + // assertEquals("0.00123456789", s); + + s = DoubleFormatter.stringValue(0.0000000000001); + // assertEquals("0.0000000000001", s); + } + + @Test + public void testPowersOfTwo() { + String twos = DoubleFormatter.stringValue(2.0); + assertEquals("2.0", twos); + + twos = DoubleFormatter.stringValue(128.0); + assertEquals("128.0", twos); + + twos = DoubleFormatter.stringValue(1048576.0); + assertEquals("1048576.0", twos); + + twos = DoubleFormatter.stringValue(1073741824.0); + assertEquals("1.073741824E9", twos); + } + + @Test + public void testSmallNumbers() { + for (double d = 1.0; d > 1.0e-200; d *= 0.75) { + String fs = DoubleFormatter.stringValue(d); + String vs = String.valueOf(d); + double rp = Double.valueOf(fs); + if (d != rp) { + // System.err.println("differs: "+d+" became "+fs+" then instead: "+rp+" diff: "+(d-rp)); + } else if (! fs.equals(vs)) { + // System.err.println("string rep differs: "+vs+" became "+fs); + } + assertEquals(d, rp, 1.0e-7*d); + } + } + + @Test + public void testVerySmallNumbers() { + for (double d = 1.0; d > 1.0e-200; d *= 0.5) { + String fs = DoubleFormatter.stringValue(d); + String vs = String.valueOf(d); + double rp = Double.valueOf(fs); + if (d != rp) { + // System.err.println("differs: "+d+" became "+fs+" then instead: "+rp+" diff: "+(d-rp)); + } else if (! fs.equals(vs)) { + // System.err.println("string rep differs: "+vs+" became "+fs); + } + assertEquals(d, rp, 1.0e-13*d); + } + } + + @Test + public void testVeryVerySmallNumbers() { + for (double d = 1.0e-200; d > 0; d *= 0.5) { + String fs = DoubleFormatter.stringValue(d); + String vs = String.valueOf(d); + double rp = Double.valueOf(fs); + if (d != rp) { + // System.err.println("differs: "+d+" became "+fs+" then instead: "+rp+" diff: "+(d-rp)); + } else if (! fs.equals(vs)) { + // System.err.println("string rep differs: "+vs+" became "+fs); + } + assertEquals(d, rp, 1.0e-13*d); + } + } + + @Test + public void testVeryBigNumbers() { + for (double d = 1.0; d < Double.POSITIVE_INFINITY; d *= 2.0) { + String fs = DoubleFormatter.stringValue(d); + String vs = String.valueOf(d); + double rp = Double.valueOf(fs); + if (d != rp) { + // System.err.println("differs: "+d+" became "+fs+" then instead: "+rp); + } else if (! fs.equals(vs)) { + // System.err.println("string rep differs: "+vs+" became "+fs); + } + assertEquals(d, rp, 1.0e-13*d); + } + + assertEquals("1.0E200", String.valueOf(1.0e+200)); + + String big = DoubleFormatter.stringValue(1.0e+200); + assertEquals("1.0E200", big); + + big = DoubleFormatter.stringValue(1.0e+298); + assertEquals("1.0E298", big); + + big = DoubleFormatter.stringValue(1.0e+299); + assertEquals("1.0E299", big); + + big = DoubleFormatter.stringValue(1.0e+300); + assertEquals("1.0E300", big); + + } + + @Test + public void testRandomNumbers() { + java.util.Random rgen = new java.util.Random(0xCafeBabe); + for (int i = 0; i < 123456; i++) { + double d = rgen.nextDouble(); + } + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/DoubleParserTestCase.java b/vespajlib/src/test/java/com/yahoo/text/DoubleParserTestCase.java new file mode 100644 index 00000000000..88e253e3425 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/DoubleParserTestCase.java @@ -0,0 +1,158 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +/** + * @author arnej27959 + */ +public class DoubleParserTestCase { + + @Test + public void testZero() { + String[] zeros = { + "0", + "0.", + ".0", + "0.0", + "0.0e0", + "0.0e99", + "0.0e+300", + "0.0e-42" + }; + for (String s : zeros) { + double d = DoubleParser.parse(s); + assertEquals(0.0, d, 0); + } + } + + @Test + public void testOne() { + String[] ones = { + "1", + "1.", + "1.0", + "+1", + "10.0e-1", + "0.1e1", + "1000.0e-3", + ".001e+3", + }; + for (String s : ones) { + System.out.println("parsing: '"+s+"' now"); + double d = DoubleParser.parse(s); + System.out.println("expected: 1.0"); + System.out.println("actual: "+d); + assertEquals(1.0, d, 0); + } + } + + @Test + public void testMinusOne() { + String[] numbers = { + "-1", + "-1.0", + "-1.", + "-1e0", + "-10e-1", + }; + for (String s : numbers) { + System.out.println("parsing: '"+s+"' now"); + double d = DoubleParser.parse(s); + System.out.println("expected: -1.0"); + System.out.println("actual: "+d); + assertEquals(-1.0, d, 0); + } + } + + @Test + public void testNanInf() { + String[] numbers = { + "NaN", + "Infinity", + "-Infinity", + "+Infinity", + "+NaN", + "-NaN" + }; + for (String s : numbers) { + System.out.println("parsing: '"+s+"' now"); + double d1 = Double.parseDouble(s); + double d2 = DoubleParser.parse(s); + long lb1 = Double.doubleToRawLongBits(d1); + long lb2 = Double.doubleToRawLongBits(d2); + assertEquals(lb1, lb2, 0); + } + } + + @Test + public void testSeven() { + String[] sevens = { + "7", + "7.", + "7.0", + "70.0e-1", + "0.7e1", + "7000.0e-3", + ".007e+3", + }; + for (String s : sevens) { + System.out.println("parsing: '"+s+"' now"); + double d = DoubleParser.parse(s); + System.out.println("expected: 7.0"); + System.out.println("actual: "+d); + assertEquals(7.0, d, 0); + } + } + + @Test + public void testVerySmallNumbers() { + String[] numbers = { + "1.e-320", + "-1.e-320", + "1.0013378241589014e-303" + }; + for (String s : numbers) { + System.out.println("parsing: '"+s+"' now"); + double d1 = Double.parseDouble(s); + double d2 = DoubleParser.parse(s); + System.out.println("expected: "+d1); + System.out.println("actual: "+d2); + assertEquals(d1, d2, 0); + } + } + + @Test + public void testRandomNumbers() { + java.util.Random rgen = new java.util.Random(0xCafeBabe); + for (int i = 0; i < 123456; i++) { + double d = rgen.nextDouble(); + int exp = rgen.nextInt(); + d *= Math.pow(1.0000006, exp); + String s = Double.toString(d); + double d2 = Double.parseDouble(s); + double d3 = DoubleParser.parse(s); + + if (d != d2) { + System.out.println("WARNING: value ["+d+"] parses as ["+d2+"] by Java"); + } + double allow = 1.0e-14 * d2; + if (allow < 0) { + allow = -allow; + } + if (d2 != d3) { + long lb2 = Double.doubleToRawLongBits(d2); + long lb3 = Double.doubleToRawLongBits(d3); + if (lb2 - lb3 > 15 || lb3 - lb2 > 15) { + System.out.println("WARNING: string '"+s+"' parses as"); + System.out.println("["+d2+"] by Java, ["+d3+"] by our method"); + System.out.println("["+Long.toHexString(lb2)+"] bits vs ["+Long.toHexString(lb3)+"]"); + System.out.println("==> "+(lb2 - lb3)+" <== diff value"); + } + } + assertEquals(d2, d3, allow); + } + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/DoubleToStringBenchmark.java b/vespajlib/src/test/java/com/yahoo/text/DoubleToStringBenchmark.java new file mode 100644 index 00000000000..2e0af153fc7 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/DoubleToStringBenchmark.java @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Ignore; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; + +/** + * @author arnej27959 + */ +public class DoubleToStringBenchmark { + + @Test + @Ignore + public void benchmarkStringConstruction() throws Exception { + List<Class<? extends Benchmark.Task>> taskList = Arrays.asList(UseStringValueOf.class, + UseDoubleFormatter.class, + UseDoubleFormatter.class, + UseStringValueOf.class, + UseStringValueOf.class, + UseDoubleFormatter.class, + UseDoubleFormatter.class, + UseStringValueOf.class, + UseDoubleFormatter.class, + UseStringValueOf.class); + + int maxThreads = 256; + int dummy = 0; + System.out.print("warmup"); + for (Class<? extends Benchmark.Task> taskClass : taskList) { + dummy += runBenchmark(maxThreads, taskClass); + System.out.print("."); + } + System.out.println(" " + dummy); + + System.out.format("%-35s", ""); + for (int numThreads = 1; numThreads <= maxThreads; numThreads *= 2) { + System.out.format("%13s t ", numThreads); + } + System.out.println(); + for (Class<? extends Benchmark.Task> taskClass : taskList) { + System.out.format("%-35s", taskClass.getSimpleName()); + for (int numThreads = 1; numThreads <= maxThreads; numThreads *= 2) { + System.out.format("%15d ", runBenchmark(numThreads, taskClass)); + } + System.out.println(); + } + } + + private long runBenchmark(int numThreads, Class<? extends Benchmark.Task> taskClass) throws Exception { + return new Benchmark.Builder() + .setNumIterationsPerThread(80000) + .setNumThreads(numThreads) + .setTaskClass(taskClass) + .build() + .run(); + } + + public static class UseStringValueOf implements Benchmark.Task { + + private long timeIt(Random randomGen, int num) { + long before = System.nanoTime(); + + String str = null; + double v = 0.0; + for (int i = 0; i < num; ++i) { + v = randomGen.nextDouble() * 1.0e-2; + str = String.valueOf(v); + } + + long after = System.nanoTime(); + assertEquals(""+v, str); + return after - before; + } + + @Override + public long run(CyclicBarrier barrier, int numIterations) throws Exception { + Random randomGen = new Random(0xDeadBeef); + barrier.await(600, TimeUnit.SECONDS); + long t1 = timeIt(randomGen, numIterations / 4); + long t2 = timeIt(randomGen, numIterations / 2); + long t3 = timeIt(randomGen, numIterations / 4); + return t2; + } + } + + public static class UseDoubleFormatter implements Benchmark.Task { + + private long timeIt(Random randomGen, int num) { + long before = System.nanoTime(); + + String str = null; + double v = 0.0; + for (int i = 0; i < num; ++i) { + v = randomGen.nextDouble() * 1.0e-2; + str = DoubleFormatter.stringValue(v); + } + + long after = System.nanoTime(); + // assertEquals(""+v, str); + return after - before; + } + + + @Override + public long run(CyclicBarrier barrier, int numIterations) throws Exception { + Random randomGen = new Random(0xDeadBeef); + barrier.await(600, TimeUnit.SECONDS); + long t1 = timeIt(randomGen, numIterations / 4); + long t2 = timeIt(randomGen, numIterations / 2); + long t3 = timeIt(randomGen, numIterations / 4); + return t2; + } + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/ForwardWriterTestCase.java b/vespajlib/src/test/java/com/yahoo/text/ForwardWriterTestCase.java new file mode 100644 index 00000000000..6984c2ec6a8 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/ForwardWriterTestCase.java @@ -0,0 +1,435 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.List; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.protect.ClassValidator; + +/** + * Check all methods forward correctly and wrap exceptions as documented. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * + */ +public class ForwardWriterTestCase { + private static final String WRITE_ABSTRACT_UTF8_ARRAY = "write(AbstractUtf8Array)"; + private static final String WRITE_BOOLEAN = "write(boolean)"; + private static final String WRITE_CHAR = "write(char)"; + private static final String WRITE_DOUBLE = "write(double)"; + private static final String WRITE_FLOAT = "write(float)"; + private static final String WRITE_CHAR_SEQUENCE = "write(CharSequence)"; + private static final String WRITE_CHAR_INT_INT = "write(char[], int, int)"; + private static final String FLUSH = "flush()"; + private static final String CLOSE = "close()"; + private static final String WRITE_STRING = "write(String)"; + private static final String WRITE_LONG = "write(long)"; + private static final String WRITE_SHORT = "write(short)"; + private static final String WRITE_BYTE = "write(byte)"; + + private static class Boom extends GenericWriter { + @Override + public GenericWriter write(final char c) throws IOException { + method(WRITE_CHAR); + final GenericWriter w = super.write(c); + explode(); + return w; + } + + @Override + public GenericWriter write(final CharSequence s) throws IOException { + method(WRITE_CHAR_SEQUENCE); + final GenericWriter w = super.write(s); + explode(); + return w; + } + + @Override + public void write(final String s) throws IOException { + method(WRITE_STRING); + super.write(s); + explode(); + } + + @Override + public GenericWriter write(final long i) throws IOException { + method(WRITE_LONG); + final GenericWriter w = super.write(i); + explode(); + return w; + } + + @Override + public void write(final int i) throws IOException { + method("write(int)"); + super.write(i); + explode(); + } + + @Override + public GenericWriter write(final short i) throws IOException { + method(WRITE_SHORT); + final GenericWriter w = super.write(i); + explode(); + return w; + } + + @Override + public GenericWriter write(final byte i) throws IOException { + method(WRITE_BYTE); + final GenericWriter w = super.write(i); + explode(); + return w; + } + + @Override + public GenericWriter write(final double i) throws IOException { + method(WRITE_DOUBLE); + final GenericWriter w = super.write(i); + explode(); + return w; + } + + @Override + public GenericWriter write(final float i) throws IOException { + method(WRITE_FLOAT); + final GenericWriter w = super.write(i); + explode(); + return w; + } + + @Override + public GenericWriter write(final boolean i) throws IOException { + method(WRITE_BOOLEAN); + final GenericWriter w = super.write(i); + explode(); + return w; + } + + @Override + public GenericWriter write(final AbstractUtf8Array v) + throws IOException { + method(WRITE_ABSTRACT_UTF8_ARRAY); + final GenericWriter w = super.write(v); + explode(); + return w; + } + + StringBuilder last = new StringBuilder(); + private boolean explode = false; + private boolean toplevel; + private String method; + + @Override + public void write(final char[] cbuf, final int off, final int len) + throws IOException { + method(WRITE_CHAR_INT_INT); + last.append(cbuf, off, len); + explode(); + } + + @Override + public void flush() throws IOException { + method(FLUSH); + explode(); + + } + + @Override + public void close() throws IOException { + method(CLOSE); + explode(); + } + + private void method(final String method) { + if (toplevel) { + this.method = method; + toplevel = false; + } + } + + private void explode() throws IOException { + if (explode) { + throw new IOException(method); + } + } + + void arm() { + explode = true; + toplevel = true; + } + } + + private Boom wrapped; + private ForwardWriter forward; + private boolean gotException; + + @Before + public void setUp() throws Exception { + wrapped = new Boom(); + forward = new ForwardWriter(wrapped); + gotException = false; + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void requireForwardWriterIsComplete() { + final List<Method> methods = ClassValidator + .unmaskedMethodsFromSuperclass(ForwardWriter.class); + assertEquals(0, methods.size()); + } + + @Test + public final void testWriteInt() { + forward.write(0x1ECD); + assertEquals("\u1ECD", wrapped.last.toString()); + wrapped.arm(); + try { + forward.write(0); + } catch (final RuntimeException e) { + assertEquals("write(int)", e.getCause().getMessage()); + gotException = true; + } + assertTrue(gotException); + } + + @Test + public final void testWriteCharArrayIntInt() { + writeCharArrayIntInt(); + assertEquals("0", wrapped.last.toString()); + wrapped.arm(); + try { + writeCharArrayIntInt(); + } catch (final RuntimeException e) { + assertEquals(WRITE_CHAR_INT_INT, e.getCause().getMessage()); + gotException = true; + } + assertTrue(gotException); + } + + public void writeCharArrayIntInt() { + forward.write(new char[] { '0' }, 0, 1); + } + + @Test + public final void testFlush() { + wrapped.arm(); + try { + forward.flush(); + } catch (final RuntimeException e) { + assertEquals(FLUSH, e.getCause().getMessage()); + gotException = true; + } + assertTrue(gotException); + } + + @Test + public final void testClose() { + wrapped.arm(); + try { + forward.close(); + } catch (final RuntimeException e) { + assertEquals(CLOSE, e.getCause().getMessage()); + gotException = true; + } + assertTrue(gotException); + } + + @Test + public final void testWriteString() { + writeString(); + assertEquals("0", wrapped.last.toString()); + wrapped.arm(); + try { + writeString(); + } catch (final RuntimeException e) { + assertEquals(WRITE_STRING, e.getCause().getMessage()); + gotException = true; + } + assertTrue(gotException); + } + + public void writeString() { + forward.write("0"); + } + + @Test + public final void testWriteCharSequence() { + writeCharSequence(); + assertEquals("0", wrapped.last.toString()); + wrapped.arm(); + try { + writeCharSequence(); + } catch (final RuntimeException e) { + assertEquals(WRITE_CHAR_SEQUENCE, e.getCause().getMessage()); + gotException = true; + } + assertTrue(gotException); + } + + public void writeCharSequence() { + forward.write((CharSequence) "0"); + } + + @Test + public final void testWriteLong() { + writeLong(); + assertEquals("0", wrapped.last.toString()); + wrapped.arm(); + try { + writeLong(); + } catch (final RuntimeException e) { + assertEquals(WRITE_LONG, e.getCause().getMessage()); + gotException = true; + } + assertTrue(gotException); + } + + public void writeLong() { + forward.write((long) 0); + } + + @Test + public final void testWriteFloat() { + writeFloat(); + assertEquals("0.0", wrapped.last.toString()); + wrapped.arm(); + try { + writeFloat(); + } catch (final RuntimeException e) { + assertEquals(WRITE_FLOAT, e.getCause().getMessage()); + gotException = true; + } + assertTrue(gotException); + } + + public void writeFloat() { + forward.write(0.0f); + } + + @Test + public final void testWriteDouble() { + writeDouble(); + assertEquals("0.0", wrapped.last.toString()); + wrapped.arm(); + try { + writeDouble(); + } catch (final RuntimeException e) { + assertEquals(WRITE_DOUBLE, e.getCause().getMessage()); + gotException = true; + } + assertTrue(gotException); + } + + public void writeDouble() { + forward.write(0.0d); + } + + @Test + public final void testWriteShort() { + writeShort(); + assertEquals("0", wrapped.last.toString()); + wrapped.arm(); + try { + writeShort(); + } catch (final RuntimeException e) { + assertEquals(WRITE_SHORT, e.getCause().getMessage()); + gotException = true; + } + assertTrue(gotException); + } + + public void writeShort() { + forward.write((short) 0); + } + + @Test + public final void testWriteChar() { + writeChar(); + assertEquals("0", wrapped.last.toString()); + wrapped.arm(); + try { + writeChar(); + } catch (final RuntimeException e) { + assertEquals(WRITE_CHAR, e.getCause().getMessage()); + gotException = true; + } + assertTrue(gotException); + } + + public void writeChar() { + forward.write('0'); + } + + @Test + public final void testWriteByte() { + writeByte(); + assertEquals("0", wrapped.last.toString()); + wrapped.arm(); + try { + writeByte(); + } catch (final RuntimeException e) { + assertEquals(WRITE_BYTE, e.getCause().getMessage()); + gotException = true; + } + assertTrue(gotException); + } + + public void writeByte() { + forward.write((byte) 0); + } + + @Test + public final void testWriteBoolean() { + writeBoolean(); + assertEquals("true", wrapped.last.toString()); + wrapped.arm(); + try { + writeBoolean(); + } catch (final RuntimeException e) { + assertEquals(WRITE_BOOLEAN, e.getCause().getMessage()); + gotException = true; + } + assertTrue(gotException); + } + + public void writeBoolean() { + forward.write(true); + } + + @Test + public final void testWriteAbstractUtf8Array() { + writeUtf8Array(); + assertEquals("0", wrapped.last.toString()); + wrapped.arm(); + try { + writeUtf8Array(); + } catch (final RuntimeException e) { + assertEquals(WRITE_ABSTRACT_UTF8_ARRAY, e.getCause().getMessage()); + gotException = true; + } + assertTrue(gotException); + + } + + public void writeUtf8Array() { + forward.write(new Utf8Array(Utf8.toBytes("0"))); + } + + @Test + public final void testGetWriter() { + assertSame(wrapped, forward.getWriter()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/text/GenericWriterTestCase.java b/vespajlib/src/test/java/com/yahoo/text/GenericWriterTestCase.java new file mode 100644 index 00000000000..619065ff7bf --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/GenericWriterTestCase.java @@ -0,0 +1,107 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import static org.junit.Assert.*; + +import java.io.IOException; + +import org.junit.Before; +import org.junit.Test; + +/** + * Completeness check for GenericWriter. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class GenericWriterTestCase { + private static class MockWriter extends GenericWriter { + private StringBuilder written = new StringBuilder(); + + @Override + public void write(char[] cbuf, int off, int len) throws IOException { + written.append(String.copyValueOf(cbuf, off, len)); + } + + @Override + public void flush() throws IOException { + } + + @Override + public void close() throws IOException { + } + } + + MockWriter mock; + + @Before + public void setUp() throws Exception { + mock = new MockWriter(); + } + + @Test + public final void testWriteInt() throws Exception { + mock.write(0xa); + assertEquals("\n", mock.written.toString()); + } + + @Test + public final void testWriteChar() throws IOException { + mock.write('\u0020'); + assertEquals(" ", mock.written.toString()); + } + + @Test + public final void testWriteCharSequence() throws IOException { + mock.write((CharSequence) "abc"); + assertEquals("abc", mock.written.toString()); + } + + @Test + public final void testWriteString() throws IOException { + mock.write("abc"); + assertEquals("abc", mock.written.toString()); + } + + @Test + public final void testWriteLong() throws IOException { + mock.write(42L); + assertEquals("42", mock.written.toString()); + } + + @Test + public final void testWriteShort() throws IOException { + mock.write((short) 42); + assertEquals("42", mock.written.toString()); + } + + @Test + public final void testWriteByte() throws IOException { + mock.write((byte) 42); + assertEquals("42", mock.written.toString()); + } + + @Test + public final void testWriteDouble() throws IOException { + mock.write(0.0d); + assertEquals("0.0", mock.written.toString()); + } + + @Test + public final void testWriteFloat() throws IOException { + mock.write(0.0f); + assertEquals("0.0", mock.written.toString()); + } + + @Test + public final void testWriteBoolean() throws IOException { + mock.write(true); + assertEquals("true", mock.written.toString()); + } + + @Test + public final void testWriteAbstractUtf8Array() throws IOException { + mock.write(new Utf8Array(Utf8.toBytes("abc"))); + assertEquals("abc", mock.written.toString()); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/HTMLTestCase.java b/vespajlib/src/test/java/com/yahoo/text/HTMLTestCase.java new file mode 100644 index 00000000000..2eab7d42aed --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/HTMLTestCase.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * @author Bjorn Borud + */ +public class HTMLTestCase { + + @Test + public void testSimpleEscape() { + assertEquals(""this <&> that"", + HTML.htmlescape("\"this <&> that\"")); + } + + @Test + public void testBunchOfEscapes() { + assertEquals( + "©®ÀÁÂÃÄÅÆÇ", + HTML.htmlescape("\u00A9\u00AE\u00C0\u00C1\u00C2\u00C3\u00C4\u00C5\u00C6\u00C7")); + + assertEquals( + "ÈÉÊËÌÍÎÏÐÑ", + HTML.htmlescape("\u00C8\u00C9\u00CA\u00CB\u00CC\u00CD\u00CE\u00CF\u00D0\u00D1")); + + assertEquals( + "ÒÓÔÕÖØÙÚÛÜ", + HTML.htmlescape("\u00D2\u00D3\u00D4\u00D5\u00D6\u00D8\u00D9\u00DA\u00DB\u00DC")); + + assertEquals( + "ÝÞßàáâãäåæ", + HTML.htmlescape("\u00DD\u00DE\u00DF\u00E0\u00E1\u00E2\u00E3\u00E4\u00E5\u00E6")); + + assertEquals( + "çèéêëìíîïì", + HTML.htmlescape("\u00E7\u00E8\u00E9\u00EA\u00EB\u00EC\u00ED\u00EE\u00EF\u00EC")); + + assertEquals( + "íîïðñòóôõö", + HTML.htmlescape("\u00ED\u00EE\u00EF\u00F0\u00F1\u00F2\u00F3\u00F4\u00F5\u00F6")); + + assertEquals( + "øùúûüýþÿ", + HTML.htmlescape("\u00F8\u00F9\u00FA\u00FB\u00FC\u00FD\u00FE\u00FF")); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/IdentifierTestCase.java b/vespajlib/src/test/java/com/yahoo/text/IdentifierTestCase.java new file mode 100644 index 00000000000..447b109983e --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/IdentifierTestCase.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 11.11.12 + * Time: 10:58 + * To change this template use File | Settings | File Templates. + */ +public class IdentifierTestCase { + @Test + public void testIdentifier() { + assertEquals(new Identifier("").toString(), ""); + assertEquals(new Identifier("a").toString(), "a"); + assertEquals(new Identifier("z").toString(), "z"); + assertEquals(new Identifier("B").toString(), "B"); + assertEquals(new Identifier("Z").toString(), "Z"); + assertEquals(new Identifier("_").toString(), "_"); + try { + assertEquals(new Identifier("0").toString(), "0"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Illegal starting character '0' of identifier '0'."); + } + try { + assertEquals(new Identifier("-").toString(), "-"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Illegal starting character '-' of identifier '-'."); + } + assertEquals(new Identifier("a0_9").toString(), "a0_9"); + assertEquals(new Identifier("a9Z_").toString(), "a9Z_"); + try { + assertEquals(new Identifier("a-b").toString(), "a-b"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Illegal character '-' of identifier 'a-b'."); + } + + } +} diff --git a/vespajlib/src/test/java/com/yahoo/text/JSONTest.java b/vespajlib/src/test/java/com/yahoo/text/JSONTest.java new file mode 100644 index 00000000000..53be5a1bda5 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/JSONTest.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class JSONTest { + + @Test + public void testMapToString() { + Map<String,Object> map = new LinkedHashMap<>(); + map.put("a \"key\"", 3); + map.put("key2", "value"); + map.put("key3", 3.3); + + assertEquals("{\"a \\\"key\\\"\":3,\"key2\":\"value\",\"key3\":3.3}", JSON.encode(map)); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/JSONWriterTestCase.java b/vespajlib/src/test/java/com/yahoo/text/JSONWriterTestCase.java new file mode 100644 index 00000000000..16d9fe65769 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/JSONWriterTestCase.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +import static org.junit.Assert.*; + +/** + * Tests the JSON writer + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +@SuppressWarnings("deprecation") +public class JSONWriterTestCase { + + @Test + public void testJSONWriter() throws IOException { + OutputStream out = new ByteArrayOutputStream(); + JSONWriter w = new JSONWriter(out); + + w.beginObject(); + + w.beginField("string").value("a string").endField(); + w.beginField("number").value(37).endField(); + w.beginField("true").value(true).endField(); + w.beginField("false").value(false).endField(); + w.beginField("null").value().endField(); + + w.beginField("object").beginObject(); + w.beginField("nested-array").beginArray().beginArrayValue().value(1).endArrayValue().endArray().endField(); + w.endObject().endField(); + + w.beginField("array").beginArray(); + w.beginArrayValue().value("item1").endArrayValue(); + w.beginArrayValue().value("item2").endArrayValue(); + w.beginArrayValue().beginObject().beginField("nested").value("item3").endField().endObject().endArrayValue(); + w.endArray().endField(); + + w.endObject(); + + assertEquals("{\"string\":\"a string\"," + + "\"number\":37," + + "\"true\":true," + + "\"false\":false," + + "\"null\":null," + + "\"object\":{\"nested-array\":[1]}," + + "\"array\":[\"item1\",\"item2\",{\"nested\":\"item3\"}]}", + out.toString()); + } + + @Test + public void testJSONWriterEmptyObject() throws IOException { + OutputStream out = new ByteArrayOutputStream(); + JSONWriter w = new JSONWriter(out); + w.beginObject(); + w.endObject(); + + assertEquals("{}",out.toString()); + } + + @Test + public void testJSONWriterEmptyArray() throws IOException { + OutputStream out = new ByteArrayOutputStream(); + JSONWriter w = new JSONWriter(out); + w.beginArray(); + w.endArray(); + + assertEquals("[]",out.toString()); + } + + @Test + public void testJSONWriterStringOnly() throws IOException { + OutputStream out = new ByteArrayOutputStream(); + JSONWriter w = new JSONWriter(out); + w.value("Hello, world!"); + + assertEquals("\"Hello, world!\"",out.toString()); + } + + @Test + public void testJSONWriterNestedArrays() throws IOException { + OutputStream out = new ByteArrayOutputStream(); + JSONWriter w = new JSONWriter(out); + w.beginArray(); + + w.beginArrayValue().beginArray(); + w.endArray().endArrayValue(); + + w.beginArrayValue().beginArray(); + w.beginArrayValue().value("hello").endArrayValue(); + w.beginArrayValue().value("world").endArrayValue(); + w.endArray().endArrayValue(); + + w.beginArrayValue().beginArray(); + w.endArray().endArrayValue(); + + w.beginArrayValue().beginArray(); + w.beginArrayValue().beginArray(); + w.endArray().endArrayValue(); + w.endArray().endArrayValue(); + + w.beginArrayValue().beginArray(); + w.endArray().endArrayValue(); + + w.endArray(); + + assertEquals("[[],[\"hello\",\"world\"],[],[[]],[]]",out.toString()); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/JsonMicroBenchmarkTestCase.java b/vespajlib/src/test/java/com/yahoo/text/JsonMicroBenchmarkTestCase.java new file mode 100644 index 00000000000..c2c1774ca1d --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/JsonMicroBenchmarkTestCase.java @@ -0,0 +1,563 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import static org.junit.Assert.*; + +import java.io.ByteArrayOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeMap; + +import com.fasterxml.jackson.core.JsonEncoding; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +@SuppressWarnings("deprecation") +public class JsonMicroBenchmarkTestCase { + + private static final long RUNTIME = 20L * 60L * 1000L; + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + enum Strategy { + VESPAJLIB, JACKSON; + } + + private static abstract class BenchFactory { + abstract Bench produce(); + } + + private static class VespajlibFactory extends BenchFactory { + + @Override + Bench produce() { + return new OutputWithWriter(); + } + + } + + private static class JacksonFactory extends BenchFactory { + @Override + Bench produce() { + return new OutputWithGenerator(); + } + } + + private static abstract class Bench implements Runnable { + public volatile long runs; + public volatile long start; + public volatile long end; + public volatile long metric; + + /** + * Object identity is used to differentiate between different implementation strategies, toString() is used to print a report. + * + * @return an object with a descriptive toString() for the implementation under test + */ + abstract Object category(); + + @Override + public final void run() { + Random random = new Random(42L); + long localBytesWritten = 0L; + long localRuns = 0; + + start = System.currentTimeMillis(); + long target = start + JsonMicroBenchmarkTestCase.RUNTIME; + + while (System.currentTimeMillis() < target) { + for (int i = 0; i < 1000; ++i) { + localBytesWritten += iterate(random); + } + localRuns += 1000L; + } + end = System.currentTimeMillis(); + runs = localRuns; + metric = localBytesWritten; + } + + abstract int iterate(Random random); + } + + private static final class OutputWithGenerator extends Bench { + + public OutputWithGenerator() { + } + + + int iterate(Random random) { + JsonGenerator generator; + ByteArrayOutputStream generatorOut = new ByteArrayOutputStream(); + try { + generator = new JsonFactory().createJsonGenerator(generatorOut, + JsonEncoding.UTF8); + } catch (IOException e) { + e.printStackTrace(); + return 0; + } + try { + serialize(generatedDoc(random), generator); + } catch (IOException e) { + e.printStackTrace(); + return 0; + } + try { + generator.close(); + } catch (IOException e) { + e.printStackTrace(); + return 0; + } + return generatorOut.toByteArray().length; + } + + static void serialize(Map<String, Object> m, JsonGenerator g) throws IOException { + g.writeStartObject(); + for (Map.Entry<String, Object> e : m.entrySet()) { + g.writeFieldName(e.getKey()); + serializeField(g, e.getValue()); + } + g.writeEndObject(); + } + + @SuppressWarnings("unchecked") + static void serializeField(JsonGenerator g, final Object value) + throws IOException { + if (value instanceof Map) { + serialize((Map<String, Object>) value, g); + } else if (value instanceof Number) { + g.writeNumber(((Number) value).intValue()); + } else if (value instanceof String) { + g.writeString((String) value); + } else if (value instanceof List) { + g.writeStartArray(); + for (Object o : (List<Object>) value) { + serializeField(g, o); + } + g.writeEndArray(); + } else { + throw new IllegalArgumentException(); + } + } + + @Override + Object category() { + return Strategy.JACKSON; + } + + } + + private static final class OutputWithWriter extends Bench { + + OutputWithWriter() { + } + + int iterate(Random random) { + ByteArrayOutputStream writerOut = new ByteArrayOutputStream(); + JSONWriter writer = new JSONWriter(writerOut); + try { + serialize(generatedDoc(random), writer); + } catch (IOException e) { + e.printStackTrace(); + return 0; + } + return writerOut.toByteArray().length; + } + + static void serialize(Map<String, Object> m, JSONWriter w) throws IOException { + w.beginObject(); + for (Map.Entry<String, Object> e : m.entrySet()) { + w.beginField(e.getKey()); + final Object value = e.getValue(); + serializeField(w, value); + w.endField(); + } + w.endObject(); + } + + @SuppressWarnings("unchecked") + static void serializeField(JSONWriter w, final Object value) + throws IOException { + if (value instanceof Map) { + serialize((Map<String, Object>) value, w); + } else if (value instanceof Number) { + w.value((Number) value); + } else if (value instanceof String) { + w.value((String) value); + } else if (value instanceof List) { + w.beginArray(); + for (Object o : (List<Object>) value) { + w.beginArrayValue(); + serializeField(w, o); + w.endArrayValue(); + } + w.endArray(); + } else { + throw new IllegalArgumentException(); + } + } + + @Override + Object category() { + return Strategy.VESPAJLIB; + } + + } + + @Test + @Ignore + public final void test() throws InterruptedException { + final OutputWithWriter forWriter = new OutputWithWriter(); + Thread writerThread = new Thread(forWriter); + final OutputWithGenerator forGenerator = new OutputWithGenerator(); + Thread generatorThread = new Thread(forGenerator); + writerThread.start(); + generatorThread.start(); + writerThread.join(); + generatorThread.join(); + System.out.println("Generator time: " + (forGenerator.end - forGenerator.start)); + System.out.println("Writer time: " + (forWriter.end - forWriter.start)); + System.out.println("Output length from generator: " + forGenerator.metric); + System.out.println("Output length from writer: " + forWriter.metric); + System.out.println("Iterations with generator: " + forGenerator.runs); + System.out.println("Iterations with writer: " + forWriter.runs); + System.out.println("Iterations/s with generator: " + ((double) forGenerator.runs / (double) (forGenerator.end - forGenerator.start)) * 1000.0d); + System.out.println("Iterations/s with writer: " + ((double) forWriter.runs / (double) (forWriter.end - forWriter.start)) * 1000.0d); + } + + @Test + @Ignore + public final void test16Threads() throws InterruptedException { + List<Thread> threads = new ArrayList<>(16); + List<Bench> benches = createBenches(8, new VespajlibFactory(), new JacksonFactory()); + + for (Bench bench : benches) { + threads.add(new Thread(bench)); + } + for (Thread t : threads) { + t.start(); + } + for (Thread t : threads) { + t.join(); + } + + System.out.println("8 Jackson threads competing with 8 VespaJLib threads."); + metrics(benches, Strategy.JACKSON); + metrics(benches, Strategy.VESPAJLIB); + } + + @Test + @Ignore + public final void test16ThreadsJacksonOnly() throws InterruptedException { + List<Thread> threads = new ArrayList<>(16); + List<Bench> benches = createBenches(16, new JacksonFactory()); + + for (Bench bench : benches) { + threads.add(new Thread(bench)); + } + for (Thread t : threads) { + t.start(); + } + for (Thread t : threads) { + t.join(); + } + + System.out.println("16 Jackson threads."); + metrics(benches, Strategy.JACKSON); + } + + @Test + @Ignore + public final void test16ThreadsVespaJlibOnly() throws InterruptedException { + List<Thread> threads = new ArrayList<>(16); + List<Bench> benches = createBenches(16, new VespajlibFactory()); + + for (Bench bench : benches) { + threads.add(new Thread(bench)); + } + for (Thread t : threads) { + t.start(); + } + for (Thread t : threads) { + t.join(); + } + + System.out.println("16 VespaJLib threads."); + metrics(benches, Strategy.VESPAJLIB); + } + + + private void metrics(List<Bench> benches, Strategy choice) { + List<Bench> chosen = new ArrayList<>(); + + for (Bench b : benches) { + if (b.category() == choice) { + chosen.add(b); + } + } + + long[] rawTime = new long[chosen.size()]; + long[] rawOutputLength = new long[chosen.size()]; + long[] rawIterations = new long[chosen.size()]; + double[] rawIterationsPerSecond = new double[chosen.size()]; + + for (int i = 0; i < chosen.size(); ++i) { + Bench b = chosen.get(i); + rawTime[i] = b.end - b.start; + rawOutputLength[i] = b.metric; + rawIterations[i] = b.runs; + rawIterationsPerSecond[i] = ((double) b.runs) / (((double) (b.end - b.start)) / 1000.0d); + } + + double avgTime = mean(rawTime); + double avgOutputLength = mean(rawOutputLength); + double avgIterations = mean(rawIterations); + double avgIterationsPerSecond = mean(rawIterationsPerSecond); + + System.out.println("For " + choice + ":"); + dumpMetric("run time", rawTime, avgTime, "s", 0.001d); + dumpMetric("output length", rawOutputLength, avgOutputLength, "bytes", 1.0d); + dumpMetric("iterations", rawIterations, avgIterations, "", 1.0d); + dumpMetric("iterations per second", rawIterationsPerSecond, avgIterationsPerSecond, "s**-1", 1.0d); + } + + private void dumpMetric(String name, long[] raw, double mean, String unit, double scale) { + System.out.println("Average " + name + ": " + mean * scale + " " + unit); + System.out.println("Mean absolute deviation of " + name + ": " + averageAbsoluteDeviationFromMean(raw, mean) * scale + " " + unit); + System.out.println("Minimum " + name + ": " + min(raw) * scale + " " + unit); + System.out.println("Maximum " + name + ": " + max(raw) * scale + " " + unit); + } + + private void dumpMetric(String name, double[] raw, double mean, String unit, double scale) { + System.out.println("Average " + name + ": " + mean * scale + " " + unit); + System.out.println("Mean absolute deviation of " + name + ": " + averageAbsoluteDeviationFromMean(raw, mean) * scale + " " + unit); + System.out.println("Minimum " + name + ": " + min(raw) * scale + " " + unit); + System.out.println("Maximum " + name + ": " + max(raw) * scale + " " + unit); + } + + private List<Bench> createBenches(int ofEach, BenchFactory... factories) { + List<Bench> l = new ArrayList<>(ofEach * factories.length); + + // note how the bench objects of different objects become intermingled, this is by design + for (int i = 0; i < ofEach; ++i) { + for (BenchFactory factory : factories) { + l.add(factory.produce()); + } + } + return l; + } + + private double mean(long[] values) { + long sum = 0L; + + // ignore overflow :) + for (long v : values) { + sum += v; + } + return ((double) sum / (double) values.length); + } + + private double mean(double[] values) { + double sum = 0L; + + for (double v : values) { + sum += v; + } + return sum / (double) values.length; + } + + private double averageAbsoluteDeviationFromMean(long[] values, double mean) { + double sum = 0.0d; + + for (long v : values) { + sum += Math.abs(mean - (double) v); + } + + return sum / (double) values.length; + } + + private double averageAbsoluteDeviationFromMean(double[] values, double mean) { + double sum = 0.0d; + + for (double v : values) { + sum += Math.abs(mean - v); + } + + return sum / (double) values.length; + } + + private long min(long[] values) { + long min = Long.MAX_VALUE; + + for (long v : values) { + min = Math.min(min, v); + } + return min; + } + + private double min(double[] values) { + double min = Double.MAX_VALUE; + + for (double v : values) { + min = Math.min(min, v); + } + return min; + } + + private long max(long[] values) { + long max = Long.MIN_VALUE; + + for (long v : values) { + max = Math.max(max, v); + } + return max; + } + + private double max(double[] values) { + double max = Double.MIN_VALUE; + + for (double v : values) { + max = Math.max(max, v); + } + return max; + } + + @SuppressWarnings("null") + @Test + @Ignore + public final void testSanity() throws IOException { + @SuppressWarnings("unused") + String a, b; + { + Random random = new Random(42L); + JsonGenerator generator = null; + ByteArrayOutputStream generatorOut = new ByteArrayOutputStream(); + try { + generator = new JsonFactory().createJsonGenerator(generatorOut, + JsonEncoding.UTF8); + } catch (IOException e) { + e.printStackTrace(); + fail(); + } + try { + OutputWithGenerator.serialize(generatedDoc(random), generator); + } catch (IOException e) { + e.printStackTrace(); + fail(); + } + try { + generator.close(); + } catch (IOException e) { + e.printStackTrace(); + fail(); + } + a = generatorOut.toString("UTF-8"); + } + { + Random random = new Random(42L); + ByteArrayOutputStream writerOut = new ByteArrayOutputStream(); + JSONWriter writer = new JSONWriter(writerOut); + try { + OutputWithWriter.serialize(generatedDoc(random), writer); + } catch (IOException e) { + e.printStackTrace(); + fail(); + } + b = writerOut.toString("UTF-8"); + } + // dumpToFile("/tmp/a", a); + // dumpToFile("/tmp/b", b); + } + + @SuppressWarnings("unused") + private void dumpToFile(String path, String b) throws IOException { + FileWriter f = new FileWriter(path); + f.write(b); + f.close(); + } + + static Map<String, Object> generatedDoc(Random random) { + return generateObject(random, 0, random.nextInt(8)); + } + + static String generateFieldName(Random random) { + int len = random.nextInt(100) + 3; + char[] base = new char[len]; + for (int i = 0; i < len; ++i) { + base[i] = (char) (random.nextInt(26) + 'a'); + } + return new String(base); + } + + static byte[] generateByteArrayPayload(Random random) { + return null; + } + + static String generateStringPayload(Random random) { + int len = random.nextInt(100) + random.nextInt(100) + random.nextInt(100) + random.nextInt(100); + char[] base = new char[len]; + for (int i = 0; i < len; ++i) { + base[i] = (char) random.nextInt(0xd800); + } + return new String(base); + } + + static Number generateInt(Random random) { + return Integer.valueOf(random.nextInt()); + } + + static List<Object> generateArray(Random random, int nesting, int maxNesting) { + int len = random.nextInt(10) + random.nextInt(10) + random.nextInt(10) + random.nextInt(10); + List<Object> list = new ArrayList<>(len); + for (int i = 0; i < len; ++i) { + list.add(generateStuff(random, nesting, maxNesting)); + } + return list; + } + + private static Object generateStuff(Random random, int nesting, int maxNesting) { + if (nesting >= maxNesting) { + return generatePrimitive(random); + } else { + final int die = random.nextInt(10); + if (die == 9) { + return generateObject(random, nesting + 1, maxNesting); + } else if (die == 8) { + return generateArray(random, nesting + 1, maxNesting); + } else { + return generatePrimitive(random); + } + } + } + + private static Object generatePrimitive(Random random) { + if (random.nextInt(2) == 0) { + return generateStringPayload(random); + } else { + return generateInt(random); + } + } + + static Map<String, Object> generateObject(Random random, int nesting, int maxNesting) { + int len = random.nextInt(5) + random.nextInt(5) + random.nextInt(5) + random.nextInt(5); + Map<String, Object> m = new TreeMap<>(); + for (int i = 0; i < len; ++i) { + m.put(generateFieldName(random), generateStuff(random, nesting, maxNesting)); + } + return m; + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/LanguageHacksTestCase.java b/vespajlib/src/test/java/com/yahoo/text/LanguageHacksTestCase.java new file mode 100644 index 00000000000..7b322e44583 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/LanguageHacksTestCase.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Test; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Test for LanguageHacks. + * $Id$ + */ +@SuppressWarnings("deprecation") +public class LanguageHacksTestCase { + + @Test + public void isCJK() { + assertFalse("NULL language", LanguageHacks.isCJK(null)); + assertTrue(LanguageHacks.isCJK("zh")); + assertFalse("Norwegian is CJK", LanguageHacks.isCJK("no")); + } + + @Test + public void yellDesegments() { + assertFalse("NULL language", LanguageHacks.yellDesegments(null)); + assertTrue(LanguageHacks.yellDesegments("de")); + assertFalse("Norwegian desegments", LanguageHacks.yellDesegments("no")); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/text/LowercaseIdentifierTestCase.java b/vespajlib/src/test/java/com/yahoo/text/LowercaseIdentifierTestCase.java new file mode 100644 index 00000000000..7d6b066a499 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/LowercaseIdentifierTestCase.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 11.11.12 + * Time: 20:54 + * To change this template use File | Settings | File Templates. + */ +public class LowercaseIdentifierTestCase { + @Test + public void testLowercaseIdentifier() { + assertEquals(new LowercaseIdentifier("").toString(), ""); + assertEquals(new LowercaseIdentifier("a").toString(), "a"); + assertEquals(new LowercaseIdentifier("z").toString(), "z"); + assertEquals(new LowercaseIdentifier("_").toString(), "_"); + try { + assertEquals(new Identifier("0").toString(), "0"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Illegal starting character '0' of identifier '0'."); + } + try { + assertEquals(new LowercaseIdentifier("Z").toString(), "z"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Illegal uppercase character 'Z' of identifier 'Z'."); + } + try { + assertEquals(new LowercaseIdentifier("aZb").toString(), "azb"); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Illegal uppercase character 'Z' of identifier 'aZb'."); + } + + + + } +} diff --git a/vespajlib/src/test/java/com/yahoo/text/LowercaseTestCase.java b/vespajlib/src/test/java/com/yahoo/text/LowercaseTestCase.java new file mode 100644 index 00000000000..420d058892b --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/LowercaseTestCase.java @@ -0,0 +1,96 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Ignore; +import org.junit.Test; + +import java.util.Locale; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.14 + */ +public class LowercaseTestCase { + + @Test + public void testAZ() { + { + String lowercase = Lowercase.toLowerCase("ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + assertThat(lowercase, equalTo("abcdefghijklmnopqrstuvwxyz")); + } + { + String lowercase = Lowercase.toLowerCase("abcdefghijklmnopqrstuvwxyz"); + assertThat(lowercase, equalTo("abcdefghijklmnopqrstuvwxyz")); + } + { + String lowercase = Lowercase.toLowerCase("AbCDEfGHIJklmnoPQRStuvwXyz"); + assertThat(lowercase, equalTo("abcdefghijklmnopqrstuvwxyz")); + } + + { + String lowercase = Lowercase.toLowerCase("@+#"); + assertThat(lowercase, equalTo("@+#")); + } + { + String lowercase = Lowercase.toLowerCase("[]"); + assertThat(lowercase, equalTo("[]")); + } + { + String lowercase = Lowercase.toLowerCase("{}"); + assertThat(lowercase, equalTo("{}")); + } + { + String lowercase = Lowercase.toLowerCase("\u00cd\u00f4"); + assertThat(lowercase, equalTo("\u00ed\u00f4")); + } + } + + @Test + @Ignore + public void performance() { + Lowercase.toLowerCase("warmup"); + String lowercaseInput = "abcdefghijklmnopqrstuvwxyz"; + String uppercaseInput = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + String mixedcaseInput = "AbCDEfGHIJklmnoPQRStuvwXyz"; + + System.err.println("Lowercase input: "); + testPerformance(lowercaseInput); + + System.err.println("Uppercase input: "); + testPerformance(uppercaseInput); + + System.err.println("Mixed-case input: "); + testPerformance(mixedcaseInput); + } + + private void testPerformance(String input) { + final int NUM = 10000000; + long elapsedTimeOwnImpl; + { + long startTimeOwnImpl = System.currentTimeMillis(); + for (int i = 0; i < NUM; i++) { + Lowercase.toLowerCase(input); + } + elapsedTimeOwnImpl = System.currentTimeMillis() - startTimeOwnImpl; + System.err.println("Own implementation: " + elapsedTimeOwnImpl); + } + + long elapsedTimeJava; + { + long startTimeJava = System.currentTimeMillis(); + for (int i = 0; i < NUM; i++) { + input.toLowerCase(Locale.ENGLISH); + } + elapsedTimeJava = System.currentTimeMillis() - startTimeJava; + System.err.println("Java's implementation: " + elapsedTimeJava); + } + + long diff = elapsedTimeJava - elapsedTimeOwnImpl; + double diffPercentage = (((double) diff) / ((double) elapsedTimeJava)) * 100.0; + System.err.println("Own implementation is " + diffPercentage + " % faster."); + + } +} diff --git a/vespajlib/src/test/java/com/yahoo/text/MapParserMicroBenchmark.java b/vespajlib/src/test/java/com/yahoo/text/MapParserMicroBenchmark.java new file mode 100644 index 00000000000..21ab4fd4309 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/MapParserMicroBenchmark.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.util.HashMap; +import java.util.Map; + +/** + * A benchmark of map parsing. + * Expected time on Jon's mac: 200 microseconds per 1k size map. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class MapParserMicroBenchmark { + + private static String generateValues(int size) { + StringBuilder b = new StringBuilder("{"); + for (int i=0; i<size; i++) + b.append("a").append(i).append(":").append(i+1).append(","); + b.setLength(b.length() - 1); + b.append("}"); + return b.toString(); + } + + public void benchmark(int repetitions,int mapSize) { + String values = generateValues(mapSize); + System.out.println("Ranking expression parsing"); + System.out.println(" warming up"); + rankingExpressionParserParse(1000, values, mapSize); + long startTime = System.currentTimeMillis(); + System.out.println( "starting ...."); + rankingExpressionParserParse(repetitions, values, mapSize); + long totalTime = System.currentTimeMillis() - startTime; + System.out.println(" Total time: " + totalTime + " ms, time per expression: " + (totalTime*1000/repetitions) + " microseconds"); + } + + private void rankingExpressionParserParse(int repetitions, String values, int expectedMapSize) { + Map<String,Double> map = new HashMap<>(); + for (int i=0; i<repetitions; i++) { + rankingExpressionParserParse(values, map); + if ( map.size()!=expectedMapSize) + throw new RuntimeException("Expected size: " + expectedMapSize + ", actual size: " + map.size()); + map.clear(); + } + } + private Map<String,Double> rankingExpressionParserParse(String values, Map<String,Double> map) { + return new DoubleMapParser().parse(values,map); + } + + public static void main(String[] args) { + new MapParserMicroBenchmark().benchmark(100*1000,1000); + } + + private static class DoubleMapParser extends MapParser<Double> { + + @Override + protected Double parseValue(String s) { + return Double.parseDouble(s); + } + + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/MapParserTestCase.java b/vespajlib/src/test/java/com/yahoo/text/MapParserTestCase.java new file mode 100644 index 00000000000..7bf11c277e1 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/MapParserTestCase.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Test; +import java.util.Map; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class MapParserTestCase { + + private static final double delta=0.0001; + + @Test + public void testEmpty() { + assertEquals(0, new DoubleMapParser().parseToMap("{}").size()); + } + + @Test + public void testPlain() { + Map<String,Double> values=new DoubleMapParser().parseToMap("{a:0.33,foo:-1.13,bar:1}"); + assertEquals(3, values.size()); + assertEquals(0.33d,values.get("a"),delta); + assertEquals(-1.13d,values.get("foo"),delta); + assertEquals(1.0d,values.get("bar"),delta); + } + + @Test + public void testNoisy() { + Map<String,Double> values=new DoubleMapParser().parseToMap(" { a:0.33, foo:-1.13,bar:1,\"key:colon,\":1.2, '}':0}"); + assertEquals(5, values.size()); + assertEquals(0.33d,values.get("a"),delta); + assertEquals(-1.13d,values.get("foo"),delta); + assertEquals(1.0d,values.get("bar"),delta); + assertEquals(1.2,values.get("key:colon,"),delta); + assertEquals(0,values.get("}"),delta); + } + + @Test + public void testInvalid() { + assertException("Missing quoted string termination","Expected a string terminated by '\"' starting at position 9 but was 'f'","{a:0.33,\"foo:1,bar:1}"); + assertException("Missing map termination","Expected a value followed by ',' or '}' starting at position 10 but was '1'","{a:0.33,b:1"); + assertException("Missing map start","Expected '{' starting at position 0 but was 'a'","a:0.33,b:1}"); + assertException("Missing comma separator","Expected a legal value from position 3 to 11 but was '0.33 b:1'","{a:0.33 b:1}"); + assertException("A single key with no value","Expected a key followed by ':' starting at position 1 but was 'f'","{foo}"); + assertException("A key with no value","Expected ':' starting at position 4 but was ','","{foo,a:2}"); + assertException("Invalid value","Expected a legal value from position 9 to 19 but was 'notanumber'","{invalid:notanumber}"); + assertException("Double key","Expected a legal value from position 3 to 6 but was 'a:1'","{a:a:1}"); + } + + private void assertException(String explanation,String exceptionString,String invalidMapString) { + try { + Map<String,Double> map=new DoubleMapParser().parseToMap(invalidMapString); + fail("Expected exception on: " + explanation + " but parsed to " + map); + } + catch (IllegalArgumentException e) { + assertEquals("Expected message on: " + explanation,exceptionString,e.getCause().getMessage()); + } + } + + public static final class DoubleMapParser extends MapParser<Double> { + + @Override + protected Double parseValue(String value) { + return Double.parseDouble(value); + } + + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/StringAppendMicroBenchmarkTest.java b/vespajlib/src/test/java/com/yahoo/text/StringAppendMicroBenchmarkTest.java new file mode 100644 index 00000000000..69d62d59be5 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/StringAppendMicroBenchmarkTest.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Test; + +/** + * Compares alternative ways of appending strings + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class StringAppendMicroBenchmarkTest { + + private static abstract class Benchmark { + + private int repetitions=10000000; + + public void execute() { + System.out.println("Executing benchmark '" + getName() + "' ..."); + append(100000); // warm-up + long start=System.currentTimeMillis(); + append(repetitions); + long duration=System.currentTimeMillis()-start; + System.out.println("Completed " + repetitions + " repetitions in " + duration + " ms\n"); + } + + private int append(int repetitions) { + String prefix="hello"; + int totalSize=0; + for (int i=0; i<repetitions; i++) { + String full=appendStrings(prefix, String.valueOf(i)); + totalSize+=full.length(); + } + return totalSize; + } + + protected abstract String getName(); + protected abstract String appendStrings(String a,String b); + + } + + private static final class PlusOperatorBenchmark extends Benchmark { + + @Override + protected String getName() { return "Plus operator"; } + + @Override + protected String appendStrings(String a, String b) { + return a+b; + } + + } + + private static final class StringConcatBenchmark extends Benchmark { + + @Override + protected String getName() { return "String concat"; } + + @Override + protected String appendStrings(String a, String b) { + return a.concat(b); + } + + } + + /** + * Make Clover shut up about this in the coverage report. + */ + @Test + public void shutUpClover() { + } + + public static void main(String[] args) { + new PlusOperatorBenchmark().execute(); // Typical number on my box with Java 7: 1000 ms + new StringConcatBenchmark().execute(); // Typical number on my box with Java 7: 1150 ms + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/StringUtilitiesTest.java b/vespajlib/src/test/java/com/yahoo/text/StringUtilitiesTest.java new file mode 100644 index 00000000000..bebee69e7e5 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/StringUtilitiesTest.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import java.util.Arrays; + +import org.junit.Test; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +public class StringUtilitiesTest { + + @Test + public void testEscape() { + assertEquals("abz019ABZ", StringUtilities.escape("abz019ABZ")); + assertEquals("\\t", StringUtilities.escape("\t")); + assertEquals("\\n", StringUtilities.escape("\n")); + assertEquals("\\r", StringUtilities.escape("\r")); + assertEquals("\\\"", StringUtilities.escape("\"")); + assertEquals("\\f", StringUtilities.escape("\f")); + assertEquals("\\\\", StringUtilities.escape("\\")); + assertEquals("\\x05", StringUtilities.escape("" + (char) 5)); + assertEquals("\\tA\\ncombined\\r\\x055test", StringUtilities.escape("\tA\ncombined\r" + ((char) 5) + "5test")); + assertEquals("A\\x20space\\x20separated\\x20string", StringUtilities.escape("A space separated string", ' ')); + } + + @Test + public void testUnescape() { + assertEquals("abz019ABZ", StringUtilities.unescape("abz019ABZ")); + assertEquals("\t", StringUtilities.unescape("\\t")); + assertEquals("\n", StringUtilities.unescape("\\n")); + assertEquals("\r", StringUtilities.unescape("\\r")); + assertEquals("\"", StringUtilities.unescape("\\\"")); + assertEquals("\f", StringUtilities.unescape("\\f")); + assertEquals("\\", StringUtilities.unescape("\\\\")); + assertEquals("" + (char) 5, StringUtilities.unescape("\\x05")); + assertEquals("\tA\ncombined\r" + ((char) 5) + "5test", StringUtilities.unescape("\\tA\\ncombined\\r\\x055test")); + assertEquals("A space separated string", StringUtilities.unescape("A\\x20space\\x20separated\\x20string")); + } + + @Test + public void testImplode() { + assertEquals(StringUtilities.implode(null, null), null); + assertEquals(StringUtilities.implode(new String[0], null), ""); + assertEquals(StringUtilities.implode(new String[] {"foo"}, null), "foo"); + assertEquals(StringUtilities.implode(new String[] {"foo"}, "asdfsdfsadfsadfasdfs"), "foo"); + assertEquals(StringUtilities.implode(new String[] {"foo", "bar"}, null), "foobar"); + assertEquals(StringUtilities.implode(new String[] {"foo", "bar"}, "\n"), "foo\nbar"); + assertEquals(StringUtilities.implode(new String[] {"foo"}, "\n"), "foo"); + assertEquals(StringUtilities.implode(new String[] {"foo", "bar", null}, "\n"), "foo\nbar\nnull"); + assertEquals(StringUtilities.implode(new String[] {"foo", "bar"}, "\n"), "foo\nbar"); + assertEquals(StringUtilities.implode(new String[] {"foo", "bar", "baz"}, null), "foobarbaz"); + + } + + @Test + public void testImplodeMultiline() { + assertEquals(StringUtilities.implodeMultiline(Arrays.asList("foo", "bar")), "foo\nbar"); + assertEquals(StringUtilities.implodeMultiline(Arrays.asList("")), ""); + assertEquals(StringUtilities.implodeMultiline(null), null); + assertEquals(StringUtilities.implodeMultiline(Arrays.asList("\n")), "\n"); + } + + @Test + public void testTruncation() { + String a = "abbc"; + assertTrue(a == StringUtilities.truncateSequencesIfNecessary(a, 2)); + assertTrue(a != StringUtilities.truncateSequencesIfNecessary(a, 1)); + assertEquals("abc", StringUtilities.truncateSequencesIfNecessary(a, 1)); + assertEquals("abc", StringUtilities.truncateSequencesIfNecessary("aabbccc", 1)); + assertEquals("abc", StringUtilities.truncateSequencesIfNecessary("abcc", 1)); + assertEquals("abc", StringUtilities.truncateSequencesIfNecessary("aabc", 1)); + assertEquals("abcb", StringUtilities.truncateSequencesIfNecessary("abcb", 1)); + assertEquals("g g g g g g g g g g\n g g g g g g g g g g\n g g g g g g g g g g", StringUtilities.truncateSequencesIfNecessary("g g g g g g g g g g\n g g g g g g g g g g\n g g g g g g g g g g", 5)); + } + + + @Test + public void testStripSuffix() { + assertThat(StringUtilities.stripSuffix("abc.def", ".def"), is("abc")); + assertThat(StringUtilities.stripSuffix("abc.def", ""), is("abc.def")); + assertThat(StringUtilities.stripSuffix("", ".def"), is("")); + assertThat(StringUtilities.stripSuffix("", ""), is("")); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/text/Utf8ArrayTestCase.java b/vespajlib/src/test/java/com/yahoo/text/Utf8ArrayTestCase.java new file mode 100644 index 00000000000..4195113e2e1 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/Utf8ArrayTestCase.java @@ -0,0 +1,167 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import static org.junit.Assert.*; + +import java.nio.ByteBuffer; + +import org.junit.Before; +import org.junit.Test; + +/** + * Check the Utf8Array API behaves as expected. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class Utf8ArrayTestCase { + private String raw; + private byte[] rawBytes; + private Utf8Array toCheck; + + @Before + public void setUp() { + raw = "0123456789"; + rawBytes = Utf8.toBytes(raw); + toCheck = new Utf8Array(rawBytes); + } + + @Test + public final void testGetByteLength() { + assertEquals(rawBytes.length, toCheck.getByteLength()); + } + + @Test + public final void testGetBytes() { + assertSame(rawBytes, toCheck.getBytes()); + } + + @Test + public final void testGetByteOffset() { + assertEquals(0, toCheck.getByteOffset()); + } + + @Test + public final void testUtf8ArrayByteArrayIntInt() { + Utf8Array otherConstructed = new Utf8Array(rawBytes, 0, rawBytes.length); + assertNotSame(rawBytes, otherConstructed.getBytes()); + assertArrayEquals(rawBytes, otherConstructed.getBytes()); + } + + @Test + public final void testUtf8ArrayByteBufferInt() { + final ByteBuffer wrapper = ByteBuffer.wrap(rawBytes); + Utf8Array otherConstructed = new Utf8Array(wrapper, wrapper.remaining()); + assertNotSame(rawBytes, otherConstructed.getBytes()); + assertArrayEquals(rawBytes, otherConstructed.getBytes()); + } + + @Test + public final void testHashCode() { + Utf8Array other = new Utf8Array(Utf8.toBytes(" a23456789")); + assertFalse(other.hashCode() == toCheck.hashCode()); + } + + @Test + public final void testWriteTo() { + ByteBuffer b = ByteBuffer.allocate(rawBytes.length * 2); + byte[] copy = new byte[rawBytes.length]; + toCheck.writeTo(b); + assertEquals(rawBytes.length, b.position()); + b.position(0); + b.limit(rawBytes.length); + b.get(copy); + assertArrayEquals(rawBytes, copy); + } + + @Test + public final void testGetByte() { + assertEquals('8', toCheck.getByte(8)); + } + + @Test + public final void testWrap() { + ByteBuffer b1 = toCheck.wrap(); + ByteBuffer b2 = ByteBuffer.wrap(rawBytes); + byte[] c1 = new byte[b1.remaining()]; + byte[] c2 = new byte[b2.remaining()]; + b1.get(c1); + b2.get(c2); + assertArrayEquals(c2, c1); + } + + @Test + public final void testIsEmpty() { + assertFalse(toCheck.isEmpty()); + assertTrue(new Utf8Array(new byte[] {}).isEmpty()); + } + + @Test + public final void testEqualsObject() { + assertTrue(toCheck.equals(raw)); + assertFalse(toCheck.equals(new Utf8Array(new byte[] {}))); + assertFalse(toCheck.equals(new Utf8Array(Utf8.toBytes(" " + raw.substring(1))))); + assertTrue(toCheck.equals(toCheck)); + assertTrue(toCheck.equals(new Utf8Array(rawBytes))); + } + + @Test + public final void testToString() { + assertEquals(raw, toCheck.toString()); + } + + @Test + public final void testCompareTo() { + assertTrue(toCheck.compareTo(new Utf8Array(new byte[] {})) > 0); + assertTrue(toCheck.compareTo(new Utf8Array(Utf8.toBytes(raw + raw))) < 0); + assertTrue(toCheck.compareTo(new Utf8Array(Utf8.toBytes(" " + raw.substring(1)))) > 0); + assertTrue(toCheck.compareTo(new Utf8Array(Utf8.toBytes("a" + raw.substring(1)))) < 0); + assertTrue(toCheck.compareTo(new Utf8Array(rawBytes)) == 0); + } + + @Test + public final void testPartial() { + final int length = 3; + final int offset = 1; + Utf8PartialArray partial = new Utf8PartialArray(rawBytes, offset, length); + assertEquals(length, partial.getByteLength()); + assertEquals(offset, partial.getByteOffset()); + byte[] expected = new byte[length]; + ByteBuffer intermediate = ByteBuffer.allocate(rawBytes.length * 2); + System.arraycopy(rawBytes, offset, expected, 0, length); + partial.writeTo(intermediate); + intermediate.flip(); + byte written[] = new byte[intermediate.remaining()]; + intermediate.get(written); + assertArrayEquals(expected, written); + } + + @Test + public final void testUtf8Strings() { + String nalle = "nalle"; + Utf8String utf = new Utf8String(new Utf8Array(Utf8.toBytes(nalle))); + assertEquals('n', utf.charAt(0)); + assertEquals(nalle.length(), utf.length()); + assertEquals("alle", utf.subSequence(1, 5)); + assertTrue(utf.equals(new Utf8String(new Utf8Array(Utf8.toBytes(nalle))))); + assertTrue(utf.equals(nalle)); + } + + @Test + public final void testAscii7bitLowercase() { + final byte [] expected = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06 ,0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16 ,0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26 ,0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36 ,0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66 ,0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76 ,0x77, 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66 ,0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76 ,0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f + }; + byte [] org = new byte[128]; + for (byte b = 0; b >= 0; b++) { + org[b] = b; + } + assertArrayEquals(expected, new Utf8Array(org).ascii7BitLowerCase().getBytes()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java b/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java new file mode 100644 index 00000000000..53ee1bb004a --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/Utf8TestCase.java @@ -0,0 +1,554 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Ignore; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.CharsetEncoder; +import java.util.Arrays; + +import static com.yahoo.text.Lowercase.toLowerCase; +import static com.yahoo.text.Utf8.calculateBytePositions; +import static com.yahoo.text.Utf8.calculateStringPositions; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class Utf8TestCase { + + private static final String TEST_STRING = "This is just sort of random mix. \u5370\u57df\u60c5\u5831\u53EF\u4EE5\u6709x\u00e9\u00e8"; + private static final int[] TEST_CODEPOINTS = {0x0, 0x7f, 0x80, 0x7ff, 0x800, 0xd7ff, 0xe000, 0xffff, 0x10000, 0x10ffff, + 0x34, 0x355, 0x2567, 0xfff, 0xe987, 0x100abc + }; + + public void dumpSome() throws java.io.IOException { + int i = 32; + int j = 3; + int cnt = 0; + while (i < 0x110000) { + if (i < 0xD800 || i >= 0xE000) ++cnt; + i += j; + ++j; + } + System.out.println("allocate "+cnt+" array entries"); + int codes[] = new int[cnt]; + i = 32; + j = 3; + cnt = 0; + while (i < 0x110000) { + if (i < 0xD800 || i >= 0xE000) codes[cnt++] = i; + i += j; + ++j; + } + assertEquals(cnt, codes.length); + System.out.println("fill "+cnt+" array entries"); + String str = new String(codes, 0, cnt); + byte[] arr = Utf8.toBytes(str); + java.io.FileOutputStream fos = new java.io.FileOutputStream("random-long-utf8.dat"); + fos.write(arr); + fos.close(); + } + + public void dumpMore() throws java.io.IOException { + java.text.Normalizer.Form form = java.text.Normalizer.Form.NFKC; + + java.io.FileOutputStream fos = new java.io.FileOutputStream("lowercase-table.dat"); + for (int i = 0; i < 0x110000; i++) { + StringBuilder b = new StringBuilder(); + b.appendCodePoint(i); + String n1 = b.toString(); + String n2 = java.text.Normalizer.normalize(b, form); + if (n1.equals(n2)) { + String l = toLowerCase(n1); + int chars = l.length(); + int codes = l.codePointCount(0, chars); + if (codes != 1) { + System.out.println("codepoint "+i+" transformed into "+codes+" codepoints: "+n1+" -> "+l); + } else { + int lc = l.codePointAt(0); + if (lc != i) { + String o = "lowercase( "+i+" )= "+lc+"\n"; + byte[] arr = Utf8.toBytes(o); + fos.write(arr); + } + } + } + } + fos.close(); + } + + @Test + public void testSimple() { + String s1 = "test"; + String s2 = "f\u00F8rst"; + String s3 = "\u00C5pen"; + byte[] b4 = { (byte) 0xE5, (byte) 0xA4, (byte) 0x89, (byte) 0xE6, + (byte) 0x85, (byte) 0x8B }; + + byte[] b1 = Utf8.toBytes(s1); + byte[] b2 = Utf8.toBytes(s2); + byte[] b3 = Utf8.toBytes(s3); + String s4 = Utf8.toString(b4); + + assertEquals('t', b1[0]); + assertEquals('e', b1[1]); + assertEquals('s', b1[2]); + assertEquals('t', b1[3]); + + assertEquals('f', b2[0]); + assertEquals((byte) 0xC3, b2[1]); + assertEquals((byte) 0xB8, b2[2]); + assertEquals('r', b2[3]); + assertEquals('s', b2[4]); + assertEquals('t', b2[5]); + + assertEquals((byte) 0xC3, b3[0]); + assertEquals((byte) 0x85, b3[1]); + assertEquals('p', b3[2]); + assertEquals('e', b3[3]); + assertEquals('n', b3[4]); + + assertEquals('\u5909', s4.charAt(0)); + assertEquals('\u614B', s4.charAt(1)); + + String ss1 = Utf8.toString(b1); + String ss2 = Utf8.toString(b2); + String ss3 = Utf8.toString(b3); + byte[] bb4 = Utf8.toBytes(s4); + + assertEquals(s1, ss1); + assertEquals(s3, ss3); + assertEquals(s2, ss2); + assertEquals(Utf8.toString(b4), Utf8.toString(bb4)); + } + + private int javaCountBytes(String str) { + byte[] octets = Utf8.toBytes(str); + return octets.length; + } + + private String makeString(int codePoint) { + char[] chars = Character.toChars(codePoint); + return String.valueOf(chars); + } + + @Test + public void testByteCounting() { + for (int c : TEST_CODEPOINTS) { + String testCharacter = makeString(c); + assertEquals(javaCountBytes(testCharacter), Utf8.byteCount(testCharacter)); + } + assertEquals(javaCountBytes(TEST_STRING), Utf8.byteCount(TEST_STRING)); + } + + @Test + public void testTotalBytes() { + //Test with a random mix of + assertEquals(1,Utf8.totalBytes((byte)0x05)); + assertEquals(4,Utf8.totalBytes((byte)0xF3)); + assertEquals(4,Utf8.totalBytes((byte)0xF0)); + assertEquals(1,Utf8.totalBytes((byte)0x7F)); + assertEquals(2,Utf8.totalBytes((byte)0xC2)); + assertEquals(3,Utf8.totalBytes((byte)0xE0)); + } + + @Test + public void testUnitCounting() { + for (int c : TEST_CODEPOINTS) { + String testCharacter = makeString(c); + byte[] utf8 = Utf8.toBytes(testCharacter); + assertEquals(testCharacter.length(), Utf8.unitCount(utf8)); + assertEquals(testCharacter.length(), Utf8.unitCount(utf8[0])); + } + byte[] stringAsUtf8 = Utf8.toBytes(TEST_STRING); + assertEquals(TEST_STRING.length(), Utf8.unitCount(stringAsUtf8)); + + + } + + @Test + public void testCumbersomeEncoding() { + String[] a = {"abc", "def", "ghi\u00e8"}; + int[] aLens = {3, 3, 5}; + CharsetEncoder ce = Utf8.getNewEncoder(); + ByteBuffer forWire = ByteBuffer.allocate(500); + + for (int i = 0; i < a.length; i++) { + forWire.putInt(aLens[i]); + Utf8.toBytes(a[i], 0, + a[i].length(), forWire, ce); + } + forWire.flip(); + int totalLimit = forWire.limit(); + for (String anA : a) { + int len = forWire.getInt(); + forWire.limit(forWire.position() + len); + String s = Utf8.toString(forWire); + assertEquals(anA, s); + forWire.limit(totalLimit); + } + assertEquals(0, forWire.remaining()); + } + + @Test + public void basic() { + String foo = "Washington"; + int[] indexes = calculateBytePositions(foo); + assertThat(indexes.length, is(foo.length() + 1)); + for (int i = 0; i < indexes.length; i++) { + assertThat(indexes[i], is(i)); + } + } + + @Test + public void decodeBasic() { + byte[] foo = Utf8.toBytes("Washington"); + int[] indexes = calculateStringPositions(foo); + assertThat(indexes.length, is(foo.length + 1)); + for (int i = 0; i < indexes.length; i++) { + assertThat(indexes[i], is(i)); + } + } + + @Test + public void highBytes() { + String foo = "\u0128st\u0200e"; + //utf-8 + // 0xC4A8 0x73 0x74 0xC880 0x65 + int[] indexes = calculateBytePositions(foo); + assertThat(indexes.length, is(foo.length() + 1)); + assertThat(indexes[0], is(0)); //128 + assertThat(indexes[1], is(2)); //s + assertThat(indexes[2], is(3)); //t + assertThat(indexes[3], is(4)); //200 + assertThat(indexes[4], is(6)); //e + } + + @Test + public void decodeHighBytes() { + byte[] foo = Utf8.toBytes("\u0128st\u0200e"); + //utf-8 + // 0xC4A8 0x73 0x74 0xC880 0x65 + int[] indexes = calculateStringPositions(foo); + assertThat(indexes.length, is(foo.length + 1)); + assertThat(indexes[0], is(0)); //128 + assertThat(indexes[1], is(0)); //128 + assertThat(indexes[2], is(1)); //s + assertThat(indexes[3], is(2)); //t + assertThat(indexes[4], is(3)); //200 + assertThat(indexes[5], is(3)); //200 + assertThat(indexes[6], is(4)); //e + } + + @Test + public void moreHighBytes() { + String foo = "\u0200\u0201\u0202abc\u0300def\u0301g\u07ff\u0800a\uffffa"; + //utf-8 + //0xC880 0xC881 0xC882 0x61 0x62 0x63 0xCC80 0x64 0x65 0x66 0xCC81 0x67 0xDFBF 0xE0A080 0x61 0xEFBFBF 0x61 + int[] indexes = calculateBytePositions(foo); + assertThat(indexes.length, is(foo.length() + 1)); + assertThat(indexes[0], is(0)); //200 + assertThat(indexes[1], is(2)); //201 + assertThat(indexes[2], is(4)); //202 + assertThat(indexes[3], is(6)); //a + assertThat(indexes[4], is(7)); //b + assertThat(indexes[5], is(8)); //c + assertThat(indexes[6], is(9)); //300 + assertThat(indexes[7], is(11)); //d + assertThat(indexes[8], is(12)); //e + assertThat(indexes[9], is(13)); //f + assertThat(indexes[10], is(14)); //301 + assertThat(indexes[11], is(16)); //g + assertThat(indexes[12], is(17)); //7ff + assertThat(indexes[13], is(19)); //800 + assertThat(indexes[14], is(22)); //a + assertThat(indexes[15], is(23)); //ffff + assertThat(indexes[16], is(26)); //a + } + + @Test + public void decodeMoreHighBytes() { + String foo = "\u0200\u0201\u0202abc\u0300def\u0301g\u07ff\u0800a\uffffa"; + //utf-8 + //0xC880 0xC881 0xC882 0x61 0x62 0x63 0xCC80 0x64 0x65 0x66 0xCC81 0x67 0xDFBF 0xE0A080 0x61 0xEFBFBF 0x61 + int[] indexes = calculateStringPositions(Utf8.toBytes(foo)); + assertThat(indexes.length, is(28)); + assertThat(indexes[0], is(0)); //200 + assertThat(indexes[1], is(0)); //200 + assertThat(indexes[2], is(1)); //201 + assertThat(indexes[3], is(1)); //201 + assertThat(indexes[4], is(2)); //202 + assertThat(indexes[5], is(2)); //202 + assertThat(indexes[6], is(3)); //a + assertThat(indexes[7], is(4)); //b + assertThat(indexes[8], is(5)); //c + assertThat(indexes[9], is(6)); //300 + assertThat(indexes[10], is(6)); //300 + assertThat(indexes[11], is(7)); //d + assertThat(indexes[12], is(8)); //e + assertThat(indexes[13], is(9)); //f + assertThat(indexes[14], is(10)); //301 + assertThat(indexes[15], is(10)); //301 + assertThat(indexes[16], is(11)); //g + assertThat(indexes[17], is(12)); //7ff + assertThat(indexes[18], is(12)); //7ff + assertThat(indexes[19], is(13)); //800 + assertThat(indexes[20], is(13)); //800 + assertThat(indexes[21], is(13)); //800 + assertThat(indexes[22], is(14)); //a + assertThat(indexes[23], is(15)); //ffff + assertThat(indexes[24], is(15)); //ffff + assertThat(indexes[25], is(15)); //ffff + assertThat(indexes[26], is(16)); //a + } + + @Test + public void testOptimisticEncoder() { + for (char i=0; i < 256; i++) { + StringBuilder sb = new StringBuilder(); + for (char c=0; c < i; c++) { + sb.append(c); + } + assertTrue(Arrays.equals(Utf8.toBytesStd(sb.toString()), Utf8.toBytes(sb.toString()))); + } + } + + @Test + public void testLong() + { + for (long l=-0x10000; l < 0x10000; l++) { + assertLongEquals(l); + } + assertLongEquals(Long.MAX_VALUE); + assertLongEquals(Long.MIN_VALUE); + } + + private void assertLongEquals(long l) { + byte [] a = Utf8.toBytes(String.valueOf(l)); + byte [] b = Utf8.toAsciiBytes(l); + if (!Arrays.equals(a, b)) { + assertTrue(Arrays.equals(a, b)); + } + } + + @Test + public void testBoolean() { + assertEquals("true", String.valueOf(true)); + assertEquals("false", String.valueOf(false)); + assertTrue(Arrays.equals(Utf8.toAsciiBytes(true), new Utf8String(String.valueOf(true)).getBytes())); + assertTrue(Arrays.equals(Utf8.toAsciiBytes(false), new Utf8String(String.valueOf(false)).getBytes())); + } + @Test + public void testInt() + { + for (int l=-0x10000; l < 0x10000; l++) { + byte [] a = Utf8.toBytes(String.valueOf(l)); + byte [] b = Utf8.toAsciiBytes(l); + if (!Arrays.equals(a, b)) { + assertTrue(Arrays.equals(a, b)); + } + } + } + @Test + public void testShort() + { + for (short l=-0x1000; l < 0x1000; l++) { + byte [] a = Utf8.toBytes(String.valueOf(l)); + byte [] b = Utf8.toAsciiBytes(l); + if (!Arrays.equals(a, b)) { + assertTrue(Arrays.equals(a, b)); + } + } + } + + @Test + public void surrogatePairs() { + String foo = "a\uD800\uDC00b"; + //unicode + //0x61 0x10000 0x62 + //utf-16 + //0x61 0xD800DC00 0x62 + //utf-8 + //0x61 0xF0908080 0x62 + int[] indexes = calculateBytePositions(foo); + assertThat(indexes.length, is(foo.length() + 1)); + assertThat(indexes[0], is(0)); //a + assertThat(indexes[1], is(1)); //10000 + assertThat(indexes[2], is(1)); //10000, second of surrogate pair + assertThat(indexes[3], is(5)); //b + } + + @Test + public void decodeSurrogatePairs() { + String foo = "a\uD800\uDC00b"; + //unicode + //0x61 0x10000 0x62 + //utf-16 + //0x61 0xD800DC00 0x62 + //utf-8 + //0x61 0xF0908080 0x62 + int[] indexes = calculateStringPositions(Utf8.toBytes(foo)); + assertThat(indexes.length, is(7)); + assertThat(indexes[0], is(0)); //a + assertThat(indexes[1], is(1)); //10000 + assertThat(indexes[2], is(1)); //10000 + assertThat(indexes[3], is(1)); //10000 + assertThat(indexes[4], is(1)); //10000 + assertThat(indexes[5], is(2)); //b + } + + @Test + public void encodeStartEndPositions() { + String foo = "abcde"; + int start = 0; + int length = foo.length(); //5 + int end = start + length; + + int[] indexes = calculateBytePositions(foo); + int byteStart = indexes[start]; + int byteEnd = indexes[end]; + int byteLength = byteEnd - byteStart; + + assertThat(byteStart, equalTo(start)); + assertThat(byteEnd, equalTo(end)); + assertThat(byteLength, equalTo(length)); + } + + @Test + public void encodeStartEndPositionsMultibyteCharsAtEnd() { + String foo = "\u0200abcde\uD800\uDC00"; + int start = 0; + int length = foo.length(); //8 + int end = start + length; + + int[] indexes = calculateBytePositions(foo); + int byteStart = indexes[start]; + int byteEnd = indexes[end]; + int byteLength = byteEnd - byteStart; + + //utf-8 + //0xC880 a b c d e 0xD800DC00 + + assertThat(byteStart, equalTo(start)); + assertThat(byteEnd, equalTo(11)); + assertThat(byteLength, equalTo(11)); + } + + @Test + public void decodeStartEndPositions() { + byte[] foo = Utf8.toBytes("abcde"); + int start = 0; + int length = foo.length; //5 + int end = start + length; + + int[] indexes = calculateStringPositions(foo); + int stringStart = indexes[start]; + int stringEnd = indexes[end]; + int stringLength = stringEnd - stringStart; + + assertThat(stringStart, equalTo(start)); + assertThat(stringEnd, equalTo(end)); + assertThat(stringLength, equalTo(length)); + } + + @Test + public void decodeStartEndPositionsMultibyteCharsAtEnd() { + byte[] foo = Utf8.toBytes("\u0200abcde\uD800\uDC00"); + int start = 0; + int length = foo.length; //11 + int end = start + length; + + int[] indexes = calculateStringPositions(foo); + int stringStart = indexes[start]; + int stringEnd = indexes[end]; + int stringLength = stringEnd - stringStart; + + //utf-8 + //0xC880 a b c d e 0xD800DC00 + + assertThat(stringStart, equalTo(start)); + assertThat(stringEnd, equalTo(8)); + assertThat(stringLength, equalTo(8)); + } + + @Test + public void emptyInputStringResultsInArrayWithSingleZero() { + byte[] empty = new byte[] {}; + int[] indexes = calculateStringPositions(empty); + assertThat(indexes.length, is(1)); + assertThat(indexes[0], is(0)); + } + + @Test + public void testEncoding() { + for (int c : TEST_CODEPOINTS) { + byte[] encoded = Utf8.encode(c); + String testCharacter = makeString(c); + byte[] utf8 = Utf8.toBytes(testCharacter); + assertArrayEquals(utf8, encoded); + } + byte[] stringAsUtf8 = Utf8.toBytes(TEST_STRING); + byte[] handEncoded = new byte[Utf8.byteCount(TEST_STRING)]; + for (int i = 0, j = 0; i < TEST_STRING.length(); i = TEST_STRING.offsetByCodePoints(i, 1)) { + j = Utf8.encode(TEST_STRING.codePointAt(i), handEncoded, j); + } + assertArrayEquals(stringAsUtf8, handEncoded); + } + + @Test + public void testStreamEncoding() throws IOException { + for (int c : TEST_CODEPOINTS) { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + Utf8.encode(c, buffer); + byte[] encoded = buffer.toByteArray(); + String testCharacter = makeString(c); + byte[] utf8 = Utf8.toBytes(testCharacter); + assertArrayEquals(utf8, encoded); + } + byte[] stringAsUtf8 = Utf8.toBytes(TEST_STRING); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + for (int i = 0; i < TEST_STRING.length(); i = TEST_STRING.offsetByCodePoints(i, 1)) { + Utf8.encode(TEST_STRING.codePointAt(i), buffer); + } + byte[] handEncoded = buffer.toByteArray(); + assertArrayEquals(stringAsUtf8, handEncoded); + } + + @Test + public void testByteBufferEncoding() { + for (int c : TEST_CODEPOINTS) { + ByteBuffer buffer = ByteBuffer.allocate(4); + Utf8.encode(c, buffer); + byte[] encoded = new byte[buffer.position()]; + buffer.flip(); + for (int i = 0; i < encoded.length; ++i) { + encoded[i] = buffer.get(); + } + String testCharacter = makeString(c); + byte[] utf8 = Utf8.toBytes(testCharacter); + assertArrayEquals(utf8, encoded); + } + byte[] stringAsUtf8 = Utf8.toBytes(TEST_STRING); + ByteBuffer buffer = ByteBuffer.allocate(TEST_STRING.length() * 4); + for (int i = 0; i < TEST_STRING.length(); i = TEST_STRING.offsetByCodePoints(i, 1)) { + Utf8.encode(TEST_STRING.codePointAt(i), buffer); + } + byte[] handEncoded = new byte[buffer.position()]; + buffer.flip(); + for (int i = 0; i < handEncoded.length; ++i) { + handEncoded[i] = buffer.get(); + } + assertArrayEquals(stringAsUtf8, handEncoded); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/text/XMLTestCase.java b/vespajlib/src/test/java/com/yahoo/text/XMLTestCase.java new file mode 100644 index 00000000000..6320efc2f11 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/XMLTestCase.java @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.Test; + +import java.io.StringReader; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class XMLTestCase { + + @Test + public void testSimple() { + String s1 = "this is a < test"; + String s2 = "this is a & test"; + String s3 = "this is a \" test"; + String s4 = "this is a <\" test"; + String s5 = "this is a low \u001F test"; + + assertEquals("this is a < test", XML.xmlEscape(s1, true)); + assertEquals("this is a & test", XML.xmlEscape(s2, true)); + + // quotes are only escaped in attributes + // + assertEquals("this is a " test", XML.xmlEscape(s3, true)); + assertEquals("this is a \" test", XML.xmlEscape(s3, false)); + + // quotes are only escaped in attributes. prevent bug + // no. 187006 from happening again! + // + assertEquals("this is a <" test", XML.xmlEscape(s4, true)); + assertEquals("this is a <\" test", XML.xmlEscape(s4, false)); + + assertEquals("this is a low \uFFFD test", XML.xmlEscape(s5, false)); + String s = XML.xmlEscape(s5, false, false); + assertEquals(0x1F, s.toCharArray()[14]); + } + + @Test + public void testInvalidUnicode() { + assertEquals("a\ufffd\ufffdb",XML.xmlEscape("a\uffff\uffffb", false)); + } + + @Test + public void testInvalidUnicodeAlongWithEscaping() { + assertEquals("a\ufffd\ufffdb&",XML.xmlEscape("a\ufffe\uffffb&", false)); + } + + @Test + public void testWhenFirstCharacterMustBeEscaped() { + assertEquals("&co", XML.xmlEscape("&co", false)); + assertEquals("\ufffd is a perfectly fine character;", + XML.xmlEscape("\u0000 is a perfectly fine character;", false)); + } + + @Test + public void testLineNoise() { + assertEquals("\ufffda\ufffd\ufffd\ufffdb&\u380c\ufb06\uD87E\uDDF2\ufffd \ufffd", + XML.xmlEscape("\u0001a\u0000\ufffe\uffffb&\u380c\ufb06\uD87E\uDDF2\uD87E \uD87E", false)); + } + + @Test + public void testZeroLength() { + assertEquals("", XML.xmlEscape("", false)); + } + + @Test + public void testAllEscaped() { + assertEquals("&\ufffd\ufffd", XML.xmlEscape("&\u0000\uffff", false)); + } + + @Test + public void testNoneEscaped() { + assertEquals("a\ud87e\uddf2\u00e5", XML.xmlEscape("a\ud87e\uddf2\u00e5", false)); + } + + @Test + public void testReturnSameIfNoQuoting() { + String a = "abc"; + String b = XML.xmlEscape(a, false); + assertSame("xmlEscape should return its input if no change is necessary.", + a, b); + } + + @Test + public void testValidAttributeNames() { + assertTrue(XML.isName(":A_a\u00C0\u00D8\u00F8\u0370\u037F\u200C\u2070\u2C00\u3001\uF900\uFDF0\uD800\uDC00")); + assertFalse(XML.isName(" ")); + assertFalse(XML.isName(": ")); + assertTrue(XML.isName("sss")); + } + + @Test + public void testExceptionContainingLineNumberAndColumnNumber() { + final String invalidXml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<foo"; + try { + XML.getDocument(new StringReader(invalidXml)); + fail("Did not get expected exception"); + } catch (IllegalArgumentException e) { + System.out.println(e.getMessage()); + assertTrue(e.getMessage().contains("Could not parse '")); + assertTrue(e.getMessage().contains("error at line 2, column 5")); + } + } +} diff --git a/vespajlib/src/test/java/com/yahoo/text/XMLWriterTestCase.java b/vespajlib/src/test/java/com/yahoo/text/XMLWriterTestCase.java new file mode 100644 index 00000000000..dc3530fdd97 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/text/XMLWriterTestCase.java @@ -0,0 +1,178 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.text; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.StringWriter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * This test is currently incomplete. Also much tested in the prelude module though. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +@SuppressWarnings("deprecation") +public class XMLWriterTestCase { + + private XMLWriter xml; + + @Before + public void setUp() { + xml = new XMLWriter(new StringWriter()); + } + + @After + public void tearDown() { + } + + @Test + public void test3Levels() { + xml.openTag("l1").openTag("l2").openTag("l3").closeTag().closeTag().closeTag(); + assertEquals( + "<l1>\n" + + "\n" + + " <l2>\n" + + " <l3/>\n" + + " </l2>\n" + + "\n" + + "</l1>\n" + , getWritten()); + } + + private String getWritten() { + xml.close(); + return xml.getWrapped().toString(); + } + + @Test + public void test3LevelsCustomFormatting() { + xml=new XMLWriter(new StringWriter(),1,-1); + xml.openTag("l1").openTag("l2").openTag("l3").closeTag().closeTag().closeTag(); + assertEquals( + "<l1>\n" + + " <l2>\n" + + " <l3/>\n" + + " </l2>\n" + + "</l1>\n" + , getWritten()); + } + + @Test + public void test4LevelsA() { + xml.openTag("l1"); + xml.openTag("l21").closeTag(); + xml.openTag("l22"); + xml.openTag("l31").openTag("l4").closeTag().closeTag(); + xml.openTag("l32").closeTag(); + xml.closeTag(); + xml.closeTag(); + assertEquals( + "<l1>\n" + + "\n" + + " <l21/>\n" + + "\n" + + " <l22>\n" + + " <l31>\n" + + " <l4/>\n" + + " </l31>\n" + + " <l32/>\n" + + " </l22>\n" + + "\n" + + "</l1>\n" + , getWritten()); + } + + @Test + public void test4LevelsB() { + xml.openTag("l1"); + xml.openTag("l21"); + xml.openTag("l31").closeTag(); + xml.openTag("l32").openTag("l4").closeTag().closeTag(); + xml.closeTag(); + xml.openTag("l22").closeTag(); + xml.closeTag(); + assertEquals( + "<l1>\n" + + "\n" + + " <l21>\n" + + " <l31/>\n" + + " <l32>\n" + + " <l4/>\n" + + " </l32>\n" + + " </l21>\n" + + "\n" + + " <l22/>\n" + + "\n" + + "</l1>\n" + , getWritten()); + } + + @Test + public void testEmpty() { + xml.openTag("l1").closeTag(); + assertEquals( + "<l1/>\n" + , getWritten()); + } + + @Test + public void checkHeader() { + xml.xmlHeader("utf-8"); + assertEquals("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n", getWritten()); + } + + @Test + public void forcedAttribute() { + xml.openTag("a").forceAttribute(new Utf8String("nalle"), "\"").closeTag(); + assertEquals("<a nalle=\""\"/>\n", getWritten()); + } + + @Test + public void attributeString() { + xml.openTag("a").attribute(new Utf8String("nalle"), new Utf8String("b")).closeTag(); + assertEquals("<a nalle=\"b\"/>\n", getWritten()); + } + + @Test + public void attributeLong() { + xml.openTag("a").attribute(new Utf8String("nalle"), 5L).closeTag(); + assertEquals("<a nalle=\"5\"/>\n", getWritten()); + } + + @Test + public void attributeBoolean() { + xml.openTag("a").attribute(new Utf8String("nalle"), true).closeTag(); + assertEquals("<a nalle=\"true\"/>\n", getWritten()); + } + + @Test + public void content() { + xml.content("a\na", false).content("a\na", true); + assertEquals("a\naa\na", getWritten()); + } + + @Test + public void escapedContent() { + xml.escapedContent("a&\na", false).escapedContent("a&\na", true); + assertEquals("a&\naa&\na", getWritten()); + } + + @Test + public void escapedAsciiContent() { + xml.escapedAsciiContent("a&\na", false).escapedAsciiContent("a&\na", true); + assertEquals("a&\naa&\na", getWritten()); + } + + @Test + public void isIn() { + assertFalse(xml.isIn("a")); + xml.openTag("a"); + assertTrue(xml.isIn("a")); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/time/WallClockSourceTestCase.java b/vespajlib/src/test/java/com/yahoo/time/WallClockSourceTestCase.java new file mode 100644 index 00000000000..e26591c9c64 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/time/WallClockSourceTestCase.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.time; + +import org.junit.Test; +import static org.junit.Assert.assertTrue; + +public class WallClockSourceTestCase { + + @Test + public void testSimple() { + long actualBefore = System.currentTimeMillis(); + WallClockSource clock = new WallClockSource(); + long nanos = clock.currentTimeNanos(); + long micros = nanos / 1000; + long millis = micros / 1000; + long actualAfter = System.currentTimeMillis(); + + assertTrue(actualBefore <= millis); + assertTrue(millis <= actualAfter); + } + + @Test + public void testWithAdjust() { + WallClockSource clock = new WallClockSource(); + long diffB = 0; + long diffA = 0; + for (int i = 0; i < 66666; i++) { + long actualB = System.currentTimeMillis(); + clock.adjust(); + long nanos = clock.currentTimeNanos(); + long actualA = System.currentTimeMillis(); + long micros = nanos / 1000; + long millis = micros / 1000; + diffB = Math.max(diffB, actualB - millis); + diffA = Math.max(diffA, millis - actualA); + // System.out.println("adj Timing values, before: "+actualB+" <= guess: "+millis+" <= after: "+actualA); + } + System.out.println("adjust test: biggest difference (beforeTime - guess): "+diffB); + System.out.println("adjust test: biggest difference (guess - afterTime): "+diffA); + assertTrue("actual time before sample must be <= wallclocksource, diff: " + diffB, diffB < 2); + assertTrue("actual time after sample must be >= wallclocksource, diff: " + diffA, diffA < 2); + } + + @Test + public void testNoAdjust() { + WallClockSource clock = new WallClockSource(); + long diffB = 0; + long diffA = 0; + for (int i = 0; i < 66666; i++) { + long actualB = System.currentTimeMillis(); + long nanos = clock.currentTimeNanos(); + long actualA = System.currentTimeMillis(); + long micros = nanos / 1000; + long millis = micros / 1000; + diffB = Math.max(diffB, actualB - millis); + diffA = Math.max(diffA, millis - actualA); + // System.out.println("noadj Timing values, before: "+actualB+" <= guess: "+millis+" <= after: "+actualA); + } + System.out.println("noadjust test: biggest difference (beforeTime - guess): "+diffB); + System.out.println("noadjust test: biggest difference (guess - afterTime): "+diffA); + assertTrue("actual time before sample must be <= wallclocksource, diff: " + diffB, diffB < 3); + assertTrue("actual time after sample must be >= wallclocksource, diff: " + diffA, diffA < 3); + } + + @Test + public void testAutoAdjust() { + WallClockSource clock = WallClockSource.get(); + long diffB = 0; + long diffA = 0; + for (int i = 0; i < 66666; i++) { + long actualB = System.currentTimeMillis(); + long nanos = clock.currentTimeNanos(); + long actualA = System.currentTimeMillis(); + long micros = nanos / 1000; + long millis = micros / 1000; + diffB = Math.max(diffB, actualB - millis); + diffA = Math.max(diffA, millis - actualA); + // System.out.println("noadj Timing values, before: "+actualB+" <= guess: "+millis+" <= after: "+actualA); + } + System.out.println("autoadjust test: biggest difference (beforeTime - guess): "+diffB); + System.out.println("autoadjust test: biggest difference (guess - afterTime): "+diffA); + assertTrue("actual time before sample must be <= wallclocksource, diff: " + diffB, diffB < 3); + assertTrue("actual time after sample must be >= wallclocksource, diff: " + diffA, diffA < 3); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/transaction/NestedTransactionTestCase.java b/vespajlib/src/test/java/com/yahoo/transaction/NestedTransactionTestCase.java new file mode 100644 index 00000000000..d75daa506b4 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/transaction/NestedTransactionTestCase.java @@ -0,0 +1,181 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.transaction; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author bratseth + */ +public class NestedTransactionTestCase { + + @Test + public void testNestedTransaction() { + NestedTransaction t = new NestedTransaction(); + t.add(new TransactionTypeB("B1"), TransactionTypeC.class); + t.add(new TransactionTypeC("C1")); + t.add(new TransactionTypeA("A1"), TransactionTypeB.class); + t.add(new TransactionTypeA("A2")); + t.add(new TransactionTypeC("C2")); + + // Add two tasks to run after commit + MutableInteger tasksRun = new MutableInteger(); + t.onCommitted(() -> { tasksRun.value++; }); + t.onCommitted(() -> { tasksRun.value++; }); + + assertEquals(3, t.transactions().size()); + assertEquals(TransactionTypeA.class, t.transactions().get(0).getClass()); + assertEquals(TransactionTypeB.class, t.transactions().get(1).getClass()); + assertEquals(TransactionTypeC.class, t.transactions().get(2).getClass()); + + assertEquals("A1", ((MockOperation)t.transactions().get(0).operations().get(0)).name); + assertEquals("A2", ((MockOperation)t.transactions().get(0).operations().get(1)).name); + assertEquals("B1", ((MockOperation)t.transactions().get(1).operations().get(0)).name); + assertEquals("C1", ((MockOperation)t.transactions().get(2).operations().get(0)).name); + assertEquals("C2", ((MockOperation)t.transactions().get(2).operations().get(1)).name); + + t.commit(); + assertTrue(((MockTransaction)t.transactions().get(0)).committed); + assertTrue(((MockTransaction)t.transactions().get(1)).committed); + assertTrue(((MockTransaction)t.transactions().get(2)).committed); + assertEquals("After commit tasks are run", 2, tasksRun.value); + } + + @Test + public void testNestedTransactionFailingOnCommit() { + NestedTransaction t = new NestedTransaction(); + t.add(new TransactionTypeC("C1")); + t.add(new TransactionTypeA("A1"), TransactionTypeB.class, FailAtCommitTransactionType.class); + t.add(new TransactionTypeA("A2")); + t.add(new FailAtCommitTransactionType("Fail"), TransactionTypeC.class); + t.add(new TransactionTypeC("C2")); + t.add(new TransactionTypeB("B1"), TransactionTypeC.class, FailAtCommitTransactionType.class); + + // Add task to run after commit + MutableInteger tasksRun = new MutableInteger(); + t.onCommitted(() -> { + tasksRun.value++; + }); + + assertEquals(4, t.transactions().size()); + assertEquals(TransactionTypeA.class, t.transactions().get(0).getClass()); + assertEquals(TransactionTypeB.class, t.transactions().get(1).getClass()); + assertEquals(FailAtCommitTransactionType.class, t.transactions().get(2).getClass()); + assertEquals(TransactionTypeC.class, t.transactions().get(3).getClass()); + + try { t.commit(); } catch (IllegalStateException expected) { } + assertTrue(((MockTransaction)t.transactions().get(0)).rolledback); + assertTrue(((MockTransaction)t.transactions().get(1)).rolledback); + assertFalse(((MockTransaction) t.transactions().get(2)).committed); + assertEquals("After commit tasks are not run", 0, tasksRun.value); + } + + @Test + public void testConflictingOrdering() { + NestedTransaction t = new NestedTransaction(); + t.add(new TransactionTypeA("A1"), TransactionTypeB.class); + t.add(new TransactionTypeB("B1"), TransactionTypeC.class); + t.add(new TransactionTypeC("C1"), TransactionTypeA.class); + try { + t.commit(); + fail("Expected exception"); + } + catch (IllegalStateException expected) { + } + } + + private static class TransactionTypeA extends MockTransaction { + public TransactionTypeA(String name) { super(name); } + } + + private static class TransactionTypeB extends MockTransaction { + public TransactionTypeB(String name) { super(name); } + } + + private static class TransactionTypeC extends MockTransaction { + public TransactionTypeC(String name) { super(name); } + } + + private static class FailAtCommitTransactionType extends MockTransaction { + public FailAtCommitTransactionType(String name) { super(name); } + @Override + public void commit() { + throw new RuntimeException(); + } + } + + private static class MockTransaction implements Transaction { + + public boolean prepared = false, committed = false, rolledback = false; + private List<Operation> operations = new ArrayList<>(); + + public MockTransaction(String name) { + operations.add(new MockOperation(name)); + } + + @Override + public Transaction add(Operation operation) { + operations.add(operation); + return this; + } + + @Override + public Transaction add(List<Operation> operation) { + operations.addAll(operation); + return this; + } + + @Override + public List<Operation> operations() { + return operations; + } + + @Override + public void prepare() { + prepared = true; + } + + @Override + public void commit() { + if ( ! prepared) + throw new IllegalStateException("Commit before prepare"); + committed = true; + } + + @Override + public void rollbackOrLog() { + if ( ! committed) + throw new IllegalStateException("Rollback before commit"); + rolledback = true; + } + + @Override + public void close() { + } + + } + + private static class MockOperation implements Transaction.Operation { + + public String name; + + public MockOperation(String name) { + this.name = name; + } + + } + + private static class MutableInteger { + + public int value = 0; + + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/vespa/objects/BigIdClass.java b/vespajlib/src/test/java/com/yahoo/vespa/objects/BigIdClass.java new file mode 100644 index 00000000000..3c1065369ca --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/vespa/objects/BigIdClass.java @@ -0,0 +1,183 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +import java.nio.ByteBuffer; + +public class BigIdClass extends Identifiable +{ + public static final int classId = registerClass(42, BigIdClass.class); + + static public final FieldBase fByte = new FieldBase("myByte"); + static public final FieldBase fShort = new FieldBase("myShort"); + static public final FieldBase fInt = new FieldBase("myInt"); + static public final FieldBase fLong = new FieldBase("myLong"); + static public final FieldBase fFloat = new FieldBase("myFloat"); + static public final FieldBase fDouble = new FieldBase("myDouble"); + static public final FieldBase fArrayOne = new FieldBase("myArrayOne"); + static public final FieldBase fArrayTwo = new FieldBase("myArrayTwo"); + static public final FieldBase fByteBuffer = new FieldBase("myByteBuffer"); + static public final FieldBase fString = new FieldBase("myString"); + static public final FieldBase fAlternate = new FieldBase("myAlternate"); + static public final FieldBase fChildOne = new FieldBase("childOne"); + static public final FieldBase fChildTwo = new FieldBase("childTwo"); + + private byte myByte = 42; + private short myShort = 4242; + private int myInt = 424242; + private long myLong = 9876543210L; + private float myFloat = 42.42f; + private double myDouble = 42.4242e-42; + + private byte[] myArrayOne = new byte[5]; + private byte[] myArrayTwo = new byte[10]; + private ByteBuffer myByteBuffer; + + private String myString = "default-value"; + private String myAlternate = "some \u2603 Utf8"; + + private Identifiable childOne = null; + private Identifiable childTwo = new FooBarIdClass(); + + @Override + public void visitMembers(ObjectVisitor visitor) { + super.visitMembers(visitor); + visitor.visit("", childOne); + visitor.visit("one", childOne); + visitor.visit("two", childTwo); + visitor.visit(null, childTwo); + visitor.visit("myArrayOne", myArrayOne); + } + + public BigIdClass() { + myArrayOne[0] = 1; + myArrayOne[1] = 2; + myArrayOne[2] = 3; + myArrayOne[3] = 4; + myArrayOne[4] = 5; + + myArrayTwo[0] = 6; + myArrayTwo[1] = 7; + myArrayTwo[2] = 8; + + myArrayTwo[9] = 9; + } + + public BigIdClass(int value) { + myByte = (byte)value; + myShort = (short)value; + myInt = value; + myLong = value; + myLong <<= 30; + myLong ^= value; + myFloat = (float)(value + 0.000001*value); + myDouble = 123456.789*value + 0.987654321*value; + myArrayOne[1] = (byte)(value >> 1); + myArrayOne[2] = (byte)(value >> 5); + myArrayOne[3] = (byte)(value >> 9); + + myArrayTwo[3] = (byte)(value >> 2); + myArrayTwo[4] = (byte)(value >> 4); + myArrayTwo[5] = (byte)(value >> 6); + myArrayTwo[6] = (byte)(value >> 8); + + myString = Integer.toString(value); + myAlternate = "a \u2603 " + Integer.toString(value) + " b"; + + childOne = new FooBarIdClass(); + childTwo = null; + } + + @Override + protected int onGetClassId() { + return classId; + } + + @Override + protected void onSerialize(Serializer buf) { + buf.putByte(fByte, myByte); + buf.putShort(fShort, myShort); + buf.putInt(fInt, myInt); + buf.putLong(fLong, myLong); + buf.putFloat(fFloat, myFloat); + buf.putDouble(fDouble, myDouble); + buf.put(fArrayOne, myArrayOne); + buf.put(fArrayTwo, myArrayTwo); + /* buf.put(fByteBuffer, myByteBuffer); */ + buf.put(fString, myString); + putUtf8(buf, myAlternate); + + serializeOptional(buf, childOne); + serializeOptional(buf, childTwo); + } + + @Override + protected void onDeserialize(Deserializer buf) { + + myByte = buf.getByte(fByte); + myShort = buf.getShort(fShort); + myInt = buf.getInt(fInt); + myLong = buf.getLong(fLong); + myFloat = buf.getFloat(fFloat); + myDouble = buf.getDouble(fDouble); + + myArrayOne = buf.getBytes(fArrayOne, 5); + myArrayTwo = buf.getBytes(fArrayTwo, 10); + + myString = buf.getString(fString); + myAlternate = getUtf8(buf); + + childOne = deserializeOptional(buf); + childTwo = deserializeOptional(buf); + } + + public boolean equals(Object other) { + if (super.equals(other) && other instanceof BigIdClass) { + boolean allEq = true; + BigIdClass o = (BigIdClass)other; + if (myByte != o.myByte) { allEq = false; } + if (myShort != o.myShort) { allEq = false; } + if (myInt != o.myInt) { allEq = false; } + if (myLong != o.myLong) { allEq = false; } + if (myFloat != o.myFloat) { allEq = false; } + if (myDouble != o.myDouble) { allEq = false; } + if (! myString.equals(o.myString)) { allEq = false; } + if (! equals(childOne, o.childOne)) { allEq = false; } + if (! equals(childTwo, o.childTwo)) { allEq = false; } + if (childTwo != null && o.childTwo == null) { allEq = false; } + return allEq; + } + return false; + } + +/*** + public boolean diff(BigIdClass o) { + boolean allEq = true; + + if (myByte != o.myByte) { System.out.println("myByte differ: "+myByte+" != "+o.myByte); allEq = false; } + if (myShort != o.myShort) { System.out.println("myShort differ: "+myShort+" != "+o.myShort); allEq = false; } + if (myInt != o.myInt) { System.out.println("myInt differ: "+myInt+" != "+o.myInt); allEq = false; } + if (myLong != o.myLong) { System.out.println("myLong differ: "+myLong+" != "+o.myLong); allEq = false; } + if (myFloat != o.myFloat) { System.out.println("myFloat differ: "+myFloat+" != "+o.myFloat); allEq = false; } + if (myDouble != o.myDouble) { System.out.println("myDouble differ: "+myDouble+" != "+o.myDouble); allEq = false; } + if (! myString.equals(o.myString)) { System.out.println("myString differ: "+myString+" != "+o.myString); allEq = false; } + if (childOne == null && o.childOne != null) { + System.err.println("childOne is null, o.childOne is: "+o.childOne); + allEq = false; + } + if (childOne != null && o.childOne == null) { + System.err.println("o.childOne is null, childOne is: "+childOne); + allEq = false; + } + if (childTwo == null && o.childTwo != null) { + System.err.println("childTwo is null, o.childTwo is: "+o.childTwo); + allEq = false; + } + if (childTwo != null && o.childTwo == null) { + System.err.println("o.childTwo is null, childTwo is: "+childTwo); + allEq = false; + } + return allEq; + } +***/ + +} diff --git a/vespajlib/src/test/java/com/yahoo/vespa/objects/FieldBaseTestCase.java b/vespajlib/src/test/java/com/yahoo/vespa/objects/FieldBaseTestCase.java new file mode 100644 index 00000000000..d60184c5616 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/vespa/objects/FieldBaseTestCase.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author arnej27959 + */ +public class FieldBaseTestCase { + + @Test + public void testFieldBaseAPI() { + String s1 = "test"; + FieldBase f1 = new FieldBase(s1); + FieldBase f2 = new FieldBase("tESt"); + FieldBase f3 = new FieldBase("TEST"); + + assertThat(f1.getName(), is(s1)); + assertThat(f1, equalTo(f1)); + assertThat(f1, equalTo(new FieldBase("test"))); + assertThat(f1, equalTo(f2)); + assertThat(f1, equalTo(f3)); + + assertThat(f1.hashCode(), equalTo(s1.hashCode())); + assertThat(f1.hashCode(), equalTo(f2.hashCode())); + assertThat(f1.hashCode(), equalTo(f3.hashCode())); + + assertThat(f1.toString(), equalTo("field test")); + + FieldBase f4 = new FieldBase("foo"); + FieldBase f5 = new FieldBase("bar"); + FieldBase f6 = new FieldBase("qux"); + + assertThat(f1, not(equalTo(f4))); + assertThat(f1, not(equalTo(f5))); + assertThat(f1, not(equalTo(f6))); + + assertThat(f1.hashCode(), not(equalTo(f4.hashCode()))); + assertThat(f1.hashCode(), not(equalTo(f5.hashCode()))); + assertThat(f1.hashCode(), not(equalTo(f6.hashCode()))); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/vespa/objects/FooBarIdClass.java b/vespajlib/src/test/java/com/yahoo/vespa/objects/FooBarIdClass.java new file mode 100644 index 00000000000..9aa5716d4ee --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/vespa/objects/FooBarIdClass.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +import java.util.List; +import java.util.ArrayList; + +public class FooBarIdClass extends Identifiable +{ + public static final int classId = registerClass(17, FooBarIdClass.class); + + private String foo = "def-foo"; + private int bar = 42; + + private List<Integer> lst = new ArrayList<>(); + + public FooBarIdClass() { + lst.add(17); + lst.add(42); + lst.add(666); + } + + @Override + protected int onGetClassId() { + return classId; + } + + @Override + public void visitMembers(ObjectVisitor visitor) { + super.visitMembers(visitor); + visitor.visit("foo", foo); + visitor.visit("bar", bar); + visitor.visit("lst", lst); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/vespa/objects/ObjectDumperTestCase.java b/vespajlib/src/test/java/com/yahoo/vespa/objects/ObjectDumperTestCase.java new file mode 100644 index 00000000000..03a03ba063f --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/vespa/objects/ObjectDumperTestCase.java @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +import org.junit.Test; + +import java.nio.ByteBuffer; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author arnej27959 + */ +public class ObjectDumperTestCase { + + @Test + public void testSimple() { + String s1 = "test"; + + ObjectDumper defOD = new ObjectDumper(); + ObjectDumper oneOD = new ObjectDumper(1); + + defOD.visit("s1", s1); + oneOD.visit("s2", s1); + + assertEquals("s1: 'test'\n", defOD.toString()); + assertEquals("s2: 'test'\n", oneOD.toString()); + } + + @Test + public void testBig() { + BigIdClass b = new BigIdClass(); + ObjectDumper oneOD = new ObjectDumper(1); + + oneOD.visit("biggie", b); + + assertThat(oneOD.toString(), equalTo( +"biggie: BigIdClass {\n"+ +" classId: 42\n"+ +" : <NULL>\n"+ +" one: <NULL>\n"+ +" two: FooBarIdClass {\n"+ +" classId: 17\n"+ +" foo: 'def-foo'\n"+ +" bar: 42\n"+ +" lst: List {\n"+ +" [0]: 17\n"+ +" [1]: 42\n"+ +" [2]: 666\n"+ +" }\n"+ +" }\n"+ +" FooBarIdClass {\n"+ +" classId: 17\n"+ +" foo: 'def-foo'\n"+ +" bar: 42\n"+ +" lst: List {\n"+ +" [0]: 17\n"+ +" [1]: 42\n"+ +" [2]: 666\n"+ +" }\n"+ +" }\n"+ +" myArrayOne: byte[] {\n"+ +" [0]: 1\n"+ +" [1]: 2\n"+ +" [2]: 3\n"+ +" [3]: 4\n"+ +" [4]: 5\n"+ +" }\n"+ +"}\n")); + + ObjectDumper defOD = new ObjectDumper(); + defOD.visit("", b); + assertThat(b.toString(), equalTo(b.toString())); + } + + @Test + public void testOne() { + SomeIdClass s3 = new SomeIdClass(); + + ObjectDumper defOD = new ObjectDumper(); + ObjectDumper oneOD = new ObjectDumper(1); + + defOD.visit("s3", s3); + oneOD.visit("s4", s3); + + assertEquals("s3: SomeIdClass {\n classId: 1234321\n}\n", defOD.toString()); + assertEquals("s4: SomeIdClass {\n classId: 1234321\n}\n", oneOD.toString()); + } + + @Test + public void testTwo() { + FooBarIdClass s5 = new FooBarIdClass(); + + ObjectDumper defOD = new ObjectDumper(); + ObjectDumper oneOD = new ObjectDumper(1); + + defOD.visit("s5", s5); + oneOD.visit("s6", s5); + + assertThat(defOD.toString(), is("s5: FooBarIdClass {\n"+ + " classId: 17\n"+ + " foo: 'def-foo'\n"+ + " bar: 42\n"+ + " lst: List {\n"+ + " [0]: 17\n"+ + " [1]: 42\n"+ + " [2]: 666\n"+ + " }\n"+ + "}\n")); + assertThat(oneOD.toString(), is("s6: FooBarIdClass {\n"+ + " classId: 17\n"+ + " foo: 'def-foo'\n"+ + " bar: 42\n"+ + " lst: List {\n"+ + " [0]: 17\n"+ + " [1]: 42\n"+ + " [2]: 666\n"+ + " }\n"+ + "}\n")); + + } + + @Test + public void testRegistry() { + assertThat(FooBarIdClass.classId, is(17)); + int x = Identifiable.registerClass(17, FooBarIdClass.class); + assertThat(x, is(17)); + boolean caught = false; + try { + x = Identifiable.registerClass(17, SomeIdClass.class); + } catch (IllegalArgumentException e) { + caught = true; + assertThat(e.getMessage(), is( +"Can not register class 'class com.yahoo.vespa.objects.SomeIdClass' with id 17,"+ +" because it already maps to class 'class com.yahoo.vespa.objects.FooBarIdClass'.")); + } + assertThat(x, is(17)); + assertThat(caught, is(true)); + + Identifiable s7 = Identifiable.createFromId(17); + ObjectDumper defOD = new ObjectDumper(); + defOD.visit("s7", s7); + assertThat(defOD.toString(), is("s7: FooBarIdClass {\n"+ + " classId: 17\n"+ + " foo: 'def-foo'\n"+ + " bar: 42\n"+ + " lst: List {\n"+ + " [0]: 17\n"+ + " [1]: 42\n"+ + " [2]: 666\n"+ + " }\n"+ + "}\n")); + + Identifiable nsi = Identifiable.createFromId(717273); + assertThat(nsi, is((Identifiable)null)); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/vespa/objects/SerializeTestCase.java b/vespajlib/src/test/java/com/yahoo/vespa/objects/SerializeTestCase.java new file mode 100644 index 00000000000..122cdf24a89 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/vespa/objects/SerializeTestCase.java @@ -0,0 +1,143 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +import com.yahoo.io.GrowableByteBuffer; +import java.nio.ByteOrder; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author arnej27959 + */ +public class SerializeTestCase { + + @Test + public void testSimple() { + SomeIdClass s = new SomeIdClass(); + BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer()); + s.serialize(buf); + buf.flip(); + s.deserialize(buf); + } + + @Test + public void testOne() { + SomeIdClass s = new SomeIdClass(); + BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer()); + s.serializeWithId(buf); + buf.flip(); + Identifiable s2 = Identifiable.create(buf); + assertThat((s2 instanceof SomeIdClass), is(true)); + } + + @Test + public void testUnderflow() { + BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer()); + buf.putByte(null, (byte)123); + buf.flip(); + boolean caught = false; + try { + byte[] val = buf.getBytes(null, 2); + } catch (IllegalArgumentException e) { + // System.out.println(e); + caught = true; + } + assertThat(caught, is(true)); + } + + @Test + public void testIdNotFound() { + BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer()); + buf.putInt(null, 717273); + buf.flip(); + boolean caught = false; + try { + Identifiable nsi = Identifiable.create(buf); + } catch (IllegalArgumentException e) { + // System.out.println(e); + caught = true; + } + assertThat(caught, is(true)); + } + + @Test + public void testOrdering() { + BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer()); + assertThat(buf.order(), is(ByteOrder.BIG_ENDIAN)); + buf.putInt(null, 0x11223344); + buf.order(ByteOrder.LITTLE_ENDIAN); + buf.putInt(null, 0x55667788); + assertThat(buf.order(), is(ByteOrder.LITTLE_ENDIAN)); + buf.flip(); + assertThat(buf.getByte(null), is((byte)0x11)); + assertThat(buf.getByte(null), is((byte)0x22)); + assertThat(buf.getShort(null), is((short)0x4433)); + buf.order(ByteOrder.BIG_ENDIAN); + assertThat(buf.getByte(null), is((byte)0x88)); + assertThat(buf.getByte(null), is((byte)0x77)); + assertThat(buf.getShort(null), is((short)0x6655)); + } + + @Test + public void testBig() { + BigIdClass dv = new BigIdClass(); + BigIdClass ov = new BigIdClass(6667666); + BigIdClass bv = new BigIdClass(123456789); + + assertThat(BigIdClass.classId, is(42)); + assertThat(dv.getClassId(), is(42)); + assertThat(ov.getClassId(), is(42)); + assertThat(bv.getClassId(), is(42)); + + assertThat(ov.equals(dv), is(false)); + assertThat(dv.equals(bv), is(false)); + assertThat(bv.equals(ov), is(false)); + + BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer()); + ov.serialize(buf); + buf.flip(); + dv.deserialize(buf); + assertThat(ov, equalTo(dv)); + assertThat(dv, equalTo(ov)); + buf = new BufferSerializer(new GrowableByteBuffer()); + bv.serializeWithId(buf); + buf.flip(); + dv.deserializeWithId(buf); + assertThat(bv, equalTo(dv)); + assertThat(dv, equalTo(bv)); + + buf = new BufferSerializer(new GrowableByteBuffer()); + SomeIdClass s = new SomeIdClass(); + assertThat(dv.equals(s), is(false)); + assertThat(ov.equals(s), is(false)); + assertThat(bv.equals(s), is(false)); + assertThat(dv.equals(new Object()), is(false)); + + s.serializeWithId(buf); + buf.flip(); + boolean caught = false; + try { + dv.deserializeWithId(buf); + } catch (IllegalArgumentException ex) { + caught = true; + // System.out.println(ex); + } + assertThat(caught, is(true)); + buf = new BufferSerializer(new GrowableByteBuffer()); + buf.putLong(null, 0x7777777777777777L); + buf.flip(); + caught = false; + try { + dv.deserializeWithId(buf); + } catch (IllegalArgumentException ex) { + caught = true; + // System.out.println(ex); + } + assertThat(caught, is(true)); + } + +} diff --git a/vespajlib/src/test/java/com/yahoo/vespa/objects/SomeIdClass.java b/vespajlib/src/test/java/com/yahoo/vespa/objects/SomeIdClass.java new file mode 100644 index 00000000000..6c24ba1367d --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/vespa/objects/SomeIdClass.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.objects; + +public class SomeIdClass extends Identifiable +{ + public static final int classId = registerClass(1234321, SomeIdClass.class); + + @Override + protected int onGetClassId() { + return classId; + } + +} |