Publish

6 files changed, 584 insertions, 0 deletions

diff --git a/vespajlib/developernotes/CharClassStats.java b/vespajlib/developernotes/CharClassStats.java
new file mode 100644
index 00000000000..359b41766a2
--- /dev/null
+++ b/vespajlib/developernotes/CharClassStats.java
@@ -0,0 +1,116 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.text;
+
+import java.util.*;
+
+public class CharClassStats {
+
+    public static class TypeStat {
+        public final int typecode;
+        public final String name;
+        public final List<Integer> codepoints = new ArrayList<Integer>();
+
+        TypeStat(int typecode) {
+            this(typecode, "[???]");
+        }
+        TypeStat(int typecode, String name) {
+            this.typecode = typecode;
+            this.name = name;
+        }
+        void addCodepoint(int codepoint) {
+            codepoints.add(codepoint);
+        }
+    }
+
+    private static void init(Map<Integer, TypeStat> map) {
+
+        TypeStat stat;
+        stat = new TypeStat(Character.COMBINING_SPACING_MARK, "COMBINING_SPACING_MARK");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.CONNECTOR_PUNCTUATION, "CONNECTOR_PUNCTUATION");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.CONTROL, "CONTROL");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.CURRENCY_SYMBOL, "CURRENCY_SYMBOL");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.DASH_PUNCTUATION, "DASH_PUNCTUATION");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.DECIMAL_DIGIT_NUMBER, "DECIMAL_DIGIT_NUMBER");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.ENCLOSING_MARK, "ENCLOSING_MARK");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.END_PUNCTUATION, "END_PUNCTUATION");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.FINAL_QUOTE_PUNCTUATION, "FINAL_QUOTE_PUNCTUATION");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.FORMAT, "FORMAT");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.INITIAL_QUOTE_PUNCTUATION, "INITIAL_QUOTE_PUNCTUATION");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.LETTER_NUMBER, "LETTER_NUMBER");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.LINE_SEPARATOR, "LINE_SEPARATOR");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.LOWERCASE_LETTER, "LOWERCASE_LETTER");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.MATH_SYMBOL, "MATH_SYMBOL");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.MODIFIER_LETTER, "MODIFIER_LETTER");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.MODIFIER_SYMBOL, "MODIFIER_SYMBOL");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.NON_SPACING_MARK, "NON_SPACING_MARK");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.OTHER_LETTER, "OTHER_LETTER");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.OTHER_NUMBER, "OTHER_NUMBER");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.OTHER_PUNCTUATION, "OTHER_PUNCTUATION");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.OTHER_SYMBOL, "OTHER_SYMBOL");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.PARAGRAPH_SEPARATOR, "PARAGRAPH_SEPARATOR");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.PRIVATE_USE, "PRIVATE_USE");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.SPACE_SEPARATOR, "SPACE_SEPARATOR");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.START_PUNCTUATION, "START_PUNCTUATION");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.SURROGATE, "SURROGATE");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.TITLECASE_LETTER, "TITLECASE_LETTER");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.UNASSIGNED, "UNASSIGNED");
+        map.put(stat.typecode, stat);
+        stat = new TypeStat(Character.UPPERCASE_LETTER, "UPPERCASE_LETTER");
+        map.put(stat.typecode, stat);
+    }
+
+    public static void main(String[] args) {
+        Map<Integer, TypeStat> map = new HashMap<Integer, TypeStat>();
+
+        init(map);
+
+        for (int codepoint = 0; codepoint <= 0x110000; codepoint++) {
+            int type = java.lang.Character.getType(codepoint);
+
+            if (! map.containsKey(type)) {
+                map.put(type, new TypeStat(type));
+            }
+            map.get(type).addCodepoint(codepoint);
+        }
+
+        int[] codes = new int[map.size()];
+        int numcodes = 0;
+        for (Integer type : map.keySet()) {
+            codes[numcodes++] = type;
+        }
+        Arrays.sort(codes);
+        for (int type : codes) {
+            TypeStat ts = map.get(type);
+            System.out.println("type "+type+" typecode="+ts.typecode+" name="+ts.name+" contains "+ts.codepoints.size()+" codepoints");
+        }
+    }
+
+}
diff --git a/vespajlib/developernotes/CopyOnWriteHashMapBenchmark.java b/vespajlib/developernotes/CopyOnWriteHashMapBenchmark.java
new file mode 100644
index 00000000000..c1cb0cc3e6c
--- /dev/null
+++ b/vespajlib/developernotes/CopyOnWriteHashMapBenchmark.java
@@ -0,0 +1,95 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.concurrent;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ * @since 5.2
+ */
+public class RcuHashMapBenchmark {
+    static class Actor implements Runnable {
+        private final CopyOnWriteHashMap<Long, Long> m;
+        private long mSum = 0;
+        private long mMissRate = 0;
+        Actor(CopyOnWriteHashMap<Long, Long> m) {
+            this.m = m;
+        }
+        @Override
+        public void run() {
+            final int NUM_UPDATES=100;
+            final long NUM_LOOKUPS=10000000;
+            final List<Long> upd = new ArrayList<Long>(NUM_UPDATES);
+            upd.add(0l);
+            long missRate = 0;
+            long sum = 0;
+            for (long i=0; i < NUM_LOOKUPS; i++) {
+                long t = i%upd.size();
+                Long v = m.get(upd.get((int)t));
+                if (v == null) {
+                    missRate++;
+                    m.put(upd.get((int)t), i);
+                    sum += i;
+                } else {
+                    sum += v;
+                }
+                if (i%(NUM_LOOKUPS/NUM_UPDATES) == 0) {
+                    upd.add((long)upd.size());
+                }
+            }
+            synchronized (this) {
+                mSum = sum;
+                mMissRate = missRate;
+            }
+        }
+        long getSum() { synchronized (this) { return mSum; } }
+        long getMissRate() { synchronized (this) { return mMissRate;} }
+    }
+    RcuHashMapBenchmark(int numThreads) {
+        CopyOnWriteHashMap<Long, Long> m = new CopyOnWriteHashMap<Long, Long>();
+        Thread[] threads = new Thread[numThreads];
+        Actor [] actors = new Actor[threads.length];
+        for (int i = 0; i < threads.length; ++i) {
+            Actor a = new Actor(m);
+            actors[i] = a;
+            threads[i] = new Thread(a);
+        }
+        runAll(threads);
+        long missRate=0;
+        long sum=0;
+        for (Actor a : actors) {
+            missRate += a.getMissRate();
+            sum += a.getSum();
+            System.out.println("Missrate: " + a.getMissRate() + " sum = " + a.getSum());
+        }
+        System.out.println("Total Missrate: " + missRate + " sum = " + sum);
+    }
+
+    private void runAll(Thread[] threads) {
+        for (Thread t : threads) {
+            t.start();
+        }
+        for (Thread t : threads) {
+            try {
+                t.join();
+            } catch (InterruptedException e) {
+                // nop
+            }
+        }
+    }
+    public static void main(String[] args) {
+        long start, end;
+        start = System.currentTimeMillis();
+        new RcuHashMapBenchmark(1);
+        end = System.currentTimeMillis();
+        System.out.println("Elapsed during warmup: " + (end - start) + " ms.");
+        for (int i=0; i < 16; i++) {
+            start = System.currentTimeMillis();
+            new RcuHashMapBenchmark(i+1);
+            end = System.currentTimeMillis();
+            System.out.println("Elapsed during " + (i+1) + " threads: " + (end - start) + " ms.");
+        }
+
+    }
+}
diff --git a/vespajlib/developernotes/ThreadLocalDirectoryBenchmark.java b/vespajlib/developernotes/ThreadLocalDirectoryBenchmark.java
new file mode 100644
index 00000000000..bc91d076e8d
--- /dev/null
+++ b/vespajlib/developernotes/ThreadLocalDirectoryBenchmark.java
@@ -0,0 +1,230 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.concurrent;
+
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Benchmark to compare ThreadLocalDirectory with java.util.concurrent's atomic
+ * variables. Very low precision since it's an adapted unit test.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class ThreadLocalDirectoryBenchmark {
+    private static final int ITERATIONS = 500000;
+    private final AtomicInteger atomicCounter = new AtomicInteger(0);
+    private volatile int volatileCounter = 0;
+    private int naiveCounter = 0;
+
+    private static class SumUpdater implements ThreadLocalDirectory.Updater<Integer, Integer> {
+
+        @Override
+        public Integer update(Integer current, Integer x) {
+            return Integer.valueOf(current.intValue() + x.intValue());
+        }
+
+        @Override
+        public Integer createGenerationInstance(Integer previous) {
+            return Integer.valueOf(0);
+        }
+    }
+
+    private static class Counter implements Runnable {
+        ThreadLocalDirectory<Integer, Integer> r;
+
+        Counter(ThreadLocalDirectory<Integer, Integer> r) {
+            this.r = r;
+        }
+
+        @Override
+        public void run() {
+            LocalInstance<Integer, Integer> s = r.getLocalInstance();
+            for (int i = 0; i < ITERATIONS; ++i) {
+                r.update(Integer.valueOf(i), s);
+            }
+        }
+    }
+
+    private static class MutableSumUpdater implements ThreadLocalDirectory.Updater<IntWrapper, IntWrapper> {
+
+        @Override
+        public IntWrapper update(IntWrapper current, IntWrapper x) {
+            current.counter += x.counter;
+            return current;
+        }
+
+        @Override
+        public IntWrapper createGenerationInstance(IntWrapper previous) {
+            return new IntWrapper();
+        }
+    }
+
+    private static class IntWrapper {
+        public int counter = 0;
+    }
+
+    private static class WrapperCounter implements Runnable {
+        ThreadLocalDirectory<IntWrapper, IntWrapper> r;
+
+        WrapperCounter(ThreadLocalDirectory<IntWrapper, IntWrapper> r) {
+            this.r = r;
+        }
+
+        @Override
+        public void run() {
+            LocalInstance<IntWrapper, IntWrapper> s = r.getLocalInstance();
+            IntWrapper w = new IntWrapper();
+            for (int i = 0; i < ITERATIONS; ++i) {
+                w.counter = i;
+                r.update(w, s);
+            }
+        }
+    }
+
+    private class AtomicCounter implements Runnable {
+        @Override
+        public void run() {
+            for (int i = 0; i < ITERATIONS; ++i) {
+                atomicCounter.addAndGet(i);
+            }
+        }
+    }
+
+    /**
+     * This just bangs on a shared volatile to give an idea of the basic cost of
+     * sharing a single variable with a memory barrier.
+     */
+    private class VolatileSillyness implements Runnable {
+
+        @Override
+        public void run() {
+            for (int i = 0; i < ITERATIONS; ++i) {
+                volatileCounter += i;
+            }
+        }
+    }
+
+    /**
+     * This just bangs on a shared to give some sort of lower bound for time
+     * elapsed.
+     */
+    private class SillySillyness implements Runnable {
+
+        @Override
+        public void run() {
+            for (int i = 0; i < ITERATIONS; ++i) {
+                naiveCounter += i;
+            }
+        }
+    }
+
+    private void sumFromMultipleThreads() {
+        SumUpdater updater = new SumUpdater();
+        ThreadLocalDirectory<Integer, Integer> s = new ThreadLocalDirectory<Integer, Integer>(updater);
+        Thread[] threads = new Thread[500];
+        for (int i = 0; i < 500; ++i) {
+            Counter c = new Counter(s);
+            threads[i] = new Thread(c);
+        }
+        runAll(threads);
+        List<Integer> measurements = s.fetch();
+        long sum = 0;
+        for (Integer i : measurements) {
+            sum += i.intValue();
+        }
+        System.out.println("Sum from all threads: " + sum);
+    }
+
+    private void sumMutableFromMultipleThreads() {
+        MutableSumUpdater updater = new MutableSumUpdater();
+        ThreadLocalDirectory<IntWrapper, IntWrapper> s = new ThreadLocalDirectory<IntWrapper, IntWrapper>(updater);
+        Thread[] threads = new Thread[500];
+        for (int i = 0; i < 500; ++i) {
+            WrapperCounter c = new WrapperCounter(s);
+            threads[i] = new Thread(c);
+        }
+        runAll(threads);
+        List<IntWrapper> measurements = s.fetch();
+        long sum = 0;
+        for (IntWrapper i : measurements) {
+            sum += i.counter;
+        }
+        System.out.println("Sum from all threads: " + sum);
+    }
+
+    private void sumAtomicFromMultipleThreads() {
+        Thread[] threads = new Thread[500];
+        for (int i = 0; i < 500; ++i) {
+            AtomicCounter c = new AtomicCounter();
+            threads[i] = new Thread(c);
+        }
+        runAll(threads);
+        System.out.println("Sum from all threads: " + atomicCounter.get());
+    }
+
+    private void overwriteVolatileFromMultipleThreads() {
+        Thread[] threads = new Thread[500];
+        for (int i = 0; i < 500; ++i) {
+            VolatileSillyness c = new VolatileSillyness();
+            threads[i] = new Thread(c);
+        }
+        runAll(threads);
+        System.out.println("Checksum from all threads: " + volatileCounter);
+    }
+
+    private void overwriteIntegerFromMultipleThreads() {
+        Thread[] threads = new Thread[500];
+        for (int i = 0; i < 500; ++i) {
+            SillySillyness c = new SillySillyness();
+            threads[i] = new Thread(c);
+        }
+        runAll(threads);
+        System.out.println("Checksum from all threads: " + volatileCounter);
+    }
+
+    private void runAll(Thread[] threads) {
+        for (Thread t : threads) {
+            t.start();
+        }
+        for (Thread t : threads) {
+            try {
+                t.join();
+            } catch (InterruptedException e) {
+                // nop
+            }
+        }
+    }
+
+    public static void main(String[] args) {
+        ThreadLocalDirectoryBenchmark benchmark = new ThreadLocalDirectoryBenchmark();
+        long end;
+        System.out.println("ThreadLocalDirectory<Integer, Integer>");
+        long start = System.currentTimeMillis();
+        benchmark.sumFromMultipleThreads();
+        end = System.currentTimeMillis();
+        System.out.println("Elapsed using threadlocals: " + (end - start) + " ms.");
+        System.out.println("AtomicInteger");
+        start = System.currentTimeMillis();
+        benchmark.sumAtomicFromMultipleThreads();
+        end = System.currentTimeMillis();
+        System.out.println("Elapsed using atomic integer: " + (end - start) + " ms.");
+        System.out.println("volatile int += volatile int");
+        start = System.currentTimeMillis();
+        benchmark.overwriteVolatileFromMultipleThreads();
+        end = System.currentTimeMillis();
+        System.out.println("Elapsed using single shared volatile: " + (end - start) + " ms.");
+        System.out.println("int += int");
+        start = System.currentTimeMillis();
+        benchmark.overwriteIntegerFromMultipleThreads();
+        end = System.currentTimeMillis();
+        System.out.println("Checksum: " + benchmark.naiveCounter);
+        System.out.println("Elapsed using shared int: " + (end - start) + " ms.");
+        System.out.println("ThreadLocalDirectory<IntWrapper, IntWrapper>");
+        start = System.currentTimeMillis();
+        benchmark.sumMutableFromMultipleThreads();
+        end = System.currentTimeMillis();
+        System.out.println("Elapsed using threadlocal with mutable int wrapper: " + (end - start) + " ms.");
+    }
+
+}
diff --git a/vespajlib/developernotes/Utf8MicroBencmark.java b/vespajlib/developernotes/Utf8MicroBencmark.java
new file mode 100644
index 00000000000..ff1ae4ce3a2
--- /dev/null
+++ b/vespajlib/developernotes/Utf8MicroBencmark.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.text;
+
+/**
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ * @since 5.2
+ */
+
+public class Utf8MicroBencmark {
+    public void benchmark(int sizeInK) {
+        String [] l = new String[1000];
+        for (int i=0; i < l.length; i++) {
+            l[i] = "typical ascii string" + i;
+        }
+        System.out.println("Warming up...");
+        utf8encode(l, 10000); // warm-up
+        utf8encodeFast(l, 10000);
+
+        long startTime, endTime, sum;
+        System.out.println("Starting benchmark ...");
+        startTime=System.currentTimeMillis();
+        sum = utf8encode(l, sizeInK);
+        endTime=System.currentTimeMillis();
+        System.out.println("Utf8 encoding " + sizeInK + "k strings took " + (endTime-startTime) + "ms generating " + sum + "bytes");
+        startTime=System.currentTimeMillis();
+        sum = utf8encodeFast(l, sizeInK);
+        endTime=System.currentTimeMillis();
+        System.out.println("Utf8 fast encoding " + sizeInK + "k strings took " + (endTime-startTime) + "ms generating " + sum + "bytes");
+    }
+
+    private long utf8encode(String [] l, int sizeInK) {
+        long sum = 0;
+        for (int i=0; i<1000*sizeInK; i++) {
+            sum += Utf8.toBytesStd(l[i%l.length]).length;
+        }
+        return sum;
+    }
+    private long utf8encodeFast(String [] l, int sizeInK) {
+        long sum = 0;
+        for (int i=0; i<1000*sizeInK; i++) {
+            sum += Utf8.toBytes(l[i%l.length]).length;
+        }
+        return sum;
+    }
+
+    public static void main(String[] args) {
+        new Utf8MicroBencmark().benchmark(10000);
+    }
+
+}
diff --git a/vespajlib/developernotes/XMLMicroBenchmark.java b/vespajlib/developernotes/XMLMicroBenchmark.java
new file mode 100644
index 00000000000..6a9d02e1c45
--- /dev/null
+++ b/vespajlib/developernotes/XMLMicroBenchmark.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.text;
+
+/**
+ * It is what it says
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class XMLMicroBenchmark {
+
+    public void benchmark(int sizeInK) {
+        System.out.println("Warming up...");
+        escapeStrings(1000); // warm-up
+
+        System.out.println("Starting benchmark...");
+        long startTime=System.currentTimeMillis();
+        escapeStrings(sizeInK);
+        long endTime=System.currentTimeMillis();
+        System.out.println("Done.\nEscaping " + sizeInK + "k strings took " + (endTime-startTime) + "ms");
+    }
+
+    private void escapeStrings(int sizeInK) {
+        for (int i=0; i<1000*sizeInK; i++) {
+            XML.xmlEscape("foobar" + i,true,true,'\u001f');
+        }
+    }
+
+    public static void main(String[] args) {
+        new XMLMicroBenchmark().benchmark(10000);
+    }
+
+}
diff --git a/vespajlib/developernotes/XMLWriterMicroBenchmark.java b/vespajlib/developernotes/XMLWriterMicroBenchmark.java
new file mode 100644
index 00000000000..67570d54ea6
--- /dev/null
+++ b/vespajlib/developernotes/XMLWriterMicroBenchmark.java
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.text;
+
+import com.yahoo.io.ByteWriter;
+
+import java.io.ByteArrayOutputStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+
+/**
+ * It is what it says
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class XMLWriterMicroBenchmark {
+
+    private final ByteArrayOutputStream output;
+    private final XMLWriter xmlWriter;
+
+    public XMLWriterMicroBenchmark(boolean optimize) {
+        // setup
+        output=new ByteArrayOutputStream();
+        Charset cs = Charset.forName("utf-8");
+        CharsetEncoder encoder = cs.newEncoder();
+        xmlWriter=new XMLWriter(new ByteWriter(output, encoder), optimize);
+    }
+
+    public void benchmark(int sizeInK,boolean verifyOutput) {
+        System.out.println("Warming up...");
+        writeStrings(1000); // warm-up
+
+        System.out.println("Starting benchmark...");
+        long startTime=System.currentTimeMillis();
+        writeStrings(sizeInK);
+        long endTime=System.currentTimeMillis();
+        System.out.println("Done.\nWriting " + sizeInK + "k strings took " + (endTime-startTime) + "ms");
+
+        if (verifyOutput) {
+            System.out.println("First 1k of output:");
+            String result=null;
+            try { result=output.toString("utf-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); }
+            System.out.println(result.substring(0,Math.min(500,result.length())));
+        }
+    }
+
+    private void writeStrings(int sizeInK) {
+        for (int i=0; i<1000*sizeInK; i++) {
+            xmlWriter.openTag("dummytag").content(i,false).closeTag();
+        }
+    }
+
+    public static void main(String[] args) {
+        System.out.println("Unoptimized: -------------------------");
+        new XMLWriterMicroBenchmark(false).benchmark(10000,false);
+        System.out.println("Optimized:   ------------------------");
+        new XMLWriterMicroBenchmark(true).benchmark(10000,false);
+    }
+
+
+}