Publish

7 files changed, 1218 insertions, 0 deletions

diff --git a/vespajlib/src/test/java/com/yahoo/net/HostNameTestCase.java b/vespajlib/src/test/java/com/yahoo/net/HostNameTestCase.java
new file mode 100644
index 00000000000..98be9f0ef6f
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/net/HostNameTestCase.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.net;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author lulf
+ */
+public class HostNameTestCase {
+    @Test
+    public void testHostnameIsFound() {
+        assertFalse(HostName.getLocalhost().isEmpty());
+    }
+}
diff --git a/vespajlib/src/test/java/com/yahoo/net/LinuxInetAddressTestCase.java b/vespajlib/src/test/java/com/yahoo/net/LinuxInetAddressTestCase.java
new file mode 100755
index 00000000000..fc1945d2d39
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/net/LinuxInetAddressTestCase.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

+package com.yahoo.net;

+

+import java.net.UnknownHostException;

+import java.net.InetAddress;

+import java.net.Inet4Address;

+

+/**

+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>

+ */

+public class LinuxInetAddressTestCase extends junit.framework.TestCase {

+

+    public void testPreferIPv4() throws UnknownHostException {

+        try {

+            // This test only works if there is at least one inet address returned.

+            InetAddress[] arr = LinuxInetAddress.getAllLocal();

+            if (arr.length > 0) {

+                // System.out.println("Got " + arr.length + " addresses.");

+

+                // And it can only make sure it is preferred if there is at least one ip v4 address.

+                boolean ipv4 = false;

+                for (int i = 0; i < arr.length; ++i) {

+                    // System.out.println("Address " + i + " is an instance of " + arr[i].getClass() + ".");

+                    if (arr[i] instanceof Inet4Address) {

+                        ipv4 = true;

+                    }

+                }

+

+                // And the only thing we test is that an ip v4 address is preferred.

+                if (ipv4) {

+                    InetAddress addr = LinuxInetAddress.getLocalHost();

+                    assertNotNull("IPv4 is prefered", addr instanceof Inet4Address);

+                }

+            }

+        }

+        catch (java.net.UnknownHostException e) {

+            // We're on vpn or have no network

+        }

+    }

+

+}

diff --git a/vespajlib/src/test/java/com/yahoo/net/URITestCase.java b/vespajlib/src/test/java/com/yahoo/net/URITestCase.java
new file mode 100644
index 00000000000..7bb2303d7bb
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/net/URITestCase.java
@@ -0,0 +1,512 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.net;
+
+
+import java.util.List;
+
+
+/**
+ * Tests the URI class
+ *
+ * @author  <a href="mailto:bratseth@fast.no">Jon S Bratseth</a>
+ */
+public class URITestCase extends junit.framework.TestCase {
+
+    public URITestCase(String name) {
+        super(name);
+    }
+
+    public void testEquality() {
+        URI one = new URI("http://www.nils.arne.com");
+        URI two = new URI("http://www.nils.arne.com");
+
+        assertEquals(one, two);
+        assertEquals(one.hashCode(), two.hashCode());
+        assertEquals("http://www.nils.arne.com/", one.toString());
+
+        assertEqualURIs(
+                "http://info.t.fast.no/art.php?sid=29&mode=thread&order=0",
+                "http://info.t.fast.no/art.php?sid=29&mode=thread&order=0");
+        assertEqualURIs("http://a/g/", "http://a/g/");
+        assertEquals("http://a/g;x?y#s",
+                new URI("http://a/g;x?y#s", true).stringValue());
+        assertEquals("http://a/g?y#s",
+                new URI("http://a/g?y#s", true).stringValue());
+        assertEqualURIs("http://a/b/c/.g", "http://a/b/c/.g");
+        assertEqualURIs("http://a/b/c/..g", "http://a/b/c/..g");
+        assertEqualURIs("http://a/b/c/g;x=1/y", "http://a/b/c/g;x=1/y");
+        assertEquals("http://a/b/c/g#s/../x",
+                new URI("http://a/b/c/g#s/../x", true).stringValue());
+        assertEquals("http://www.strange_host.com/b",
+                new URI("http://www.strange_host.com/b", true).stringValue());
+    }
+
+    public void testOpaque() {
+        URI uri = new URI("mailto:knut");
+
+        assertEquals("mailto:knut", uri.toString());
+        assertTrue(uri.isOpaque());
+    }
+
+    public void testValid() {
+        assertTrue(
+                new URI("http://www.one.com/isValid?even=if&theres=args").isValid());
+        assertTrue(
+                !new URI("http://www.one.com/isValid?even=if&theres=args").isOpaque());
+
+        assertTrue(!(new URI("not\\uri?", false, true).isValid()));
+
+        assertTrue(new URI("http://www.strange_host.com/b").isValid());
+        assertTrue(!new URI("http://www.strange_host.com/b").isOpaque());
+    }
+
+    public void testSorting() {
+        URI first = new URI("http://aisfirst.kanoo.com");
+        URI second = new URI("www.thentheresw.com");
+
+        assertTrue(first.compareTo(second) < 0);
+        assertTrue(second.compareTo(second) == 0);
+        assertTrue(second.compareTo(first) > 1);
+    }
+
+    public void testHost() {
+        assertEquals("a.b.c", new URI("http://A.B.C:567").getHost());
+        assertEquals("www.kanoo.com",
+                new URI("www.kanoo.com/foo", false, true).getHost());
+        assertEquals("a.b.c", new URI("http://a.b.C/foo").getHost());
+        assertEquals("a.b.c", new URI("http://a.b.C").getHost());
+        assertEquals("a", new URI("http://A").getHost());
+        assertEquals("a", new URI("http://A:80").getHost());
+    }
+
+    public void testUnfragmenting() {
+        assertEquals("http://www.sng.no/a/b/dee?kanoos&at=nught#chapter3",
+                new URI("http://www.sng.no/a/b/cee/../dee?kanoos&at=nught#chapter3", true).stringValue());
+        assertEquals("http://www.sng.no/a/b/dee?kanoos&at=nught",
+                new URI("http://www.sng.no/a/b/cee/../dee?kanoos&at=nught#chapter3", false).stringValue());
+    }
+
+    public void testNormalizing() {
+        // Abbreviation resolving heuristics
+        assertEquals("http://www.a.b/c",
+                new URI("www.a.b/c", false, true).toString());
+        assertEquals("file://x:\\a", new URI("x:\\a", false, true).toString());
+        assertEquals("file://c:/a", new URI("c:/a", false, true).toString());
+
+        // RFC 2396 normalizing
+        assertEqualURIs("http://a/c/d", "http://a/b/../c/d");
+        assertEqualURIs("http://a/b", "http://a/./b");
+
+        // FAST normalizing
+        assertEqualURIs("http://a/", "  http://a  ");
+        assertEqualURIs("http://a/%e6;m%e5;ha%f8;l", "http://a/\u00E6m\u00E5ha\u00F8l");
+        assertEqualURIs("http://a/&b", "http://a/&amp;b");
+        assertEqualURIs("http://a/", "http://A");
+        assertEqualURIs("http://a/", "http://a:80");
+        assertEqualURIs("https://a/", "https://a:443");
+        assertEqualURIs("http://a/", "http://a.");
+        assertEqualURIs("http://a/b", "http://a//b");
+        assertEqualURIs("http://a/b/", "http://A/b/");
+        assertEqualURIs("http://a/b/", "http://a./b/");
+        assertEqualURIs("http://a/", "http://a/b/../");
+        assertEqualURIs("http://a/../", "http://a/b/../a/../../");
+        assertEqualURIs("http://a/", "http://a/b/../");
+        assertEqualURIs("http://a/b/c/d", "http://a/b/c/d");
+        assertEqualURIs("http://a/b/c", "http://a/b/c#kanoo");
+
+        // Everything combined
+        assertEquals("http://www.a.b/m%e5;l/&/%f8;l&&/",
+                new URI("   WWW.a.B:80//m\u00E5l/.//&amp;/./\u00F8l&amp;&amp;/foo/../upp/./..", true, true).toString());
+    }
+
+    public void testParemeterAdding() {
+        assertEquals("http://a/?knug=zagg",
+                new URI("http://a/").addParameter("knug", "zagg").stringValue());
+        assertEquals("http://a/b?knug=zagg&fjukk=barra",
+                new URI("http://a/b?knug=zagg").addParameter("fjukk", "barra").stringValue());
+    }
+
+    private void assertEqualURIs(String fasit, String test) {
+        assertEquals(fasit, new URI(test).toString());
+    }
+
+    public void testDepth() {
+        assertEquals(0, new URI("test:hit").getDepth());
+        assertEquals(0, new URI("test://hit").getDepth());
+        assertEquals(0, new URI("test://hit/").getDepth());
+        assertEquals(1, new URI("test://hit.test/hello  ").getDepth());
+        assertEquals(1, new URI("test://hit.test/hello/").getDepth());
+        assertEquals(0, new URI("test:// ").getDepth());
+        assertEquals(0, new URI("test:///").getDepth());
+        assertEquals(1, new URI("test:////").getDepth());
+        assertEquals(2, new URI("test://hit.test/hello/test2/").getDepth());
+    }
+
+    public void testURLEmpty() {
+        URI uri = new URI("", true);
+        assertTrue(uri.isValid());
+        assertNull(uri.getScheme());
+        assertNull(uri.getHost());
+        assertNull(uri.getDomain());
+        assertNull(uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertNull(uri.getPath());
+        assertNull(uri.getFilename());
+        assertNull(uri.getExtension());
+        assertNull(uri.getParams());
+        assertNull(uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLDot() {
+        URI uri = new URI(".", true);
+        assertTrue(uri.isValid());
+        assertNull(uri.getScheme());
+        assertNull(uri.getHost());
+        assertNull(uri.getDomain());
+        assertNull(uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertNull(uri.getPath());  //differs from FastS_URL, "."
+        assertNull(uri.getFilename());  //differs from FastS_URL, "."
+        assertNull(uri.getExtension());
+        assertNull(uri.getParams());
+        assertNull(uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLDotDot() {
+        URI uri = new URI("..", true);
+        assertTrue(uri.isValid());
+        assertNull(uri.getScheme());
+        assertNull(uri.getHost());
+        assertNull(uri.getDomain());
+        assertNull(uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertNull(uri.getPath());  //differs from FastS_URL, ".."
+        assertNull(uri.getFilename());  //differs from FastS_URL, ".."
+        assertNull(uri.getExtension());
+        assertNull(uri.getParams());
+        assertNull(uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLUninett() {
+        URI uri = new URI("http://180.uninett.no/servlet/online.Bransje", true);
+        assertTrue(uri.isValid());
+        assertEquals("http", uri.getScheme());
+        assertEquals("180.uninett.no", uri.getHost());
+        assertEquals("uninett.no", uri.getDomain());
+        assertEquals("no", uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertEquals("/servlet/online.Bransje", uri.getPath());
+        assertEquals("online.Bransje", uri.getFilename());
+        assertEquals("Bransje", uri.getExtension());
+        assertNull(uri.getParams());
+        assertNull(uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLUnderdusken() {
+        URI uri = new URI("http://www.underdusken.no", true);
+        assertTrue(uri.isValid());
+        assertEquals("http", uri.getScheme());
+        assertEquals("www.underdusken.no", uri.getHost());
+        assertEquals("underdusken.no", uri.getDomain());
+        assertEquals("no", uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertEquals("", uri.getPath());
+        assertEquals("", uri.getFilename());
+        assertNull(uri.getExtension());
+        assertNull(uri.getParams());
+        assertNull(uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLUnderduskenUholdbar() {
+        URI uri =
+                new URI("http://www.underdusken.no/?page=dusker/html/0008/Uholdbar.html", true);
+        assertTrue(uri.isValid());
+        assertEquals("http", uri.getScheme());
+        assertEquals("www.underdusken.no", uri.getHost());
+        assertEquals("underdusken.no", uri.getDomain());
+        assertEquals("no", uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertEquals("/", uri.getPath());
+        assertEquals("", uri.getFilename());
+        assertNull(uri.getExtension());
+        assertNull(uri.getParams());
+        assertEquals("page=dusker/html/0008/Uholdbar.html", uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLUniKarlsruhe() {
+        URI uri = new URI("http://www.uni-karlsruhe.de/~ig25/ssh-faq/", true);
+        assertTrue(uri.isValid());
+        assertEquals("http", uri.getScheme());
+        assertEquals("www.uni-karlsruhe.de", uri.getHost());
+        assertEquals("uni-karlsruhe.de", uri.getDomain());
+        assertEquals("de", uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertEquals("/~ig25/ssh-faq/", uri.getPath());
+        assertEquals("", uri.getFilename());
+        assertNull(uri.getExtension());
+        assertNull(uri.getParams());
+        assertNull(uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLDetteErEn() {
+        URI uri = new URI("https://dette.er.en:2020/~janie/index.htm?param1=q&param2=r", true);
+        assertTrue(uri.isValid());
+        assertEquals("https", uri.getScheme());
+        assertEquals("dette.er.en", uri.getHost());
+        assertEquals("er.en", uri.getDomain());
+        assertEquals("en", uri.getMainTld());
+        assertEquals(2020, uri.getPort());
+        assertEquals("/~janie/index.htm", uri.getPath());
+        assertEquals("index.htm", uri.getFilename());
+        assertEquals("htm", uri.getExtension());
+        assertNull(uri.getParams());
+        assertEquals("param1=q&param2=r", uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLSonyCoUk() {
+        URI uri = new URI("http://www.sony.co.uk/", true);
+        assertTrue(uri.isValid());
+        assertEquals("http", uri.getScheme());
+        assertEquals("www.sony.co.uk", uri.getHost());
+        assertEquals("sony.co.uk", uri.getDomain());
+        assertEquals("uk", uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertEquals("/", uri.getPath());
+        assertEquals("", uri.getFilename());
+        assertNull(uri.getExtension());
+        assertNull(uri.getParams());
+        assertNull(uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLSonyCoUk2() {
+        URI uri = new URI("http://sony.co.uk/", true);
+        assertTrue(uri.isValid());
+        assertEquals("http", uri.getScheme());
+        assertEquals("sony.co.uk", uri.getHost());
+        //TODO: Fix when tldlist is implemented:
+        //assertEquals("sony.co.uk", uri.getDomain());
+        assertEquals("co.uk", uri.getDomain());
+        assertEquals("uk", uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertEquals("/", uri.getPath());
+        assertEquals("", uri.getFilename());
+        assertNull(uri.getExtension());
+        assertNull(uri.getParams());
+        assertNull(uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLSomehostSomedomain() {
+        URI uri = new URI("http://somehost.somedomain/this!is!it/boom", true);
+        assertTrue(uri.isValid());
+        assertEquals("http", uri.getScheme());
+        assertEquals("somehost.somedomain", uri.getHost());
+        assertEquals("somehost.somedomain", uri.getDomain());
+        assertEquals("somedomain", uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertEquals("/this!is!it/boom", uri.getPath());
+        assertEquals("boom", uri.getFilename());
+        assertNull(uri.getExtension());
+        assertNull(uri.getParams());
+        assertNull(uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLTestCom() {
+        URI uri = new URI("http://test.com/index.htm?p1=q%20test&p2=r%10d", true);
+        assertTrue(uri.isValid());
+        assertEquals("http", uri.getScheme());
+        assertEquals("test.com", uri.getHost());
+        assertEquals("test.com", uri.getDomain());
+        assertEquals("com", uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertEquals("/index.htm", uri.getPath());
+        assertEquals("index.htm", uri.getFilename());
+        assertEquals("htm", uri.getExtension());
+        assertNull(uri.getParams());
+        assertEquals("p1=q%20test&p2=r%10d", uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLArthur() {
+        URI uri = new URI("http://arthur/qm/images/qm1.gif", true);
+        assertTrue(uri.isValid());
+        assertEquals("http", uri.getScheme());
+        assertEquals("arthur", uri.getHost());
+        assertEquals("arthur", uri.getDomain());
+        assertNull(uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertEquals("/qm/images/qm1.gif", uri.getPath());
+        assertEquals("qm1.gif", uri.getFilename());
+        assertEquals("gif", uri.getExtension());
+        assertNull(uri.getParams());
+        assertNull(uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLFooCom() {
+        URI uri = new URI("http://foo.com/ui;.gif", true);
+        assertTrue(uri.isValid());
+        assertEquals("http", uri.getScheme());
+        assertEquals("foo.com", uri.getHost());
+        assertEquals("foo.com", uri.getDomain());
+        assertEquals("com", uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertEquals("/ui;.gif", uri.getPath());
+        assertEquals("ui", uri.getFilename());
+        assertNull(uri.getExtension());
+        assertEquals(".gif", uri.getParams());
+        assertNull(uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLFooCom2() {
+        URI uri = new URI("http://foo.com/ui;par1=1/par2=2", true);
+        assertTrue(uri.isValid());
+        assertEquals("http", uri.getScheme());
+        assertEquals("foo.com", uri.getHost());
+        assertEquals("foo.com", uri.getDomain());
+        assertEquals("com", uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertEquals("/ui;par1=1/par2=2", uri.getPath());
+        assertEquals("ui", uri.getFilename());
+        assertNull(uri.getExtension());
+        assertEquals("par1=1/par2=2", uri.getParams());
+        assertNull(uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testURLFooNo() {
+        URI uri = new URI(
+                "http://www.foo.no:8080/path/filename.ext;par1=hello/par2=world?query=test#fragment", true);
+        assertTrue(uri.isValid());
+        assertEquals("http", uri.getScheme());
+        assertEquals("www.foo.no", uri.getHost());
+        assertEquals("foo.no", uri.getDomain());
+        assertEquals("no", uri.getMainTld());
+        assertEquals(8080, uri.getPort());
+        assertEquals("/path/filename.ext;par1=hello/par2=world", uri.getPath());
+        assertEquals("filename.ext", uri.getFilename());
+        assertEquals("ext", uri.getExtension());
+        assertEquals("par1=hello/par2=world", uri.getParams());
+        assertEquals("query=test", uri.getQuery());
+        assertEquals("fragment", uri.getFragment());
+    }
+
+    public void testURLAmpersand() {
+        URI uri = new URI("http://canonsarang.com/zboard/data/gallery04/HU&BANG.jpg", true);
+        assertTrue(uri.isValid());
+        assertEquals("http", uri.getScheme());
+        assertEquals("canonsarang.com", uri.getHost());
+        assertEquals("canonsarang.com", uri.getDomain());
+        assertEquals("com", uri.getMainTld());
+        assertEquals(-1, uri.getPort());
+        assertEquals("/zboard/data/gallery04/HU&BANG.jpg", uri.getPath());
+        assertEquals("HU&BANG.jpg", uri.getFilename());
+        assertEquals("jpg", uri.getExtension());
+        assertNull(uri.getParams());
+        assertNull(uri.getQuery());
+        assertNull(uri.getFragment());
+    }
+
+    public void testQMark() {
+        URI uri = new URI("http://foobar/?");
+        assertTrue(uri.isValid());
+        assertEquals("http", uri.getScheme());
+        assertEquals("foobar", uri.getHost());
+        assertEquals("", uri.getQuery());
+    }
+
+    public void testTokenization() {
+        URI uri = new URI("http://this.i_s:5000/wo_ho;ba-lo?gobo#banana", true);
+        List<URI.Token> tokens = uri.tokenize();
+        URI.Token token;
+
+        token = tokens.get(0);
+        assertEquals("http", token.getToken());
+        assertEquals(URI.URLContext.URL_SCHEME, token.getContext());
+
+        token = tokens.get(1);
+        assertEquals("this", token.getToken());
+        assertEquals(URI.URLContext.URL_HOST, token.getContext());
+
+        token = tokens.get(2);
+        assertEquals("i_s", token.getToken());
+        assertEquals(URI.URLContext.URL_HOST, token.getContext());
+
+        token = tokens.get(3);
+        assertEquals("5000", token.getToken());
+        assertEquals(URI.URLContext.URL_PORT, token.getContext());
+
+        token = tokens.get(4);
+        assertEquals("wo_ho", token.getToken());
+        assertEquals(URI.URLContext.URL_PATH, token.getContext());
+
+        token = tokens.get(5);
+        assertEquals("ba-lo", token.getToken());
+        assertEquals(URI.URLContext.URL_PATH, token.getContext());
+
+        token = tokens.get(6);
+        assertEquals("gobo", token.getToken());
+        assertEquals(URI.URLContext.URL_QUERY, token.getContext());
+
+        token = tokens.get(7);
+        assertEquals("banana", token.getToken());
+        assertEquals(URI.URLContext.URL_FRAGMENT, token.getContext());
+
+        try {
+            tokens.get(8);
+            fail();
+        } catch (IndexOutOfBoundsException ioobe) {
+        }
+    }
+
+        // Error reported int bug #2466528
+        public void testFileURIEmptyHost() {
+                URI uri = new URI("file:///C:/Inetpub/wwwroot/DW_SHORTCUTS.htm");
+        List<URI.Token> tokens = uri.tokenize();
+        URI.Token token;
+        token = tokens.get(0);
+        assertEquals("file", token.getToken());
+        assertEquals(URI.URLContext.URL_SCHEME, token.getContext());
+
+        token = tokens.get(1);
+        assertEquals("localhost", token.getToken());
+        assertEquals(URI.URLContext.URL_HOST, token.getContext());
+
+        token = tokens.get(2);
+        assertEquals("C", token.getToken());
+        assertEquals(URI.URLContext.URL_PATH, token.getContext());
+
+        token = tokens.get(3);
+        assertEquals("Inetpub", token.getToken());
+        assertEquals(URI.URLContext.URL_PATH, token.getContext());
+
+        token = tokens.get(4);
+        assertEquals("wwwroot", token.getToken());
+        assertEquals(URI.URLContext.URL_PATH, token.getContext());
+
+        token = tokens.get(5);
+        assertEquals("DW_SHORTCUTS", token.getToken());
+        assertEquals(URI.URLContext.URL_PATH, token.getContext());
+
+        token = tokens.get(6);
+        assertEquals("htm", token.getToken());
+        assertEquals(URI.URLContext.URL_PATH, token.getContext());
+
+        try {
+            tokens.get(7);
+            fail();
+        } catch (IndexOutOfBoundsException ioobe) {
+        }
+        }
+}
diff --git a/vespajlib/src/test/java/com/yahoo/net/UriToolsTestCase.java b/vespajlib/src/test/java/com/yahoo/net/UriToolsTestCase.java
new file mode 100644
index 00000000000..9a17fa341be
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/net/UriToolsTestCase.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.net;
+
+import static org.junit.Assert.*;
+
+import java.net.URISyntaxException;
+
+import org.junit.Test;
+
+/**
+ * Check validity of the URI helper methods.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class UriToolsTestCase {
+
+    private static final String SEARCH_QUERY = "/search/?query=sddocname:music#trick";
+
+    @Test
+    public final void testRawRequest() throws URISyntaxException {
+        java.net.URI uri = new java.net.URI("http://localhost:" + 8080 + SEARCH_QUERY);
+        assertEquals(SEARCH_QUERY, UriTools.rawRequest(uri));
+    }
+
+}
diff --git a/vespajlib/src/test/java/com/yahoo/net/UrlTestCase.java b/vespajlib/src/test/java/com/yahoo/net/UrlTestCase.java
new file mode 100644
index 00000000000..7cf99cf2c5a
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/net/UrlTestCase.java
@@ -0,0 +1,192 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.net;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class UrlTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        Url url = Url.fromString("scheme://user:pass@host:69/path?query#fragment");
+        assertEquals("scheme://user:pass@host:69/path?query#fragment", url.toString());
+        assertEquals("scheme", url.getScheme());
+        assertEquals(0, url.getSchemeBegin());
+        assertEquals(6, url.getSchemeEnd());
+        assertEquals("user", url.getUserInfo());
+        assertEquals(9, url.getUserInfoBegin());
+        assertEquals(13, url.getUserInfoEnd());
+        assertEquals("pass", url.getPassword());
+        assertEquals(14, url.getPasswordBegin());
+        assertEquals(18, url.getPasswordEnd());
+        assertEquals("host", url.getHost());
+        assertEquals(19, url.getHostBegin());
+        assertEquals(23, url.getHostEnd());
+        assertEquals("69", url.getPortString());
+        assertEquals(24, url.getPortBegin());
+        assertEquals(26, url.getPortEnd());
+        assertEquals(Integer.valueOf(69), url.getPort());
+        assertEquals("/path", url.getPath());
+        assertEquals(26, url.getPathBegin());
+        assertEquals(31, url.getPathEnd());
+        assertEquals("query", url.getQuery());
+        assertEquals(32, url.getQueryBegin());
+        assertEquals(37, url.getQueryEnd());
+        assertEquals("fragment", url.getFragment());
+        assertEquals(38, url.getFragmentBegin());
+        assertEquals(46, url.getFragmentEnd());
+    }
+
+    @Test
+    public void requireThatOffsetsAreNeverOutOfBounds() {
+        Url url = Url.fromString("http:");
+        assertEquals(0, url.getSchemeBegin());
+        assertEquals(4, url.getSchemeEnd());
+        assertEquals(5, url.getUserInfoBegin());
+        assertEquals(5, url.getUserInfoEnd());
+        assertEquals(5, url.getPasswordBegin());
+        assertEquals(5, url.getPasswordEnd());
+        assertEquals(5, url.getHostBegin());
+        assertEquals(5, url.getHostEnd());
+        assertEquals(5, url.getPortBegin());
+        assertEquals(5, url.getPortEnd());
+        assertEquals(5, url.getPathBegin());
+        assertEquals(5, url.getPathEnd());
+        assertEquals(5, url.getQueryBegin());
+        assertEquals(5, url.getQueryEnd());
+        assertEquals(5, url.getFragmentBegin());
+        assertEquals(5, url.getFragmentEnd());
+
+        url = Url.fromString("//host");
+        assertEquals(0, url.getSchemeBegin());
+        assertEquals(0, url.getSchemeEnd());
+        assertEquals(2, url.getUserInfoBegin());
+        assertEquals(2, url.getUserInfoEnd());
+        assertEquals(2, url.getPasswordBegin());
+        assertEquals(2, url.getPasswordEnd());
+        assertEquals(2, url.getHostBegin());
+        assertEquals(6, url.getHostEnd());
+        assertEquals(6, url.getPortBegin());
+        assertEquals(6, url.getPortEnd());
+        assertEquals(6, url.getPathBegin());
+        assertEquals(6, url.getPathEnd());
+        assertEquals(6, url.getQueryBegin());
+        assertEquals(6, url.getQueryEnd());
+        assertEquals(6, url.getFragmentBegin());
+        assertEquals(6, url.getFragmentEnd());
+    }
+
+    @Test
+    public void requireThatCommonSchemesCanBeParsed() {
+        assertParse("ftp://ftp.is.co.za/rfc/rfc1808.txt",
+                    "ftp", null, null, "ftp.is.co.za", null, "/rfc/rfc1808.txt", null, null);
+        assertParse("http://www.ietf.org/rfc/rfc 2396.txt",
+                    "http", null, null, "www.ietf.org", null, "/rfc/rfc 2396.txt", null, null);
+        assertParse("ldap://[2001:db8::7]/c=GB?objectClass?one",
+                    "ldap", null, null, "2001:db8::7", null, "/c=GB", "objectClass?one", null);
+        assertParse("mailto:John.Doe@example.com",
+                    "mailto", null, null, null, null, "John.Doe@example.com", null, null);
+        assertParse("news:comp.infosystems.www.servers.unix",
+                    "news", null, null, null, null, "comp.infosystems.www.servers.unix", null, null);
+        assertParse("tel:+1-816-555-1212",
+                    "tel", null, null, null, null, "+1-816-555-1212", null, null);
+        assertParse("telnet://192.0.2.16:80/",
+                    "telnet", null, null, "192.0.2.16", 80, "/", null, null);
+        assertParse("urn:oasis:names:specification:docbook:dtd:xml:4.1.2",
+                    "urn", null, null, null, null, "oasis:names:specification:docbook:dtd:xml:4.1.2", null, null);
+    }
+
+    @Test
+    public void requireThatAllComponentsCanBeParsed() {
+        assertParse("scheme:",
+                    "scheme", null, null, null, null, null, null, null);
+        assertParse("scheme://",
+                    "scheme", null, null, null, null, "//", null, null);
+        assertParse("scheme://host",
+                    "scheme", null, null, "host", null, null, null, null);
+        try {
+            assertParse("scheme://host:foo",
+                        null, null, null, null, null, null, null, null);
+            fail();
+        } catch (NumberFormatException e) {
+            // expected
+        }
+        assertParse("scheme://host:69",
+                    "scheme", null, null, "host", 69, null, null, null);
+        assertParse("scheme://user@host:69",
+                    "scheme", "user", null, "host", 69, null, null, null);
+        assertParse("scheme://user:pass@host:69",
+                    "scheme", "user", "pass", "host", 69, null, null, null);
+        assertParse("scheme://user:pass@host:69",
+                    "scheme", "user", "pass", "host", 69, null, null, null);
+        assertParse("scheme://user:pass@host:69/",
+                    "scheme", "user", "pass", "host", 69, "/", null, null);
+        assertParse("scheme://user:pass@host:69/path",
+                    "scheme", "user", "pass", "host", 69, "/path", null, null);
+        assertParse("scheme://user:pass@host:69/path?query",
+                    "scheme", "user", "pass", "host", 69, "/path", "query", null);
+        assertParse("scheme://user:pass@host:69/path?query#fragment",
+                    "scheme", "user", "pass", "host", 69, "/path", "query", "fragment");
+        assertParse("scheme://user@host:69/path?query#fragment",
+                    "scheme", "user", null, "host", 69, "/path", "query", "fragment");
+        assertParse("scheme://host:69/path?query#",
+                    "scheme", null, null, "host", 69, "/path", "query", null);
+        assertParse("scheme://host:69/path?query#fragment",
+                    "scheme", null, null, "host", 69, "/path", "query", "fragment");
+        assertParse("scheme://host/path?query#fragment",
+                    "scheme", null, null, "host", null, "/path", "query", "fragment");
+        assertParse("scheme:///path?query#fragment",
+                    "scheme", null, null, null, null, "///path", "query", "fragment");
+        assertParse("scheme://?query#fragment",
+                    "scheme", null, null, null, null, "//", "query", "fragment");
+        assertParse("scheme://#fragment",
+                    "scheme", null, null, null, null, "//", null, "fragment");
+    }
+
+    @Test
+    public void requireThatIPv6CanBeParsed() {
+        assertParse("http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]",
+                    "http", null, null, "2001:0db8:85a3:0000:0000:8a2e:0370:7334", null, null, null, null);
+        assertParse("http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]/path",
+                    "http", null, null, "2001:0db8:85a3:0000:0000:8a2e:0370:7334", null, "/path", null, null);
+
+        assertParse("http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:80",
+                    "http", null, null, "2001:0db8:85a3:0000:0000:8a2e:0370:7334", 80, null, null, null);
+        assertParse("http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:80/path",
+                    "http", null, null, "2001:0db8:85a3:0000:0000:8a2e:0370:7334", 80, "/path", null, null);
+    }
+
+    private static void assertParse(String input, String scheme, String userInfo, String password, String host,
+                                    Integer port, String path, String query, String fragment)
+    {
+        Url urlA = Url.fromString(input);
+        assertEquals("Image", input, urlA.toString());
+        assertUrl(urlA, scheme, userInfo, password, host, port, path, query, fragment);
+
+        Url urlB = new Url(urlA.getScheme(), urlA.getUserInfo(), urlA.getPassword(), urlA.getHost(), urlA.getPort(),
+                           urlA.getPath(), urlA.getQuery(), urlA.getFragment());
+        assertUrl(urlB, scheme, userInfo, password, host, port, path, query, fragment);
+
+        Url urlC = Url.fromString(urlB.toString());
+        assertEquals(urlB, urlC);
+        assertUrl(urlC, scheme, userInfo, password, host, port, path, query, fragment);
+    }
+
+    private static void assertUrl(Url url, String scheme, String userInfo, String password, String host, Integer port,
+                                  String path, String query, String fragment)
+    {
+        assertEquals("Scheme", scheme, url.getScheme());
+        assertEquals("User", userInfo, url.getUserInfo());
+        assertEquals("Password", password, url.getPassword());
+        assertEquals("Host", host, url.getHost());
+        assertEquals("Port", port, url.getPort());
+        assertEquals("Path", path, url.getPath());
+        assertEquals("Query", query, url.getQuery());
+        assertEquals("Fragment", fragment, url.getFragment());
+    }
+}
diff --git a/vespajlib/src/test/java/com/yahoo/net/UrlTokenTestCase.java b/vespajlib/src/test/java/com/yahoo/net/UrlTokenTestCase.java
new file mode 100644
index 00000000000..ca42a701655
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/net/UrlTokenTestCase.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.net;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class UrlTokenTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        UrlToken token = new UrlToken(UrlToken.Type.FRAGMENT, 69, "foo", "bar");
+        assertEquals(UrlToken.Type.FRAGMENT, token.getType());
+        assertEquals(69, token.getOffset());
+        assertEquals(3, token.getLength());
+        assertEquals("foo", token.getOrig());
+        assertEquals("bar", token.getTerm());
+    }
+
+    @Test
+    public void requireThatTypeCanNotBeNull() {
+        try {
+            new UrlToken(null, 0, "foo", "bar");
+            fail();
+        } catch (NullPointerException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatOrigAndTermCanBeNull() {
+        UrlToken token = new UrlToken(UrlToken.Type.SCHEME, 0, null, "foo");
+        assertNull(token.getOrig());
+        assertEquals("foo", token.getTerm());
+
+        token = new UrlToken(UrlToken.Type.SCHEME, 0, "foo", null);
+        assertEquals("foo", token.getOrig());
+        assertNull(token.getTerm());
+
+        token = new UrlToken(UrlToken.Type.SCHEME, 0, null, null);
+        assertNull(token.getOrig());
+        assertNull(token.getTerm());
+    }
+}
diff --git a/vespajlib/src/test/java/com/yahoo/net/UrlTokenizerTestCase.java b/vespajlib/src/test/java/com/yahoo/net/UrlTokenizerTestCase.java
new file mode 100644
index 00000000000..d192c0901a6
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/net/UrlTokenizerTestCase.java
@@ -0,0 +1,385 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.net;
+
+import org.junit.Test;
+
+import java.util.*;
+
+import static org.junit.Assert.*;
+
+import static com.yahoo.text.Lowercase.toLowerCase;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class UrlTokenizerTestCase {
+
+    @Test
+    public void requireThatAllTokenCharactersAreAccepted() {
+        assertTerms("a", "a");
+        assertTerms("aa", "aa");
+        assertTerms("aaa", "aaa");
+        for (int c = Character.MIN_VALUE; c < Character.MAX_VALUE; ++c) {
+            if (c == '%') {
+                continue; // escape
+            }
+            String img = String.format("a%ca", c);
+            if ((c >= '0' && c <= '9') ||
+                (c >= 'a' && c <= 'z') ||
+                (c >= 'A' && c <= 'Z') ||
+                (c == '-' || c == '_'))
+            {
+                assertTerms(img, toLowerCase(img));
+            } else {
+                assertTerms(img, "a", "a");
+            }
+        }
+    }
+
+    @Test
+    public void requireThatUrlCanBeTokenized() {
+        assertTokenize("",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, null, "http"),
+                       new UrlToken(UrlToken.Type.PORT, 0, null, "80"));
+        assertTokenize("scheme:",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"));
+        assertTokenize("scheme://host",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.HOST, 9, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 9, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 13, null, UrlTokenizer.TERM_ENDHOST));
+        assertTokenize("scheme://user@host",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"),
+                       new UrlToken(UrlToken.Type.HOST, 14, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 14, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 18, null, UrlTokenizer.TERM_ENDHOST));
+        assertTokenize("scheme://user:pass@host",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 19, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST));
+        assertTokenize("scheme://user:pass@host:69",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 19, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 24, "69", "69"));
+        assertTokenize("scheme://user:pass@host:69/path",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 19, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 24, "69", "69"),
+                       new UrlToken(UrlToken.Type.PATH, 27, "path", "path"));
+        assertTokenize("scheme://user:pass@host:69/path?query",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 19, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 24, "69", "69"),
+                       new UrlToken(UrlToken.Type.PATH, 27, "path", "path"),
+                       new UrlToken(UrlToken.Type.QUERY, 32, "query", "query"));
+        assertTokenize("scheme://user:pass@host:69/path?query#fragment",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 19, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 24, "69", "69"),
+                       new UrlToken(UrlToken.Type.PATH, 27, "path", "path"),
+                       new UrlToken(UrlToken.Type.QUERY, 32, "query", "query"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 38, "fragment", "fragment"));
+    }
+
+    @Test
+    public void requireThatComponentsCanHaveMultipleTokens() {
+        assertTokenize("sch+eme://us+er:pa+ss@ho+st:69/pa/th?que+ry#frag+ment",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "sch", "sch"),
+                       new UrlToken(UrlToken.Type.SCHEME, 4, "eme", "eme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 10, "us", "us"),
+                       new UrlToken(UrlToken.Type.USERINFO, 13, "er", "er"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 16, "pa", "pa"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 19, "ss", "ss"),
+                       new UrlToken(UrlToken.Type.HOST, 22, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 22, "ho", "ho"),
+                       new UrlToken(UrlToken.Type.HOST, 25, "st", "st"),
+                       new UrlToken(UrlToken.Type.HOST, 27, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 28, "69", "69"),
+                       new UrlToken(UrlToken.Type.PATH, 31, "pa", "pa"),
+                       new UrlToken(UrlToken.Type.PATH, 34, "th", "th"),
+                       new UrlToken(UrlToken.Type.QUERY, 37, "que", "que"),
+                       new UrlToken(UrlToken.Type.QUERY, 41, "ry", "ry"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 44, "frag", "frag"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 49, "ment", "ment"));
+    }
+    @Test
+    public void requireThatSequencesOfDelimitersAreCollapsed() {
+        assertTokenize("sch++eme://us++er:pa++ss@ho++st:69/pa/th?que++ry#frag++ment",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "sch", "sch"),
+                       new UrlToken(UrlToken.Type.SCHEME, 5, "eme", "eme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 11, "us", "us"),
+                       new UrlToken(UrlToken.Type.USERINFO, 15, "er", "er"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 18, "pa", "pa"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 22, "ss", "ss"),
+                       new UrlToken(UrlToken.Type.HOST, 25, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 25, "ho", "ho"),
+                       new UrlToken(UrlToken.Type.HOST, 29, "st", "st"),
+                       new UrlToken(UrlToken.Type.HOST, 31, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 32, "69", "69"),
+                       new UrlToken(UrlToken.Type.PATH, 35, "pa", "pa"),
+                       new UrlToken(UrlToken.Type.PATH, 38, "th", "th"),
+                       new UrlToken(UrlToken.Type.QUERY, 41, "que", "que"),
+                       new UrlToken(UrlToken.Type.QUERY, 46, "ry", "ry"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 49, "frag", "frag"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 55, "ment", "ment"));
+    }
+
+    @Test
+    public void requireThatIPv6CanBeTokenized() {
+        assertTokenize("scheme://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.HOST, 10, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 10, "2001", "2001"),
+                       new UrlToken(UrlToken.Type.HOST, 15, "0db8", "0db8"),
+                       new UrlToken(UrlToken.Type.HOST, 20, "85a3", "85a3"),
+                       new UrlToken(UrlToken.Type.HOST, 25, "0000", "0000"),
+                       new UrlToken(UrlToken.Type.HOST, 30, "0000", "0000"),
+                       new UrlToken(UrlToken.Type.HOST, 35, "8a2e", "8a2e"),
+                       new UrlToken(UrlToken.Type.HOST, 40, "0370", "0370"),
+                       new UrlToken(UrlToken.Type.HOST, 45, "7334", "7334"),
+                       new UrlToken(UrlToken.Type.HOST, 49, null, UrlTokenizer.TERM_ENDHOST));
+    }
+
+    @Test
+    public void requireThatTermsAreLowerCased() {
+        assertTokenize("SCHEME://USER:PASS@HOST:69/PATH?QUERY#FRAGMENT",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "SCHEME", "scheme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 9, "USER", "user"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 14, "PASS", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 19, "HOST", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 24, "69", "69"),
+                       new UrlToken(UrlToken.Type.PATH, 27, "PATH", "path"),
+                       new UrlToken(UrlToken.Type.QUERY, 32, "QUERY", "query"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 38, "FRAGMENT", "fragment"));
+    }
+
+    @Test
+    public void requireThatEscapedCharsAreDecoded() {
+        assertTokenize("sch%65me://%75ser:p%61ss@h%6fst:69/p%61th?q%75ery#fr%61gment",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "sch%65me", "scheme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 11, "%75ser", "user"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 18, "p%61ss", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 25, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 25, "h%6fst", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 31, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 32, "69", "69"),
+                       new UrlToken(UrlToken.Type.PATH, 35, "p%61th", "path"),
+                       new UrlToken(UrlToken.Type.QUERY, 42, "q%75ery", "query"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 50, "fr%61gment", "fragment"));
+    }
+
+    @Test
+    public void requireThatDecodedCharsAreLowerCased() {
+        assertTokenize("sch%45me://%55ser:p%41ss@h%4fst:69/p%41th?q%55ery#fr%41gment",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "sch%45me", "scheme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 11, "%55ser", "user"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 18, "p%41ss", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 25, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 25, "h%4fst", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 31, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 32, "69", "69"),
+                       new UrlToken(UrlToken.Type.PATH, 35, "p%41th", "path"),
+                       new UrlToken(UrlToken.Type.QUERY, 42, "q%55ery", "query"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 50, "fr%41gment", "fragment"));
+    }
+
+    @Test
+    public void requireThatDecodedCharsCanSplitTokens() {
+        assertTokenize("sch%2beme://us%2ber:pa%2bss@ho%2bst:69/pa/th?que%2bry#frag%2bment",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "sch", "sch"),
+                       new UrlToken(UrlToken.Type.SCHEME, 6, "eme", "eme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 12, "us", "us"),
+                       new UrlToken(UrlToken.Type.USERINFO, 17, "er", "er"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 20, "pa", "pa"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 25, "ss", "ss"),
+                       new UrlToken(UrlToken.Type.HOST, 28, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 28, "ho", "ho"),
+                       new UrlToken(UrlToken.Type.HOST, 33, "st", "st"),
+                       new UrlToken(UrlToken.Type.HOST, 35, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 36, "69", "69"),
+                       new UrlToken(UrlToken.Type.PATH, 39, "pa", "pa"),
+                       new UrlToken(UrlToken.Type.PATH, 42, "th", "th"),
+                       new UrlToken(UrlToken.Type.QUERY, 45, "que", "que"),
+                       new UrlToken(UrlToken.Type.QUERY, 51, "ry", "ry"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 54, "frag", "frag"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 61, "ment", "ment"));
+    }
+
+    @Test
+    public void requireThatSchemeCanBeGuessed() {
+        assertTokenize("//host:80",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, null, "http"),
+                       new UrlToken(UrlToken.Type.HOST, 2, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 2, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 6, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 7, "80", "80"));
+    }
+
+    @Test
+    public void requireThatHostCanBeGuessed() {
+        assertTokenize("file:/path",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "file", "file"),
+                       new UrlToken(UrlToken.Type.HOST, 4, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 4, null, "localhost"),
+                       new UrlToken(UrlToken.Type.HOST, 4, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PATH, 6, "path", "path"));
+    }
+
+    @Test
+    public void requireThatPortCanBeGuessed() {
+        assertTokenize("http://host",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "http", "http"),
+                       new UrlToken(UrlToken.Type.HOST, 7, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 7, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 11, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 11, null, "80"));
+    }
+
+    @Test
+    public void requireThatComponentsAreOptional() {
+        assertTokenize("scheme", "user", "pass", "host", 99, "/path", "query", "fragment",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 19, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 24, "99", "99"),
+                       new UrlToken(UrlToken.Type.PATH, 27, "path", "path"),
+                       new UrlToken(UrlToken.Type.QUERY, 32, "query", "query"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 38, "fragment", "fragment"));
+        assertTokenize(null, "user", "pass", "host", 99, "/path", "query", "fragment",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, null, "http"),
+                       new UrlToken(UrlToken.Type.USERINFO, 2, "user", "user"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 7, "pass", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 12, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 12, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 16, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 17, "99", "99"),
+                       new UrlToken(UrlToken.Type.PATH, 20, "path", "path"),
+                       new UrlToken(UrlToken.Type.QUERY, 25, "query", "query"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 31, "fragment", "fragment"));
+        assertTokenize("scheme", null, "pass", "host", 99, "/path", "query", "fragment",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 10, "pass", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 15, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 15, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 20, "99", "99"),
+                       new UrlToken(UrlToken.Type.PATH, 23, "path", "path"),
+                       new UrlToken(UrlToken.Type.QUERY, 28, "query", "query"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 34, "fragment", "fragment"));
+        assertTokenize("scheme", null, null, "host", 99, "/path", "query", "fragment",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.HOST, 9, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 9, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 13, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 14, "99", "99"),
+                       new UrlToken(UrlToken.Type.PATH, 17, "path", "path"),
+                       new UrlToken(UrlToken.Type.QUERY, 22, "query", "query"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 28, "fragment", "fragment"));
+        assertTokenize("scheme", null, null, null, 99, "/path", "query", "fragment",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.PORT, 8, "99", "99"),
+                       new UrlToken(UrlToken.Type.PATH, 11, "path", "path"),
+                       new UrlToken(UrlToken.Type.QUERY, 16, "query", "query"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 22, "fragment", "fragment"));
+        assertTokenize("scheme", "user", "pass", "host", null, "/path", "query", "fragment",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 19, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PATH, 24, "path", "path"),
+                       new UrlToken(UrlToken.Type.QUERY, 29, "query", "query"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 35, "fragment", "fragment"));
+        assertTokenize("scheme", "user", "pass", "host", 99, null, "query", "fragment",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 19, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 24, "99", "99"),
+                       new UrlToken(UrlToken.Type.QUERY, 27, "query", "query"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 33, "fragment", "fragment"));
+        assertTokenize("scheme", "user", "pass", "host", 99, "/path", null, "fragment",
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 19, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 24, "99", "99"),
+                       new UrlToken(UrlToken.Type.PATH, 27, "path", "path"),
+                       new UrlToken(UrlToken.Type.FRAGMENT, 32, "fragment", "fragment"));
+        assertTokenize("scheme", "user", "pass", "host", 99, "/path", "query", null,
+                       new UrlToken(UrlToken.Type.SCHEME, 0, "scheme", "scheme"),
+                       new UrlToken(UrlToken.Type.USERINFO, 9, "user", "user"),
+                       new UrlToken(UrlToken.Type.PASSWORD, 14, "pass", "pass"),
+                       new UrlToken(UrlToken.Type.HOST, 19, null, UrlTokenizer.TERM_STARTHOST),
+                       new UrlToken(UrlToken.Type.HOST, 19, "host", "host"),
+                       new UrlToken(UrlToken.Type.HOST, 23, null, UrlTokenizer.TERM_ENDHOST),
+                       new UrlToken(UrlToken.Type.PORT, 24, "99", "99"),
+                       new UrlToken(UrlToken.Type.PATH, 27, "path", "path"),
+                       new UrlToken(UrlToken.Type.QUERY, 32, "query", "query"));
+    }
+
+    private static void assertTokenize(String scheme, String userInfo, String password, String host, Integer port,
+                                       String path, String query, String fragment, UrlToken... expected)
+    {
+        assertTokenize(new Url(scheme, userInfo, password, host, port, path, query, fragment), expected);
+    }
+
+    private static void assertTokenize(String url, UrlToken... expected) {
+        assertTokenize(Url.fromString(url), expected);
+    }
+
+    private static void assertTokenize(Url url, UrlToken... expected) {
+        Iterator<UrlToken> expectedIt = Arrays.asList(expected).iterator();
+        Iterator<UrlToken> actualIt = new UrlTokenizer(url).tokenize().iterator();
+        while (expectedIt.hasNext()) {
+            assertTrue(actualIt.hasNext());
+            assertEquals(expectedIt.next(), actualIt.next());
+        }
+        assertFalse(expectedIt.hasNext());
+        assertFalse(actualIt.hasNext());
+    }
+
+    private static void assertTerms(String img, String... expected) {
+        List<UrlToken> actual = new LinkedList<>();
+        UrlTokenizer.addTokens(actual, UrlToken.Type.PATH, 0, img, true);
+
+        Iterator<String> expectedIt = Arrays.asList(expected).iterator();
+        Iterator<UrlToken> actualIt = actual.iterator();
+        while (expectedIt.hasNext()) {
+            assertTrue(actualIt.hasNext());
+            assertEquals(expectedIt.next(), actualIt.next().getTerm());
+        }
+        assertFalse(expectedIt.hasNext());
+        assertFalse(actualIt.hasNext());
+    }
+}