Publish

195 files changed, 17644 insertions, 0 deletions

diff --git a/indexinglanguage/.gitignore b/indexinglanguage/.gitignore
new file mode 100644
index 00000000000..b19abb01892
--- /dev/null
+++ b/indexinglanguage/.gitignore
@@ -0,0 +1,14 @@
+*.iml
+*.ipr
+*.iws
+.classpath
+.project
+archive
+build
+classes
+lib/*
+libexec
+log
+target
+testLogs
+/pom.xml.build
diff --git a/indexinglanguage/OWNERS b/indexinglanguage/OWNERS
new file mode 100644
index 00000000000..7ae1acb1be9
--- /dev/null
+++ b/indexinglanguage/OWNERS
@@ -0,0 +1 @@
+geirst
diff --git a/indexinglanguage/README b/indexinglanguage/README
new file mode 100644
index 00000000000..e56a0c1f1dd
--- /dev/null
+++ b/indexinglanguage/README
@@ -0,0 +1 @@
+Indexing Language interpreter
diff --git a/indexinglanguage/pom.xml b/indexinglanguage/pom.xml
new file mode 100644
index 00000000000..a4328091d35
--- /dev/null
+++ b/indexinglanguage/pom.xml
@@ -0,0 +1,112 @@
+<?xml version="1.0"?>
+<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>com.yahoo.vespa</groupId>
+    <artifactId>parent</artifactId>
+    <version>6-SNAPSHOT</version>
+    <relativePath>../parent/pom.xml</relativePath>
+  </parent>
+  <artifactId>indexinglanguage</artifactId>
+  <packaging>jar</packaging>
+  <version>6-SNAPSHOT</version>
+  <name>indexinglanguage</name>
+  <description>Interpreter for the Indexing Language</description>
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>vespajlib</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>document</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>linguistics</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>predicate-search-core</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+      <version>1.4</version>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <compilerArgs>
+            <arg>-Xlint:unchecked</arg>
+            <arg>-Xlint:deprecation</arg>
+          </compilerArgs>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <configuration>
+          <descriptorRefs>
+            <descriptorRef>jar-with-dependencies</descriptorRef>
+          </descriptorRefs>
+        </configuration>
+        <executions>
+          <execution>
+            <id>make-assembly</id>
+            <phase>package</phase>
+            <!-- append to the packaging phase. -->
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>javacc-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>javacc</id>
+            <goals>
+              <goal>javacc</goal>
+            </goals>
+            <configuration>
+              <lookAhead>1</lookAhead>
+              <isStatic>false</isStatic>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-install-plugin</artifactId>
+        <configuration>
+          <updateReleaseInfo>true</updateReleaseInfo>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/AdapterFactory.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/AdapterFactory.java
new file mode 100644
index 00000000000..349555ee3fc
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/AdapterFactory.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentUpdate;
+
+import java.util.List;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public interface AdapterFactory {
+
+    public DocumentAdapter newDocumentAdapter(Document doc);
+
+    public List<UpdateAdapter> newUpdateAdapterList(DocumentUpdate upd);
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/DocumentAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/DocumentAdapter.java
new file mode 100644
index 00000000000..93d848fe0ca
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/DocumentAdapter.java
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.Document;
+import com.yahoo.vespa.indexinglanguage.expressions.FieldValueAdapter;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public interface DocumentAdapter extends FieldValueAdapter {
+
+    public Document getFullOutput();
+
+    public Document getUpdatableOutput();
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java
new file mode 100644
index 00000000000..0653fba6e86
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java
@@ -0,0 +1,136 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@SuppressWarnings({ "UnusedDeclaration" })
+public abstract class ExpressionConverter implements Cloneable {
+
+    public final Expression convert(Expression exp) {
+        if (exp == null) {
+            return null;
+        }
+        if (shouldConvert(exp)) {
+            return doConvert(exp);
+        }
+        if (!(exp instanceof CompositeExpression)) {
+            return exp;
+        }
+        try {
+            // The class.getMethod here takes 8% of the cpu time in reading the SSBE application package
+            // TODO: Implement double dispatch through visitor instead?
+            return (Expression)ExpressionConverter.class.getMethod("innerConvert", exp.getClass()).invoke(this, exp);
+        } catch (IllegalAccessException | NoSuchMethodException e) {
+            throw new UnsupportedOperationException(exp.getClass().getName(), e);
+        } catch (InvocationTargetException e) {
+            Throwable t = e.getTargetException();
+            throw t instanceof RuntimeException ? (RuntimeException)t : new RuntimeException(t);
+        }
+    }
+
+    public Expression innerConvert(ArithmeticExpression exp) {
+        return new ArithmeticExpression(convert(exp.getLeftHandSide()),
+                                        exp.getOperator(),
+                                        convert(exp.getRightHandSide()));
+    }
+
+    public Expression innerConvert(CatExpression exp) {
+        List<Expression> lst = new LinkedList<>();
+        for (Expression innerExp : exp) {
+            Expression next = convert(innerExp);
+            if (next != null) {
+                lst.add(next);
+            }
+        }
+        return new CatExpression(lst);
+    }
+
+    public Expression innerConvert(ForEachExpression exp) {
+        return new ForEachExpression(convert(exp.getInnerExpression()));
+    }
+
+    public Expression innerConvert(GuardExpression exp) {
+        return new GuardExpression(convert(exp.getInnerExpression()));
+    }
+
+    public Expression innerConvert(IfThenExpression exp) {
+        return new IfThenExpression(branch().convert(exp.getLeftHandSide()),
+                                    exp.getComparator(),
+                                    branch().convert(exp.getRightHandSide()),
+                                    branch().convert(exp.getIfTrueExpression()),
+                                    branch().convert(exp.getIfFalseExpression()));
+    }
+
+    public Expression innerConvert(ParenthesisExpression exp) {
+        return new ParenthesisExpression(convert(exp.getInnerExpression()));
+    }
+
+    public Expression innerConvert(ScriptExpression exp) {
+        List<StatementExpression> lst = new LinkedList<>();
+        for (Expression innerExp : exp) {
+            StatementExpression next = (StatementExpression)branch().convert(innerExp);
+            if (next != null) {
+                lst.add(next);
+            }
+        }
+        return new ScriptExpression(lst);
+    }
+
+    public Expression innerConvert(SelectInputExpression exp) {
+        List<Pair<String, Expression>> cases = new LinkedList<>();
+        for (Pair<String, Expression> pair : exp.getCases()) {
+            cases.add(new Pair<>(pair.getFirst(), branch().convert(pair.getSecond())));
+        }
+        return new SelectInputExpression(cases);
+    }
+
+    public Expression innerConvert(StatementExpression exp) {
+        List<Expression> lst = new LinkedList<>();
+        for (Expression innerExp : exp) {
+            Expression next = convert(innerExp);
+            if (next != null) {
+                lst.add(next);
+            }
+        }
+        return new StatementExpression(lst);
+    }
+
+    public Expression innerConvert(SwitchExpression exp) {
+        Map<String, Expression> cases = new HashMap<>();
+        for (Map.Entry<String, Expression> entry : exp.getCases().entrySet()) {
+            Expression next = branch().convert(entry.getValue());
+            if (next != null) {
+                cases.put(entry.getKey(), next);
+            }
+        }
+        return new SwitchExpression(cases, branch().convert(exp.getDefaultExpression()));
+    }
+
+    protected ExpressionConverter branch() {
+        return this;
+    }
+
+    @Override
+    @SuppressWarnings("CloneDoesntDeclareCloneNotSupportedException")
+    public ExpressionConverter clone() {
+        try {
+            return (ExpressionConverter)super.clone();
+        } catch (CloneNotSupportedException e) {
+            throw new UnsupportedOperationException(e);
+        }
+    }
+
+    protected abstract boolean shouldConvert(Expression exp);
+
+    protected abstract Expression doConvert(Expression exp);
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizer.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizer.java
new file mode 100644
index 00000000000..1f38b402e62
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizer.java
@@ -0,0 +1,114 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Optimizes expressions by removing expressions that have no effect.
+ * Typical examples are statements that come before a statement that
+ * generates a new execution value without regard for the existing one.
+ */
+public class ExpressionOptimizer extends ExpressionConverter {
+    @Override
+    protected boolean shouldConvert(Expression exp) {
+        return exp instanceof StatementExpression;
+    }
+
+    @Override
+    protected Expression doConvert(Expression exp) {
+        return optimizeStatement((StatementExpression) exp);
+    }
+
+    private Expression optimizeStatement(StatementExpression statement) {
+        List<Expression> expressionList = new ArrayList<>();
+        List<Expression> candidateList = new ArrayList<>();
+        for (Expression exp : statement) {
+            if (ignoresInput(exp)) {
+                candidateList.clear();
+            }
+            candidateList.add(convert(exp));
+            if (hasSideEffect(exp)) {
+                expressionList.addAll(candidateList);
+                candidateList.clear();
+            }
+        }
+        expressionList.addAll(candidateList);
+        return new StatementExpression(expressionList);
+    }
+
+    static boolean hasSideEffect(Expression exp) {
+        HasSideEffectVisitor visitor = new HasSideEffectVisitor();
+        visitor.visit(exp);
+        return visitor.hasSideEffect;
+    }
+
+    private static class HasSideEffectVisitor extends ExpressionVisitor {
+
+        boolean hasSideEffect = false;
+
+        @Override
+        protected void doVisit(Expression exp) {
+            hasSideEffect |= exp instanceof OutputExpression ||
+                             exp instanceof SetVarExpression ||
+                             exp instanceof EchoExpression;
+        }
+    }
+
+    static boolean ignoresInput(Expression exp) {
+        if (exp instanceof SwitchExpression || exp instanceof ScriptExpression) {
+            return false;  // Switch and script never ignores input.
+        }
+        if (exp instanceof CompositeExpression) {
+            return new IgnoresInputVisitor().ignoresInput(exp);
+        }
+        if (exp instanceof RandomExpression) {
+            return ((RandomExpression)exp).getMaxValue() != null;
+        }
+        return exp instanceof InputExpression ||
+               exp instanceof NowExpression ||
+               exp instanceof SetValueExpression ||
+               exp instanceof HostNameExpression ||
+               exp instanceof GetVarExpression;
+    }
+
+    private static class IgnoresInputVisitor extends ExpressionConverter {
+        private boolean ignoresInput = true;
+        private Expression root = null;
+
+        public boolean ignoresInput(Expression root) {
+            this.root = root;
+            convert(root);
+            return ignoresInput;
+        }
+
+        @Override
+        protected boolean shouldConvert(Expression exp) {
+            if (!ignoresInput) {
+                return true;  // Answer found, skip ahead
+            }
+            if (exp == root) {
+                return false;  // Skip root, check children
+            }
+            if (exp instanceof StatementExpression) {
+                for (Expression expression : (StatementExpression) exp) {
+                    if (ExpressionOptimizer.ignoresInput(expression)) {
+                        return true;  // Skip children
+                    }
+                }
+                ignoresInput = false;
+                return true;  // Answer found, skip children
+            }
+            ignoresInput &= ExpressionOptimizer.ignoresInput(exp);
+            return true;  // Children already checked. Skip them.
+        }
+
+        @Override
+        protected Expression doConvert(Expression exp) {
+            return exp;
+        }
+
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcher.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcher.java
new file mode 100644
index 00000000000..c24cfcf4f09
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcher.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ExpressionSearcher<T extends Expression> {
+
+    private final Class<T> searchFor;
+
+    public ExpressionSearcher(Class<T> searchFor) {
+        this.searchFor = searchFor;
+    }
+
+    public boolean containedIn(Expression searchIn) {
+        return searchIn(searchIn) != null;
+    }
+
+    public T searchIn(Expression searchIn) {
+        MyConverter searcher = new MyConverter();
+        searcher.convert(searchIn);
+        return searcher.found;
+    }
+
+    private class MyConverter extends ExpressionConverter {
+
+        T found = null;
+
+        @Override
+        protected boolean shouldConvert(Expression exp) {
+            if (searchFor.isInstance(exp)) {
+                found = searchFor.cast(exp);
+                return true; // terminate search
+            }
+            return false;
+        }
+
+        @Override
+        protected Expression doConvert(Expression exp) {
+            return exp;
+        }
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitor.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitor.java
new file mode 100644
index 00000000000..5cb73b7fba0
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitor.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class ExpressionVisitor {
+
+    private final MyConverter converter = new MyConverter();
+
+    public void visit(Expression exp) {
+        converter.convert(exp);
+    }
+
+    protected abstract void doVisit(Expression exp);
+
+    private class MyConverter extends ExpressionConverter {
+
+        @Override
+        protected boolean shouldConvert(Expression exp) {
+            doVisit(exp);
+            return false;
+        }
+
+        @Override
+        protected Expression doConvert(Expression exp) {
+            throw new AssertionError();
+        }
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateAdapter.java
new file mode 100644
index 00000000000..5d3fcd24727
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateAdapter.java
@@ -0,0 +1,126 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.FieldValueAdapter;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class FieldPathUpdateAdapter implements UpdateAdapter {
+
+    private final DocumentAdapter adapter;
+    private final FieldPathUpdate update;
+
+    public FieldPathUpdateAdapter(DocumentAdapter documentAdapter, FieldPathUpdate fieldUpdate) {
+        adapter = documentAdapter;
+        update = fieldUpdate;
+    }
+
+    @Override
+    public Expression getExpression(Expression expression) {
+        return expression;
+    }
+
+    @Override
+    public DocumentUpdate getOutput() {
+        Document doc = adapter.getFullOutput();
+        DocumentUpdate upd = new DocumentUpdate(doc.getDataType(), doc.getId());
+        createUpdatesAt(new ArrayList<>(), adapter.getUpdatableOutput(), 0, upd);
+        return upd;
+    }
+
+    @Override
+    public DataType getInputType(Expression exp, String fieldName) {
+        return adapter.getInputType(exp, fieldName);
+    }
+
+    @Override
+    public FieldValue getInputValue(String fieldName) {
+        return adapter.getInputValue(fieldName);
+    }
+
+    @Override
+    public FieldValue getInputValue(FieldPath fieldPath) {
+        return adapter.getInputValue(fieldPath);
+    }
+
+    @Override
+    public void tryOutputType(Expression exp, String fieldName, DataType valueType) {
+        adapter.tryOutputType(exp, fieldName, valueType);
+    }
+
+    @Override
+    public FieldValueAdapter setOutputValue(Expression exp, String fieldName, FieldValue fieldValue) {
+        return adapter.setOutputValue(exp, fieldName, fieldValue);
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    private void createUpdatesAt(List<FieldPathEntry> path, FieldValue value, int idx, DocumentUpdate out) {
+        FieldPath updatePath = update.getFieldPath();
+        if (idx < updatePath.size()) {
+            FieldPathEntry pathEntry = updatePath.get(idx);
+            FieldPathEntry.Type type = pathEntry.getType();
+            if (type == FieldPathEntry.Type.STRUCT_FIELD) {
+                if (!(value instanceof StructuredFieldValue)) {
+                    throw new IllegalArgumentException("Expected structured field value, got " +
+                                                       value.getClass().getName() + ".");
+                }
+                for (Iterator<Map.Entry<Field, FieldValue>> it = ((StructuredFieldValue)value).iterator(); it.hasNext();) {
+                    Map.Entry<Field, FieldValue> structEntry = it.next();
+                    List<FieldPathEntry> nextPath = new ArrayList<>(path);
+                    nextPath.add(FieldPathEntry.newStructFieldEntry(structEntry.getKey()));
+                    createUpdatesAt(nextPath, structEntry.getValue(), idx + 1, out);
+                }
+            } else if (type == FieldPathEntry.Type.MAP_KEY) {
+                if (value instanceof WeightedSet) {
+                    WeightedSet wset = (WeightedSet)value;
+                    for (Iterator<FieldValue> it = wset.fieldValueIterator(); it.hasNext();) {
+                        FieldValue wsetEntry = it.next();
+                        List<FieldPathEntry> nextPath = new ArrayList<>(path);
+                        nextPath.add(FieldPathEntry.newMapLookupEntry(wsetEntry, DataType.INT));
+                        createUpdatesAt(nextPath, new IntegerFieldValue(wset.get(wsetEntry)), idx + 1, out);
+                    }
+                } else if (value instanceof MapFieldValue) {
+                    MapFieldValue<FieldValue, FieldValue> map = (MapFieldValue)value;
+                    for (Map.Entry<FieldValue, FieldValue> entry : map.entrySet()) {
+                        List<FieldPathEntry> nextPath = new ArrayList<>(path);
+                        FieldValue nextVal = entry.getValue();
+                        nextPath.add(FieldPathEntry.newMapLookupEntry(entry.getKey(), nextVal.getDataType()));
+                        createUpdatesAt(nextPath, nextVal, idx + 1, out);
+                    }
+                } else {
+                    throw new IllegalArgumentException("Expected map or weighted set, got " +
+                                                       value.getClass().getName() + ".");
+                }
+            } else {
+                path.add(pathEntry);
+                createUpdatesAt(new ArrayList<>(path), value, idx + 1, out);
+            }
+        } else if (update instanceof AddFieldPathUpdate) {
+            if (!(value instanceof Array)) {
+                throw new IllegalStateException("Expected array, got " +
+                                                value.getClass().getName() + ".");
+            }
+            out.addFieldPathUpdate(new AddFieldPathUpdate(update.getDocumentType(), new FieldPath(path).toString(),
+                                                          update.getOriginalWhereClause(), (Array)value));
+        } else if (update instanceof AssignFieldPathUpdate) {
+            out.addFieldPathUpdate(new AssignFieldPathUpdate(update.getDocumentType(), new FieldPath(path).toString(),
+                                                             update.getOriginalWhereClause(), value));
+        } else if (update instanceof RemoveFieldPathUpdate) {
+            out.addFieldPathUpdate(new RemoveFieldPathUpdate(update.getDocumentType(), new FieldPath(path).toString(),
+                                                             update.getOriginalWhereClause()));
+        }
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateHelper.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateHelper.java
new file mode 100644
index 00000000000..361b54e3015
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateHelper.java
@@ -0,0 +1,75 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.FieldPathEntry;
+import com.yahoo.document.datatypes.FieldPathIteratorHandler;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class FieldPathUpdateHelper {
+
+    public static boolean isComplete(FieldPathUpdate update) {
+        if (!(update instanceof AssignFieldPathUpdate)) {
+            return false;
+        }
+        for (FieldPathEntry entry : update.getFieldPath()) {
+            switch (entry.getType()) {
+            case STRUCT_FIELD:
+            case MAP_ALL_KEYS:
+            case MAP_ALL_VALUES:
+                continue;
+            case ARRAY_INDEX:
+            case MAP_KEY:
+            case VARIABLE:
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public static void applyUpdate(FieldPathUpdate update, Document doc) {
+        if (update instanceof AddFieldPathUpdate) {
+            update.applyTo(doc);
+        } else if (update instanceof AssignFieldPathUpdate) {
+            AssignFieldPathUpdate assign = (AssignFieldPathUpdate)update;
+            boolean createMissingPath = assign.getCreateMissingPath();
+            boolean removeIfZero = assign.getRemoveIfZero();
+            assign.setCreateMissingPath(true);
+            assign.setRemoveIfZero(false);
+
+            assign.applyTo(doc);
+
+            assign.setCreateMissingPath(createMissingPath);
+            assign.setRemoveIfZero(removeIfZero);
+        } else if (update instanceof RemoveFieldPathUpdate) {
+            doc.iterateNested(update.getFieldPath(), 0, new MyHandler());
+        }
+    }
+
+    public static Document newPartialDocument(DocumentId docId, FieldPathUpdate update) {
+        Document doc = new Document(update.getDocumentType(), docId);
+        applyUpdate(update, doc);
+        return doc;
+    }
+
+    private static class MyHandler extends FieldPathIteratorHandler {
+
+        @Override
+        public ModificationStatus doModify(FieldValue fv) {
+            return ModificationStatus.MODIFIED;
+        }
+
+        @Override
+        public boolean createMissingPath() {
+            return true;
+        }
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateAdapter.java
new file mode 100644
index 00000000000..cf88dc74ed0
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateAdapter.java
@@ -0,0 +1,231 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.update.*;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.FieldValueAdapter;
+
+import java.util.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class FieldUpdateAdapter implements UpdateAdapter {
+
+    private final DocumentAdapter adapter;
+    private final Builder builder;
+    private final Expression optimizedExpression;
+
+    private FieldUpdateAdapter(Expression optimizedExpression, DocumentAdapter adapter, Builder builder) {
+        this.adapter = adapter;
+        this.builder = builder;
+        this.optimizedExpression = optimizedExpression;
+    }
+
+    @Override
+    public DocumentUpdate getOutput() {
+        Document doc = adapter.getUpdatableOutput();
+        DocumentUpdate upd = new DocumentUpdate(doc.getDataType(), doc.getId());
+        for (Iterator<Map.Entry<Field, FieldValue>> it = doc.iterator(); it.hasNext();) {
+            Map.Entry<Field, FieldValue> entry = it.next();
+            Field field = entry.getKey();
+            if (field.getName().equals("sddocname")) {
+                continue;
+            }
+            FieldUpdate fieldUpd = FieldUpdate.create(field);
+            fieldUpd.addValueUpdates(builder.build(entry.getValue()));
+            if (!fieldUpd.isEmpty()) {
+                upd.addFieldUpdate(fieldUpd);
+            }
+        }
+        return upd.isEmpty() ? null : upd;
+    }
+
+    @Override
+    public Expression getExpression(Expression expression) {
+        return optimizedExpression != null ? optimizedExpression : expression;
+    }
+
+    @Override
+    public DataType getInputType(Expression exp, String fieldName) {
+        return adapter.getInputType(exp, fieldName);
+    }
+
+    @Override
+    public FieldValue getInputValue(String fieldName) {
+        return adapter.getInputValue(fieldName);
+    }
+    @Override
+    public FieldValue getInputValue(FieldPath fieldPath) { return adapter.getInputValue(fieldPath); }
+
+    @Override
+    public void tryOutputType(Expression exp, String fieldName, DataType valueType) {
+        adapter.tryOutputType(exp, fieldName, valueType);
+    }
+
+    @Override
+    public FieldValueAdapter setOutputValue(Expression exp, String fieldName, FieldValue fieldValue) {
+        return adapter.setOutputValue(exp, fieldName, fieldValue);
+    }
+
+    public static FieldUpdateAdapter fromPartialUpdate(DocumentAdapter documentAdapter, ValueUpdate valueUpdate) {
+        return new FieldUpdateAdapter(null, documentAdapter, new PartialBuilder(valueUpdate));
+    }
+    public static FieldUpdateAdapter fromPartialUpdate(Expression expression, DocumentAdapter documentAdapter, ValueUpdate valueUpdate) {
+        return new FieldUpdateAdapter(expression, documentAdapter, new PartialBuilder(valueUpdate));
+    }
+
+    public static FieldUpdateAdapter fromCompleteUpdate(DocumentAdapter documentAdapter) {
+        return new FieldUpdateAdapter(null, documentAdapter, new CompleteBuilder());
+    }
+
+    private interface Builder {
+
+        List<ValueUpdate> build(FieldValue val);
+    }
+
+    private static class PartialBuilder implements Builder {
+
+        final ValueUpdate update;
+
+        PartialBuilder(ValueUpdate update) {
+            this.update = update;
+        }
+
+        @Override
+        public List<ValueUpdate> build(FieldValue val) {
+            return createValueUpdates(val, update);
+        }
+
+        @SuppressWarnings({ "unchecked" })
+        List<ValueUpdate> createValueUpdates(FieldValue val, ValueUpdate upd) {
+            List<ValueUpdate> lst = new ArrayList<>();
+            if (upd instanceof ClearValueUpdate) {
+                lst.add(new ClearValueUpdate());
+            } else if (upd instanceof AssignValueUpdate) {
+                lst.add(new AssignValueUpdate(val));
+            } else if (upd instanceof AddValueUpdate) {
+                if (val instanceof Array) {
+                    lst.addAll(createAddValueUpdateForArray((Array)val, ((AddValueUpdate)upd).getWeight()));
+                } else if (val instanceof WeightedSet) {
+                    lst.addAll(createAddValueUpdateForWset((WeightedSet)val));
+                } else {
+                    // do nothing
+                }
+            } else if (upd instanceof ArithmeticValueUpdate) {
+                lst.add(upd); // leave arithmetics alone
+            } else if (upd instanceof RemoveValueUpdate) {
+                if (val instanceof Array) {
+                    lst.addAll(createRemoveValueUpdateForEachElement(((Array)val).fieldValueIterator()));
+                } else if (val instanceof WeightedSet) {
+                    lst.addAll(createRemoveValueUpdateForEachElement(((WeightedSet)val).fieldValueIterator()));
+                } else {
+                    // do nothing
+                }
+            } else if (upd instanceof MapValueUpdate) {
+                if (val instanceof Array) {
+                    lst.addAll(createMapValueUpdatesForArray((Array)val, (MapValueUpdate)upd));
+                } else if (val instanceof MapFieldValue) {
+                    throw new UnsupportedOperationException("Can not map into a " + val.getClass().getName() + ".");
+                } else if (val instanceof StructuredFieldValue) {
+                    lst.addAll(createMapValueUpdatesForStruct((StructuredFieldValue)val, (MapValueUpdate)upd));
+                } else if (val instanceof WeightedSet) {
+                    lst.addAll(createMapValueUpdatesForWset((WeightedSet)val, (MapValueUpdate)upd));
+                } else {
+                    // do nothing
+                }
+            } else {
+                throw new UnsupportedOperationException(
+                        "Value update type " + upd.getClass().getName() + " not supported.");
+            }
+            return lst;
+        }
+
+        @SuppressWarnings({ "unchecked" })
+        private List<ValueUpdate> createAddValueUpdateForArray(Array arr, int weight) {
+            List<ValueUpdate> ret = new ArrayList<>(arr.size());
+            for (Iterator<FieldValue> it = arr.fieldValueIterator(); it.hasNext(); ) {
+                ret.add(new AddValueUpdate(it.next(), weight));
+            }
+            return ret;
+        }
+
+        @SuppressWarnings({ "unchecked" })
+        private List<ValueUpdate> createAddValueUpdateForWset(WeightedSet wset) {
+            List<ValueUpdate> ret = new ArrayList<>(wset.size());
+            for (Iterator<FieldValue> it = wset.fieldValueIterator(); it.hasNext(); ) {
+                FieldValue key = it.next();
+                ret.add(new AddValueUpdate(key, wset.get(key)));
+            }
+            return ret;
+        }
+
+        @SuppressWarnings({ "unchecked" })
+        private List<ValueUpdate> createRemoveValueUpdateForEachElement(Iterator<FieldValue> it) {
+            List<ValueUpdate> ret = new ArrayList<>();
+            while (it.hasNext()) {
+                ret.add(new RemoveValueUpdate(it.next()));
+            }
+            return ret;
+        }
+
+        @SuppressWarnings({ "unchecked" })
+        private List<ValueUpdate> createMapValueUpdatesForArray(Array arr, MapValueUpdate upd) {
+            List<ValueUpdate> ret = new ArrayList<>();
+            for (Iterator<FieldValue> it = arr.fieldValueIterator(); it.hasNext();) {
+                FieldValue childVal = it.next();
+                for (ValueUpdate childUpd : createValueUpdates(childVal, upd.getUpdate())) {
+                    ret.add(new MapValueUpdate(childVal, childUpd));
+                }
+            }
+            return ret;
+        }
+
+        @SuppressWarnings({ "unchecked" })
+        private List<ValueUpdate> createMapValueUpdatesForStruct(StructuredFieldValue struct, MapValueUpdate upd) {
+            List<ValueUpdate> ret = new ArrayList<>();
+            for (Iterator<Map.Entry<Field, FieldValue>> it = struct.iterator(); it.hasNext();) {
+                Map.Entry<Field, FieldValue> entry = it.next();
+                for (ValueUpdate childUpd : createValueUpdates(entry.getValue(), upd.getUpdate())) {
+                    ret.add(new MapValueUpdate(new StringFieldValue(entry.getKey().getName()), childUpd));
+                }
+            }
+            return ret;
+        }
+
+        @SuppressWarnings({ "unchecked" })
+        private List<ValueUpdate> createMapValueUpdatesForWset(WeightedSet wset, MapValueUpdate upd) {
+            List<ValueUpdate> ret = new ArrayList<>();
+            for (Iterator<FieldValue> it = wset.fieldValueIterator(); it.hasNext();) {
+                FieldValue childVal = it.next();
+                for (ValueUpdate childUpd : createValueUpdates(new IntegerFieldValue(wset.get(childVal)),
+                                                               upd.getUpdate()))
+                {
+                    ret.add(new MapValueUpdate(childVal, childUpd));
+                }
+            }
+            return ret;
+        }
+    }
+
+    private static class CompleteBuilder extends PartialBuilder {
+
+        static final ValueUpdate nullMap = new MapValueUpdate(null, null);
+        static final ValueUpdate nullAssign = new AssignValueUpdate(null);
+
+        CompleteBuilder() {
+            super(null);
+        }
+
+        @Override
+        List<ValueUpdate> createValueUpdates(FieldValue val, ValueUpdate upd) {
+            if (val instanceof StructuredFieldValue) {
+                return super.createValueUpdates(val, nullMap);
+            } else {
+                return super.createValueUpdates(val, nullAssign);
+            }
+        }
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateHelper.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateHelper.java
new file mode 100644
index 00000000000..7412b7171d1
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateHelper.java
@@ -0,0 +1,95 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.update.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class FieldUpdateHelper {
+
+    public static boolean isComplete(Field field, ValueUpdate update) {
+        if (update instanceof AssignValueUpdate) {
+            return true;
+        }
+        if (!(update instanceof MapValueUpdate)) {
+            return false;
+        }
+        DataType fieldType = field.getDataType();
+        if (!(fieldType instanceof StructuredDataType)) {
+            return false;
+        }
+        field = ((StructuredDataType)fieldType).getField(String.valueOf(update.getValue()));
+        if (field == null) {
+            return false;
+        }
+        return isComplete(field, ((MapValueUpdate)update).getUpdate());
+    }
+
+    public static void applyUpdate(Field field, ValueUpdate update, Document doc) {
+        doc.setFieldValue(field, createFieldValue(field.getDataType().createFieldValue(), update));
+    }
+
+    public static Document newPartialDocument(DocumentType docType, DocumentId docId, Field field, ValueUpdate update) {
+        Document doc = new Document(docType, docId);
+        applyUpdate(field, update, doc);
+        return doc;
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    private static FieldValue createFieldValue(FieldValue val, ValueUpdate upd) {
+        if (upd instanceof ClearValueUpdate) {
+            return val;
+        } else if (upd instanceof AssignValueUpdate) {
+            val.assign(upd.getValue());
+            return val;
+        } else if (upd instanceof AddValueUpdate) {
+            if (val instanceof Array) {
+                ((Array)val).add(upd.getValue());
+            } else if (val instanceof WeightedSet) {
+                ((WeightedSet)val).put(upd.getValue(), ((AddValueUpdate)upd).getWeight());
+            }
+            return val;
+        } else if (upd instanceof ArithmeticValueUpdate) {
+            if (((ArithmeticValueUpdate)upd).getOperator() == ArithmeticValueUpdate.Operator.DIV &&
+                ((ArithmeticValueUpdate)upd).getOperand().doubleValue() == 0) {
+                throw new IllegalArgumentException("Division by zero.");
+            }
+            val.assign(upd.getValue());
+            return val;
+        } else if (upd instanceof RemoveValueUpdate) {
+            if (val instanceof Array) {
+                ((Array)val).add(upd.getValue());
+            } else if (val instanceof WeightedSet) {
+                ((WeightedSet)val).put(upd.getValue(), 1);
+            }
+            return val;
+        } else if (upd instanceof MapValueUpdate) {
+            if (val instanceof Array) {
+                return createFieldValue(val, ((MapValueUpdate)upd).getUpdate());
+            } else if (val instanceof MapFieldValue) {
+                throw new UnsupportedOperationException("Can not map into a " + val.getClass().getName() + ".");
+            } else if (val instanceof StructuredFieldValue) {
+                Field field = ((StructuredFieldValue)val).getField(String.valueOf(upd.getValue()));
+                if (field == null) {
+                    throw new IllegalArgumentException("Field '" + upd.getValue() + "' not found.");
+                }
+                ((StructuredFieldValue)val).setFieldValue(field, createFieldValue(field.getDataType().createFieldValue(),
+                                                                                  ((MapValueUpdate)upd).getUpdate()));
+                return val;
+            } else if (val instanceof WeightedSet) {
+                FieldValue weight = createFieldValue(new IntegerFieldValue(), ((MapValueUpdate)upd).getUpdate());
+                if (!(weight instanceof IntegerFieldValue)) {
+                    throw new IllegalArgumentException("Expected integer, got " + weight.getClass().getName() + ".");
+                }
+                ((WeightedSet)val).put(upd.getValue(), ((IntegerFieldValue)weight).getInteger());
+                return val;
+            } else {
+                throw new IllegalArgumentException("Expected multi-value data type, got " + val.getDataType().getName() + ".");
+            }
+        }
+        throw new UnsupportedOperationException("Value update type " + upd.getClass().getName() + " not supported.");
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldValueConverter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldValueConverter.java
new file mode 100644
index 00000000000..59d5de4a965
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldValueConverter.java
@@ -0,0 +1,167 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.*;
+
+import java.util.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class FieldValueConverter {
+
+    @SuppressWarnings({ "unchecked" })
+    public final FieldValue convert(FieldValue value) {
+        if (value == null) {
+            return null;
+        }
+        if (shouldConvert(value)) {
+            return doConvert(value);
+        }
+        if (value instanceof Array) {
+            return convertArray((Array)value);
+        }
+        if (value instanceof MapFieldValue) {
+            return convertMap((MapFieldValue)value);
+        }
+        if (value instanceof WeightedSet) {
+            return convertWset((WeightedSet)value);
+        }
+        if (value instanceof StructuredFieldValue) {
+            return convertStructured((StructuredFieldValue)value);
+        }
+        return value;
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    private FieldValue convertArray(Array val) {
+        List<FieldValue> next = new LinkedList<FieldValue>();
+        DataType nextType = null;
+        for (Iterator<FieldValue> it = val.fieldValueIterator(); it.hasNext();) {
+            FieldValue prevVal = it.next();
+            FieldValue nextVal = convert(prevVal);
+            if (nextVal == null) {
+                continue;
+            }
+            if (nextType == null) {
+                nextType = nextVal.getDataType();
+            } else if (!nextType.isValueCompatible(nextVal)) {
+                throw new IllegalArgumentException("Expected " + nextType.getName() + ", got " +
+                                                   nextVal.getDataType().getName() + ".");
+            }
+            next.add(nextVal);
+        }
+        if (nextType == null) {
+            return null;
+        }
+        Array ret = DataType.getArray(nextType).createFieldValue();
+        for (FieldValue nextVal : next) {
+            ret.add(nextVal);
+        }
+        return ret;
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    private FieldValue convertMap(MapFieldValue<FieldValue, FieldValue> val) {
+        Map<FieldValue, FieldValue> next = new LinkedHashMap<FieldValue, FieldValue>();
+        DataType nextKeyType = null, nextValType = null;
+        for (Map.Entry<FieldValue, FieldValue> entry : val.entrySet()) {
+            FieldValue prevKey = entry.getKey();
+            FieldValue nextKey = convert(prevKey);
+            if (nextKey == null) {
+                continue;
+            }
+            if (nextKeyType == null) {
+                nextKeyType = nextKey.getDataType();
+            } else if (!nextKeyType.isValueCompatible(nextKey)) {
+                throw new IllegalArgumentException("Expected " + nextKeyType.getName() + ", got " +
+                                                   nextKey.getDataType().getName() + ".");
+            }
+            FieldValue prevVal = entry.getValue();
+            FieldValue nextVal = convert(prevVal);
+            if (nextVal == null) {
+                continue;
+            }
+            if (nextValType == null) {
+                nextValType = nextVal.getDataType();
+            } else if (!nextValType.isValueCompatible(nextVal)) {
+                throw new IllegalArgumentException("Expected " + nextValType.getName() + ", got " +
+                                                   nextVal.getDataType().getName() + ".");
+            }
+            next.put(nextKey, nextVal);
+        }
+        if (nextKeyType == null || nextValType == null) {
+            return null;
+        }
+        MapFieldValue ret = DataType.getMap(nextKeyType, nextValType).createFieldValue();
+        for (Map.Entry<FieldValue, FieldValue> entry : next.entrySet()) {
+            ret.put(entry.getKey(), entry.getValue());
+        }
+        return ret;
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    private FieldValue convertWset(WeightedSet val) {
+        Map<FieldValue, Integer> next = new LinkedHashMap<FieldValue, Integer>();
+        DataType nextType = null;
+        for (Iterator<FieldValue> it = val.fieldValueIterator(); it.hasNext();) {
+            FieldValue prevKey = it.next();
+            Integer prevVal = val.get(prevKey);
+
+            FieldValue nextKey = convert(prevKey);
+            if (nextKey == null) {
+                continue;
+            }
+            if (nextType == null) {
+                nextType = nextKey.getDataType();
+            } else if (!nextType.isValueCompatible(nextKey)) {
+                throw new IllegalArgumentException("Expected " + nextType.getName() + ", got " +
+                                                   nextKey.getDataType().getName() + ".");
+            }
+            next.put(nextKey, prevVal);
+        }
+        if (nextType == null) {
+            return null;
+        }
+        WeightedSet ret = DataType.getWeightedSet(nextType, val.getDataType().createIfNonExistent(),
+                                                  val.getDataType().removeIfZero()).createFieldValue();
+        for (Map.Entry<FieldValue, Integer> entry : next.entrySet()) {
+            ret.put(entry.getKey(), entry.getValue());
+        }
+        return ret;
+    }
+
+    private FieldValue convertStructured(StructuredFieldValue val) {
+        StructuredFieldValue ret = val.getDataType().createFieldValue();
+        for (Iterator<Map.Entry<Field, FieldValue>> it = val.iterator(); it.hasNext();) {
+            Map.Entry<Field, FieldValue> entry = it.next();
+            FieldValue prev = entry.getValue();
+            FieldValue next = convert(prev);
+            if (next == null) {
+                continue;
+            }
+            ret.setFieldValue(entry.getKey(), next);
+        }
+        return ret;
+    }
+
+    /**
+     * Returns whether or not the given {@link FieldValue} should be converted. If this method returns <em>false</em>,
+     * the converter will proceed to traverse the value itself to see if its internal can be converted.
+     *
+     * @param value The value to check.
+     * @return True to convert, false to traverse.
+     */
+    protected abstract boolean shouldConvert(FieldValue value);
+
+    /**
+     * Converts the given value. It is IMPERATIVE that the implementation of this method DOES NOT mutate the given
+     * {@link FieldValue} in place, as that can cause SERIOUS inconsistencies in the parent structures.
+     *
+     * @param value The value to convert.
+     * @return The value to replace the old.
+     */
+    protected abstract FieldValue doConvert(FieldValue value);
+}
\ No newline at end of file
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ScriptParser.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ScriptParser.java
new file mode 100644
index 00000000000..4643bb05b08
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ScriptParser.java
@@ -0,0 +1,78 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.javacc.FastCharStream;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
+import com.yahoo.vespa.indexinglanguage.parser.CharStream;
+import com.yahoo.vespa.indexinglanguage.parser.IndexingParser;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+import com.yahoo.vespa.indexinglanguage.parser.TokenMgrError;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public final class ScriptParser {
+
+    public static Expression parseExpression(ScriptParserContext config) throws ParseException {
+        return parse(config, new ParserMethod<Expression>() {
+
+            @Override
+            public Expression call(IndexingParser parser) throws ParseException {
+                return parser.root();
+            }
+        });
+    }
+
+    public static ScriptExpression parseScript(ScriptParserContext config) throws ParseException {
+        return parse(config, new ParserMethod<ScriptExpression>() {
+
+            @Override
+            public ScriptExpression call(IndexingParser parser) throws ParseException {
+                return parser.script();
+            }
+        });
+    }
+
+    public static StatementExpression parseStatement(ScriptParserContext config) throws ParseException {
+        return parse(config, new ParserMethod<StatementExpression>() {
+
+            @Override
+            public StatementExpression call(IndexingParser parser) throws ParseException {
+                try {
+                    return parser.statement();
+                }
+                catch (TokenMgrError e) {
+                    throw new ParseException(e.getMessage());
+                }
+            }
+        });
+    }
+
+    private static interface ParserMethod<T extends Expression> {
+
+        T call(IndexingParser parser) throws ParseException;
+    }
+
+    private static <T extends Expression> T parse(ScriptParserContext config, ParserMethod<T> method)
+            throws ParseException {
+        CharStream input = config.getInputStream();
+        IndexingParser parser = new IndexingParser(input);
+        parser.setAnnotatorConfig(config.getAnnotatorConfig());
+        parser.setDefaultFieldName(config.getDefaultFieldName());
+        parser.setLinguistics(config.getLinguistcs());
+        try {
+            return method.call(parser);
+        } catch (ParseException e) {
+            if (!(input instanceof FastCharStream)) {
+                throw e;
+            }
+            throw new ParseException(((FastCharStream)input).formatException(e.getMessage()));
+        } finally {
+            if (parser.token != null && parser.token.next != null) {
+                input.backup(parser.token.next.image.length());
+            }
+        }
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ScriptParserContext.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ScriptParserContext.java
new file mode 100644
index 00000000000..a79ac557754
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ScriptParserContext.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
+import com.yahoo.vespa.indexinglanguage.parser.CharStream;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ScriptParserContext {
+
+    private AnnotatorConfig annotatorConfig = new AnnotatorConfig();
+    private Linguistics linguistics;
+    private String defaultFieldName = null;
+    private CharStream inputStream = null;
+
+    public ScriptParserContext(Linguistics linguistics) {
+        this.linguistics = linguistics;
+    }
+
+    public AnnotatorConfig getAnnotatorConfig() {
+        return annotatorConfig;
+    }
+
+    public ScriptParserContext setAnnotatorConfig(AnnotatorConfig config) {
+        annotatorConfig = new AnnotatorConfig(config);
+        return this;
+    }
+
+    public Linguistics getLinguistcs() {
+        return linguistics;
+    }
+
+    public ScriptParserContext setLinguistics(Linguistics linguistics) {
+        this.linguistics = linguistics;
+        return this;
+    }
+
+    public String getDefaultFieldName() {
+        return defaultFieldName;
+    }
+
+    public ScriptParserContext setDefaultFieldName(String name) {
+        defaultFieldName = name;
+        return this;
+    }
+
+    public CharStream getInputStream() {
+        return inputStream;
+    }
+
+    public ScriptParserContext setInputStream(CharStream stream) {
+        inputStream = stream;
+        return this;
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java
new file mode 100644
index 00000000000..2a00161080e
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.*;
+import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.ValueUpdate;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SimpleAdapterFactory implements AdapterFactory {
+    public static class SelectExpression {
+        public Expression selectExpression(DocumentType documentType, String fieldName) {
+            return null;
+        }
+    }
+    private final SelectExpression expressionSelector;
+
+    public SimpleAdapterFactory() {
+        this(new SelectExpression());
+    }
+    public SimpleAdapterFactory(SelectExpression expressionSelector) {
+        this.expressionSelector = expressionSelector;
+    }
+
+    @Override
+    public DocumentAdapter newDocumentAdapter(Document doc) {
+        return newDocumentAdapter(doc, false);
+    }
+
+    public DocumentAdapter newDocumentAdapter(Document doc, boolean isUpdate) {
+        if (isUpdate) {
+            return new SimpleDocumentAdapter(doc);
+        }
+        return new SimpleDocumentAdapter(doc, doc);
+    }
+
+    @Override
+    public List<UpdateAdapter> newUpdateAdapterList(DocumentUpdate upd) {
+        List<UpdateAdapter> ret = new ArrayList<>();
+        DocumentType docType = upd.getDocumentType();
+        DocumentId docId = upd.getId();
+        Document complete = new Document(docType, upd.getId());
+        for (FieldPathUpdate fieldUpd : upd) {
+            if (FieldPathUpdateHelper.isComplete(fieldUpd)) {
+                FieldPathUpdateHelper.applyUpdate(fieldUpd, complete);
+            } else {
+                Document partial = FieldPathUpdateHelper.newPartialDocument(docId, fieldUpd);
+                ret.add(new FieldPathUpdateAdapter(newDocumentAdapter(partial, true), fieldUpd));
+            }
+        }
+        for (FieldUpdate fieldUpd : upd.getFieldUpdates()) {
+            Field field = fieldUpd.getField();
+            for (ValueUpdate valueUpd : fieldUpd.getValueUpdates()) {
+                if (FieldUpdateHelper.isComplete(field, valueUpd)) {
+                    FieldUpdateHelper.applyUpdate(field, valueUpd, complete);
+                } else {
+                    Document partial = FieldUpdateHelper.newPartialDocument(docType, docId, field, valueUpd);
+                    ret.add(FieldUpdateAdapter.fromPartialUpdate(expressionSelector.selectExpression(docType, field.getName()),newDocumentAdapter(partial, true), valueUpd));
+                }
+            }
+        }
+        ret.add(FieldUpdateAdapter.fromCompleteUpdate(newDocumentAdapter(complete, true)));
+        return ret;
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapter.java
new file mode 100644
index 00000000000..77914ec9035
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapter.java
@@ -0,0 +1,90 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.Field;
+import com.yahoo.document.FieldPath;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.VerificationException;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SimpleDocumentAdapter implements DocumentAdapter {
+
+    private final Document input;
+    private final Document output;
+
+    public SimpleDocumentAdapter(Document input) {
+        this(input, new Document(input.getDataType(), input.getId()));
+    }
+
+    public SimpleDocumentAdapter(Document input, Document output) {
+        this.input = input;
+        this.output = output;
+    }
+
+    @Override
+    public Document getFullOutput() {
+        return output;
+    }
+
+    @Override
+    public Document getUpdatableOutput() {
+        return output;
+    }
+
+    @Override
+    public DataType getInputType(Expression exp, String fieldName) {
+        try {
+            return input.getDataType().buildFieldPath(fieldName).getResultingDataType();
+        } catch (IllegalArgumentException e) {
+            throw new VerificationException(exp, "Input field '" + fieldName + "' not found.");
+        }
+    }
+
+    @Override
+    public FieldValue getInputValue(String fieldName) {
+        try {
+            return input.getRecursiveValue(fieldName);
+        } catch (IllegalArgumentException e) {
+            return null;
+        }
+    }
+
+    @Override
+    public FieldValue getInputValue(FieldPath fieldPath) {
+        try {
+            return input.getRecursiveValue(fieldPath);
+        } catch (IllegalArgumentException e) {
+            return null;
+        }
+    }
+
+    @Override
+    public void tryOutputType(Expression exp, String fieldName, DataType valueType) {
+        Field field = output.getDataType().getField(fieldName);
+        if (field == null) {
+            throw new VerificationException(exp, "Field '" + fieldName + "' not found.");
+        }
+        DataType fieldType = field.getDataType();
+        if (!fieldType.isAssignableFrom(valueType)) {
+            throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to field '" +
+                                                 fieldName + "' which is " + fieldType.getName() + ".");
+        }
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    @Override
+    public SimpleDocumentAdapter setOutputValue(Expression exp, String fieldName, FieldValue fieldValue) {
+        Field field = output.getField(fieldName);
+        if (field == null) {
+            throw new IllegalArgumentException("Field '" + fieldName + "' not found in document type '" +
+                                               output.getDataType().getName() + "'.");
+        }
+        output.setFieldValue(field, fieldValue);
+        return this;
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/StringFieldConverter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/StringFieldConverter.java
new file mode 100644
index 00000000000..69631c20f95
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/StringFieldConverter.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class StringFieldConverter extends FieldValueConverter {
+
+    @Override
+    protected final boolean shouldConvert(FieldValue value) {
+        return value.getDataType().equals(DataType.STRING);
+    }
+
+    @Override
+    protected final FieldValue doConvert(FieldValue value) {
+        return doConvert((StringFieldValue)value);
+    }
+
+    protected abstract FieldValue doConvert(StringFieldValue value);
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/TypedExpressionConverter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/TypedExpressionConverter.java
new file mode 100644
index 00000000000..9e174001aef
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/TypedExpressionConverter.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class TypedExpressionConverter<T extends Expression> extends ExpressionConverter {
+
+    private final Class<T> expClass;
+
+    public TypedExpressionConverter(Class<T> expClass) {
+        this.expClass = expClass;
+    }
+
+    @Override
+    protected final boolean shouldConvert(Expression exp) {
+        return expClass.isInstance(exp);
+    }
+
+    @Override
+    protected final Expression doConvert(Expression exp) {
+        return typedConvert(expClass.cast(exp));
+    }
+
+    protected abstract Expression typedConvert(T exp);
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/UpdateAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/UpdateAdapter.java
new file mode 100644
index 00000000000..f8d08efce24
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/UpdateAdapter.java
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.FieldValueAdapter;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public interface UpdateAdapter extends FieldValueAdapter {
+
+    public DocumentUpdate getOutput();
+    public Expression getExpression(Expression expression);
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java
new file mode 100644
index 00000000000..05fbf860295
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
+
+/**
+ * Inserts a "newTransform()" before expressions that "requiresTransform()"
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class ValueTransformProvider extends ExpressionConverter {
+
+    private final Class<? extends Expression> transformClass;
+    private boolean transformed = false;
+    private boolean duplicate = false;
+
+    public ValueTransformProvider(Class<? extends Expression> transformClass) {
+        this.transformClass = transformClass;
+    }
+
+    @Override
+    protected final ExpressionConverter branch() {
+        return clone();
+    }
+
+    @Override
+    protected final boolean shouldConvert(Expression exp) {
+        if (transformClass.isInstance(exp)) {
+            if (transformed) {
+                duplicate = true;
+                return true;
+            }
+            transformed = true;
+            return false;
+        }
+        if (exp.createdOutputType() != null) {
+            transformed = false;
+            return false;
+        }
+        if (!requiresTransform(exp)) {
+            return false;
+        }
+        if (transformed) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    protected final Expression doConvert(Expression exp) {
+        if (duplicate) {
+            duplicate = false;
+            return null;
+        }
+        transformed = true;
+        return new StatementExpression(newTransform(), exp);
+    }
+
+    protected abstract boolean requiresTransform(Expression exp);
+
+    protected abstract Expression newTransform();
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java
new file mode 100644
index 00000000000..ce56597cc86
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java
@@ -0,0 +1,220 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.NumericDataType;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.vespa.objects.ObjectOperation;
+import com.yahoo.vespa.objects.ObjectPredicate;
+
+import java.math.BigDecimal;
+import java.math.MathContext;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ArithmeticExpression extends CompositeExpression {
+
+    public enum Operator {
+
+        ADD(1, "+"),
+        SUB(1, "-"),
+        MUL(0, "*"),
+        DIV(0, "/"),
+        MOD(0, "%");
+
+        private final int precedence;
+        private final String img;
+
+        Operator(int precedence, String img) {
+            this.precedence = precedence;
+            this.img = img;
+        }
+
+        public boolean precedes(Operator op) {
+            return precedence <= op.precedence;
+        }
+
+        @Override
+        public String toString() {
+            return img;
+        }
+    }
+
+    private final Expression lhs;
+    private final Operator op;
+    private final Expression rhs;
+
+    public ArithmeticExpression(Expression lhs, Operator op, Expression rhs) {
+        lhs.getClass(); // throws NullPointerException
+        op.getClass();
+        rhs.getClass();
+        this.lhs = lhs;
+        this.op = op;
+        this.rhs = rhs;
+    }
+
+    public Expression getLeftHandSide() {
+        return lhs;
+    }
+
+    public Operator getOperator() {
+        return op;
+    }
+
+    public Expression getRightHandSide() {
+        return rhs;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        FieldValue input = ctx.getValue();
+        ctx.setValue(evaluate(ctx.setValue(input).execute(lhs).getValue(),
+                              ctx.setValue(input).execute(rhs).getValue()));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        DataType input = ctx.getValue();
+        ctx.setValue(evaluate(ctx.setValue(input).execute(lhs).getValue(),
+                              ctx.setValue(input).execute(rhs).getValue()));
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        DataType lhsType = lhs.requiredInputType();
+        DataType rhsType = rhs.requiredInputType();
+        if (lhsType == null) {
+            return rhsType;
+        }
+        if (rhsType == null) {
+            return lhsType;
+        }
+        if (!lhsType.equals(rhsType)) {
+            throw new VerificationException(this, "Operands require conflicting input types, " +
+                                                  lhsType.getName() + " vs " + rhsType.getName() + ".");
+        }
+        return lhsType;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public String toString() {
+        return lhs + " " + op + " " + rhs;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof ArithmeticExpression)) {
+            return false;
+        }
+        ArithmeticExpression exp = (ArithmeticExpression)obj;
+        if (!lhs.equals(exp.lhs)) {
+            return false;
+        }
+        if (!op.equals(exp.op)) {
+            return false;
+        }
+        if (!rhs.equals(exp.rhs)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + lhs.hashCode() + op.hashCode() + rhs.hashCode();
+    }
+
+    private DataType evaluate(DataType lhs, DataType rhs) {
+        if (lhs == null || rhs == null) {
+            throw new VerificationException(this, "Attempting to perform arithmetic on a null value.");
+        }
+        if (!(lhs instanceof NumericDataType) ||
+            !(rhs instanceof NumericDataType))
+        {
+            throw new VerificationException(this, "Attempting to perform unsupported arithmetic: [" +
+                                                  lhs.getName() + "] " + op + " [" + rhs.getName() + "]");
+        }
+        if (lhs == DataType.FLOAT || lhs == DataType.DOUBLE ||
+            rhs == DataType.FLOAT || rhs == DataType.DOUBLE)
+        {
+            if (lhs == DataType.DOUBLE || rhs == DataType.DOUBLE) {
+                return DataType.DOUBLE;
+            }
+            return DataType.FLOAT;
+        }
+        if (lhs == DataType.LONG || rhs == DataType.LONG) {
+            return DataType.LONG;
+        }
+        return DataType.INT;
+    }
+
+    private FieldValue evaluate(FieldValue lhs, FieldValue rhs) {
+        if (lhs == null || rhs == null) {
+            return null;
+        }
+        if (!(lhs instanceof NumericFieldValue) ||
+            !(rhs instanceof NumericFieldValue))
+        {
+            throw new IllegalArgumentException("Unsupported operation: [" + lhs.getDataType().getName() + "] " +
+                                               op + " [" + rhs.getDataType().getName() + "]");
+        }
+        BigDecimal lhsVal = asBigDecimal((NumericFieldValue)lhs);
+        BigDecimal rhsVal = asBigDecimal((NumericFieldValue)rhs);
+        switch (op) {
+        case ADD:
+            return createFieldValue(lhs, rhs, lhsVal.add(rhsVal));
+        case SUB:
+            return createFieldValue(lhs, rhs, lhsVal.subtract(rhsVal));
+        case MUL:
+            return createFieldValue(lhs, rhs, lhsVal.multiply(rhsVal));
+        case DIV:
+            return createFieldValue(lhs, rhs, lhsVal.divide(rhsVal, MathContext.DECIMAL64));
+        case MOD:
+            return createFieldValue(lhs, rhs, lhsVal.remainder(rhsVal));
+        }
+        throw new IllegalStateException("Unsupported operation: " + op);
+    }
+
+    private FieldValue createFieldValue(FieldValue lhs, FieldValue rhs, BigDecimal val) {
+        if (lhs instanceof FloatFieldValue || lhs instanceof DoubleFieldValue ||
+            rhs instanceof FloatFieldValue || rhs instanceof DoubleFieldValue)
+        {
+            if (lhs instanceof DoubleFieldValue || rhs instanceof DoubleFieldValue) {
+                return new DoubleFieldValue(val.doubleValue());
+            }
+            return new FloatFieldValue(val.floatValue());
+        }
+        if (lhs instanceof LongFieldValue || rhs instanceof LongFieldValue) {
+            return new LongFieldValue(val.longValue());
+        }
+        return new IntegerFieldValue(val.intValue());
+    }
+
+    public static BigDecimal asBigDecimal(NumericFieldValue value) {
+        if (value instanceof ByteFieldValue) {
+            return BigDecimal.valueOf(((ByteFieldValue)value).getByte());
+        } else if (value instanceof DoubleFieldValue) {
+            return BigDecimal.valueOf(((DoubleFieldValue)value).getDouble());
+        } else if (value instanceof FloatFieldValue) {
+            return BigDecimal.valueOf(((FloatFieldValue)value).getFloat());
+        } else if (value instanceof IntegerFieldValue) {
+            return BigDecimal.valueOf(((IntegerFieldValue)value).getInteger());
+        } else if (value instanceof LongFieldValue) {
+            return BigDecimal.valueOf(((LongFieldValue)value).getLong());
+        }
+        throw new IllegalArgumentException("Unsupported numeric field value type '" +
+                                           value.getClass().getName() + "'.");
+    }
+
+    @Override
+    public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) {
+        lhs.select(predicate, operation);
+        rhs.select(predicate, operation);
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpression.java
new file mode 100644
index 00000000000..fe3708a7b31
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpression.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class AttributeExpression extends OutputExpression {
+
+    public AttributeExpression(String fieldName) {
+        super("attribute", fieldName);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return super.equals(obj) && obj instanceof AttributeExpression;
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java
new file mode 100644
index 00000000000..d40005bcba1
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.LongFieldValue;
+import org.apache.commons.codec.binary.Base64;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Base64DecodeExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        String input = String.valueOf(ctx.getValue());
+        if (input.isEmpty()) {
+            ctx.setValue(new LongFieldValue(Long.MIN_VALUE));
+            return;
+        }
+        if (input.length() > 12) {
+            throw new NumberFormatException("Base64 value '" + input + "' is out of range.");
+        }
+        byte[] decoded = Base64.decodeBase64(input);
+        if (decoded == null || decoded.length == 0) {
+            throw new NumberFormatException("Illegal base64 value '" + input + "'.");
+        }
+        long output = 0;
+        for (int i = decoded.length; --i >= 0;) {
+            output = (output << 8) + (((int)decoded[i]) & 0xff);
+        }
+        ctx.setValue(new LongFieldValue(output));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.LONG;
+    }
+
+    @Override
+    public String toString() {
+        return "base64decode";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof Base64DecodeExpression;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java
new file mode 100644
index 00000000000..b29971449e4
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.apache.commons.codec.binary.Base64;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Base64EncodeExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        long input = ((LongFieldValue)ctx.getValue()).getLong();
+        byte[] output = new byte[8];
+        for (int i = 0; i < output.length; ++i) {
+            output[i] = (byte)(input & 0xffL);
+            input >>>= 8;
+        }
+        String encoded = new Base64(0).encodeToString(output);
+        ctx.setValue(new StringFieldValue(encoded));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.LONG;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public String toString() {
+        return "base64encode";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof Base64EncodeExpression)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java
new file mode 100644
index 00000000000..fe7771f83ec
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java
@@ -0,0 +1,163 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+
+import java.util.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class CatExpression extends ExpressionList<Expression> {
+
+    public CatExpression(Expression... lst) {
+        super(Arrays.asList(lst));
+    }
+
+    public CatExpression(Collection<? extends Expression> lst) {
+        super(lst);
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        FieldValue input = ctx.getValue();
+        DataType inputType = input != null ? input.getDataType() : null;
+        VerificationContext ver = new VerificationContext(ctx);
+        List<FieldValue> values = new LinkedList<>();
+        List<DataType> types = new LinkedList<>();
+        for (Expression exp : this) {
+            FieldValue val = ctx.setValue(input).execute(exp).getValue();
+            values.add(val);
+
+            DataType type;
+            if (val != null) {
+                type = val.getDataType();
+            } else {
+                type = ver.setValue(inputType).execute(this).getValue();
+            }
+            types.add(type);
+        }
+        DataType type = resolveOutputType(types);
+        ctx.setValue(type == DataType.STRING ? asString(values) : asCollection(type, values));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        DataType input = ctx.getValue();
+        List<DataType> types = new LinkedList<>();
+        for (Expression exp : this) {
+            DataType val = ctx.setValue(input).execute(exp).getValue();
+            types.add(val);
+            if (val == null) {
+                throw new VerificationException(this, "Attempting to concatenate a null value (" + exp + ").");
+            }
+        }
+        ctx.setValue(resolveOutputType(types));
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        DataType prev = null;
+        for (Expression exp : this) {
+            DataType next = exp.requiredInputType();
+            if (next == null) {
+                // ignore
+            } else if (prev == null) {
+                prev = next;
+            } else if (!prev.isAssignableFrom(next)) {
+                throw new VerificationException(this, "Operands require conflicting input types, " +
+                                                      prev.getName() + " vs " + next.getName() + ".");
+            }
+        }
+        return prev;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder ret = new StringBuilder();
+        for (Iterator<Expression> it = iterator(); it.hasNext();) {
+            ret.append(it.next());
+            if (it.hasNext()) {
+                ret.append(" . ");
+            }
+        }
+        return ret.toString();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return super.equals(obj) && obj instanceof CatExpression;
+    }
+
+    private static DataType resolveOutputType(List<DataType> types) {
+        DataType ret = null;
+        for (DataType type : types) {
+            if (!(type instanceof CollectionDataType)) {
+                return DataType.STRING;
+            }
+            if (ret == null) {
+                ret = type;
+            } else if (!ret.isAssignableFrom(type)) {
+                return DataType.STRING;
+            }
+        }
+        return ret;
+    }
+
+    private static FieldValue asString(List<FieldValue> outputs) {
+        StringBuilder ret = new StringBuilder();
+        for (FieldValue val : outputs) {
+            if (val == null) {
+                return null;
+            }
+            ret.append(val.toString());
+        }
+        return new StringFieldValue(ret.toString());
+    }
+
+    private static FieldValue asCollection(DataType type, List<FieldValue> values) {
+        if (type instanceof ArrayDataType) {
+            return asArray((ArrayDataType)type, values);
+        } else if (type instanceof WeightedSetDataType) {
+            return asWset((WeightedSetDataType)type, values);
+        } else {
+            throw new UnsupportedOperationException(type.getName());
+        }
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    private static FieldValue asArray(ArrayDataType arrType, List<FieldValue> values) {
+        Array out = arrType.createFieldValue();
+        for (FieldValue val : values) {
+            if (val == null) {
+                continue;
+            }
+            out.addAll((Array)val);
+        }
+        return out;
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    private static FieldValue asWset(WeightedSetDataType wsetType, List<FieldValue> values) {
+        WeightedSet out = wsetType.createFieldValue();
+        for (FieldValue val : values) {
+            if (val == null) {
+                continue;
+            }
+            out.putAll((WeightedSet)val);
+        }
+        return out;
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateExpression.java
new file mode 100644
index 00000000000..7fb4951bfd9
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateExpression.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ClearStateExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.clear();
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.clear();
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return null;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        return "clear_state";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof ClearStateExpression;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java
new file mode 100644
index 00000000000..a25a3b1533b
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class CompositeExpression extends Expression {
+
+    protected static String toScriptBlock(Expression exp) {
+        if (exp instanceof ScriptExpression) {
+            return exp.toString();
+        }
+        if (exp instanceof StatementExpression) {
+            return new ScriptExpression((StatementExpression)exp).toString();
+        }
+        return new ScriptExpression(new StatementExpression(exp)).toString();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EchoExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EchoExpression.java
new file mode 100644
index 00000000000..67f0c2faef4
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EchoExpression.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+
+import java.io.PrintStream;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class EchoExpression extends Expression {
+
+    private final PrintStream out;
+
+    public EchoExpression() {
+        this(System.out);
+    }
+
+    public EchoExpression(PrintStream out) {
+        this.out = out;
+    }
+
+    public PrintStream getOutputStream() {
+        return out;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        out.println(String.valueOf(ctx.getValue()));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        // empty
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        return "echo";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof EchoExpression)) {
+            return false;
+        }
+        EchoExpression rhs = (EchoExpression)obj;
+        if (out != rhs.out) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + out.hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java
new file mode 100644
index 00000000000..be70291cb70
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.annotation.*;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.language.process.TokenType;
+
+import static com.yahoo.language.LinguisticsCase.toLowerCase;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ExactExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        StringFieldValue input = (StringFieldValue)ctx.getValue();
+        if (input.getString().isEmpty()) {
+            return;
+        }
+        StringFieldValue output = input.clone();
+        ctx.setValue(output);
+
+        String prev = output.getString();
+        String next = toLowerCase(prev);
+
+        SpanList root = new SpanList();
+        SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS, root);
+        SpanNode node = new Span(0, prev.length());
+        tree.annotate(node, new Annotation(AnnotationTypes.TERM,
+                                           next.equals(prev) ? null : new StringFieldValue(next)));
+        tree.annotate(node, new Annotation(AnnotationTypes.TOKEN_TYPE,
+                                           new IntegerFieldValue(TokenType.ALPHABETIC.getValue())));
+        root.add(node);
+        output.setSpanTree(tree);
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        // empty
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        return "exact";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof ExactExpression;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java
new file mode 100644
index 00000000000..24527a61901
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java
@@ -0,0 +1,130 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.FieldPath;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.language.Language;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.detect.Detection;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ExecutionContext implements FieldTypeAdapter, FieldValueAdapter, Cloneable {
+
+    private final Map<String, FieldValue> variables = new HashMap<>();
+    private final FieldValueAdapter adapter;
+    private FieldValue value;
+    private Language language;
+
+    public ExecutionContext() {
+        this(null);
+    }
+
+    public ExecutionContext(FieldValueAdapter adapter) {
+        this.adapter = adapter;
+        this.language = Language.UNKNOWN;
+    }
+
+    public ExecutionContext execute(Expression exp) {
+        if (exp != null) {
+            exp.execute(this);
+        }
+        return this;
+    }
+
+    @Override
+    public DataType getInputType(Expression exp, String fieldName) {
+        return adapter.getInputType(exp, fieldName);
+    }
+
+    @Override
+    public FieldValue getInputValue(String fieldName) {
+        if (adapter == null) {
+            throw new IllegalStateException("Can not get field '" + fieldName + "' because adapter is null.");
+        }
+        return adapter.getInputValue(fieldName);
+    }
+
+    @Override
+    public FieldValue getInputValue(FieldPath fieldPath) {
+        if (adapter == null) {
+            throw new IllegalStateException("Can not get field '" + fieldPath + "' because adapter is null.");
+        }
+        return adapter.getInputValue(fieldPath);
+    }
+
+    @Override
+    public void tryOutputType(Expression exp, String fieldName, DataType valueType) {
+        adapter.tryOutputType(exp, fieldName, valueType);
+    }
+
+    @Override
+    public ExecutionContext setOutputValue(Expression exp, String fieldName, FieldValue fieldValue) {
+        if (adapter == null) {
+            throw new IllegalStateException("Can not set field '" + fieldName + "' because adapter is null.");
+        }
+        adapter.setOutputValue(exp, fieldName, fieldValue);
+        return this;
+    }
+
+    public FieldValueAdapter getAdapter() {
+        return adapter;
+    }
+
+    public FieldValue getVariable(String name) {
+        return variables.get(name);
+    }
+
+    public ExecutionContext setVariable(String name, FieldValue value) {
+        variables.put(name, value);
+        return this;
+    }
+
+    public Language getLanguage() {
+        return language;
+    }
+
+    public ExecutionContext setLanguage(Language language) {
+        language.getClass();
+        this.language = language;
+        return this;
+    }
+
+    public Language resolveLanguage(Linguistics linguistics) {
+        if (language != null && language != Language.UNKNOWN) {
+            return language;
+        }
+        if (linguistics == null) {
+            return Language.ENGLISH;
+        }
+        Detection detection = linguistics.getDetector().detect(String.valueOf(value), null);
+        if (detection == null) {
+            return Language.ENGLISH;
+        }
+        Language detected = detection.getLanguage();
+        if (detected == Language.UNKNOWN) {
+            return Language.ENGLISH;
+        }
+        return detected;
+    }
+
+    public FieldValue getValue() {
+        return value;
+    }
+
+    public ExecutionContext setValue(FieldValue value) {
+        this.value = value;
+        return this;
+    }
+
+    public ExecutionContext clear() {
+        variables.clear();
+        value = null;
+        return this;
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java
new file mode 100644
index 00000000000..f94a100b31d
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java
@@ -0,0 +1,217 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.*;
+import com.yahoo.vespa.indexinglanguage.parser.IndexingInput;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+import com.yahoo.vespa.objects.Selectable;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class Expression extends Selectable {
+
+    public final FieldValue execute(FieldValue val) {
+        return execute(new ExecutionContext().setValue(val));
+    }
+
+    public final Document execute(AdapterFactory factory, Document doc) {
+        return execute(factory.newDocumentAdapter(doc));
+    }
+
+    public final Document execute(DocumentAdapter adapter) {
+        execute((FieldValueAdapter)adapter);
+        return adapter.getFullOutput();
+    }
+
+    public static DocumentUpdate execute(Expression expression, AdapterFactory factory, DocumentUpdate upd) {
+        DocumentUpdate ret = null;
+        for (UpdateAdapter adapter : factory.newUpdateAdapterList(upd)) {
+            DocumentUpdate output = adapter.getExpression(expression).execute(adapter);
+            if (output == null) {
+                // ignore
+            } else if (ret != null) {
+                ret.addAll(output);
+            } else {
+                ret = output;
+            }
+        }
+        if (ret != null) {
+            ret.setCreateIfNonExistent(upd.getCreateIfNonExistent());
+        }
+        return ret;
+    }
+
+    public final DocumentUpdate execute(UpdateAdapter adapter) {
+        execute((FieldValueAdapter)adapter);
+        return adapter.getOutput();
+    }
+
+    public final FieldValue execute(FieldValueAdapter adapter) {
+        return execute(new ExecutionContext(adapter));
+    }
+
+    public final FieldValue execute(ExecutionContext ctx) {
+        DataType inputType = requiredInputType();
+        if (inputType != null) {
+            FieldValue input = ctx.getValue();
+            if (input == null) {
+                return null;
+            }
+            if (!inputType.isValueCompatible(input)) {
+                throw new IllegalArgumentException("Expression '" + this + "' expected " + inputType.getName() +
+                                                   " input, got " + input.getDataType().getName() + ".");
+            }
+        }
+        doExecute(ctx);
+        DataType outputType = createdOutputType();
+        if (outputType != null) {
+            FieldValue output = ctx.getValue();
+            if (output != null && !outputType.isValueCompatible(output)) {
+                throw new IllegalStateException("Expression '" + this + "' expected " + outputType.getName() +
+                                                " output, got " + output.getDataType().getName() + ".");
+            }
+        }
+        return ctx.getValue();
+    }
+
+    protected abstract void doExecute(ExecutionContext ctx);
+
+    public final DataType verify() {
+        return verify(new VerificationContext());
+    }
+
+    public final DataType verify(DataType val) {
+        return verify(new VerificationContext().setValue(val));
+    }
+
+    public final Document verify(Document doc) {
+        return verify(new SimpleAdapterFactory(), doc);
+    }
+
+    public final Document verify(AdapterFactory factory, Document doc) {
+        return verify(factory.newDocumentAdapter(doc));
+    }
+
+    public final Document verify(DocumentAdapter adapter) {
+        verify((FieldTypeAdapter)adapter);
+        return adapter.getFullOutput();
+    }
+
+    public final DocumentUpdate verify(DocumentUpdate upd) {
+        return verify(new SimpleAdapterFactory(), upd);
+    }
+
+    public final DocumentUpdate verify(AdapterFactory factory, DocumentUpdate upd) {
+        DocumentUpdate ret = null;
+        for (UpdateAdapter adapter : factory.newUpdateAdapterList(upd)) {
+            DocumentUpdate output = verify(adapter);
+            if (output == null) {
+                // ignore
+            } else if (ret != null) {
+                ret.addAll(output);
+            } else {
+                ret = output;
+            }
+        }
+        return ret;
+    }
+
+    public final DocumentUpdate verify(UpdateAdapter adapter) {
+        verify((FieldTypeAdapter)adapter);
+        return adapter.getOutput();
+    }
+
+    public final DataType verify(FieldTypeAdapter adapter) {
+        return verify(new VerificationContext(adapter));
+    }
+
+    public final DataType verify(VerificationContext ctx) {
+//        System.err.println("enter_verify(exp = '" + this + "', req = " + requiredInputType(ctx) +
+//                           ", in = " + ctx.getValue() + ")");
+        DataType inputType = requiredInputType();
+        if (inputType != null) {
+            DataType input = ctx.getValue();
+            if (input == null) {
+                throw new VerificationException(this, "Expected " + inputType.getName() + " input, got null.");
+            }
+            if (input.getPrimitiveType() == UnresolvedDataType.INSTANCE) {
+                throw new VerificationException(this, "Failed to resolve input type.");
+            }
+            if (!inputType.isAssignableFrom(input)) {
+                throw new VerificationException(this, "Expected " + inputType.getName() + " input, got " +
+                                                      input.getName() + ".");
+            }
+        }
+        doVerify(ctx);
+        DataType outputType = createdOutputType();
+//        System.err.println("exit_verify(exp = '" + this + "', req = " + createdOutputType(ctx) +
+//                           ", out = " + ctx.getValue() + ")");
+        if (outputType != null) {
+            DataType output = ctx.getValue();
+            if (output == null) {
+                throw new VerificationException(this, "Expected " + outputType.getName() + " output, got null.");
+            }
+            if (output.getPrimitiveType() == UnresolvedDataType.INSTANCE) {
+                throw new VerificationException(this, "Failed to resolve output type.");
+            }
+            if (!outputType.isAssignableFrom(output)) {
+                throw new VerificationException(this, "Expected " + outputType.getName() + " output, got " +
+                                                      output.getName() + ".");
+            }
+        }
+        return ctx.getValue();
+    }
+
+    protected abstract void doVerify(VerificationContext ctx);
+
+    public abstract DataType requiredInputType();
+
+    public abstract DataType createdOutputType();
+
+    /** Creates an expression with simple lingustics for testing */
+    public static Expression fromString(String expression) throws ParseException {
+        return fromString(expression, new SimpleLinguistics());
+    }
+
+    public static Expression fromString(String expression, Linguistics linguistics) throws ParseException {
+        return newInstance(new ScriptParserContext(linguistics).setInputStream(new IndexingInput(expression)));
+    }
+
+    public static Expression newInstance(ScriptParserContext context) throws ParseException {
+        return ScriptParser.parseExpression(context);
+    }
+
+    protected static boolean equals(Object lhs, Object rhs) {
+        if (lhs == null) {
+            if (rhs != null) {
+                return false;
+            }
+        } else {
+            if (rhs == null) {
+                return false;
+            }
+            if (!lhs.equals(rhs)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    // Convenience For testing
+    public static Document execute(Expression expression, Document doc) {
+        return expression.execute(new SimpleAdapterFactory(), doc);
+    }
+    public static final DocumentUpdate execute(Expression expression, DocumentUpdate upd) {
+        return expression.execute(expression, new SimpleAdapterFactory(), upd);
+    }
+    public final FieldValue execute() {
+        return execute(new ExecutionContext());
+    }
+
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java
new file mode 100644
index 00000000000..aec0b2ada28
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DocumentType;
+import com.yahoo.vespa.objects.ObjectOperation;
+import com.yahoo.vespa.objects.ObjectPredicate;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class ExpressionList<T extends Expression> extends CompositeExpression implements Iterable<T> {
+
+    private final List<T> expressions = new LinkedList<T>();
+
+    protected ExpressionList() {
+        // empty
+    }
+
+    protected ExpressionList(Iterable<? extends T> lst) {
+        for (T exp : lst) {
+            this.expressions.add(exp);
+        }
+    }
+
+    public int size() {
+        return expressions.size();
+    }
+
+    public T get(int idx) {
+        return expressions.get(idx);
+    }
+
+    public boolean isEmpty() {
+        return expressions.isEmpty();
+    }
+
+    public List<T> asList() {
+        return Collections.unmodifiableList(expressions);
+    }
+
+    @Override
+    public Iterator<T> iterator() {
+        return expressions.iterator();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof ExpressionList)) {
+            return false;
+        }
+        ExpressionList rhs = (ExpressionList)obj;
+        if (!expressions.equals(rhs.expressions)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + expressions.hashCode();
+    }
+
+    @Override
+    public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) {
+        for (T exp : expressions) {
+            exp.select(predicate, operation);
+        }
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldTypeAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldTypeAdapter.java
new file mode 100644
index 00000000000..a7f42178bcf
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldTypeAdapter.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public interface FieldTypeAdapter {
+
+    public DataType getInputType(Expression exp, String fieldName);
+
+    public void tryOutputType(Expression exp, String fieldName, DataType valueType);
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldValueAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldValueAdapter.java
new file mode 100644
index 00000000000..89ed4e005f1
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldValueAdapter.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.FieldPath;
+import com.yahoo.document.datatypes.FieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public interface FieldValueAdapter extends FieldTypeAdapter {
+
+    public FieldValue getInputValue(String fieldName);
+    public FieldValue getInputValue(FieldPath fieldPath);
+
+    public FieldValueAdapter setOutputValue(Expression exp, String fieldName, FieldValue fieldValue);
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java
new file mode 100644
index 00000000000..915092f5e02
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java
@@ -0,0 +1,89 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.annotation.*;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+
+import java.util.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class FlattenExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        StringFieldValue input = (StringFieldValue)ctx.getValue();
+        SpanTree tree = input.getSpanTree(SpanTrees.LINGUISTICS);
+        Map<Integer, List<String>> map = new HashMap<>();
+        for (Annotation anno : tree) {
+            SpanNode span = anno.getSpanNode();
+            if (span == null) {
+                continue;
+            }
+            if (anno.getType() != AnnotationTypes.TERM) {
+                continue;
+            }
+            FieldValue val = anno.getFieldValue();
+            String str;
+            if (val instanceof StringFieldValue) {
+                str = ((StringFieldValue)val).getString();
+            } else {
+                str = input.getString().substring(span.getFrom(), span.getTo());
+            }
+            Integer pos = span.getTo();
+            List<String> entry = map.get(pos);
+            if (entry == null) {
+                entry = new LinkedList<>();
+                map.put(pos, entry);
+            }
+            entry.add(str);
+        }
+        String inputVal = String.valueOf(input);
+        StringBuilder output = new StringBuilder();
+        for (int i = 0, len = inputVal.length(); i <= len; ++i) {
+            List<String> entry = map.get(i);
+            if (entry != null) {
+                Collections.sort(entry);
+                output.append(entry);
+            }
+            if (i < len) {
+                output.append(inputVal.charAt(i));
+            }
+        }
+        ctx.setValue(new StringFieldValue(output.toString()));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public String toString() {
+        return "flatten";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof FlattenExpression;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java
new file mode 100644
index 00000000000..a74f02e9a4a
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java
@@ -0,0 +1,136 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.vespa.indexinglanguage.FieldValueConverter;
+import com.yahoo.vespa.objects.ObjectOperation;
+import com.yahoo.vespa.objects.ObjectPredicate;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ForEachExpression extends CompositeExpression {
+
+    private final Expression exp;
+
+    public ForEachExpression(Expression exp) {
+        this.exp = exp;
+    }
+
+    public Expression getInnerExpression() {
+        return exp;
+    }
+
+    @Override
+    protected void doExecute(final ExecutionContext ctx) {
+        FieldValue input = ctx.getValue();
+        if (input instanceof Array || input instanceof WeightedSet) {
+            FieldValue next = new MyConverter(ctx, exp).convert(input);
+            if (next == null) {
+                VerificationContext vctx = new VerificationContext(ctx);
+                vctx.setValue(input.getDataType()).execute(this);
+                next = vctx.getValue().createFieldValue();
+            }
+            ctx.setValue(next);
+        } else if (input instanceof Struct) {
+            ctx.setValue(new MyConverter(ctx, exp).convert(input));
+        } else {
+            throw new IllegalArgumentException("Expected Array, Struct or WeightedSet input, got " +
+                                               input.getDataType().getName() + ".");
+        }
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        DataType input = ctx.getValue();
+        if (input instanceof ArrayDataType || input instanceof WeightedSetDataType) {
+            ctx.setValue(((CollectionDataType)input).getNestedType()).execute(exp);
+            if (input instanceof ArrayDataType) {
+                ctx.setValue(DataType.getArray(ctx.getValue()));
+            } else {
+                WeightedSetDataType wset = (WeightedSetDataType)input;
+                ctx.setValue(DataType.getWeightedSet(ctx.getValue(), wset.createIfNonExistent(), wset.removeIfZero()));
+            }
+        } else if (input instanceof StructDataType) {
+            for (Field field : ((StructDataType)input).getFields()) {
+                DataType fieldType = field.getDataType();
+                DataType valueType = ctx.setValue(fieldType).execute(exp).getValue();
+                if (!fieldType.isAssignableFrom(valueType)) {
+                    throw new VerificationException(this, "Expected " + fieldType.getName() + " output, got " +
+                                                          valueType.getName() + ".");
+                }
+            }
+            ctx.setValue(input);
+        } else {
+            throw new VerificationException(this, "Expected Array, Struct or WeightedSet input, got " +
+                                                  input.getName() + ".");
+        }
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        if (exp.createdOutputType() == null) {
+            return null;
+        }
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public String toString() {
+        return "for_each { " + exp + " }";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof ForEachExpression)) {
+            return false;
+        }
+        ForEachExpression rhs = (ForEachExpression)obj;
+        if (!exp.equals(rhs.exp)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + exp.hashCode();
+    }
+
+    private static final class MyConverter extends FieldValueConverter {
+
+        final ExecutionContext context;
+        final Expression expression;
+        int depth = 0;
+
+        MyConverter(ExecutionContext context, Expression expression) {
+            this.context = context;
+            this.expression = expression;
+        }
+
+        @Override
+        protected boolean shouldConvert(FieldValue value) {
+            return ++depth > 1;
+        }
+
+        @Override
+        protected FieldValue doConvert(FieldValue value) {
+            context.setValue(value).execute(expression);
+            return context.getValue();
+        }
+    }
+
+    @Override
+    public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) {
+        select(exp, predicate, operation);
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java
new file mode 100644
index 00000000000..6cc06953437
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java
@@ -0,0 +1,84 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructuredDataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class GetFieldExpression extends Expression {
+
+    private final String fieldName;
+
+    public GetFieldExpression(String fieldName) {
+        this.fieldName = fieldName;
+    }
+
+    public String getFieldName() {
+        return fieldName;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        FieldValue input = ctx.getValue();
+        if (!(input instanceof StructuredFieldValue)) {
+            throw new IllegalArgumentException("Expected structured input, got " + input.getDataType().getName() + ".");
+        }
+        StructuredFieldValue struct = (StructuredFieldValue)input;
+        Field field = struct.getField(fieldName);
+        if (field == null) {
+            throw new IllegalArgumentException("Field '" + fieldName + "' not found.");
+        }
+        ctx.setValue(struct.getFieldValue(field));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        DataType input = ctx.getValue();
+        if (!(input instanceof StructuredDataType)) {
+            throw new VerificationException(this, "Expected structured input, got " + input.getName() + ".");
+        }
+        Field field = ((StructuredDataType)input).getField(fieldName);
+        if (field == null) {
+            throw new VerificationException(this, "Field '" + fieldName + "' not found.");
+        }
+        ctx.setValue(field.getDataType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public String toString() {
+        return "get_field " + fieldName;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof GetFieldExpression)) {
+            return false;
+        }
+        GetFieldExpression rhs = (GetFieldExpression)obj;
+        if (!fieldName.equals(rhs.fieldName)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + fieldName.hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java
new file mode 100644
index 00000000000..34f0139037b
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class GetVarExpression extends Expression {
+
+    private final String varName;
+
+    public GetVarExpression(String varName) {
+        this.varName = varName;
+    }
+
+    public String getVariableName() {
+        return varName;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setValue(ctx.getVariable(varName));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        DataType input = ctx.getVariable(varName);
+        if (input == null) {
+            throw new VerificationException(this, "Variable '" + varName + "' not found.");
+        }
+        ctx.setValue(input);
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return null;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public String toString() {
+        return "get_var " + varName;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof GetVarExpression)) {
+            return false;
+        }
+        GetVarExpression rhs = (GetVarExpression)obj;
+        if (!varName.equals(rhs.varName)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + varName.hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java
new file mode 100644
index 00000000000..b7a49e938b5
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java
@@ -0,0 +1,96 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.UpdateAdapter;
+import com.yahoo.vespa.objects.ObjectOperation;
+import com.yahoo.vespa.objects.ObjectPredicate;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class GuardExpression extends CompositeExpression {
+
+    private final Expression exp;
+    private final boolean shouldExecute;
+
+    public GuardExpression(Expression exp) {
+        this.exp = exp;
+        shouldExecute = shouldExecute(exp);
+    }
+
+    public Expression getInnerExpression() {
+        return exp;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        if (!shouldExecute && ctx.getAdapter() instanceof UpdateAdapter) {
+            ctx.setValue(null);
+        } else {
+            exp.execute(ctx);
+        }
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        exp.verify(ctx);
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return exp.requiredInputType();
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return exp.createdOutputType();
+    }
+
+    @Override
+    public String toString() {
+        return "guard " + toScriptBlock(exp);
+    }
+
+    @Override
+    public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) {
+         select(exp, predicate, operation);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof GuardExpression)) {
+            return false;
+        }
+        GuardExpression rhs = (GuardExpression)obj;
+        if (!exp.equals(rhs.exp)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + exp.hashCode();
+    }
+
+    private static boolean shouldExecute(Expression exp) {
+        ExecutionGuard guard = new ExecutionGuard();
+        guard.visit(exp);
+        return guard.shouldExecute;
+    }
+
+    private static class ExecutionGuard extends ExpressionVisitor {
+
+        boolean shouldExecute = false;
+
+        @Override
+        protected void doVisit(Expression exp) {
+            if (exp instanceof InputExpression || exp instanceof SetLanguageExpression) {
+                shouldExecute = true;
+            }
+        }
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java
new file mode 100644
index 00000000000..11e58883b27
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.LongFieldValue;
+
+import java.math.BigInteger;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class HexDecodeExpression extends Expression {
+
+    private static final BigInteger ULONG_MAX = new BigInteger("18446744073709551616");
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        String input = String.valueOf(ctx.getValue());
+        if (input.isEmpty()) {
+            ctx.setValue(new LongFieldValue(Long.MIN_VALUE));
+            return;
+        }
+        BigInteger output;
+        try {
+            output = new BigInteger(input, 16);
+        } catch (NumberFormatException e) {
+            throw new NumberFormatException("Illegal hex value '" + input + "'.");
+        }
+        if (output.bitLength() > 64) {
+            throw new NumberFormatException("Hex value '" + input + "' is out of range.");
+        }
+        if (output.compareTo(BigInteger.ZERO) == 1 && output.bitLength() == 64) {
+            output = output.subtract(ULONG_MAX); // flip to negative
+        }
+        ctx.setValue(new LongFieldValue(output.longValue()));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.LONG;
+    }
+
+    @Override
+    public String toString() {
+        return "hexdecode";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof HexDecodeExpression;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java
new file mode 100644
index 00000000000..80c7b95308f
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class HexEncodeExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        long input = ((LongFieldValue)ctx.getValue()).getLong();
+        ctx.setValue(new StringFieldValue(Long.toHexString(input)));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.LONG;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public String toString() {
+        return "hexencode";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof HexEncodeExpression)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java
new file mode 100644
index 00000000000..30007090b78
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.StringFieldValue;
+
+import java.net.InetAddress;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class HostNameExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        try {
+            ctx.setValue(new StringFieldValue(normalizeHostName(InetAddress.getLocalHost().getHostName())));
+        } catch (java.net.UnknownHostException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return null;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public String toString() {
+        return "hostname";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof HostNameExpression)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+
+    public static String normalizeHostName(String hostName) {
+        int pos = hostName.indexOf('.');
+        return pos < 0 ? hostName : hostName.substring(0, pos);
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java
new file mode 100644
index 00000000000..eb7e1c9a005
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java
@@ -0,0 +1,216 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.NumericFieldValue;
+import com.yahoo.vespa.objects.ObjectOperation;
+import com.yahoo.vespa.objects.ObjectPredicate;
+
+import java.math.BigDecimal;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class IfThenExpression extends CompositeExpression {
+
+    public static enum Comparator {
+        EQ("=="),
+        NE("!="),
+        LT("<"),
+        LE("<="),
+        GT(">"),
+        GE(">=");
+
+        private final String img;
+
+        private Comparator(String img) {
+            this.img = img;
+        }
+
+        @Override
+        public String toString() {
+            return img;
+        }
+    }
+
+    private final Expression lhs;
+    private final Comparator cmp;
+    private final Expression rhs;
+    private final Expression ifTrue;
+    private final Expression ifFalse;
+
+    public IfThenExpression(Expression lhs, Comparator cmp, Expression rhs, Expression ifTrue) {
+        this(lhs, cmp, rhs, ifTrue, null);
+    }
+
+    public IfThenExpression(Expression lhs, Comparator cmp, Expression rhs, Expression ifTrue, Expression ifFalse) {
+        this.lhs = lhs;
+        this.cmp = cmp;
+        this.rhs = rhs;
+        this.ifTrue = ifTrue;
+        this.ifFalse = ifFalse;
+    }
+
+    public Expression getLeftHandSide() {
+        return lhs;
+    }
+
+    public Comparator getComparator() {
+        return cmp;
+    }
+
+    public Expression getRightHandSide() {
+        return rhs;
+    }
+
+    public Expression getIfTrueExpression() {
+        return ifTrue;
+    }
+
+    public Expression getIfFalseExpression() {
+        return ifFalse;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        FieldValue input = ctx.getValue();
+        FieldValue lhsVal = ctx.setValue(input).execute(lhs).getValue();
+        if (lhsVal == null) {
+            ctx.setValue(null);
+            return;
+        }
+        FieldValue rhsVal = ctx.setValue(input).execute(rhs).getValue();
+        if (rhsVal == null) {
+            ctx.setValue(null);
+            return;
+        }
+        ctx.setValue(input);
+        if (isTrue(lhsVal, cmp, rhsVal)) {
+            ifTrue.execute(ctx);
+        } else if (ifFalse != null) {
+            ifFalse.execute(ctx);
+        }
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        DataType input = ctx.getValue();
+        ctx.setValue(input).execute(lhs);
+        ctx.setValue(input).execute(rhs);
+        ctx.setValue(input).execute(ifTrue);
+        ctx.setValue(input).execute(ifFalse);
+        ctx.setValue(input);
+    }
+
+    @Override
+    public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) {
+        select(lhs, predicate, operation);
+        select(rhs, predicate, operation);
+        select(ifTrue, predicate, operation);
+        select(ifFalse, predicate, operation);
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        DataType input = null;
+        input = resolveRequiredInputType(input, lhs.requiredInputType());
+        input = resolveRequiredInputType(input, rhs.requiredInputType());
+        input = resolveRequiredInputType(input, ifTrue.requiredInputType());
+        if (ifFalse != null) {
+            input = resolveRequiredInputType(input, ifFalse.requiredInputType());
+        }
+        return input;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder ret = new StringBuilder();
+        ret.append("if (").append(lhs).append(" ").append(cmp).append(" ").append(rhs).append(") ");
+        ret.append(toScriptBlock(ifTrue));
+        if (ifFalse != null) {
+            ret.append(" else ").append(toScriptBlock(ifFalse));
+        }
+        return ret.toString();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof IfThenExpression)) {
+            return false;
+        }
+        IfThenExpression exp = (IfThenExpression)obj;
+        if (!lhs.equals(exp.lhs)) {
+            return false;
+        }
+        if (!cmp.equals(exp.cmp)) {
+            return false;
+        }
+        if (!rhs.equals(exp.rhs)) {
+            return false;
+        }
+        if (!ifTrue.equals(exp.ifTrue)) {
+            return false;
+        }
+        if (!equals(ifFalse, exp.ifFalse)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int ret = getClass().hashCode() + lhs.hashCode() + cmp.hashCode() + rhs.hashCode() + ifTrue.hashCode();
+        if (ifFalse != null) {
+            ret += ifFalse.hashCode();
+        }
+        return ret;
+    }
+
+    private DataType resolveRequiredInputType(DataType prev, DataType next) {
+        if (next == null) {
+            return prev;
+        }
+        if (prev == null) {
+            return next;
+        }
+        if (!prev.equals(next)) {
+            throw new VerificationException(this, "Operands require conflicting input types, " +
+                                                  prev.getName() + " vs " + next.getName() + ".");
+        }
+        return prev;
+    }
+
+    private static boolean isTrue(FieldValue lhs, Comparator cmp, FieldValue rhs) {
+        int res;
+        if (lhs instanceof NumericFieldValue && rhs instanceof NumericFieldValue) {
+            BigDecimal lhsVal = ArithmeticExpression.asBigDecimal((NumericFieldValue)lhs);
+            BigDecimal rhsVal = ArithmeticExpression.asBigDecimal((NumericFieldValue)rhs);
+            res = lhsVal.compareTo(rhsVal);
+        } else {
+            res = lhs.compareTo(rhs);
+        }
+        switch (cmp) {
+        case EQ:
+            return res == 0;
+        case NE:
+            return res != 0;
+        case GT:
+            return res > 0;
+        case GE:
+            return res >= 0;
+        case LT:
+            return res < 0;
+        case LE:
+            return res <= 0;
+        default:
+            throw new UnsupportedOperationException(cmp.toString());
+        }
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IndexExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IndexExpression.java
new file mode 100644
index 00000000000..eb7d79abe10
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IndexExpression.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class IndexExpression extends OutputExpression {
+
+    public IndexExpression(String fieldName) {
+        super("index", fieldName);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return super.equals(obj) && obj instanceof IndexExpression;
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java
new file mode 100644
index 00000000000..29957e440a4
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java
@@ -0,0 +1,115 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.FieldPath;
+import com.yahoo.vespa.objects.ObjectOperation;
+import com.yahoo.vespa.objects.ObjectPredicate;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class InputExpression extends Expression {
+
+    private final String fieldName;
+    private FieldPath fieldPath;
+
+    public InputExpression(String fieldName) {
+        this.fieldName = fieldName;
+    }
+
+    public String getFieldName() {
+        return fieldName;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx)
+    {
+        if (fieldPath != null) {
+            ctx.setValue(ctx.getInputValue(fieldPath));
+        } else {
+            ctx.setValue(ctx.getInputValue(fieldName));
+        }
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        DataType val = ctx.getInputType(this, fieldName);
+        if (val == null) {
+            throw new VerificationException(this, "Field '" + fieldName + "' not found.");
+        }
+        ctx.setValue(val);
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return null;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public String toString() {
+        return "input" + (fieldName != null ? " " + fieldName : "");
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof InputExpression)) {
+            return false;
+        }
+        InputExpression rhs = (InputExpression)obj;
+        if (!equals(fieldName, rhs.fieldName)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + (fieldName != null ? fieldName.hashCode() : 0);
+    }
+
+    public static class FieldPathOptimizer implements ObjectOperation, ObjectPredicate {
+        private final DocumentType documentType;
+
+        public FieldPathOptimizer(DocumentType documentType) {
+            this.documentType = documentType;
+        }
+
+        @Override
+        public void execute(Object obj) {
+            InputExpression exp = (InputExpression) obj;
+            exp.fieldPath = documentType.buildFieldPath(exp.getFieldName());
+        }
+
+        @Override
+        public boolean check(Object obj) {
+            return obj instanceof InputExpression;
+        }
+    }
+
+    public static class InputFieldNameExtractor implements ObjectOperation, ObjectPredicate {
+        private List<String> inputFieldNames = new ArrayList<>(1);
+
+        public List<String> getInputFieldNames() { return inputFieldNames; }
+
+        @Override
+        public void execute(Object obj) {
+            inputFieldNames.add(((InputExpression) obj).getFieldName());
+        }
+
+        @Override
+        public boolean check(Object obj) {
+            return obj instanceof InputExpression;
+        }
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java
new file mode 100644
index 00000000000..b440c2e68c8
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.text.StringUtilities;
+
+import java.util.Iterator;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class JoinExpression extends Expression {
+
+    private final String delimiter;
+
+    public JoinExpression(String delimiter) {
+        this.delimiter = delimiter;
+    }
+
+    public String getDelimiter() {
+        return delimiter;
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        FieldValue input = ctx.getValue();
+        if (!(input instanceof Array)) {
+            throw new IllegalArgumentException("Expected Array input, got " + input.getDataType().getName() + ".");
+        }
+        StringBuilder output = new StringBuilder();
+        for (Iterator<FieldValue> it = ((Array)input).fieldValueIterator(); it.hasNext(); ) {
+            output.append(String.valueOf(it.next()));
+            if (it.hasNext()) {
+                output.append(delimiter);
+            }
+        }
+        ctx.setValue(new StringFieldValue(output.toString()));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        DataType input = ctx.getValue();
+        if (!(input instanceof ArrayDataType)) {
+            throw new VerificationException(this, "Expected Array input, got " + input.getName() + ".");
+        }
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public String toString() {
+        return "join \"" + StringUtilities.escape(delimiter, '"') + "\"";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof JoinExpression)) {
+            return false;
+        }
+        JoinExpression rhs = (JoinExpression)obj;
+        if (!delimiter.equals(rhs.delimiter)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + delimiter.hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseExpression.java
new file mode 100644
index 00000000000..272e2c3a3e5
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseExpression.java
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.StringFieldValue;
+
+import static com.yahoo.language.LinguisticsCase.toLowerCase;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LowerCaseExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setValue(new StringFieldValue(toLowerCase(String.valueOf(ctx.getValue()))));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public String toString() {
+        return "lowercase";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof LowerCaseExpression)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java
new file mode 100644
index 00000000000..f648a0e38e4
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;

+

+import java.util.LinkedList;

+import java.util.List;

+import java.util.Stack;

+

+/**

+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>

+ */

+public class MathResolver {

+

+    private final List<Item> items = new LinkedList<>();

+

+    public void push(ArithmeticExpression.Operator op, Expression exp) {

+        op.getClass(); // throws NullPointerException

+        if (items.isEmpty() && op != ArithmeticExpression.Operator.ADD) {

+            throw new IllegalArgumentException("First item in an arithmetic operation must be an addition.");

+        }

+        items.add(new Item(op, exp));

+    }

+

+    public Expression resolve() {

+        Stack<Item> stack = new Stack<>();

+        stack.push(items.remove(0));

+        while (!items.isEmpty()) {

+            Item item = items.remove(0);

+            while (stack.size() > 1 && stack.peek().op.precedes(item.op)) {

+                pop(stack);

+            }

+            stack.push(item);

+        }

+        while (stack.size() > 1) {

+            pop(stack);

+        }

+        return stack.remove(0).exp;

+    }

+

+    private void pop(Stack<Item> stack) {

+        Item rhs = stack.pop();

+        Item lhs = stack.peek();

+        lhs.exp = new ArithmeticExpression(lhs.exp, rhs.op, rhs.exp);

+    }

+

+    private static class Item {

+

+        final ArithmeticExpression.Operator op;

+        Expression exp;

+

+        Item(ArithmeticExpression.Operator op, Expression exp) {

+            this.op = op;

+            this.exp = exp;

+        }

+    }

+}
\ No newline at end of file
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java
new file mode 100644
index 00000000000..c5b68e6c978
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java
@@ -0,0 +1,109 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.annotation.*;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.process.GramSplitter;
+import com.yahoo.language.process.TokenType;
+import com.yahoo.vespa.indexinglanguage.linguistics.LinguisticsAnnotator;
+
+import java.util.Iterator;
+
+/**
+ * A filter which splits incoming text into n-grams
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class NGramExpression extends Expression {
+
+    private final Linguistics linguistics;
+    private final int gramSize;
+
+    /**
+     * Creates an executable ngram expression
+     *
+     * @param linguistics  the gram splitter to use, or null if this is used for representation and will not be executed
+     * @param gramSize the gram size
+     */
+    public NGramExpression(Linguistics linguistics, int gramSize) {
+        this.linguistics = linguistics;
+        this.gramSize = gramSize;
+    }
+
+    public Linguistics getLinguistics() {
+        return linguistics;
+    }
+
+    public int getGramSize() {
+        return gramSize;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        StringFieldValue input = (StringFieldValue)ctx.getValue();
+        SpanList spanList = input.setSpanTree(new SpanTree(SpanTrees.LINGUISTICS)).spanList();
+        int lastPosition = 0;
+        for (Iterator<GramSplitter.Gram> it = linguistics.getGramSplitter().split(input.getString(), gramSize); it.hasNext();) {
+            GramSplitter.Gram gram = it.next();
+            // if there is a gap before this gram, then annotate the gram as punctuation
+            // (technically it may be of various types, but it does not matter - we just
+            // need to annotate it somehow (as a non-term) to make sure it is added to the summary)
+            if (lastPosition < gram.getStart()) {
+                typedSpan(lastPosition, gram.getStart() - lastPosition, TokenType.PUNCTUATION, spanList);
+            }
+
+            // annotate gram as a word term
+            String gramString = gram.extractFrom(input.getString());
+            typedSpan(gram.getStart(), gram.getLength(), TokenType.ALPHABETIC, spanList).
+                    annotate(LinguisticsAnnotator.lowerCaseTermAnnotation(gramString, gramString));
+
+            lastPosition = gram.getStart() + gram.getLength();
+        }
+        // handle punctuation at the end
+        if (lastPosition < input.toString().length()) {
+            typedSpan(lastPosition, input.toString().length() - lastPosition, TokenType.PUNCTUATION, spanList);
+        }
+    }
+
+    private Span typedSpan(int from, int length, TokenType tokenType, SpanList spanList) {
+        return (Span)spanList.span(from, length).annotate(AnnotationTypes.TOKEN_TYPE, tokenType.getValue());
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        // empty
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        return "ngram " + gramSize;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof NGramExpression)) return false;
+
+        NGramExpression rhs = (NGramExpression)obj;
+        if (linguistics != rhs.linguistics) return false;
+        if (gramSize != rhs.gramSize) return false;
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + gramSize;
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java
new file mode 100644
index 00000000000..ebdd99b2cfa
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.process.Transformer;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class NormalizeExpression extends Expression {
+
+    private final Linguistics linguistics;
+
+    public NormalizeExpression(Linguistics linguistics) {
+        this.linguistics = linguistics;
+    }
+
+    public Linguistics getLinguistics() {
+        return linguistics;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext context) {
+        Transformer transformer = linguistics.getTransformer();
+        context.setValue(new StringFieldValue(transformer.accentDrop(String.valueOf(context.getValue()),
+                                                                     context.resolveLanguage(linguistics))));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext context) {
+        context.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public String toString() {
+        return "normalize";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof NormalizeExpression)) {
+            return false;
+        }
+        NormalizeExpression rhs = (NormalizeExpression)obj;
+        if (linguistics != rhs.linguistics) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NowExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NowExpression.java
new file mode 100644
index 00000000000..75fb355580c
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NowExpression.java
@@ -0,0 +1,83 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.LongFieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class NowExpression extends Expression {
+
+    private final Timer timer;
+
+    public NowExpression() {
+        this(SystemTimer.INSTANCE);
+    }
+
+    public NowExpression(Timer timer) {
+        this.timer = timer;
+    }
+
+    public Timer getTimer() {
+        return timer;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setValue(new LongFieldValue(timer.currentTimeSeconds()));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return null;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.LONG;
+    }
+
+    @Override
+    public String toString() {
+        return "now";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof NowExpression)) {
+            return false;
+        }
+        NowExpression rhs = (NowExpression)obj;
+        if (timer != rhs.timer) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + timer.hashCode();
+    }
+
+    public static interface Timer {
+
+        public long currentTimeSeconds();
+    }
+
+    private static class SystemTimer implements Timer {
+
+        static final Timer INSTANCE = new SystemTimer();
+
+        @Override
+        public long currentTimeSeconds() {
+            return System.currentTimeMillis() / 1000;
+        }
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java
new file mode 100644
index 00000000000..838fb0b3c3b
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.predicate.Predicate;
+import com.yahoo.search.predicate.optimization.AndOrSimplifier;
+import com.yahoo.search.predicate.optimization.BooleanSimplifier;
+import com.yahoo.search.predicate.optimization.ComplexNodeTransformer;
+import com.yahoo.search.predicate.optimization.NotNodeReorderer;
+import com.yahoo.search.predicate.optimization.PredicateOptions;
+import com.yahoo.search.predicate.optimization.PredicateProcessor;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class OptimizePredicateExpression extends Expression {
+
+    private final PredicateProcessor optimizer;
+
+    public OptimizePredicateExpression() {
+        this(new PredicateOptimizer());
+    }
+
+    OptimizePredicateExpression(PredicateProcessor optimizer) {
+        this.optimizer = optimizer;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        PredicateFieldValue predicate = ((PredicateFieldValue)ctx.getValue()).clone();
+        IntegerFieldValue arity = (IntegerFieldValue)ctx.getVariable("arity");
+        LongFieldValue lower_bound = (LongFieldValue)ctx.getVariable("lower_bound");
+        LongFieldValue upper_bound = (LongFieldValue)ctx.getVariable("upper_bound");
+        Long lower = lower_bound != null? lower_bound.getLong() : null;
+        Long upper = upper_bound != null? upper_bound.getLong() : null;
+        PredicateOptions options = new PredicateOptions(arity.getInteger(), lower, upper);
+        predicate.setPredicate(optimizer.process(predicate.getPredicate(), options));
+        ctx.setValue(predicate);
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        checkVariable(ctx, "arity", DataType.INT, true);
+        checkVariable(ctx, "lower_bound", DataType.LONG, false);
+        checkVariable(ctx, "upper_bound", DataType.LONG, false);
+        ctx.setValue(DataType.PREDICATE);
+    }
+
+    private void checkVariable(VerificationContext ctx, String var, DataType type, boolean required) {
+        DataType input = ctx.getVariable(var);
+        if (input == null) {
+            if (required) {
+                throw new VerificationException(this, "Variable '" + var + "' must be set.");
+            }
+        } else if (input != type) {
+            throw new VerificationException(this, "Variable '" + var + "' must have type " + type.getName() + ".");
+        }
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.PREDICATE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        return "optimize_predicate";
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof OptimizePredicateExpression;
+    }
+
+    private static class PredicateOptimizer implements PredicateProcessor {
+
+        private final ComplexNodeTransformer complexNodeTransformer = new ComplexNodeTransformer();
+        private final BooleanSimplifier booleanSimplifier = new BooleanSimplifier();
+        private final AndOrSimplifier andOrSimplifier = new AndOrSimplifier();
+        private final NotNodeReorderer notNodeReorderer = new NotNodeReorderer();
+
+        @Override
+        public Predicate process(Predicate predicate, PredicateOptions options) {
+            Predicate processedPredicate = complexNodeTransformer.process(predicate, options);
+            processedPredicate = booleanSimplifier.process(processedPredicate, options);
+            processedPredicate = andOrSimplifier.process(processedPredicate, options);
+            return notNodeReorderer.process(processedPredicate, options);
+        }
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java
new file mode 100644
index 00000000000..063ec3a8955
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class OutputExpression extends Expression {
+
+    private final String image;
+    private final String fieldName;
+
+    public OutputExpression(String image, String fieldName) {
+        this.image = image;
+        this.fieldName = fieldName;
+    }
+
+    public String getFieldName() {
+        return fieldName;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setOutputValue(this, fieldName, ctx.getValue());
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.tryOutputType(this, fieldName, ctx.getValue());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        return image + (fieldName != null ? " " + fieldName : "");
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof OutputExpression)) {
+            return false;
+        }
+        OutputExpression rhs = (OutputExpression)obj;
+        if (!equals(fieldName, rhs.fieldName)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + (fieldName != null ? fieldName.hashCode() : 0);
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java
new file mode 100644
index 00000000000..57d833a0748
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.vespa.objects.ObjectOperation;
+import com.yahoo.vespa.objects.ObjectPredicate;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ParenthesisExpression extends CompositeExpression {
+
+    private final Expression innerExp;
+
+    public ParenthesisExpression(Expression innerExp) {
+        this.innerExp = innerExp;
+    }
+
+    public Expression getInnerExpression() {
+        return innerExp;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        innerExp.execute(ctx);
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        innerExp.verify(ctx);
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return innerExp.requiredInputType();
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return innerExp.createdOutputType();
+    }
+
+    @Override
+    public String toString() {
+        return "(" + innerExp + ")";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof ParenthesisExpression)) {
+            return false;
+        }
+        ParenthesisExpression rhs = (ParenthesisExpression)obj;
+        if (!innerExp.equals(rhs.innerExp)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + innerExp.hashCode();
+    }
+
+    @Override
+    public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) {
+        select(innerExp, predicate, operation);
+    }
+
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/PassthroughExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/PassthroughExpression.java
new file mode 100644
index 00000000000..427c777db5a
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/PassthroughExpression.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+/**
+ * An output expression which has no search side effects.
+ *
+ * @author steinar
+ */
+public class PassthroughExpression extends OutputExpression {
+
+    private static final String PASSTHROUGH = "passthrough";
+
+    public PassthroughExpression(String fieldName) {
+        super(PASSTHROUGH, fieldName);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return super.equals(obj) && obj instanceof PassthroughExpression;
+    }
+}
\ No newline at end of file
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/RandomExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/RandomExpression.java
new file mode 100644
index 00000000000..8a5cfc1bb88
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/RandomExpression.java
@@ -0,0 +1,76 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class RandomExpression extends Expression {
+
+    private final Integer max;
+
+    public RandomExpression() {
+        this(null);
+    }
+
+    public RandomExpression(Integer max) {
+        this.max = max;
+    }
+
+    public Integer getMaxValue() {
+        return max;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        int max;
+        if (this.max != null) {
+            max = this.max;
+        } else {
+            max = Integer.parseInt(String.valueOf(ctx.getValue()));
+        }
+        ctx.setValue(new IntegerFieldValue(ThreadLocalRandom.current().nextInt(max)));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return null;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.INT;
+    }
+
+    @Override
+    public String toString() {
+        return "random" + (max != null ? " " + max : "");
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof RandomExpression)) {
+            return false;
+        }
+        RandomExpression rhs = (RandomExpression)obj;
+        if (!equals(max, rhs.max)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java
new file mode 100644
index 00000000000..79ac7590cac
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.ScriptParser;
+import com.yahoo.vespa.indexinglanguage.ScriptParserContext;
+import com.yahoo.vespa.indexinglanguage.parser.IndexingInput;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ScriptExpression extends ExpressionList<StatementExpression> {
+
+    public ScriptExpression() {
+        super();
+    }
+
+    public ScriptExpression(StatementExpression... lst) {
+        super(Arrays.asList(lst));
+    }
+
+    public ScriptExpression(Collection<? extends StatementExpression> lst) {
+        super(lst);
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        FieldValue input = ctx.getValue();
+        for (Expression exp : this) {
+            ctx.setValue(input).execute(exp);
+        }
+        ctx.setValue(input);
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        DataType input = ctx.getValue();
+        for (Expression exp : this) {
+            ctx.setValue(input).execute(exp);
+        }
+        ctx.setValue(input);
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        DataType prev = null;
+        for (Expression exp : this) {
+            DataType next = exp.requiredInputType();
+            if (prev == null) {
+                prev = next;
+            } else if (next != null && !prev.isAssignableFrom(next)) {
+                throw new VerificationException(this, "Statements require conflicting input types, " +
+                                                      prev.getName() + " vs " + next.getName() + ".");
+            }
+        }
+        return prev;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder ret = new StringBuilder();
+        ret.append("{ ");
+        for (Iterator<StatementExpression> it = iterator(); it.hasNext();) {
+            ret.append(it.next()).append(";");
+            if (it.hasNext()) {
+                ret.append(" ");
+            }
+        }
+        ret.append(" }");
+        return ret.toString();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return super.equals(obj) && obj instanceof ScriptExpression;
+    }
+
+    /** Creates an expression with simple lingustics for testing */
+    public static ScriptExpression fromString(String expression) throws ParseException {
+        return fromString(expression, new SimpleLinguistics());
+    }
+
+    public static ScriptExpression fromString(String expression, Linguistics linguistics) throws ParseException {
+        return newInstance(new ScriptParserContext(linguistics).setInputStream(new IndexingInput(expression)));
+    }
+
+    public static ScriptExpression newInstance(ScriptParserContext config) throws ParseException {
+        return ScriptParser.parseScript(config);
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java
new file mode 100644
index 00000000000..8663dcf7a9d
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.vespa.objects.ObjectOperation;
+import com.yahoo.vespa.objects.ObjectPredicate;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SelectInputExpression extends CompositeExpression {
+
+    private final List<Pair<String, Expression>> cases;
+
+    @SafeVarargs
+    public SelectInputExpression(Pair<String, Expression>... cases) {
+        this(Arrays.asList(cases));
+    }
+
+    public SelectInputExpression(List<Pair<String, Expression>> cases) {
+        this.cases = cases;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        FieldValue input = ctx.getValue();
+        for (Pair<String, Expression> entry : cases) {
+            FieldValue val = ctx.getInputValue(entry.getFirst());
+            if (val != null) {
+                ctx.setValue(val).execute(entry.getSecond());
+                break;
+            }
+        }
+        ctx.setValue(input);
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        DataType input = ctx.getValue();
+        for (Pair<String, Expression> entry : cases) {
+            DataType val = ctx.getInputType(this, entry.getFirst());
+            if (val == null) {
+                throw new VerificationException(this, "Field '" + entry.getFirst() + "' not found.");
+            }
+            ctx.setValue(val).execute(entry.getSecond());
+        }
+        ctx.setValue(input);
+    }
+
+    @Override
+    public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) {
+        for (Pair<String, Expression> entry : cases) {
+            select(entry.getSecond(), predicate, operation);
+        }
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return null;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return null;
+    }
+
+    public List<Pair<String, Expression>> getCases() {
+        return Collections.unmodifiableList(cases);
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder ret = new StringBuilder();
+        ret.append("select_input { ");
+        for (Pair<String, Expression> entry : cases) {
+            ret.append(entry.getFirst()).append(": ");
+            Expression exp = entry.getSecond();
+            ret.append(exp).append("; ");
+        }
+        ret.append("}");
+        return ret.toString();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof SelectInputExpression)) {
+            return false;
+        }
+        SelectInputExpression rhs = (SelectInputExpression)obj;
+        if (!cases.equals(rhs.cases)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + cases.hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageExpression.java
new file mode 100644
index 00000000000..35f02ccb05b
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageExpression.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.language.Language;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SetLanguageExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setLanguage(Language.fromLanguageTag(String.valueOf(ctx.getValue())));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        // empty
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        return "set_language";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof SetLanguageExpression)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueExpression.java
new file mode 100644
index 00000000000..edc1fcfa9ee
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueExpression.java
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.text.StringUtilities;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SetValueExpression extends Expression {
+
+    private final FieldValue value;
+
+    public SetValueExpression(FieldValue value) {
+        value.getClass(); // throws NullPointerException
+        this.value = value;
+    }
+
+    public FieldValue getValue() {
+        return value;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setValue(value);
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(value.getDataType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return null;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return value.getDataType();
+    }
+
+    @Override
+    public String toString() {
+        if (value instanceof StringFieldValue) {
+            return "\"" + StringUtilities.escape(value.toString(), '"') + "\"";
+        }
+        if (value instanceof LongFieldValue) {
+            return value.toString() + "L";
+        }
+        return value.toString();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof SetValueExpression)) {
+            return false;
+        }
+        SetValueExpression rhs = (SetValueExpression)obj;
+        if (!value.equals(rhs.value)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + value.hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java
new file mode 100644
index 00000000000..d92831fd6b3
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SetVarExpression extends Expression {
+
+    private final String varName;
+
+    public SetVarExpression(String varName) {
+        this.varName = varName;
+    }
+
+    public String getVariableName() {
+        return varName;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setVariable(varName, ctx.getValue());
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        DataType next = ctx.getValue();
+        DataType prev = ctx.getVariable(varName);
+        if (prev != null && !prev.equals(next)) {
+            throw new VerificationException(this, "Attempting to assign conflicting types to variable '" + varName +
+                                                  "', " + prev.getName() + " vs " + next.getName() + ".");
+        }
+        ctx.setVariable(varName, next);
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        return "set_var " + varName;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof SetVarExpression)) {
+            return false;
+        }
+        SetVarExpression rhs = (SetVarExpression)obj;
+        if (!varName.equals(rhs.varName)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + varName.hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SplitExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SplitExpression.java
new file mode 100644
index 00000000000..c2cfcd0cd0e
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SplitExpression.java
@@ -0,0 +1,76 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.text.StringUtilities;
+
+import java.util.regex.Pattern;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SplitExpression extends Expression {
+
+    private final Pattern splitPattern;
+
+    public SplitExpression(String splitString) {
+        this.splitPattern = Pattern.compile(splitString);
+    }
+
+    public Pattern getSplitPattern() {
+        return splitPattern;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        String input = String.valueOf(ctx.getValue());
+        Array<StringFieldValue> output = new Array<>(DataType.getArray(DataType.STRING));
+        if (!input.isEmpty()) {
+            String[] splits = splitPattern.split(input);
+            for (String split : splits) {
+                output.add(new StringFieldValue(split));
+            }
+        }
+        ctx.setValue(output);
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.getArray(DataType.STRING);
+    }
+
+    @Override
+    public String toString() {
+        return "split \"" + StringUtilities.escape(splitPattern.toString(), '"') + "\"";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof SplitExpression)) {
+            return false;
+        }
+        SplitExpression rhs = (SplitExpression)obj;
+        if (!splitPattern.toString().equals(rhs.splitPattern.toString())) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + splitPattern.toString().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java
new file mode 100644
index 00000000000..012ca87ed37
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java
@@ -0,0 +1,111 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.ScriptParser;
+import com.yahoo.vespa.indexinglanguage.ScriptParserContext;
+import com.yahoo.vespa.indexinglanguage.parser.IndexingInput;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class StatementExpression extends ExpressionList<Expression> {
+
+    public StatementExpression(Expression... lst) {
+        this(Arrays.asList(lst));
+    }
+
+    public StatementExpression(Iterable<Expression> lst) {
+        super(filterList(lst));
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        for (Expression exp : this) {
+            ctx.execute(exp);
+        }
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        for (Expression exp : this) {
+            ctx.execute(exp);
+        }
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        for (Expression exp : this) {
+            DataType type = exp.requiredInputType();
+            if (type != null) {
+                return type;
+            }
+            type = exp.createdOutputType();
+            if (type != null) {
+                return null;
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        for (int i = size(); --i >= 0; ) {
+            DataType type = get(i).createdOutputType();
+            if (type != null) {
+                return type;
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder ret = new StringBuilder();
+        for (Iterator<Expression> it = iterator(); it.hasNext();) {
+            ret.append(it.next());
+            if (it.hasNext()) {
+                ret.append(" | ");
+            }
+        }
+        return ret.toString();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return super.equals(obj) && obj instanceof StatementExpression;
+    }
+
+    /** Creates an expression with simple lingustics for testing */
+    public static StatementExpression fromString(String expression) throws ParseException {
+        return fromString(expression, new SimpleLinguistics());
+    }
+
+    public static StatementExpression fromString(String expression, Linguistics linguistics) throws ParseException {
+        return newInstance(new ScriptParserContext(linguistics).setInputStream(new IndexingInput(expression)));
+    }
+
+    public static StatementExpression newInstance(ScriptParserContext config) throws ParseException {
+        return ScriptParser.parseStatement(config);
+    }
+
+    private static List<Expression> filterList(Iterable<Expression> lst) {
+        List<Expression> ret = new LinkedList<>();
+        for (Expression exp : lst) {
+            if (exp instanceof StatementExpression) {
+                ret.addAll(filterList((StatementExpression)exp));
+            } else if (exp != null) {
+                ret.add(exp);
+            }
+        }
+        return ret;
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringExpression.java
new file mode 100644
index 00000000000..bb4351bee77
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringExpression.java
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.StringFieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SubstringExpression extends Expression {
+
+    private final int from;
+    private final int to;
+
+    public SubstringExpression(int from, int to) {
+        if (from < 0 || to < 0 || to < from) {
+            throw new IndexOutOfBoundsException();
+        }
+        this.from = from;
+        this.to = to;
+    }
+
+    public int getFrom() {
+        return from;
+    }
+
+    public int getTo() {
+        return to;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        String input = String.valueOf(ctx.getValue());
+        int len = input.length();
+        if (from >= len) {
+            input = "";
+        } else if (to >= len) {
+            input = input.substring(from);
+        } else {
+            input = input.substring(from, to);
+        }
+        ctx.setValue(new StringFieldValue(input));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public String toString() {
+        return "substring " + from + " " + to;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof SubstringExpression)) {
+            return false;
+        }
+        SubstringExpression rhs = (SubstringExpression)obj;
+        if (from != rhs.from) {
+            return false;
+        }
+        if (to != rhs.to) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() +
+               Integer.valueOf(from).hashCode() +
+               Integer.valueOf(to).hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SummaryExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SummaryExpression.java
new file mode 100644
index 00000000000..20c8173ffe2
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SummaryExpression.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SummaryExpression extends OutputExpression {
+
+    public SummaryExpression(String fieldName) {
+        super("summary", fieldName);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return super.equals(obj) && obj instanceof SummaryExpression;
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java
new file mode 100644
index 00000000000..21f633c3aba
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java
@@ -0,0 +1,137 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.text.StringUtilities;
+import com.yahoo.vespa.objects.ObjectOperation;
+import com.yahoo.vespa.objects.ObjectPredicate;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SwitchExpression extends CompositeExpression {
+
+    private final Map<String, Expression> cases = new LinkedHashMap<>();
+    private final Expression defaultExp;
+
+    public <T extends Expression> SwitchExpression(Map<String, T> cases) {
+        this(cases, null);
+    }
+
+    public <T extends Expression> SwitchExpression(Map<String, T> cases, Expression defaultExp) {
+        this.defaultExp = defaultExp;
+        for (Map.Entry<String, T> entry : cases.entrySet()) {
+            this.cases.put(entry.getKey(), entry.getValue());
+        }
+    }
+
+    public boolean isEmpty() {
+        return defaultExp == null && cases.isEmpty();
+    }
+
+    public Map<String, Expression> getCases() {
+        return Collections.unmodifiableMap(cases);
+    }
+
+    public Expression getDefaultExpression() {
+        return defaultExp;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        FieldValue input = ctx.getValue();
+        Expression exp = null;
+        if (input != null) {
+            if (!(input instanceof StringFieldValue)) {
+                throw new IllegalArgumentException("Expected " + DataType.STRING.getName() + " input, got " +
+                                                   input.getDataType().getName() + ".");
+            }
+            exp = cases.get(String.valueOf(input));
+        }
+        if (exp == null) {
+            exp = defaultExp;
+        }
+        if (exp != null) {
+            exp.execute(ctx);
+        }
+        ctx.setValue(input);
+    }
+
+    @Override
+    public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) {
+        select(defaultExp, predicate, operation);
+        for (Expression exp : cases.values()) {
+            select(exp, predicate, operation);
+        }
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        DataType input = ctx.getValue();
+        if (input == null) {
+            throw new VerificationException(this, "Expected " + DataType.STRING.getName() + " input, got null.");
+        }
+        if (input != DataType.STRING) {
+            throw new VerificationException(this, "Expected " + DataType.STRING.getName() + " input, got " +
+                                                  input.getName() + ".");
+        }
+        for (Expression exp : cases.values()) {
+            ctx.setValue(input).execute(exp);
+        }
+        ctx.setValue(input).execute(defaultExp);
+        ctx.setValue(input);
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return null;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder ret = new StringBuilder();
+        ret.append("switch { ");
+        for (Map.Entry<String, Expression> entry : cases.entrySet()) {
+            ret.append("case \"").append(StringUtilities.escape(entry.getKey(), '"')).append("\": ");
+            Expression exp = entry.getValue();
+            ret.append(exp).append("; ");
+        }
+        if (defaultExp != null) {
+            ret.append("default: ").append(defaultExp).append("; ");
+        }
+        ret.append("}");
+        return ret.toString();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof SwitchExpression)) {
+            return false;
+        }
+        SwitchExpression rhs = (SwitchExpression)obj;
+        if (!cases.equals(rhs.cases)) {
+            return false;
+        }
+        if (!equals(defaultExp, rhs.defaultExp)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + (defaultExp != null ? defaultExp.hashCode() : 0);
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ThisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ThisExpression.java
new file mode 100644
index 00000000000..8ad37b98f4c
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ThisExpression.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ThisExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        // empty
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        // empty
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public String toString() {
+        return "this";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof ThisExpression;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayExpression.java
new file mode 100644
index 00000000000..197ed431955
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayExpression.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToArrayExpression extends Expression {
+
+    @SuppressWarnings({ "unchecked" })
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        FieldValue input = ctx.getValue();
+        DataType inputType = input.getDataType();
+
+        ArrayDataType outputType = DataType.getArray(inputType);
+        Array output = outputType.createFieldValue();
+        output.add(input);
+
+        ctx.setValue(output);
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(DataType.getArray(ctx.getValue()));
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public String toString() {
+        return "to_array";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof ToArrayExpression;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteExpression.java
new file mode 100644
index 00000000000..cd01f8251b0
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteExpression.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.ByteFieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToByteExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setValue(new ByteFieldValue(Byte.valueOf(String.valueOf(ctx.getValue()))));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.BYTE;
+    }
+
+    @Override
+    public String toString() {
+        return "to_byte";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof ToByteExpression;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleExpression.java
new file mode 100644
index 00000000000..fb54aefe696
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleExpression.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToDoubleExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setValue(new DoubleFieldValue(Double.valueOf(String.valueOf(ctx.getValue()))));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.DOUBLE;
+    }
+
+    @Override
+    public String toString() {
+        return "to_double";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof ToDoubleExpression;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatExpression.java
new file mode 100644
index 00000000000..ebd866abfa9
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatExpression.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.FloatFieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToFloatExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setValue(new FloatFieldValue(Float.valueOf(String.valueOf(ctx.getValue()))));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.FLOAT;
+    }
+
+    @Override
+    public String toString() {
+        return "to_float";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof ToFloatExpression;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerExpression.java
new file mode 100644
index 00000000000..0c900c7756a
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerExpression.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToIntegerExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setValue(new IntegerFieldValue(Integer.valueOf(String.valueOf(ctx.getValue()))));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.INT;
+    }
+
+    @Override
+    public String toString() {
+        return "to_int";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof ToIntegerExpression;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongExpression.java
new file mode 100644
index 00000000000..63b8b446437
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongExpression.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.LongFieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToLongExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setValue(new LongFieldValue(Long.valueOf(String.valueOf(ctx.getValue()))));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.LONG;
+    }
+
+    @Override
+    public String toString() {
+        return "to_long";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof ToLongExpression;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionExpression.java
new file mode 100644
index 00000000000..2b89e05b7b2
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionExpression.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.PositionDataType;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToPositionExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setValue(PositionDataType.fromString(String.valueOf(ctx.getValue())));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return PositionDataType.INSTANCE;
+    }
+
+    @Override
+    public String toString() {
+        return "to_pos";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof ToPositionExpression;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringExpression.java
new file mode 100644
index 00000000000..0db289a6b6b
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringExpression.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToStringExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setValue(new StringFieldValue(String.valueOf(ctx.getValue())));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public String toString() {
+        return "to_string";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof ToStringExpression;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetExpression.java
new file mode 100644
index 00000000000..f15bac00031
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetExpression.java
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToWsetExpression extends Expression {
+
+    private final Boolean createIfNonExistent;
+    private final Boolean removeIfZero;
+
+    public ToWsetExpression(boolean createIfNonExistent, boolean removeIfZero) {
+        this.createIfNonExistent = createIfNonExistent;
+        this.removeIfZero = removeIfZero;
+    }
+
+    public boolean getCreateIfNonExistent() {
+        return createIfNonExistent;
+    }
+
+    public boolean getRemoveIfZero() {
+        return removeIfZero;
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        FieldValue input = ctx.getValue();
+        DataType inputType = input.getDataType();
+
+        WeightedSetDataType outputType = DataType.getWeightedSet(inputType, createIfNonExistent, removeIfZero);
+        WeightedSet output = outputType.createFieldValue();
+        output.add(input);
+
+        ctx.setValue(output);
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(DataType.getWeightedSet(ctx.getValue(), createIfNonExistent, removeIfZero));
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public String toString() {
+        return "to_wset" +
+               (createIfNonExistent ? " create_if_non_existent" : "") +
+               (removeIfZero ? " remove_if_zero" : "");
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof ToWsetExpression)) {
+            return false;
+        }
+        ToWsetExpression rhs = (ToWsetExpression)obj;
+        if (createIfNonExistent != rhs.createIfNonExistent) {
+            return false;
+        }
+        if (removeIfZero != rhs.removeIfZero) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() +
+               Boolean.valueOf(createIfNonExistent).hashCode() +
+               Boolean.valueOf(removeIfZero).hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java
new file mode 100644
index 00000000000..272fbde342c
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java
@@ -0,0 +1,92 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.language.Language;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.process.StemMode;
+import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
+import com.yahoo.vespa.indexinglanguage.linguistics.LinguisticsAnnotator;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class TokenizeExpression extends Expression {
+
+    private final Linguistics linguistics;
+    private final AnnotatorConfig config;
+
+    public TokenizeExpression(Linguistics linguistics, AnnotatorConfig config) {
+        this.linguistics = linguistics;
+        this.config = config;
+    }
+
+    public Linguistics getLinguistics() {
+        return linguistics;
+    }
+
+    public AnnotatorConfig getConfig() {
+        return config;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext context) {
+        StringFieldValue output = ((StringFieldValue)context.getValue()).clone();
+        context.setValue(output);
+
+        AnnotatorConfig cfg = new AnnotatorConfig(config);
+        Language lang = context.resolveLanguage(linguistics);
+        if (lang != null) {
+            cfg.setLanguage(lang);
+        }
+        LinguisticsAnnotator annotator = new LinguisticsAnnotator(linguistics, cfg);
+        annotator.annotate(output);
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        // empty
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder ret = new StringBuilder();
+        ret.append("tokenize");
+        if (config.getRemoveAccents()) {
+            ret.append(" normalize");
+        }
+        if (config.getStemMode() != StemMode.NONE) {
+            ret.append(" stem:\""+config.getStemMode()+"\"");
+        }
+        return ret.toString();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof TokenizeExpression)) {
+            return false;
+        }
+        TokenizeExpression rhs = (TokenizeExpression)obj;
+        if (!config.equals(rhs.config)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() + config.hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java
new file mode 100644
index 00000000000..d544b9218e4
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.StringFieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class TrimExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        ctx.setValue(new StringFieldValue(String.valueOf(ctx.getValue()).trim()));
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.STRING;
+    }
+
+    @Override
+    public String toString() {
+        return "trim";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof TrimExpression)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedDataType.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedDataType.java
new file mode 100644
index 00000000000..ae0649e5f87
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedDataType.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.PrimitiveDataType;
+import com.yahoo.document.datatypes.FieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+final class UnresolvedDataType extends PrimitiveDataType {
+
+    public static final UnresolvedDataType INSTANCE = new UnresolvedDataType();
+
+    private UnresolvedDataType() {
+        super("any", -69, UnresolvedFieldValue.class, UnresolvedFieldValue.getFactory());
+    }
+
+    @Override
+    public boolean isValueCompatible(FieldValue value) {
+        return value != null;
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValue.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValue.java
new file mode 100644
index 00000000000..e4216b2b5c7
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValue.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.PrimitiveDataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlStream;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class UnresolvedFieldValue extends FieldValue {
+    private static class Factory extends PrimitiveDataType.Factory {
+        public FieldValue create() {
+            return new UnresolvedFieldValue();
+        }
+    }
+    public static PrimitiveDataType.Factory getFactory() { return new Factory(); }
+    @Override
+    public DataType getDataType() {
+        return UnresolvedDataType.INSTANCE;
+    }
+
+    @Override
+    public void printXml(XmlStream xml) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void clear() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void assign(Object o) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void serialize(Field field, FieldWriter writer) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void deserialize(Field field, FieldReader reader) {
+        throw new UnsupportedOperationException();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContext.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContext.java
new file mode 100644
index 00000000000..fc6415419a6
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContext.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class VerificationContext implements FieldTypeAdapter, Cloneable {
+
+    private final Map<String, DataType> variables = new HashMap<String, DataType>();
+    private final FieldTypeAdapter adapter;
+    private DataType value;
+
+    public VerificationContext() {
+        this.adapter = null;
+    }
+
+    public VerificationContext(FieldTypeAdapter adapter) {
+        this.adapter = adapter;
+    }
+
+    public VerificationContext execute(Expression exp) {
+        if (exp != null) {
+            exp.verify(this);
+        }
+        return this;
+    }
+
+    @Override
+    public DataType getInputType(Expression exp, String fieldName) {
+        return adapter.getInputType(exp, fieldName);
+    }
+
+    @Override
+    public void tryOutputType(Expression exp, String fieldName, DataType valueType) {
+        adapter.tryOutputType(exp, fieldName, valueType);
+    }
+
+    public DataType getVariable(String name) {
+        return variables.get(name);
+    }
+
+    public VerificationContext setVariable(String name, DataType value) {
+        variables.put(name, value);
+        return this;
+    }
+
+    public DataType getValue() {
+        return value;
+    }
+
+    public VerificationContext setValue(DataType value) {
+        this.value = value;
+        return this;
+    }
+
+    public VerificationContext clear() {
+        variables.clear();
+        value = null;
+        return this;
+    }
+}
+
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationException.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationException.java
new file mode 100644
index 00000000000..c46fe38520c
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationException.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class VerificationException extends RuntimeException {
+
+    private final Expression exp;
+
+    public VerificationException(Expression exp, String msg) {
+        super(msg);
+        this.exp = exp;
+    }
+
+    public Expression getExpression() {
+        return exp;
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getName() + ": For expression '" + exp + "': " + getMessage();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveExpression.java
new file mode 100644
index 00000000000..cdf795ea7d5
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveExpression.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.geo.ZCurve;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ZCurveExpression extends Expression {
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        Struct input = ((Struct)ctx.getValue());
+        Integer x = getFieldValue(input, PositionDataType.FIELD_X);
+        Integer y = getFieldValue(input, PositionDataType.FIELD_Y);
+        if (x != null && y != null) {
+            ctx.setValue(new LongFieldValue(ZCurve.encode(x, y)));
+        } else {
+            ctx.setValue(null);
+        }
+    }
+
+    private static Integer getFieldValue(Struct struct, String fieldName) {
+        IntegerFieldValue val = (IntegerFieldValue)struct.getFieldValue(fieldName);
+        return val != null ? val.getInteger() : null;
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        ctx.setValue(createdOutputType());
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return PositionDataType.INSTANCE;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return DataType.LONG;
+    }
+
+    @Override
+    public String toString() {
+        return "zcurve";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof ZCurveExpression)) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode();
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/package-info.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/package-info.java
new file mode 100644
index 00000000000..790a58ce651
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java
new file mode 100644
index 00000000000..547ba687490
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java
@@ -0,0 +1,106 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.linguistics;

+

+import com.yahoo.language.Language;

+import com.yahoo.language.process.StemMode;

+import com.yahoo.vespa.configdefinition.IlscriptsConfig;

+

+/**

+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>

+ */

+public class AnnotatorConfig implements Cloneable {

+

+    private Language language;

+    private StemMode stemMode;

+    private boolean removeAccents;

+    private int maxTermOccurences;

+

+    public static final int DEFAULT_MAX_TERM_OCCURRENCES;

+

+    static {

+        IlscriptsConfig defaults = new IlscriptsConfig(new IlscriptsConfig.Builder());

+        DEFAULT_MAX_TERM_OCCURRENCES = defaults.maxtermoccurrences();

+    }

+

+    public AnnotatorConfig() {

+        language = Language.ENGLISH;

+        stemMode = StemMode.NONE;

+        removeAccents = false;

+        maxTermOccurences = DEFAULT_MAX_TERM_OCCURRENCES;

+    }

+

+    public AnnotatorConfig(AnnotatorConfig rhs) {

+        language = rhs.language;

+        stemMode = rhs.stemMode;

+        removeAccents = rhs.removeAccents;

+        maxTermOccurences = rhs.maxTermOccurences;

+    }

+

+    public Language getLanguage() {

+        return language;

+    }

+

+    public AnnotatorConfig setLanguage(Language language) {

+        this.language = language;

+        return this;

+    }

+

+    public StemMode getStemMode() {

+        return stemMode;

+    }

+

+    public AnnotatorConfig setStemMode(StemMode stemMode) {

+        this.stemMode = stemMode;

+        return this;

+    }

+

+    public AnnotatorConfig setStemMode(String name) {

+        this.stemMode = StemMode.valueOf(name);

+        return this;

+    }

+

+    public boolean getRemoveAccents() {

+        return removeAccents;

+    }

+

+    public AnnotatorConfig setRemoveAccents(boolean removeAccents) {

+        this.removeAccents = removeAccents;

+        return this;

+    }

+

+    public int getMaxTermOccurrences() {

+        return maxTermOccurences;

+    }

+

+    public AnnotatorConfig setMaxTermOccurrences(int maxTermCount) {

+        this.maxTermOccurences = maxTermCount;

+        return this;

+    }

+

+    @Override

+    public boolean equals(Object obj) {

+        if (!(obj instanceof AnnotatorConfig)) {

+            return false;

+        }

+        AnnotatorConfig rhs = (AnnotatorConfig)obj;

+        if (!language.equals(rhs.language)) {

+            return false;

+        }

+        if (!stemMode.equals(rhs.stemMode)) {

+            return false;

+        }

+        if (removeAccents != rhs.removeAccents) {

+            return false;

+        }

+        if (maxTermOccurences != rhs.maxTermOccurences) {

+            return false;

+        }

+        return true;

+    }

+

+    @Override

+    public int hashCode() {

+        return getClass().hashCode() + language.hashCode() + stemMode.hashCode() +

+               Boolean.valueOf(removeAccents).hashCode() + maxTermOccurences;

+    }

+}

diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java
new file mode 100644
index 00000000000..70d6168ec22
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java
@@ -0,0 +1,164 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.linguistics;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.yahoo.document.annotation.*;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.process.StemMode;
+import com.yahoo.language.process.Token;
+import com.yahoo.language.process.Tokenizer;
+
+import static com.yahoo.language.LinguisticsCase.toLowerCase;
+
+/**
+ * This is a tool for adding {@link AnnotationTypes} type annotations to {@link StringFieldValue} objects.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LinguisticsAnnotator {
+
+    private final Linguistics factory;
+    private final AnnotatorConfig config;
+
+    private static class TermOccurrences {
+        final Map<String, Integer> termOccurrences = new HashMap<>();
+        final int maxOccurrences;
+
+        public TermOccurrences(int maxOccurences) {
+            this.maxOccurrences = maxOccurences;
+        }
+
+        boolean termCountBelowLimit(String term) {
+            String lowerCasedTerm = toLowerCase(term);
+            int occurences = termOccurrences.getOrDefault(lowerCasedTerm, 0);
+            if (occurences >= maxOccurrences) {
+                return false;
+            }
+
+            termOccurrences.put(lowerCasedTerm, occurences + 1);
+            return true;
+        }
+    }
+
+    /**
+     * Constructs a new instance of this annotator.
+     *
+     * @param factory the linguistics factory to use when annotating
+     * @param config  the linguistics config to use
+     */
+    public LinguisticsAnnotator(Linguistics factory, AnnotatorConfig config) {
+        this.factory = factory;
+        this.config = config;
+    }
+
+    /**
+     * Annotates the given string with the appropriate linguistics annotations.
+     *
+     * @param text the text to annotate
+     * @return whether or not anything was annotated
+     */
+    public boolean annotate(StringFieldValue text) {
+        if (text.getSpanTree(SpanTrees.LINGUISTICS) != null) return true;  // Already annotated with LINGUISTICS.
+
+        Tokenizer tokenizer = factory.getTokenizer();
+        Iterable<Token> tokens = tokenizer.tokenize(text.getString(), config.getLanguage(), config.getStemMode(),
+                                                    config.getRemoveAccents());
+        TermOccurrences termOccurrences = new TermOccurrences(config.getMaxTermOccurrences());
+        SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS);
+        for (Token token : tokens) {
+            addAnnotationSpan(text.getString(), tree.spanList(), tokenizer, token, config.getStemMode(), termOccurrences);
+        }
+
+        if (tree.numAnnotations() == 0) return false;
+        text.setSpanTree(tree);
+        return true;
+    }
+
+    /**
+     * Creates a TERM annotation which has the lowercase value as annotation (only) if it is different from the
+     * original.
+     *
+     * @param termToLowerCase The term to lower case.
+     * @param origTerm        The original term.
+     * @return the created TERM annotation.
+     */
+    public static Annotation lowerCaseTermAnnotation(String termToLowerCase, String origTerm) {
+        String annotationValue = toLowerCase(termToLowerCase);
+        if (annotationValue.equals(origTerm)) {
+            return new Annotation(AnnotationTypes.TERM);
+        }
+        return new Annotation(AnnotationTypes.TERM, new StringFieldValue(annotationValue));
+    }
+
+    private static void addAnnotation(Span here, String term, String orig, TermOccurrences termOccurrences) {
+        if (termOccurrences.termCountBelowLimit(term)) {
+            here.annotate(lowerCaseTermAnnotation(term, orig));
+        }
+    }
+
+    private static void addAnnotationSpan(String input, SpanList parent, Tokenizer tokenizer, Token token, StemMode mode, TermOccurrences termOccurrences) {
+        if (!token.isSpecialToken()) {
+            if (token.getNumComponents() > 0) {
+                for (int i = 0; i < token.getNumComponents(); ++i) {
+                    addAnnotationSpan(input, parent, tokenizer, token.getComponent(i), mode, termOccurrences);
+                }
+                return;
+            }
+            if (!token.isIndexable()) {
+                return;
+            }
+        }
+        String orig = token.getOrig();
+        int pos = (int)token.getOffset();
+        if (pos >= input.length()) {
+            throw new IllegalArgumentException("Token '" + orig + "' has offset " + pos + ", which is outside the " +
+                                               "bounds of the input string; " + input);
+        }
+        int len = orig.length();
+        if (pos + len > input.length()) {
+            throw new IllegalArgumentException("Token '" + orig + "' has offset " + pos + ", which makes it overflow " +
+                                               "the bounds of the input string; " + input);
+        }
+        if (mode == StemMode.ALL) {
+            Span where = parent.span(pos, len);
+            String lowercasedOrig = toLowerCase(orig);
+            addAnnotation(where, orig, orig, termOccurrences);
+
+            String lowercasedTerm = lowercasedOrig;
+            String term = token.getTokenString();
+            if (term != null) {
+                term = tokenizer.getReplacementTerm(term);
+            }
+            if (term != null) {
+                lowercasedTerm = toLowerCase(term);
+            }
+            if (! lowercasedOrig.equals(lowercasedTerm)) {
+                addAnnotation(where, term, orig, termOccurrences);
+            }
+            for (int i = 0; i < token.getNumStems(); i++) {
+                String stem = token.getStem(i);
+                String lowercasedStem = toLowerCase(stem);
+                if (! (lowercasedOrig.equals(lowercasedStem)
+                       || lowercasedTerm.equals(lowercasedStem)))
+                {
+                    addAnnotation(where, stem, orig, termOccurrences);
+                }
+            }
+        } else {
+            String term = token.getTokenString();
+            if (term != null) {
+                term = tokenizer.getReplacementTerm(term);
+            }
+            if (term == null || term.trim().isEmpty()) {
+                return;
+            }
+            if (termOccurrences.termCountBelowLimit(term))  {
+                parent.span(pos, len).annotate(lowerCaseTermAnnotation(term, token.getOrig()));
+            }
+        }
+    }
+
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/package-info.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/package-info.java
new file mode 100644
index 00000000000..7d8c8103852
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.indexinglanguage.linguistics;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/package-info.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/package-info.java
new file mode 100644
index 00000000000..24261328410
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/IndexingInput.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/IndexingInput.java
new file mode 100644
index 00000000000..de9d26547da
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/IndexingInput.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.parser;

+

+import com.yahoo.javacc.FastCharStream;

+

+/**

+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>

+ */

+public final class IndexingInput extends FastCharStream implements CharStream {

+

+    public IndexingInput(String input) {

+        super(input);

+    }

+}

diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/package-info.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/package-info.java
new file mode 100644
index 00000000000..2c222f66069
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.vespa.indexinglanguage.parser;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/predicate/package-info.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/predicate/package-info.java
new file mode 100644
index 00000000000..77050a64119
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/predicate/package-info.java
@@ -0,0 +1,3 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@com.yahoo.osgi.annotation.ExportPackage
+package com.yahoo.vespa.indexinglanguage.predicate;
diff --git a/indexinglanguage/src/main/javacc/IndexingParser.jj b/indexinglanguage/src/main/javacc/IndexingParser.jj
new file mode 100644
index 00000000000..a7b17d81c83
--- /dev/null
+++ b/indexinglanguage/src/main/javacc/IndexingParser.jj
@@ -0,0 +1,822 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// --------------------------------------------------------------------------------
+//
+// JavaCC options.
+//
+// --------------------------------------------------------------------------------
+options {
+    CACHE_TOKENS = false;
+    DEBUG_PARSER = false;
+    ERROR_REPORTING = true;
+    STATIC = false;
+    USER_CHAR_STREAM = true;
+}
+
+// --------------------------------------------------------------------------------
+//
+// Parser body.
+//
+// --------------------------------------------------------------------------------
+PARSER_BEGIN(IndexingParser)
+
+package com.yahoo.vespa.indexinglanguage.parser;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.List;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.LinkedHashMap;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.text.StringUtilities;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
+import com.yahoo.language.process.StemMode;
+import com.yahoo.language.Linguistics;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ * @version $Id$
+ */
+public class IndexingParser {
+
+    private String defaultFieldName;
+    private Linguistics linguistics;
+    private AnnotatorConfig annotatorCfg;
+
+    public IndexingParser(String str) {
+        this(new IndexingInput(str));
+    }
+
+    public IndexingParser setDefaultFieldName(String fieldName) {
+        defaultFieldName = fieldName;
+        return this;
+    }
+
+    public IndexingParser setLinguistics(Linguistics linguistics) {
+        this.linguistics = linguistics;
+        return this;
+    }
+
+    public IndexingParser setAnnotatorConfig(AnnotatorConfig cfg) {
+        annotatorCfg = cfg;
+        return this;
+    }
+
+    private static FieldValue parseDouble(String str) {
+        return new DoubleFieldValue(new BigDecimal(str).doubleValue());
+    }
+
+    private static FieldValue parseFloat(String str) {
+        if (str.endsWith("f") || str.endsWith("F")) {
+            str = str.substring(0, str.length() - 1);
+        }
+        return new FloatFieldValue(new BigDecimal(str).floatValue());
+    }
+
+    private static FieldValue parseInteger(String str) {
+        if (str.startsWith("0x")) {
+            return new IntegerFieldValue(new BigInteger(str.substring(2), 16).intValue());
+        } else {
+            return new IntegerFieldValue(new BigInteger(str).intValue());
+        }
+    }
+
+    private static FieldValue parseLong(String str) {
+        if (str.endsWith("l") || str.endsWith("L")) {
+            str = str.substring(0, str.length() - 1);
+        }
+        if (str.startsWith("0x")) {
+            return new LongFieldValue(new BigInteger(str.substring(2), 16).longValue());
+        } else {
+            return new LongFieldValue(new BigInteger(str).longValue());
+        }
+    }
+}
+
+PARSER_END(IndexingParser)
+
+SKIP :
+{
+    " " | "\t" | "\r" | "\f"
+}
+
+SPECIAL_TOKEN :
+{
+    <COMMENT: "#" (~["\n","\r"])* >
+}
+
+TOKEN :
+{
+    <INTEGER: (["0"-"9"])+ | ("0" ["x","X"] (["0"-"9","a"-"f","A"-"F"])+)> |
+    <LONG: <INTEGER> ["l","L"]> |
+    <DOUBLE: (["0"-"9"])+ ("." (["0"-"9"])*)? (["e","E"] (["+","-"])? (["0"-"9"])+)?> |
+    <FLOAT: <DOUBLE> ["f", "F"]>
+}
+
+TOKEN :
+{
+    <NL: "\n"> |
+    <ADD: "+"> |
+    <SUB: "-"> |
+    <MUL: "*"> |
+    <DIV: "/"> |
+    <MOD: "%"> |
+    <EQ: "=="> |
+    <NE: "!="> |
+    <LT: "<"> |
+    <LE: "<="> |
+    <GT: ">"> |
+    <GE: ">="> |
+    <PIPE: "|"> |
+    <LCURLY: "{"> |
+    <RCURLY: "}"> |
+    <LPAREN: "("> |
+    <RPAREN: ")"> |
+    <DOT: "."> |
+    <COMMA: ","> |
+    <COLON: ":"> |
+    <SCOLON: ";"> |
+    <STRING: ("\"" (~["\""] | "\\\"")* "\"") |
+             ("'" (~["'"] | "\\'")* "'")> |
+    <ATTRIBUTE: "attribute"> |
+    <BASE64_DECODE: "base64decode"> |
+    <BASE64_ENCODE: "base64encode"> |
+    <CASE: "case"> |
+    <CASE_DEFAULT: "default"> |
+    <CLEAR_STATE: "clear_state"> |
+    <CREATE_IF_NON_EXISTENT: "create_if_non_existent"> |
+    <ECHO: "echo"> |
+    <ELSE: "else"> |
+    <EXACT: "exact"> |
+    <FLATTEN: "flatten"> |
+    <FOR_EACH: "for_each"> |
+    <GET_FIELD: "get_field"> |
+    <GET_VAR: "get_var"> |
+    <GUARD: "guard"> |
+    <HEX_DECODE: "hexdecode"> |
+    <HEX_ENCODE: "hexencode"> |
+    <HOST_NAME: "hostname"> |
+    <IF: "if"> |
+    <INDEX: "index"> |
+    <INPUT: "input"> |
+    <JOIN: "join"> |
+    <LOWER_CASE: "lowercase"> |
+    <NGRAM: "ngram"> |
+    <NORMALIZE: "normalize"> |
+    <NOW: "now"> |
+    <OPTIMIZE_PREDICATE: "optimize_predicate"> |
+    <PASSTHROUGH: "passthrough"> |
+    <RANDOM: "random"> |
+    <REMOVE_IF_ZERO: "remove_if_zero"> |
+    <SELECT_INPUT: "select_input"> |
+    <SET_LANGUAGE: "set_language"> |
+    <SET_VAR: "set_var"> |
+    <SPLIT: "split"> |
+    <STEM: "stem"> |
+    <SUBSTRING: "substring"> |
+    <SUMMARY: "summary"> |
+    <SWITCH: "switch"> |
+    <THIS: "this"> |
+    <TOKENIZE: "tokenize"> |
+    <TO_ARRAY: "to_array"> |
+    <TO_BYTE: "to_byte"> |
+    <TO_DOUBLE: "to_double"> |
+    <TO_FLOAT: "to_float"> |
+    <TO_INT: "to_int"> |
+    <TO_LONG: "to_long"> |
+    <TO_POS: "to_pos"> |
+    <TO_STRING: "to_string"> |
+    <TO_WSET: "to_wset"> |
+    <TRIM: "trim"> |
+    <ZCURVE: "zcurve"> |
+    <IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-"])*>
+}
+
+// --------------------------------------------------------------------------------
+//
+// Production rules.
+//
+// --------------------------------------------------------------------------------
+
+Expression root() :
+{
+    Expression exp;
+}
+{
+    ( exp = statement() [ <SCOLON> ] )
+    {
+      while (exp instanceof ExpressionList && ((ExpressionList)exp).size() == 1) exp = ((ExpressionList)exp).get(0);
+      return exp;
+    }
+}
+
+ScriptExpression script() :
+{
+    StatementExpression exp;
+    List<StatementExpression> lst = new LinkedList<StatementExpression>();
+}
+{
+    (   <LCURLY> nl()   exp = statement() { lst.add(exp); } nl()
+      ( <SCOLON> nl() [ exp = statement() { lst.add(exp); } nl() ] )* <RCURLY> )
+    { return new ScriptExpression(lst); }
+}
+
+StatementExpression statement() :
+{
+    Expression exp;
+    List<Expression> lst = new LinkedList<Expression>();
+}
+{
+    ( exp = expression() { lst.add(exp); } ( <PIPE> nl() exp = expression() { lst.add(exp); } )* )
+    { return new StatementExpression(lst); }
+}
+
+Expression expression() :
+{
+    Expression exp;
+    List<Expression> lst = new LinkedList<Expression>();
+}
+{
+    ( exp = math() { lst.add(exp); } ( <DOT> exp = math() { lst.add(exp); } )* )
+    { return lst.size() == 1 ? exp : new CatExpression(lst); }
+}
+
+Expression math() :
+{
+    ArithmeticExpression.Operator op = ArithmeticExpression.Operator.ADD;
+    MathResolver math = new MathResolver();
+    Expression exp;
+}
+{
+    (     exp = value() { math.push(op, exp);                     }
+      ( ( <ADD>         { op = ArithmeticExpression.Operator.ADD; } |
+          <DIV>         { op = ArithmeticExpression.Operator.DIV; } |
+          <MOD>         { op = ArithmeticExpression.Operator.MOD; } |
+          <MUL>         { op = ArithmeticExpression.Operator.MUL; } |
+          <SUB>         { op = ArithmeticExpression.Operator.SUB; } )
+          exp = value() { math.push(op, exp);                     } )* )
+    { return math.resolve(); }
+}
+
+Expression value() :
+{
+    Expression val;
+}
+{
+    ( val = attributeExp()          |
+      val = base64DecodeExp()       |
+      val = base64EncodeExp()       |
+      val = clearStateExp()         |
+      val = echoExp()               |
+      val = exactExp()              |
+      val = flattenExp()            |
+      val = forEachExp()            |
+      val = getFieldExp()           |
+      val = getVarExp()             |
+      val = guardExp()              |
+      val = hexDecodeExp()          |
+      val = hexEncodeExp()          |
+      val = hostNameExp()           |
+      val = ifThenExp()             |
+      val = indexExp()              |
+      val = inputExp()              |
+      val = joinExp()               |
+      val = lowerCaseExp()          |
+      val = ngramExp()              |
+      val = normalizeExp()          |
+      val = nowExp()                |
+      val = optimizePredicateExp()  |
+      val = passthroughExp()        |
+      val = randomExp()             |
+      val = script()                |
+      val = selectInputExp()        |
+      val = setLanguageExp()        |
+      val = setValueExp()           |
+      val = setVarExp()             |
+      val = splitExp()              |
+      val = substringExp()          |
+      val = summaryExp()            |
+      val = switchExp()             |
+      val = thisExp()               |
+      val = tokenizeExp()           |
+      val = toArrayExp()            |
+      val = toByteExp()             |
+      val = toDoubleExp()           |
+      val = toFloatExp()            |
+      val = toIntExp()              |
+      val = toLongExp()             |
+      val = toPosExp()              |
+      val = toStringExp()           |
+      val = toWsetExp()             |
+      val = trimExp()               |
+      val = zcurveExp()             |
+      ( <LPAREN> val = statement() <RPAREN> { val = new ParenthesisExpression(val); } ) )
+    { return val; }
+}
+
+Expression attributeExp() :
+{
+    String val = defaultFieldName;
+}
+{
+    ( <ATTRIBUTE> [ val = fieldName() ] )
+    { return new AttributeExpression(val); }
+}
+
+Expression base64DecodeExp() : { }
+{
+    ( <BASE64_DECODE> )
+    { return new Base64DecodeExpression(); }
+}
+
+Expression base64EncodeExp() : { }
+{
+    ( <BASE64_ENCODE> )
+    { return new Base64EncodeExpression(); }
+}
+
+Expression clearStateExp() : { }
+{
+    ( <CLEAR_STATE> )
+    { return new ClearStateExpression(); }
+}
+
+Expression echoExp() : { }
+{
+    ( <ECHO> )
+    { return new EchoExpression(); }
+}
+
+Expression exactExp() : { }
+{
+    ( <EXACT> )
+    { return new ExactExpression(); }
+}
+
+Expression flattenExp() : { }
+{
+    ( <FLATTEN> )
+    { return new FlattenExpression(); }
+}
+
+Expression forEachExp() :
+{
+    Expression val;
+}
+{
+    ( <FOR_EACH> <LCURLY> nl() val = statement() nl() <RCURLY> )
+    { return new ForEachExpression(val); }
+}
+
+Expression getFieldExp() :
+{
+    String val;
+}
+{
+    ( <GET_FIELD> val = identifier() )
+    { return new GetFieldExpression(val); }
+}
+
+Expression getVarExp() :
+{
+    String val;
+}
+{
+    ( <GET_VAR> val = identifier() )
+    { return new GetVarExpression(val); }
+}
+
+Expression guardExp() :
+{
+    Expression val;
+}
+{
+    ( <GUARD> val = script() )
+    { return new GuardExpression(val); }
+}
+
+Expression hexDecodeExp() : { }
+{
+    ( <HEX_DECODE> )
+    { return new HexDecodeExpression(); }
+}
+
+Expression hexEncodeExp() : { }
+{
+    ( <HEX_ENCODE> )
+    { return new HexEncodeExpression(); }
+}
+
+Expression hostNameExp() : { }
+{
+    ( <HOST_NAME> )
+    { return new HostNameExpression(); }
+}
+
+Expression ifThenExp() :
+{
+    Expression lhs, rhs, ifTrue, ifFalse = null;
+    IfThenExpression.Comparator cmp;
+}
+{
+    ( <IF> <LPAREN> lhs = expression() cmp = ifThenCmp() rhs = expression() <RPAREN>
+      ifTrue = script() [ <ELSE> ifFalse = script() ] )
+    { return new IfThenExpression(lhs, cmp, rhs, ifTrue, ifFalse); }
+}
+
+IfThenExpression.Comparator ifThenCmp() :
+{
+    IfThenExpression.Comparator val = null;
+}
+{
+    ( <EQ> { val = IfThenExpression.Comparator.EQ; } |
+      <NE> { val = IfThenExpression.Comparator.NE; } |
+      <LE> { val = IfThenExpression.Comparator.LE; } |
+      <LT> { val = IfThenExpression.Comparator.LT; } |
+      <GE> { val = IfThenExpression.Comparator.GE; } |
+      <GT> { val = IfThenExpression.Comparator.GT; } )
+    { return val; }
+}
+
+Expression indexExp() :
+{
+    String val = defaultFieldName;
+}
+{
+    ( <INDEX> [ val = fieldName() ] )
+    { return new IndexExpression(val); }
+}
+
+Expression inputExp() :
+{
+    String val = defaultFieldName;
+}
+{
+    ( <INPUT> [ val = identifier() ] )
+    { return new InputExpression(val); }
+}
+
+Expression joinExp() :
+{
+    String val;
+}
+{
+    ( <JOIN> val = string() )
+    { return new JoinExpression(val); }
+}
+
+Expression lowerCaseExp() : { }
+{
+    ( <LOWER_CASE> )
+    { return new LowerCaseExpression(); }
+}
+
+Expression ngramExp() :
+{
+    int gramSize;
+}
+{
+    ( <NGRAM> gramSize = integer() )
+    { return new NGramExpression(linguistics, gramSize); }
+}
+
+Expression normalizeExp() : { }
+{
+    ( <NORMALIZE> )
+    { return new NormalizeExpression(linguistics); }
+}
+
+Expression nowExp() : { }
+{
+    ( <NOW> )
+    { return new NowExpression(); }
+}
+
+Expression optimizePredicateExp() : { }
+{
+    ( <OPTIMIZE_PREDICATE> )
+    { return new OptimizePredicateExpression(); }
+}
+
+Expression passthroughExp() :
+{
+    String val = defaultFieldName;
+}
+{
+    ( <PASSTHROUGH> [ val = fieldName() ] )
+    { return new PassthroughExpression(val); }
+}
+
+Expression randomExp() :
+{
+    Integer val = null;
+}
+{
+    ( <RANDOM> [ LOOKAHEAD(2) val = integer() ] )
+    { return new RandomExpression(val); }
+}
+
+Expression selectInputExp() :
+{
+    List<Pair<String, Expression>> cases = new LinkedList<Pair<String, Expression>>();
+    Expression exp;
+    String str;
+}
+{
+    ( <SELECT_INPUT> <LCURLY> nl() ( str = identifier() <COLON> exp = statement() <SCOLON> nl()
+      { cases.add(new Pair<String, Expression>(str, exp)); } )+ <RCURLY> )
+    { return new SelectInputExpression(cases); }
+}
+
+Expression setLanguageExp() : { }
+{
+    ( <SET_LANGUAGE> )
+    { return new SetLanguageExpression(); }
+}
+
+Expression setValueExp() :
+{
+    FieldValue val;
+}
+{
+    ( val = fieldValue() )
+    { return new SetValueExpression(val); }
+}
+
+Expression setVarExp() :
+{
+    String val;
+}
+{
+    ( <SET_VAR> val = identifier() )
+    { return new SetVarExpression(val); }
+}
+
+Expression splitExp() :
+{
+    String val;
+}
+{
+    ( <SPLIT> val = string() )
+    { return new SplitExpression(val); }
+}
+
+Expression substringExp() :
+{
+    long from, to;
+}
+{
+    ( <SUBSTRING> from = integer() to = integer() )
+    { return new SubstringExpression((int)from, (int)to); }
+}
+
+Expression summaryExp() :
+{
+    String val = defaultFieldName;
+}
+{
+    ( <SUMMARY> [ val = fieldName() ] )
+    { return new SummaryExpression(val); }
+}
+
+Expression switchExp() :
+{
+    Map<String, Expression> cases = new LinkedHashMap<String, Expression>();
+    Expression exp, defaultExp = null;
+    String str;
+}
+{
+    ( <SWITCH> <LCURLY> nl()
+      ( <CASE> str = string() <COLON> exp = statement() { cases.put(str, exp); } <SCOLON> nl() )+
+      [ <CASE_DEFAULT> <COLON> defaultExp = statement() <SCOLON> nl() ]
+      <RCURLY> )
+    { return new SwitchExpression(cases, defaultExp); }
+}
+
+Expression thisExp() : { }
+{
+    ( <THIS> )
+    { return new ThisExpression(); }
+}
+
+Expression tokenizeExp() :
+{
+    AnnotatorConfig cfg = annotatorCfg;
+}
+{
+    ( <TOKENIZE> [ cfg = tokenizeCfg() ] )
+    { return new TokenizeExpression(linguistics, cfg); }
+}
+
+AnnotatorConfig tokenizeCfg() :
+{
+    AnnotatorConfig val = new AnnotatorConfig(annotatorCfg);
+    String str = "SHORTEST";
+}
+{
+    ( <STEM> ( <COLON> str = string() ) ? { val.setStemMode(str); } |
+      <NORMALIZE> { val.setRemoveAccents(true); } )+
+    { return val; }
+}
+
+Expression toArrayExp() : { }
+{
+    ( <TO_ARRAY> )
+    { return new ToArrayExpression(); }
+}
+
+Expression toByteExp() : { }
+{
+    ( <TO_BYTE> )
+    { return new ToByteExpression(); }
+}
+
+Expression toDoubleExp() : { }
+{
+    ( <TO_DOUBLE> )
+    { return new ToDoubleExpression(); }
+}
+
+Expression toFloatExp() : { }
+{
+    ( <TO_FLOAT> )
+    { return new ToFloatExpression(); }
+}
+
+Expression toIntExp() : { }
+{
+    ( <TO_INT> )
+    { return new ToIntegerExpression(); }
+}
+
+Expression toLongExp() : { }
+{
+    ( <TO_LONG> )
+    { return new ToLongExpression(); }
+}
+
+Expression toPosExp() : { }
+{
+    ( <TO_POS> )
+    { return new ToPositionExpression(); }
+}
+
+Expression toStringExp() : { }
+{
+    ( <TO_STRING> )
+    { return new ToStringExpression(); }
+}
+
+Expression toWsetExp() :
+{
+    boolean createIfNonExistent = false;
+    boolean removeIfZero = false;
+}
+{
+    ( <TO_WSET> ( <CREATE_IF_NON_EXISTENT> { createIfNonExistent = true; } |
+                  <REMOVE_IF_ZERO>         { removeIfZero = true;        } )* )
+    { return new ToWsetExpression(createIfNonExistent, removeIfZero); }
+}
+
+Expression trimExp() : { }
+{
+    ( <TRIM> )
+    { return new TrimExpression(); }
+}
+
+Expression zcurveExp() : { }
+{
+    ( <ZCURVE> )
+    { return new ZCurveExpression(); }
+}
+
+String identifier() :
+{
+    String val;
+}
+{
+    ( val = string()             |
+      ( <ATTRIBUTE>              |
+        <BASE64_DECODE>          |
+        <BASE64_ENCODE>          |
+        <CASE>                   |
+        <CASE_DEFAULT>           |
+        <CLEAR_STATE>            |
+        <CREATE_IF_NON_EXISTENT> |
+        <ECHO>                   |
+        <EXACT>                  |
+        <ELSE>                   |
+        <FLATTEN>                |
+        <FOR_EACH>               |
+        <GET_FIELD>              |
+        <GET_VAR>                |
+        <GUARD>                  |
+        <HEX_DECODE>             |
+        <HEX_ENCODE>             |
+        <HOST_NAME>              |
+        <IDENTIFIER>             |
+        <IF>                     |
+        <INDEX>                  |
+        <INPUT>                  |
+        <JOIN>                   |
+        <LOWER_CASE>             |
+        <NGRAM>                  |
+        <NORMALIZE>              |
+        <NOW>                    |
+        <OPTIMIZE_PREDICATE>     |
+        <PASSTHROUGH>            |
+        <RANDOM>                 |
+        <REMOVE_IF_ZERO>         |
+        <SELECT_INPUT>           |
+        <SET_LANGUAGE>           |
+        <SET_VAR>                |
+        <SPLIT>                  |
+        <STEM>                   |
+        <SUBSTRING>              |
+        <SUMMARY>                |
+        <SWITCH>                 |
+        <THIS>                   |
+        <TO_ARRAY>               |
+        <TO_DOUBLE>              |
+        <TO_FLOAT>               |
+        <TO_INT>                 |
+        <TO_LONG>                |
+        <TO_POS>                 |
+        <TO_STRING>              |
+        <TO_WSET>                |
+        <TOKENIZE>               |
+        <TRIM>                   |
+        <ZCURVE>                 ) { val = token.image; } )
+    { return val; }
+}
+
+
+String fieldName() :
+{
+    StringBuilder builder = new StringBuilder();
+    String str;
+}
+{
+    ( str = identifier() { builder.append(str); } (
+      LOOKAHEAD(2) <DOT> { builder.append(token.image); }
+      str = identifier() { builder.append(str); } )* )
+    { return builder.toString(); }
+}
+
+FieldValue fieldValue() :
+{
+    FieldValue val;
+}
+{
+    ( val = numericValue() | val = stringValue() )
+    { return val; }
+}
+
+FieldValue numericValue() :
+{
+    FieldValue val;
+    String pre = "";
+}
+{
+    ( [ <ADD> | <SUB> { pre = "-";                             } ]
+      ( <DOUBLE>      { val = parseDouble(pre + token.image);  } |
+        <FLOAT>       { val = parseFloat(pre + token.image);   } |
+        <INTEGER>     { val = parseInteger(pre + token.image); } |
+        <LONG>        { val = parseLong(pre + token.image);    } ) )
+    { return val; }
+}
+
+FieldValue stringValue() :
+{
+    String val;
+}
+{
+    ( val = string() )
+    { return new StringFieldValue(val); }
+}
+
+String string() : { }
+{
+    ( <STRING> )
+    { return StringUtilities.unescape(token.image.substring(1, token.image.length() - 1)); }
+}
+
+int integer() :
+{
+    String pre = "";
+    int val;
+}
+{
+    ( [ <ADD> | <SUB> { pre = "-";                                 } ]
+      <INTEGER>       { val = Integer.parseInt(pre + token.image); } )
+    { return val; }
+}
+
+void nl() : { }
+{
+    ( <NL> )*
+}
+
diff --git a/indexinglanguage/src/test/cfg/attributes.cfg b/indexinglanguage/src/test/cfg/attributes.cfg
new file mode 100644
index 00000000000..bf9ccd94453
--- /dev/null
+++ b/indexinglanguage/src/test/cfg/attributes.cfg
@@ -0,0 +1,69 @@
+attribute[22]
+attribute[0].name sales
+attribute[0].datatype INT32
+attribute[0].collectiontype SINGLE
+attribute[1].name pto
+attribute[1].datatype INT32
+attribute[1].collectiontype SINGLE
+attribute[2].name mid
+attribute[2].datatype INT32
+attribute[2].collectiontype ARRAY
+attribute[3].name ew
+attribute[3].datatype STRING
+attribute[3].collectiontype SINGLE
+attribute[4].name weight
+attribute[4].datatype FLOAT
+attribute[4].collectiontype SINGLE
+attribute[5].name bgnpfrom
+attribute[5].datatype FLOAT
+attribute[5].collectiontype SINGLE
+attribute[6].name artist
+attribute[6].datatype STRING
+attribute[6].collectiontype SINGLE
+attribute[7].name artistspid
+attribute[7].datatype STRING
+attribute[7].collectiontype WEIGHTEDSET
+attribute[8].name artistspid2
+attribute[8].datatype FLOAT
+attribute[8].collectiontype WEIGHTEDSET
+attribute[9].name title
+attribute[9].datatype STRING
+attribute[9].collectiontype SINGLE
+attribute[10].name newestedition
+attribute[10].datatype UINT32
+attribute[10].collectiontype SINGLE
+attribute[11].name year
+attribute[11].datatype INT32
+attribute[11].collectiontype ARRAY
+attribute[12].name endyear
+attribute[12].datatype INT32
+attribute[12].collectiontype ARRAY
+attribute[13].name did
+attribute[13].datatype INT32
+attribute[13].collectiontype SINGLE
+attribute[14].name cbid
+attribute[14].datatype INT32
+attribute[14].collectiontype SINGLE
+attribute[15].name noupdate
+attribute[15].datatype STRING
+attribute[15].collectiontype SINGLE
+attribute[15].noupdate false
+attribute[16].name noupdate2
+attribute[16].datatype STRING
+attribute[16].collectiontype SINGLE
+attribute[16].noupdate false
+attribute[17].name multiposition2d_position
+attribute[17].datatype INT64
+attribute[17].collectiontype ARRAY
+attribute[18].name extracategories
+attribute[18].datatype STRING
+attribute[18].collectiontype ARRAY
+attribute[19].name default_fieldlength
+attribute[19].datatype UINT32
+attribute[19].collectiontype ARRAY
+attribute[20].name fmt_fieldlength
+attribute[20].datatype UINT32
+attribute[20].collectiontype SINGLE
+attribute[21].name categories_fieldlength
+attribute[21].datatype UINT32
+attribute[21].collectiontype SINGLE
diff --git a/indexinglanguage/src/test/cfg/documentmanager.cfg b/indexinglanguage/src/test/cfg/documentmanager.cfg
new file mode 100644
index 00000000000..f93fbf75dbf
--- /dev/null
+++ b/indexinglanguage/src/test/cfg/documentmanager.cfg
@@ -0,0 +1,534 @@
+datatype[20]
+datatype[0].id -1245117006
+datatype[0].arraytype[1]
+datatype[0].arraytype[0].datatype 0
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[1].id 1328286588
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[1]
+datatype[1].weightedsettype[0].datatype 2
+datatype[1].weightedsettype[0].createifnonexistant false
+datatype[1].weightedsettype[0].removeifzero false
+datatype[1].structtype[0]
+datatype[1].documenttype[0]
+datatype[2].id 1325751891
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[1]
+datatype[2].weightedsettype[0].datatype 1
+datatype[2].weightedsettype[0].createifnonexistant false
+datatype[2].weightedsettype[0].removeifzero false
+datatype[2].structtype[0]
+datatype[2].documenttype[0]
+datatype[3].id -1486737430
+datatype[3].arraytype[1]
+datatype[3].arraytype[0].datatype 2
+datatype[3].weightedsettype[0]
+datatype[3].structtype[0]
+datatype[3].documenttype[0]
+datatype[4].id -1910204744
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name music.header
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[39]
+datatype[4].structtype[0].field[0].name bgndata
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[0].datatype 2
+datatype[4].structtype[0].field[1].name sales
+datatype[4].structtype[0].field[1].id[0]
+datatype[4].structtype[0].field[1].datatype 0
+datatype[4].structtype[0].field[2].name pto
+datatype[4].structtype[0].field[2].id[0]
+datatype[4].structtype[0].field[2].datatype 0
+datatype[4].structtype[0].field[3].name keys
+datatype[4].structtype[0].field[3].id[0]
+datatype[4].structtype[0].field[3].datatype 2
+datatype[4].structtype[0].field[4].name mid
+datatype[4].structtype[0].field[4].id[0]
+datatype[4].structtype[0].field[4].datatype -1245117006
+datatype[4].structtype[0].field[5].name ew
+datatype[4].structtype[0].field[5].id[0]
+datatype[4].structtype[0].field[5].datatype 2
+datatype[4].structtype[0].field[6].name surl
+datatype[4].structtype[0].field[6].id[0]
+datatype[4].structtype[0].field[6].datatype 2
+datatype[4].structtype[0].field[7].name userrate
+datatype[4].structtype[0].field[7].id[0]
+datatype[4].structtype[0].field[7].datatype 0
+datatype[4].structtype[0].field[8].name pid
+datatype[4].structtype[0].field[8].id[0]
+datatype[4].structtype[0].field[8].datatype 2
+datatype[4].structtype[0].field[9].name weight
+datatype[4].structtype[0].field[9].id[0]
+datatype[4].structtype[0].field[9].datatype 1
+datatype[4].structtype[0].field[10].name url
+datatype[4].structtype[0].field[10].id[0]
+datatype[4].structtype[0].field[10].datatype 2
+datatype[4].structtype[0].field[11].name isbn
+datatype[4].structtype[0].field[11].id[0]
+datatype[4].structtype[0].field[11].datatype 2
+datatype[4].structtype[0].field[12].name fmt
+datatype[4].structtype[0].field[12].id[0]
+datatype[4].structtype[0].field[12].datatype 2
+datatype[4].structtype[0].field[13].name albumid
+datatype[4].structtype[0].field[13].id[0]
+datatype[4].structtype[0].field[13].datatype 2
+datatype[4].structtype[0].field[14].name disp_song
+datatype[4].structtype[0].field[14].id[0]
+datatype[4].structtype[0].field[14].datatype 2
+datatype[4].structtype[0].field[15].name song
+datatype[4].structtype[0].field[15].id[0]
+datatype[4].structtype[0].field[15].datatype 2
+datatype[4].structtype[0].field[16].name pfrom
+datatype[4].structtype[0].field[16].id[0]
+datatype[4].structtype[0].field[16].datatype 0
+datatype[4].structtype[0].field[17].name bgnpfrom
+datatype[4].structtype[0].field[17].id[0]
+datatype[4].structtype[0].field[17].datatype 1
+datatype[4].structtype[0].field[18].name categories
+datatype[4].structtype[0].field[18].id[0]
+datatype[4].structtype[0].field[18].datatype 2
+datatype[4].structtype[0].field[19].name data
+datatype[4].structtype[0].field[19].id[0]
+datatype[4].structtype[0].field[19].datatype 2
+datatype[4].structtype[0].field[20].name numreview
+datatype[4].structtype[0].field[20].id[0]
+datatype[4].structtype[0].field[20].datatype 0
+datatype[4].structtype[0].field[21].name bgnsellers
+datatype[4].structtype[0].field[21].id[0]
+datatype[4].structtype[0].field[21].datatype 0
+datatype[4].structtype[0].field[22].name image
+datatype[4].structtype[0].field[22].id[0]
+datatype[4].structtype[0].field[22].datatype 2
+datatype[4].structtype[0].field[23].name artist
+datatype[4].structtype[0].field[23].id[0]
+datatype[4].structtype[0].field[23].datatype 2
+datatype[4].structtype[0].field[24].name artistspid
+datatype[4].structtype[0].field[24].id[0]
+datatype[4].structtype[0].field[24].datatype 1328286588
+datatype[4].structtype[0].field[25].name artistspid2
+datatype[4].structtype[0].field[25].id[0]
+datatype[4].structtype[0].field[25].datatype 1325751891
+datatype[4].structtype[0].field[26].name artistspid3
+datatype[4].structtype[0].field[26].id[0]
+datatype[4].structtype[0].field[26].datatype 1328286588
+datatype[4].structtype[0].field[27].name title
+datatype[4].structtype[0].field[27].id[0]
+datatype[4].structtype[0].field[27].datatype 2
+datatype[4].structtype[0].field[28].name newestedition
+datatype[4].structtype[0].field[28].id[0]
+datatype[4].structtype[0].field[28].datatype 0
+datatype[4].structtype[0].field[29].name bgnpto
+datatype[4].structtype[0].field[29].id[0]
+datatype[4].structtype[0].field[29].datatype 2
+datatype[4].structtype[0].field[30].name year
+datatype[4].structtype[0].field[30].id[0]
+datatype[4].structtype[0].field[30].datatype -1245117006
+datatype[4].structtype[0].field[31].name endyear
+datatype[4].structtype[0].field[31].id[0]
+datatype[4].structtype[0].field[31].datatype -1245117006
+datatype[4].structtype[0].field[32].name did
+datatype[4].structtype[0].field[32].id[0]
+datatype[4].structtype[0].field[32].datatype 0
+datatype[4].structtype[0].field[33].name scorekey
+datatype[4].structtype[0].field[33].id[0]
+datatype[4].structtype[0].field[33].datatype 0
+datatype[4].structtype[0].field[34].name cbid
+datatype[4].structtype[0].field[34].id[0]
+datatype[4].structtype[0].field[34].datatype 0
+datatype[4].structtype[0].field[35].name titles
+datatype[4].structtype[0].field[35].id[0]
+datatype[4].structtype[0].field[35].datatype -1486737430
+datatype[4].structtype[0].field[36].name noupdate
+datatype[4].structtype[0].field[36].id[0]
+datatype[4].structtype[0].field[36].datatype 2
+datatype[4].structtype[0].field[37].name noupdate2
+datatype[4].structtype[0].field[37].id[0]
+datatype[4].structtype[0].field[37].datatype 2
+datatype[4].structtype[0].field[38].name multiposition2d
+datatype[4].structtype[0].field[38].id[0]
+datatype[4].structtype[0].field[38].datatype -1486737430
+datatype[4].documenttype[0]
+datatype[5].id 993120973
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name music.body
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].field[0]
+datatype[5].documenttype[0]
+datatype[6].id 1412693671
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[0]
+datatype[6].documenttype[1]
+datatype[6].documenttype[0].name music
+datatype[6].documenttype[0].version 0
+datatype[6].documenttype[0].inherits[0]
+datatype[6].documenttype[0].headerstruct -1910204744
+datatype[6].documenttype[0].bodystruct 993120973
+datatype[7].id -1801920207
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name music_summary.header
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[40]
+datatype[7].structtype[0].field[0].name distance
+datatype[7].structtype[0].field[0].id[0]
+datatype[7].structtype[0].field[0].datatype 0
+datatype[7].structtype[0].field[1].name sddocname
+datatype[7].structtype[0].field[1].id[0]
+datatype[7].structtype[0].field[1].datatype 2
+datatype[7].structtype[0].field[2].name bgndata
+datatype[7].structtype[0].field[2].id[0]
+datatype[7].structtype[0].field[2].datatype 2
+datatype[7].structtype[0].field[3].name sales
+datatype[7].structtype[0].field[3].id[0]
+datatype[7].structtype[0].field[3].datatype 0
+datatype[7].structtype[0].field[4].name pto
+datatype[7].structtype[0].field[4].id[0]
+datatype[7].structtype[0].field[4].datatype 0
+datatype[7].structtype[0].field[5].name mid
+datatype[7].structtype[0].field[5].id[0]
+datatype[7].structtype[0].field[5].datatype 2
+datatype[7].structtype[0].field[6].name ew
+datatype[7].structtype[0].field[6].id[0]
+datatype[7].structtype[0].field[6].datatype 2
+datatype[7].structtype[0].field[7].name surl
+datatype[7].structtype[0].field[7].id[0]
+datatype[7].structtype[0].field[7].datatype 2
+datatype[7].structtype[0].field[8].name userrate
+datatype[7].structtype[0].field[8].id[0]
+datatype[7].structtype[0].field[8].datatype 0
+datatype[7].structtype[0].field[9].name pid
+datatype[7].structtype[0].field[9].id[0]
+datatype[7].structtype[0].field[9].datatype 2
+datatype[7].structtype[0].field[10].name weight
+datatype[7].structtype[0].field[10].id[0]
+datatype[7].structtype[0].field[10].datatype 1
+datatype[7].structtype[0].field[11].name url
+datatype[7].structtype[0].field[11].id[0]
+datatype[7].structtype[0].field[11].datatype 2
+datatype[7].structtype[0].field[12].name isbn
+datatype[7].structtype[0].field[12].id[0]
+datatype[7].structtype[0].field[12].datatype 2
+datatype[7].structtype[0].field[13].name fmt
+datatype[7].structtype[0].field[13].id[0]
+datatype[7].structtype[0].field[13].datatype 2
+datatype[7].structtype[0].field[14].name albumid
+datatype[7].structtype[0].field[14].id[0]
+datatype[7].structtype[0].field[14].datatype 2
+datatype[7].structtype[0].field[15].name disp_song
+datatype[7].structtype[0].field[15].id[0]
+datatype[7].structtype[0].field[15].datatype 2
+datatype[7].structtype[0].field[16].name song
+datatype[7].structtype[0].field[16].id[0]
+datatype[7].structtype[0].field[16].datatype 2
+datatype[7].structtype[0].field[17].name pfrom
+datatype[7].structtype[0].field[17].id[0]
+datatype[7].structtype[0].field[17].datatype 0
+datatype[7].structtype[0].field[18].name bgnpfrom
+datatype[7].structtype[0].field[18].id[0]
+datatype[7].structtype[0].field[18].datatype 1
+datatype[7].structtype[0].field[19].name categories
+datatype[7].structtype[0].field[19].id[0]
+datatype[7].structtype[0].field[19].datatype 2
+datatype[7].structtype[0].field[20].name data
+datatype[7].structtype[0].field[20].id[0]
+datatype[7].structtype[0].field[20].datatype 2
+datatype[7].structtype[0].field[21].name numreview
+datatype[7].structtype[0].field[21].id[0]
+datatype[7].structtype[0].field[21].datatype 0
+datatype[7].structtype[0].field[22].name bgnsellers
+datatype[7].structtype[0].field[22].id[0]
+datatype[7].structtype[0].field[22].datatype 0
+datatype[7].structtype[0].field[23].name image
+datatype[7].structtype[0].field[23].id[0]
+datatype[7].structtype[0].field[23].datatype 2
+datatype[7].structtype[0].field[24].name artist
+datatype[7].structtype[0].field[24].id[0]
+datatype[7].structtype[0].field[24].datatype 2
+datatype[7].structtype[0].field[25].name artistspid
+datatype[7].structtype[0].field[25].id[0]
+datatype[7].structtype[0].field[25].datatype 2
+datatype[7].structtype[0].field[26].name artistspid3
+datatype[7].structtype[0].field[26].id[0]
+datatype[7].structtype[0].field[26].datatype 2
+datatype[7].structtype[0].field[27].name title
+datatype[7].structtype[0].field[27].id[0]
+datatype[7].structtype[0].field[27].datatype 2
+datatype[7].structtype[0].field[28].name newestedition
+datatype[7].structtype[0].field[28].id[0]
+datatype[7].structtype[0].field[28].datatype 0
+datatype[7].structtype[0].field[29].name bgnpto
+datatype[7].structtype[0].field[29].id[0]
+datatype[7].structtype[0].field[29].datatype 2
+datatype[7].structtype[0].field[30].name year
+datatype[7].structtype[0].field[30].id[0]
+datatype[7].structtype[0].field[30].datatype 2
+datatype[7].structtype[0].field[31].name endyear
+datatype[7].structtype[0].field[31].id[0]
+datatype[7].structtype[0].field[31].datatype 2
+datatype[7].structtype[0].field[32].name did
+datatype[7].structtype[0].field[32].id[0]
+datatype[7].structtype[0].field[32].datatype 0
+datatype[7].structtype[0].field[33].name scorekey
+datatype[7].structtype[0].field[33].id[0]
+datatype[7].structtype[0].field[33].datatype 0
+datatype[7].structtype[0].field[34].name cbid
+datatype[7].structtype[0].field[34].id[0]
+datatype[7].structtype[0].field[34].datatype 0
+datatype[7].structtype[0].field[35].name titles
+datatype[7].structtype[0].field[35].id[0]
+datatype[7].structtype[0].field[35].datatype 2
+datatype[7].structtype[0].field[36].name ranklog
+datatype[7].structtype[0].field[36].id[0]
+datatype[7].structtype[0].field[36].datatype 2
+datatype[7].structtype[0].field[37].name rankfeatures
+datatype[7].structtype[0].field[37].id[0]
+datatype[7].structtype[0].field[37].datatype 2
+datatype[7].structtype[0].field[38].name summaryfeatures
+datatype[7].structtype[0].field[38].id[0]
+datatype[7].structtype[0].field[38].datatype 2
+datatype[7].structtype[0].field[39].name documentid
+datatype[7].structtype[0].field[39].id[0]
+datatype[7].structtype[0].field[39].datatype 2
+datatype[7].documenttype[0]
+datatype[8].id -1728551034
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name music_summary.body
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[0]
+datatype[8].documenttype[0]
+datatype[9].id 1601149518
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[0]
+datatype[9].documenttype[1]
+datatype[9].documenttype[0].name music_summary
+datatype[9].documenttype[0].version 0
+datatype[9].documenttype[0].inherits[0]
+datatype[9].documenttype[0].headerstruct -1801920207
+datatype[9].documenttype[0].bodystruct -1728551034
+datatype[10].id 1509154821
+datatype[10].arraytype[0]
+datatype[10].weightedsettype[0]
+datatype[10].structtype[1]
+datatype[10].structtype[0].name music_index.header
+datatype[10].structtype[0].version 0
+datatype[10].structtype[0].field[19]
+datatype[10].structtype[0].field[0].name sddocname
+datatype[10].structtype[0].field[0].id[0]
+datatype[10].structtype[0].field[0].datatype -1486737430
+datatype[10].structtype[0].field[1].name sales
+datatype[10].structtype[0].field[1].id[0]
+datatype[10].structtype[0].field[1].datatype -1245117006
+datatype[10].structtype[0].field[2].name pto
+datatype[10].structtype[0].field[2].id[0]
+datatype[10].structtype[0].field[2].datatype -1245117006
+datatype[10].structtype[0].field[3].name keys
+datatype[10].structtype[0].field[3].id[0]
+datatype[10].structtype[0].field[3].datatype -1486737430
+datatype[10].structtype[0].field[4].name mid
+datatype[10].structtype[0].field[4].id[0]
+datatype[10].structtype[0].field[4].datatype -1245117006
+datatype[10].structtype[0].field[5].name ew
+datatype[10].structtype[0].field[5].id[0]
+datatype[10].structtype[0].field[5].datatype -1486737430
+datatype[10].structtype[0].field[6].name fmt
+datatype[10].structtype[0].field[6].id[0]
+datatype[10].structtype[0].field[6].datatype -1486737430
+datatype[10].structtype[0].field[7].name song
+datatype[10].structtype[0].field[7].id[0]
+datatype[10].structtype[0].field[7].datatype -1486737430
+datatype[10].structtype[0].field[8].name categories
+datatype[10].structtype[0].field[8].id[0]
+datatype[10].structtype[0].field[8].datatype -1486737430
+datatype[10].structtype[0].field[9].name artist
+datatype[10].structtype[0].field[9].id[0]
+datatype[10].structtype[0].field[9].datatype -1486737430
+datatype[10].structtype[0].field[10].name artistspid3
+datatype[10].structtype[0].field[10].id[0]
+datatype[10].structtype[0].field[10].datatype -1486737430
+datatype[10].structtype[0].field[11].name title
+datatype[10].structtype[0].field[11].id[0]
+datatype[10].structtype[0].field[11].datatype -1486737430
+datatype[10].structtype[0].field[12].name newestedition
+datatype[10].structtype[0].field[12].id[0]
+datatype[10].structtype[0].field[12].datatype -1245117006
+datatype[10].structtype[0].field[13].name year
+datatype[10].structtype[0].field[13].id[0]
+datatype[10].structtype[0].field[13].datatype -1245117006
+datatype[10].structtype[0].field[14].name endyear
+datatype[10].structtype[0].field[14].id[0]
+datatype[10].structtype[0].field[14].datatype -1245117006
+datatype[10].structtype[0].field[15].name did
+datatype[10].structtype[0].field[15].id[0]
+datatype[10].structtype[0].field[15].datatype -1245117006
+datatype[10].structtype[0].field[16].name scorekey
+datatype[10].structtype[0].field[16].id[0]
+datatype[10].structtype[0].field[16].datatype -1245117006
+datatype[10].structtype[0].field[17].name cbid
+datatype[10].structtype[0].field[17].id[0]
+datatype[10].structtype[0].field[17].datatype -1245117006
+datatype[10].structtype[0].field[18].name titles
+datatype[10].structtype[0].field[18].id[0]
+datatype[10].structtype[0].field[18].datatype -1486737430
+datatype[10].documenttype[0]
+datatype[11].id -1997730982
+datatype[11].arraytype[0]
+datatype[11].weightedsettype[0]
+datatype[11].structtype[1]
+datatype[11].structtype[0].name music_index.body
+datatype[11].structtype[0].version 0
+datatype[11].structtype[0].field[0]
+datatype[11].documenttype[0]
+datatype[12].id 2108744186
+datatype[12].arraytype[0]
+datatype[12].weightedsettype[0]
+datatype[12].structtype[0]
+datatype[12].documenttype[1]
+datatype[12].documenttype[0].name music_index
+datatype[12].documenttype[0].version 0
+datatype[12].documenttype[0].inherits[0]
+datatype[12].documenttype[0].headerstruct 1509154821
+datatype[12].documenttype[0].bodystruct -1997730982
+datatype[13].id 58874399
+datatype[13].arraytype[1]
+datatype[13].arraytype[0].datatype 4
+datatype[13].weightedsettype[0]
+datatype[13].structtype[0]
+datatype[13].documenttype[0]
+datatype[14].id -1497398149
+datatype[14].arraytype[0]
+datatype[14].weightedsettype[0]
+datatype[14].structtype[1]
+datatype[14].structtype[0].name music_attribute.header
+datatype[14].structtype[0].version 0
+datatype[14].structtype[0].field[22]
+datatype[14].structtype[0].field[0].name sales
+datatype[14].structtype[0].field[0].id[0]
+datatype[14].structtype[0].field[0].datatype 0
+datatype[14].structtype[0].field[1].name pto
+datatype[14].structtype[0].field[1].id[0]
+datatype[14].structtype[0].field[1].datatype 0
+datatype[14].structtype[0].field[2].name mid
+datatype[14].structtype[0].field[2].id[0]
+datatype[14].structtype[0].field[2].datatype -1245117006
+datatype[14].structtype[0].field[3].name ew
+datatype[14].structtype[0].field[3].id[0]
+datatype[14].structtype[0].field[3].datatype 2
+datatype[14].structtype[0].field[4].name weight
+datatype[14].structtype[0].field[4].id[0]
+datatype[14].structtype[0].field[4].datatype 1
+datatype[14].structtype[0].field[5].name bgnpfrom
+datatype[14].structtype[0].field[5].id[0]
+datatype[14].structtype[0].field[5].datatype 1
+datatype[14].structtype[0].field[6].name artist
+datatype[14].structtype[0].field[6].id[0]
+datatype[14].structtype[0].field[6].datatype 2
+datatype[14].structtype[0].field[7].name artistspid
+datatype[14].structtype[0].field[7].id[0]
+datatype[14].structtype[0].field[7].datatype 1328286588
+datatype[14].structtype[0].field[8].name artistspid2
+datatype[14].structtype[0].field[8].id[0]
+datatype[14].structtype[0].field[8].datatype 1325751891
+datatype[14].structtype[0].field[9].name title
+datatype[14].structtype[0].field[9].id[0]
+datatype[14].structtype[0].field[9].datatype 2
+datatype[14].structtype[0].field[10].name newestedition
+datatype[14].structtype[0].field[10].id[0]
+datatype[14].structtype[0].field[10].datatype 0
+datatype[14].structtype[0].field[11].name year
+datatype[14].structtype[0].field[11].id[0]
+datatype[14].structtype[0].field[11].datatype -1245117006
+datatype[14].structtype[0].field[12].name endyear
+datatype[14].structtype[0].field[12].id[0]
+datatype[14].structtype[0].field[12].datatype -1245117006
+datatype[14].structtype[0].field[13].name did
+datatype[14].structtype[0].field[13].id[0]
+datatype[14].structtype[0].field[13].datatype 0
+datatype[14].structtype[0].field[14].name cbid
+datatype[14].structtype[0].field[14].id[0]
+datatype[14].structtype[0].field[14].datatype 0
+datatype[14].structtype[0].field[15].name noupdate
+datatype[14].structtype[0].field[15].id[0]
+datatype[14].structtype[0].field[15].datatype 2
+datatype[14].structtype[0].field[16].name noupdate2
+datatype[14].structtype[0].field[16].id[0]
+datatype[14].structtype[0].field[16].datatype 2
+datatype[14].structtype[0].field[17].name multiposition2d_position
+datatype[14].structtype[0].field[17].id[0]
+datatype[14].structtype[0].field[17].datatype 58874399
+datatype[14].structtype[0].field[18].name extracategories
+datatype[14].structtype[0].field[18].id[0]
+datatype[14].structtype[0].field[18].datatype -1486737430
+datatype[14].structtype[0].field[19].name default_fieldlength
+datatype[14].structtype[0].field[19].id[0]
+datatype[14].structtype[0].field[19].datatype -1245117006
+datatype[14].structtype[0].field[20].name fmt_fieldlength
+datatype[14].structtype[0].field[20].id[0]
+datatype[14].structtype[0].field[20].datatype 0
+datatype[14].structtype[0].field[21].name categories_fieldlength
+datatype[14].structtype[0].field[21].id[0]
+datatype[14].structtype[0].field[21].datatype 0
+datatype[14].documenttype[0]
+datatype[15].id 1243829584
+datatype[15].arraytype[0]
+datatype[15].weightedsettype[0]
+datatype[15].structtype[1]
+datatype[15].structtype[0].name music_attribute.body
+datatype[15].structtype[0].version 0
+datatype[15].structtype[0].field[0]
+datatype[15].documenttype[0]
+datatype[16].id 1990571588
+datatype[16].arraytype[0]
+datatype[16].weightedsettype[0]
+datatype[16].structtype[0]
+datatype[16].documenttype[1]
+datatype[16].documenttype[0].name music_attribute
+datatype[16].documenttype[0].version 0
+datatype[16].documenttype[0].inherits[0]
+datatype[16].documenttype[0].headerstruct -1497398149
+datatype[16].documenttype[0].bodystruct 1243829584
+datatype[17].id -592896846
+datatype[17].arraytype[0]
+datatype[17].weightedsettype[0]
+datatype[17].structtype[1]
+datatype[17].structtype[0].name indexingdocument.header
+datatype[17].structtype[0].version 0
+datatype[17].structtype[0].field[3]
+datatype[17].structtype[0].field[0].name index
+datatype[17].structtype[0].field[0].id[0]
+datatype[17].structtype[0].field[0].datatype 8
+datatype[17].structtype[0].field[1].name summary
+datatype[17].structtype[0].field[1].id[0]
+datatype[17].structtype[0].field[1].datatype 8
+datatype[17].structtype[0].field[2].name attribute
+datatype[17].structtype[0].field[2].id[0]
+datatype[17].structtype[0].field[2].datatype 8
+datatype[17].documenttype[0]
+datatype[18].id -2093772985
+datatype[18].arraytype[0]
+datatype[18].weightedsettype[0]
+datatype[18].structtype[1]
+datatype[18].structtype[0].name indexingdocument.body
+datatype[18].structtype[0].version 0
+datatype[18].structtype[0].field[0]
+datatype[18].documenttype[0]
+datatype[19].id -1831281171
+datatype[19].arraytype[0]
+datatype[19].weightedsettype[0]
+datatype[19].structtype[0]
+datatype[19].documenttype[1]
+datatype[19].documenttype[0].name indexingdocument
+datatype[19].documenttype[0].version 0
+datatype[19].documenttype[0].inherits[0]
+datatype[19].documenttype[0].headerstruct -592896846
+datatype[19].documenttype[0].bodystruct -2093772985
diff --git a/indexinglanguage/src/test/cfg/documentmanager_inherit.cfg b/indexinglanguage/src/test/cfg/documentmanager_inherit.cfg
new file mode 100644
index 00000000000..428c9049212
--- /dev/null
+++ b/indexinglanguage/src/test/cfg/documentmanager_inherit.cfg
@@ -0,0 +1,216 @@
+datatype[19]
+datatype[0].id 2006483754
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name newssummary.header
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[4]
+datatype[0].structtype[0].field[0].name uri
+datatype[0].structtype[0].field[0].id[0]
+datatype[0].structtype[0].field[0].datatype 2
+datatype[0].structtype[0].field[1].name where
+datatype[0].structtype[0].field[1].id[0]
+datatype[0].structtype[0].field[1].datatype 2
+datatype[0].structtype[0].field[2].name title
+datatype[0].structtype[0].field[2].id[0]
+datatype[0].structtype[0].field[2].datatype 2
+datatype[0].structtype[0].field[3].name weight
+datatype[0].structtype[0].field[3].id[0]
+datatype[0].structtype[0].field[3].datatype 1
+datatype[0].documenttype[0]
+datatype[1].id -2059783233
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name newssummary.body
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[0]
+datatype[1].documenttype[0]
+datatype[2].id -756330891
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].name newssummary
+datatype[2].documenttype[0].version 0
+datatype[2].documenttype[0].inherits[0]
+datatype[2].documenttype[0].headerstruct 2006483754
+datatype[2].documenttype[0].bodystruct -2059783233
+datatype[3].id 2010790819
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name newssummary_summary.header
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[6]
+datatype[3].structtype[0].field[0].name sddocname
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[0].datatype 2
+datatype[3].structtype[0].field[1].name uri
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].field[1].datatype 2
+datatype[3].structtype[0].field[2].name title
+datatype[3].structtype[0].field[2].id[0]
+datatype[3].structtype[0].field[2].datatype 2
+datatype[3].structtype[0].field[3].name weight
+datatype[3].structtype[0].field[3].id[0]
+datatype[3].structtype[0].field[3].datatype 1
+datatype[3].structtype[0].field[4].name ranklog
+datatype[3].structtype[0].field[4].id[0]
+datatype[3].structtype[0].field[4].datatype 2
+datatype[3].structtype[0].field[5].name documentid
+datatype[3].structtype[0].field[5].id[0]
+datatype[3].structtype[0].field[5].datatype 2
+datatype[3].documenttype[0]
+datatype[4].id 760329848
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name newssummary_summary.body
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[0]
+datatype[4].documenttype[0]
+datatype[5].id -1535558628
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[0]
+datatype[5].documenttype[1]
+datatype[5].documenttype[0].name newssummary_summary
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].inherits[0]
+datatype[5].documenttype[0].headerstruct 2010790819
+datatype[5].documenttype[0].bodystruct 760329848
+datatype[6].id -1486737430
+datatype[6].arraytype[1]
+datatype[6].arraytype[0].datatype 2
+datatype[6].weightedsettype[0]
+datatype[6].structtype[0]
+datatype[6].documenttype[0]
+datatype[7].id -296931593
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name newssummary_index.header
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[2]
+datatype[7].structtype[0].field[0].name sddocname
+datatype[7].structtype[0].field[0].id[0]
+datatype[7].structtype[0].field[0].datatype -1486737430
+datatype[7].structtype[0].field[1].name title
+datatype[7].structtype[0].field[1].id[0]
+datatype[7].structtype[0].field[1].datatype -1486737430
+datatype[7].documenttype[0]
+datatype[8].id -2066649396
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name newssummary_index.body
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[0]
+datatype[8].documenttype[0]
+datatype[9].id 1957994312
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[0]
+datatype[9].documenttype[1]
+datatype[9].documenttype[0].name newssummary_index
+datatype[9].documenttype[0].version 0
+datatype[9].documenttype[0].inherits[0]
+datatype[9].documenttype[0].headerstruct -296931593
+datatype[9].documenttype[0].bodystruct -2066649396
+datatype[10].id -1089205651
+datatype[10].arraytype[0]
+datatype[10].weightedsettype[0]
+datatype[10].structtype[1]
+datatype[10].structtype[0].name newssummary_attribute.header
+datatype[10].structtype[0].version 0
+datatype[10].structtype[0].field[1]
+datatype[10].structtype[0].field[0].name weight
+datatype[10].structtype[0].field[0].id[0]
+datatype[10].structtype[0].field[0].datatype 1
+datatype[10].documenttype[0]
+datatype[11].id 761573314
+datatype[11].arraytype[0]
+datatype[11].weightedsettype[0]
+datatype[11].structtype[1]
+datatype[11].structtype[0].name newssummary_attribute.body
+datatype[11].structtype[0].version 0
+datatype[11].structtype[0].field[0]
+datatype[11].documenttype[0]
+datatype[12].id -1613882222
+datatype[12].arraytype[0]
+datatype[12].weightedsettype[0]
+datatype[12].structtype[0]
+datatype[12].documenttype[1]
+datatype[12].documenttype[0].name newssummary_attribute
+datatype[12].documenttype[0].version 0
+datatype[12].documenttype[0].inherits[0]
+datatype[12].documenttype[0].headerstruct -1089205651
+datatype[12].documenttype[0].bodystruct 761573314
+datatype[13].id 2098419674
+datatype[13].arraytype[0]
+datatype[13].weightedsettype[0]
+datatype[13].structtype[1]
+datatype[13].structtype[0].name newsarticle.header
+datatype[13].structtype[0].version 0
+datatype[13].structtype[0].field[1]
+datatype[13].structtype[0].field[0].name city
+datatype[13].structtype[0].field[0].id[0]
+datatype[13].structtype[0].field[0].datatype 2
+datatype[13].documenttype[0]
+datatype[14].id 197293167
+datatype[14].arraytype[0]
+datatype[14].weightedsettype[0]
+datatype[14].structtype[1]
+datatype[14].structtype[0].name newsarticle.body
+datatype[14].structtype[0].version 0
+datatype[14].structtype[0].field[0]
+datatype[14].documenttype[0]
+datatype[15].id -1710661691
+datatype[15].arraytype[0]
+datatype[15].weightedsettype[0]
+datatype[15].structtype[0]
+datatype[15].documenttype[1]
+datatype[15].documenttype[0].name newsarticle
+datatype[15].documenttype[0].version 0
+datatype[15].documenttype[0].inherits[1]
+datatype[15].documenttype[0].inherits[0].name newssummary
+datatype[15].documenttype[0].inherits[0].version 0
+datatype[15].documenttype[0].headerstruct 2098419674
+datatype[15].documenttype[0].bodystruct 197293167
+datatype[16].id -592896846
+datatype[16].arraytype[0]
+datatype[16].weightedsettype[0]
+datatype[16].structtype[1]
+datatype[16].structtype[0].name indexingdocument.header
+datatype[16].structtype[0].version 0
+datatype[16].structtype[0].field[3]
+datatype[16].structtype[0].field[0].name index
+datatype[16].structtype[0].field[0].id[0]
+datatype[16].structtype[0].field[0].datatype 8
+datatype[16].structtype[0].field[1].name summary
+datatype[16].structtype[0].field[1].id[0]
+datatype[16].structtype[0].field[1].datatype 8
+datatype[16].structtype[0].field[2].name attribute
+datatype[16].structtype[0].field[2].id[0]
+datatype[16].structtype[0].field[2].datatype 8
+datatype[16].documenttype[0]
+datatype[17].id -2093772985
+datatype[17].arraytype[0]
+datatype[17].weightedsettype[0]
+datatype[17].structtype[1]
+datatype[17].structtype[0].name indexingdocument.body
+datatype[17].structtype[0].version 0
+datatype[17].structtype[0].field[0]
+datatype[17].documenttype[0]
+datatype[18].id -1831281171
+datatype[18].arraytype[0]
+datatype[18].weightedsettype[0]
+datatype[18].structtype[0]
+datatype[18].documenttype[1]
+datatype[18].documenttype[0].name indexingdocument
+datatype[18].documenttype[0].version 0
+datatype[18].documenttype[0].inherits[0]
+datatype[18].documenttype[0].headerstruct -592896846
+datatype[18].documenttype[0].bodystruct -2093772985
diff --git a/indexinglanguage/src/test/cfg/exactmatch/documentmanager.cfg b/indexinglanguage/src/test/cfg/exactmatch/documentmanager.cfg
new file mode 100644
index 00000000000..6a4cc7e7d37
--- /dev/null
+++ b/indexinglanguage/src/test/cfg/exactmatch/documentmanager.cfg
@@ -0,0 +1,167 @@
+datatype[16]
+datatype[0].id 125394903
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name exactmatch.header
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].field[1]
+datatype[0].structtype[0].field[0].name field3
+datatype[0].structtype[0].field[0].id[0]
+datatype[0].structtype[0].field[0].datatype 2
+datatype[0].documenttype[0]
+datatype[1].id 588535724
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name exactmatch.body
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[0]
+datatype[1].documenttype[0]
+datatype[2].id -21255576
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].name exactmatch
+datatype[2].documenttype[0].version 0
+datatype[2].documenttype[0].inherits[0]
+datatype[2].documenttype[0].headerstruct 125394903
+datatype[2].documenttype[0].bodystruct 588535724
+datatype[3].id 694033232
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[1]
+datatype[3].structtype[0].name exactmatch_summary.header
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].field[4]
+datatype[3].structtype[0].field[0].name sddocname
+datatype[3].structtype[0].field[0].id[0]
+datatype[3].structtype[0].field[0].datatype 2
+datatype[3].structtype[0].field[1].name field3
+datatype[3].structtype[0].field[1].id[0]
+datatype[3].structtype[0].field[1].datatype 2
+datatype[3].structtype[0].field[2].name ranklog
+datatype[3].structtype[0].field[2].id[0]
+datatype[3].structtype[0].field[2].datatype 2
+datatype[3].structtype[0].field[3].name documentid
+datatype[3].structtype[0].field[3].id[0]
+datatype[3].structtype[0].field[3].datatype 2
+datatype[3].documenttype[0]
+datatype[4].id -273441435
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name exactmatch_summary.body
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[0]
+datatype[4].documenttype[0]
+datatype[5].id 280229135
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[0]
+datatype[5].documenttype[1]
+datatype[5].documenttype[0].name exactmatch_summary
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].inherits[0]
+datatype[5].documenttype[0].headerstruct 694033232
+datatype[5].documenttype[0].bodystruct -273441435
+datatype[6].id -1486737430
+datatype[6].arraytype[1]
+datatype[6].arraytype[0].datatype 2
+datatype[6].weightedsettype[0]
+datatype[6].structtype[0]
+datatype[6].documenttype[0]
+datatype[7].id -1330702876
+datatype[7].arraytype[0]
+datatype[7].weightedsettype[0]
+datatype[7].structtype[1]
+datatype[7].structtype[0].name exactmatch_index.header
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].field[2]
+datatype[7].structtype[0].field[0].name sddocname
+datatype[7].structtype[0].field[0].id[0]
+datatype[7].structtype[0].field[0].datatype -1486737430
+datatype[7].structtype[0].field[1].name field3
+datatype[7].structtype[0].field[1].id[0]
+datatype[7].structtype[0].field[1].datatype -1486737430
+datatype[7].documenttype[0]
+datatype[8].id 1248472313
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name exactmatch_index.body
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[0]
+datatype[8].documenttype[0]
+datatype[9].id -1843463941
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[0]
+datatype[9].documenttype[1]
+datatype[9].documenttype[0].name exactmatch_index
+datatype[9].documenttype[0].version 0
+datatype[9].documenttype[0].inherits[0]
+datatype[9].documenttype[0].headerstruct -1330702876
+datatype[9].documenttype[0].bodystruct 1248472313
+datatype[10].id 522105562
+datatype[10].arraytype[0]
+datatype[10].weightedsettype[0]
+datatype[10].structtype[1]
+datatype[10].structtype[0].name exactmatch_attribute.header
+datatype[10].structtype[0].version 0
+datatype[10].structtype[0].field[0]
+datatype[10].documenttype[0]
+datatype[11].id -555184273
+datatype[11].arraytype[0]
+datatype[11].weightedsettype[0]
+datatype[11].structtype[1]
+datatype[11].structtype[0].name exactmatch_attribute.body
+datatype[11].structtype[0].version 0
+datatype[11].structtype[0].field[0]
+datatype[11].documenttype[0]
+datatype[12].id -398564155
+datatype[12].arraytype[0]
+datatype[12].weightedsettype[0]
+datatype[12].structtype[0]
+datatype[12].documenttype[1]
+datatype[12].documenttype[0].name exactmatch_attribute
+datatype[12].documenttype[0].version 0
+datatype[12].documenttype[0].inherits[0]
+datatype[12].documenttype[0].headerstruct 522105562
+datatype[12].documenttype[0].bodystruct -555184273
+datatype[13].id -592896846
+datatype[13].arraytype[0]
+datatype[13].weightedsettype[0]
+datatype[13].structtype[1]
+datatype[13].structtype[0].name indexingdocument.header
+datatype[13].structtype[0].version 0
+datatype[13].structtype[0].field[3]
+datatype[13].structtype[0].field[0].name index
+datatype[13].structtype[0].field[0].id[0]
+datatype[13].structtype[0].field[0].datatype -1843463941
+datatype[13].structtype[0].field[1].name summary
+datatype[13].structtype[0].field[1].id[0]
+datatype[13].structtype[0].field[1].datatype 280229135
+datatype[13].structtype[0].field[2].name attribute
+datatype[13].structtype[0].field[2].id[0]
+datatype[13].structtype[0].field[2].datatype -398564155
+datatype[13].documenttype[0]
+datatype[14].id -2093772985
+datatype[14].arraytype[0]
+datatype[14].weightedsettype[0]
+datatype[14].structtype[1]
+datatype[14].structtype[0].name indexingdocument.body
+datatype[14].structtype[0].version 0
+datatype[14].structtype[0].field[0]
+datatype[14].documenttype[0]
+datatype[15].id -1831281171
+datatype[15].arraytype[0]
+datatype[15].weightedsettype[0]
+datatype[15].structtype[0]
+datatype[15].documenttype[1]
+datatype[15].documenttype[0].name indexingdocument
+datatype[15].documenttype[0].version 0
+datatype[15].documenttype[0].inherits[0]
+datatype[15].documenttype[0].headerstruct -592896846
+datatype[15].documenttype[0].bodystruct -2093772985
diff --git a/indexinglanguage/src/test/cfg/exactmatch/indexingdocument.cfg b/indexinglanguage/src/test/cfg/exactmatch/indexingdocument.cfg
new file mode 100644
index 00000000000..761bd4ff532
--- /dev/null
+++ b/indexinglanguage/src/test/cfg/exactmatch/indexingdocument.cfg
@@ -0,0 +1,20 @@
+indexingdoc[1]
+indexingdoc[0].name exactmatch
+indexingdoc[0].source[1]
+indexingdoc[0].source[0] exactmatch
+indexingdoc[0].context[2]
+indexingdoc[0].context[0].fullname sddocname.sddocname
+indexingdoc[0].context[0].shortname sddocname
+indexingdoc[0].context[0].datatype 2
+indexingdoc[0].context[1].fullname field3.field3
+indexingdoc[0].context[1].shortname field3
+indexingdoc[0].context[1].datatype 2
+indexingdoc[0].summary[4]
+indexingdoc[0].summary[0].name sddocname
+indexingdoc[0].summary[0].datatype 2
+indexingdoc[0].summary[1].name field3
+indexingdoc[0].summary[1].datatype 2
+indexingdoc[0].summary[2].name ranklog
+indexingdoc[0].summary[2].datatype 2
+indexingdoc[0].summary[3].name documentid
+indexingdoc[0].summary[3].datatype 2
diff --git a/indexinglanguage/src/test/cfg/fileio/documentmanager.cfg b/indexinglanguage/src/test/cfg/fileio/documentmanager.cfg
new file mode 100644
index 00000000000..bd7a5645dd8
--- /dev/null
+++ b/indexinglanguage/src/test/cfg/fileio/documentmanager.cfg
@@ -0,0 +1,425 @@
+datatype[17]
+datatype[0].id -1216487824
+datatype[0].arraytype[1]
+datatype[0].arraytype[0].datatype 0
+datatype[0].weightedsettype[0]
+datatype[0].structtype[0]
+datatype[0].documenttype[0]
+datatype[1].id -1910204744
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name music.header
+datatype[1].structtype[0].version 0
+datatype[1].structtype[0].field[33]
+datatype[1].structtype[0].field[0].name bgndata
+datatype[1].structtype[0].field[0].id[0]
+datatype[1].structtype[0].field[0].datatype 2
+datatype[1].structtype[0].field[1].name sales
+datatype[1].structtype[0].field[1].id[0]
+datatype[1].structtype[0].field[1].datatype 0
+datatype[1].structtype[0].field[2].name pto
+datatype[1].structtype[0].field[2].id[0]
+datatype[1].structtype[0].field[2].datatype 0
+datatype[1].structtype[0].field[3].name keys
+datatype[1].structtype[0].field[3].id[0]
+datatype[1].structtype[0].field[3].datatype 2
+datatype[1].structtype[0].field[4].name mid
+datatype[1].structtype[0].field[4].id[0]
+datatype[1].structtype[0].field[4].datatype -1216487824
+datatype[1].structtype[0].field[5].name ew
+datatype[1].structtype[0].field[5].id[0]
+datatype[1].structtype[0].field[5].datatype 2
+datatype[1].structtype[0].field[6].name surl
+datatype[1].structtype[0].field[6].id[0]
+datatype[1].structtype[0].field[6].datatype 2
+datatype[1].structtype[0].field[7].name userrate
+datatype[1].structtype[0].field[7].id[0]
+datatype[1].structtype[0].field[7].datatype 0
+datatype[1].structtype[0].field[8].name pid
+datatype[1].structtype[0].field[8].id[0]
+datatype[1].structtype[0].field[8].datatype 2
+datatype[1].structtype[0].field[9].name weight
+datatype[1].structtype[0].field[9].id[0]
+datatype[1].structtype[0].field[9].datatype 1
+datatype[1].structtype[0].field[10].name url
+datatype[1].structtype[0].field[10].id[0]
+datatype[1].structtype[0].field[10].datatype 2
+datatype[1].structtype[0].field[11].name isbn
+datatype[1].structtype[0].field[11].id[0]
+datatype[1].structtype[0].field[11].datatype 2
+datatype[1].structtype[0].field[12].name fmt
+datatype[1].structtype[0].field[12].id[0]
+datatype[1].structtype[0].field[12].datatype 2
+datatype[1].structtype[0].field[13].name albumid
+datatype[1].structtype[0].field[13].id[0]
+datatype[1].structtype[0].field[13].datatype 2
+datatype[1].structtype[0].field[14].name disp_song
+datatype[1].structtype[0].field[14].id[0]
+datatype[1].structtype[0].field[14].datatype 2
+datatype[1].structtype[0].field[15].name song
+datatype[1].structtype[0].field[15].id[0]
+datatype[1].structtype[0].field[15].datatype 2
+datatype[1].structtype[0].field[16].name pfrom
+datatype[1].structtype[0].field[16].id[0]
+datatype[1].structtype[0].field[16].datatype 0
+datatype[1].structtype[0].field[17].name bgnpfrom
+datatype[1].structtype[0].field[17].id[0]
+datatype[1].structtype[0].field[17].datatype 1
+datatype[1].structtype[0].field[18].name categories
+datatype[1].structtype[0].field[18].id[0]
+datatype[1].structtype[0].field[18].datatype 2
+datatype[1].structtype[0].field[19].name data
+datatype[1].structtype[0].field[19].id[0]
+datatype[1].structtype[0].field[19].datatype 2
+datatype[1].structtype[0].field[20].name numreview
+datatype[1].structtype[0].field[20].id[0]
+datatype[1].structtype[0].field[20].datatype 0
+datatype[1].structtype[0].field[21].name bgnsellers
+datatype[1].structtype[0].field[21].id[0]
+datatype[1].structtype[0].field[21].datatype 0
+datatype[1].structtype[0].field[22].name image
+datatype[1].structtype[0].field[22].id[0]
+datatype[1].structtype[0].field[22].datatype 2
+datatype[1].structtype[0].field[23].name artist
+datatype[1].structtype[0].field[23].id[0]
+datatype[1].structtype[0].field[23].datatype 2
+datatype[1].structtype[0].field[24].name artistspid
+datatype[1].structtype[0].field[24].id[0]
+datatype[1].structtype[0].field[24].datatype 18
+datatype[1].structtype[0].field[25].name title
+datatype[1].structtype[0].field[25].id[0]
+datatype[1].structtype[0].field[25].datatype 2
+datatype[1].structtype[0].field[26].name newestedition
+datatype[1].structtype[0].field[26].id[0]
+datatype[1].structtype[0].field[26].datatype 0
+datatype[1].structtype[0].field[27].name bgnpto
+datatype[1].structtype[0].field[27].id[0]
+datatype[1].structtype[0].field[27].datatype 2
+datatype[1].structtype[0].field[28].name year
+datatype[1].structtype[0].field[28].id[0]
+datatype[1].structtype[0].field[28].datatype -1216487824
+datatype[1].structtype[0].field[29].name did
+datatype[1].structtype[0].field[29].id[0]
+datatype[1].structtype[0].field[29].datatype 0
+datatype[1].structtype[0].field[30].name scorekey
+datatype[1].structtype[0].field[30].id[0]
+datatype[1].structtype[0].field[30].datatype 0
+datatype[1].structtype[0].field[31].name cbid
+datatype[1].structtype[0].field[31].id[0]
+datatype[1].structtype[0].field[31].datatype 0
+datatype[1].structtype[0].field[32].name indexingdocument
+datatype[1].structtype[0].field[32].id[0]
+datatype[1].structtype[0].field[32].datatype 8
+datatype[1].documenttype[0]
+datatype[2].id 993120973
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[1]
+datatype[2].structtype[0].name music.body
+datatype[2].structtype[0].version 0
+datatype[2].structtype[0].field[0]
+datatype[2].documenttype[0]
+datatype[3].id 1412693671
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[0]
+datatype[3].documenttype[1]
+datatype[3].documenttype[0].name music
+datatype[3].documenttype[0].version 0
+datatype[3].documenttype[0].inherits[0]
+datatype[3].documenttype[0].headerstruct -1910204744
+datatype[3].documenttype[0].bodystruct 993120973
+datatype[4].id -1801920207
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name music_summary.header
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].field[32]
+datatype[4].structtype[0].field[0].name sddocname
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[0].datatype 2
+datatype[4].structtype[0].field[1].name bgndata
+datatype[4].structtype[0].field[1].id[0]
+datatype[4].structtype[0].field[1].datatype 2
+datatype[4].structtype[0].field[2].name sales
+datatype[4].structtype[0].field[2].id[0]
+datatype[4].structtype[0].field[2].datatype 0
+datatype[4].structtype[0].field[3].name pto
+datatype[4].structtype[0].field[3].id[0]
+datatype[4].structtype[0].field[3].datatype 0
+datatype[4].structtype[0].field[4].name mid
+datatype[4].structtype[0].field[4].id[0]
+datatype[4].structtype[0].field[4].datatype 0
+datatype[4].structtype[0].field[5].name ew
+datatype[4].structtype[0].field[5].id[0]
+datatype[4].structtype[0].field[5].datatype 2
+datatype[4].structtype[0].field[6].name surl
+datatype[4].structtype[0].field[6].id[0]
+datatype[4].structtype[0].field[6].datatype 2
+datatype[4].structtype[0].field[7].name userrate
+datatype[4].structtype[0].field[7].id[0]
+datatype[4].structtype[0].field[7].datatype 0
+datatype[4].structtype[0].field[8].name pid
+datatype[4].structtype[0].field[8].id[0]
+datatype[4].structtype[0].field[8].datatype 2
+datatype[4].structtype[0].field[9].name weight
+datatype[4].structtype[0].field[9].id[0]
+datatype[4].structtype[0].field[9].datatype 1
+datatype[4].structtype[0].field[10].name url
+datatype[4].structtype[0].field[10].id[0]
+datatype[4].structtype[0].field[10].datatype 2
+datatype[4].structtype[0].field[11].name isbn
+datatype[4].structtype[0].field[11].id[0]
+datatype[4].structtype[0].field[11].datatype 2
+datatype[4].structtype[0].field[12].name fmt
+datatype[4].structtype[0].field[12].id[0]
+datatype[4].structtype[0].field[12].datatype 2
+datatype[4].structtype[0].field[13].name albumid
+datatype[4].structtype[0].field[13].id[0]
+datatype[4].structtype[0].field[13].datatype 2
+datatype[4].structtype[0].field[14].name disp_song
+datatype[4].structtype[0].field[14].id[0]
+datatype[4].structtype[0].field[14].datatype 2
+datatype[4].structtype[0].field[15].name song
+datatype[4].structtype[0].field[15].id[0]
+datatype[4].structtype[0].field[15].datatype 2
+datatype[4].structtype[0].field[16].name pfrom
+datatype[4].structtype[0].field[16].id[0]
+datatype[4].structtype[0].field[16].datatype 0
+datatype[4].structtype[0].field[17].name bgnpfrom
+datatype[4].structtype[0].field[17].id[0]
+datatype[4].structtype[0].field[17].datatype 1
+datatype[4].structtype[0].field[18].name categories
+datatype[4].structtype[0].field[18].id[0]
+datatype[4].structtype[0].field[18].datatype 2
+datatype[4].structtype[0].field[19].name data
+datatype[4].structtype[0].field[19].id[0]
+datatype[4].structtype[0].field[19].datatype 2
+datatype[4].structtype[0].field[20].name numreview
+datatype[4].structtype[0].field[20].id[0]
+datatype[4].structtype[0].field[20].datatype 0
+datatype[4].structtype[0].field[21].name bgnsellers
+datatype[4].structtype[0].field[21].id[0]
+datatype[4].structtype[0].field[21].datatype 0
+datatype[4].structtype[0].field[22].name image
+datatype[4].structtype[0].field[22].id[0]
+datatype[4].structtype[0].field[22].datatype 2
+datatype[4].structtype[0].field[23].name artist
+datatype[4].structtype[0].field[23].id[0]
+datatype[4].structtype[0].field[23].datatype 2
+datatype[4].structtype[0].field[24].name title
+datatype[4].structtype[0].field[24].id[0]
+datatype[4].structtype[0].field[24].datatype 2
+datatype[4].structtype[0].field[25].name newestedition
+datatype[4].structtype[0].field[25].id[0]
+datatype[4].structtype[0].field[25].datatype 0
+datatype[4].structtype[0].field[26].name bgnpto
+datatype[4].structtype[0].field[26].id[0]
+datatype[4].structtype[0].field[26].datatype 2
+datatype[4].structtype[0].field[27].name year
+datatype[4].structtype[0].field[27].id[0]
+datatype[4].structtype[0].field[27].datatype 0
+datatype[4].structtype[0].field[28].name did
+datatype[4].structtype[0].field[28].id[0]
+datatype[4].structtype[0].field[28].datatype 0
+datatype[4].structtype[0].field[29].name scorekey
+datatype[4].structtype[0].field[29].id[0]
+datatype[4].structtype[0].field[29].datatype 0
+datatype[4].structtype[0].field[30].name cbid
+datatype[4].structtype[0].field[30].id[0]
+datatype[4].structtype[0].field[30].datatype 0
+datatype[4].structtype[0].field[31].name ranklog
+datatype[4].structtype[0].field[31].id[0]
+datatype[4].structtype[0].field[31].datatype 2
+datatype[4].documenttype[0]
+datatype[5].id -1728551034
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name music_summary.body
+datatype[5].structtype[0].version 0
+datatype[5].structtype[0].field[0]
+datatype[5].documenttype[0]
+datatype[6].id 1601149518
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[0]
+datatype[6].documenttype[1]
+datatype[6].documenttype[0].name music_summary
+datatype[6].documenttype[0].version 0
+datatype[6].documenttype[0].inherits[0]
+datatype[6].documenttype[0].headerstruct -1801920207
+datatype[6].documenttype[0].bodystruct -1728551034
+datatype[7].id 1000775434
+datatype[7].arraytype[1]
+datatype[7].arraytype[0].datatype 2
+datatype[7].weightedsettype[0]
+datatype[7].structtype[0]
+datatype[7].documenttype[0]
+datatype[8].id 1509154821
+datatype[8].arraytype[0]
+datatype[8].weightedsettype[0]
+datatype[8].structtype[1]
+datatype[8].structtype[0].name music_index.header
+datatype[8].structtype[0].version 0
+datatype[8].structtype[0].field[16]
+datatype[8].structtype[0].field[0].name sddocname
+datatype[8].structtype[0].field[0].id[0]
+datatype[8].structtype[0].field[0].datatype 1000775434
+datatype[8].structtype[0].field[1].name sales
+datatype[8].structtype[0].field[1].id[0]
+datatype[8].structtype[0].field[1].datatype -1216487824
+datatype[8].structtype[0].field[2].name pto
+datatype[8].structtype[0].field[2].id[0]
+datatype[8].structtype[0].field[2].datatype -1216487824
+datatype[8].structtype[0].field[3].name keys
+datatype[8].structtype[0].field[3].id[0]
+datatype[8].structtype[0].field[3].datatype 1000775434
+datatype[8].structtype[0].field[4].name mid
+datatype[8].structtype[0].field[4].id[0]
+datatype[8].structtype[0].field[4].datatype -1216487824
+datatype[8].structtype[0].field[5].name ew
+datatype[8].structtype[0].field[5].id[0]
+datatype[8].structtype[0].field[5].datatype 1000775434
+datatype[8].structtype[0].field[6].name fmt
+datatype[8].structtype[0].field[6].id[0]
+datatype[8].structtype[0].field[6].datatype 1000775434
+datatype[8].structtype[0].field[7].name song
+datatype[8].structtype[0].field[7].id[0]
+datatype[8].structtype[0].field[7].datatype 1000775434
+datatype[8].structtype[0].field[8].name categories
+datatype[8].structtype[0].field[8].id[0]
+datatype[8].structtype[0].field[8].datatype 1000775434
+datatype[8].structtype[0].field[9].name artist
+datatype[8].structtype[0].field[9].id[0]
+datatype[8].structtype[0].field[9].datatype 1000775434
+datatype[8].structtype[0].field[10].name title
+datatype[8].structtype[0].field[10].id[0]
+datatype[8].structtype[0].field[10].datatype 1000775434
+datatype[8].structtype[0].field[11].name newestedition
+datatype[8].structtype[0].field[11].id[0]
+datatype[8].structtype[0].field[11].datatype -1216487824
+datatype[8].structtype[0].field[12].name year
+datatype[8].structtype[0].field[12].id[0]
+datatype[8].structtype[0].field[12].datatype -1216487824
+datatype[8].structtype[0].field[13].name did
+datatype[8].structtype[0].field[13].id[0]
+datatype[8].structtype[0].field[13].datatype -1216487824
+datatype[8].structtype[0].field[14].name scorekey
+datatype[8].structtype[0].field[14].id[0]
+datatype[8].structtype[0].field[14].datatype -1216487824
+datatype[8].structtype[0].field[15].name cbid
+datatype[8].structtype[0].field[15].id[0]
+datatype[8].structtype[0].field[15].datatype -1216487824
+datatype[8].documenttype[0]
+datatype[9].id -1997730982
+datatype[9].arraytype[0]
+datatype[9].weightedsettype[0]
+datatype[9].structtype[1]
+datatype[9].structtype[0].name music_index.body
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].field[0]
+datatype[9].documenttype[0]
+datatype[10].id 2108744186
+datatype[10].arraytype[0]
+datatype[10].weightedsettype[0]
+datatype[10].structtype[0]
+datatype[10].documenttype[1]
+datatype[10].documenttype[0].name music_index
+datatype[10].documenttype[0].version 0
+datatype[10].documenttype[0].inherits[0]
+datatype[10].documenttype[0].headerstruct 1509154821
+datatype[10].documenttype[0].bodystruct -1997730982
+datatype[11].id -1497398149
+datatype[11].arraytype[0]
+datatype[11].weightedsettype[0]
+datatype[11].structtype[1]
+datatype[11].structtype[0].name music_attribute.header
+datatype[11].structtype[0].version 0
+datatype[11].structtype[0].field[10]
+datatype[11].structtype[0].field[0].name sales
+datatype[11].structtype[0].field[0].id[0]
+datatype[11].structtype[0].field[0].datatype 0
+datatype[11].structtype[0].field[1].name pto
+datatype[11].structtype[0].field[1].id[0]
+datatype[11].structtype[0].field[1].datatype 0
+datatype[11].structtype[0].field[2].name mid
+datatype[11].structtype[0].field[2].id[0]
+datatype[11].structtype[0].field[2].datatype -1216487824
+datatype[11].structtype[0].field[3].name weight
+datatype[11].structtype[0].field[3].id[0]
+datatype[11].structtype[0].field[3].datatype 1
+datatype[11].structtype[0].field[4].name bgnpfrom
+datatype[11].structtype[0].field[4].id[0]
+datatype[11].structtype[0].field[4].datatype 1
+datatype[11].structtype[0].field[5].name artistspid
+datatype[11].structtype[0].field[5].id[0]
+datatype[11].structtype[0].field[5].datatype 18
+datatype[11].structtype[0].field[6].name newestedition
+datatype[11].structtype[0].field[6].id[0]
+datatype[11].structtype[0].field[6].datatype 0
+datatype[11].structtype[0].field[7].name year
+datatype[11].structtype[0].field[7].id[0]
+datatype[11].structtype[0].field[7].datatype -1216487824
+datatype[11].structtype[0].field[8].name did
+datatype[11].structtype[0].field[8].id[0]
+datatype[11].structtype[0].field[8].datatype 0
+datatype[11].structtype[0].field[9].name cbid
+datatype[11].structtype[0].field[9].id[0]
+datatype[11].structtype[0].field[9].datatype 0
+datatype[11].documenttype[0]
+datatype[12].id 1243829584
+datatype[12].arraytype[0]
+datatype[12].weightedsettype[0]
+datatype[12].structtype[1]
+datatype[12].structtype[0].name music_attribute.body
+datatype[12].structtype[0].version 0
+datatype[12].structtype[0].field[0]
+datatype[12].documenttype[0]
+datatype[13].id 1990571588
+datatype[13].arraytype[0]
+datatype[13].weightedsettype[0]
+datatype[13].structtype[0]
+datatype[13].documenttype[1]
+datatype[13].documenttype[0].name music_attribute
+datatype[13].documenttype[0].version 0
+datatype[13].documenttype[0].inherits[0]
+datatype[13].documenttype[0].headerstruct -1497398149
+datatype[13].documenttype[0].bodystruct 1243829584
+datatype[14].id -592896846
+datatype[14].arraytype[0]
+datatype[14].weightedsettype[0]
+datatype[14].structtype[1]
+datatype[14].structtype[0].name indexingdocument.header
+datatype[14].structtype[0].version 0
+datatype[14].structtype[0].field[3]
+datatype[14].structtype[0].field[0].name index
+datatype[14].structtype[0].field[0].id[0]
+datatype[14].structtype[0].field[0].datatype 8
+datatype[14].structtype[0].field[1].name summary
+datatype[14].structtype[0].field[1].id[0]
+datatype[14].structtype[0].field[1].datatype 8
+datatype[14].structtype[0].field[2].name attribute
+datatype[14].structtype[0].field[2].id[0]
+datatype[14].structtype[0].field[2].datatype 8
+datatype[14].documenttype[0]
+datatype[15].id -2093772985
+datatype[15].arraytype[0]
+datatype[15].weightedsettype[0]
+datatype[15].structtype[1]
+datatype[15].structtype[0].name indexingdocument.body
+datatype[15].structtype[0].version 0
+datatype[15].structtype[0].field[0]
+datatype[15].documenttype[0]
+datatype[16].id -1831281171
+datatype[16].arraytype[0]
+datatype[16].weightedsettype[0]
+datatype[16].structtype[0]
+datatype[16].documenttype[1]
+datatype[16].documenttype[0].name indexingdocument
+datatype[16].documenttype[0].version 0
+datatype[16].documenttype[0].inherits[0]
+datatype[16].documenttype[0].headerstruct -592896846
+datatype[16].documenttype[0].bodystruct -2093772985
diff --git a/indexinglanguage/src/test/cfg/fileio/indexingdocument.cfg b/indexinglanguage/src/test/cfg/fileio/indexingdocument.cfg
new file mode 100644
index 00000000000..d41a56f65fb
--- /dev/null
+++ b/indexinglanguage/src/test/cfg/fileio/indexingdocument.cfg
@@ -0,0 +1,166 @@
+indexingdoc[1]
+indexingdoc[0].name music
+indexingdoc[0].source[1]
+indexingdoc[0].source[0] music
+indexingdoc[0].context[16]
+indexingdoc[0].context[0].fullname sddocname.sddocname
+indexingdoc[0].context[0].shortname sddocname
+indexingdoc[0].context[0].datatype 2
+indexingdoc[0].context[1].fullname sales.sales
+indexingdoc[0].context[1].shortname sales
+indexingdoc[0].context[1].datatype 0
+indexingdoc[0].context[2].fullname pto.pto
+indexingdoc[0].context[2].shortname pto
+indexingdoc[0].context[2].datatype 0
+indexingdoc[0].context[3].fullname default.keys
+indexingdoc[0].context[3].shortname keys
+indexingdoc[0].context[3].datatype 2
+indexingdoc[0].context[4].fullname mid.mid
+indexingdoc[0].context[4].shortname mid
+indexingdoc[0].context[4].datatype 0
+indexingdoc[0].context[5].fullname default.ew
+indexingdoc[0].context[5].shortname ew
+indexingdoc[0].context[5].datatype 2
+indexingdoc[0].context[6].fullname fmt.fmt
+indexingdoc[0].context[6].shortname fmt
+indexingdoc[0].context[6].datatype 2
+indexingdoc[0].context[7].fullname default.song
+indexingdoc[0].context[7].shortname song
+indexingdoc[0].context[7].datatype 2
+indexingdoc[0].context[8].fullname categories.categories
+indexingdoc[0].context[8].shortname categories
+indexingdoc[0].context[8].datatype 2
+indexingdoc[0].context[9].fullname default.artist
+indexingdoc[0].context[9].shortname artist
+indexingdoc[0].context[9].datatype 2
+indexingdoc[0].context[10].fullname default.title
+indexingdoc[0].context[10].shortname title
+indexingdoc[0].context[10].datatype 2
+indexingdoc[0].context[11].fullname newestedition.newestedition
+indexingdoc[0].context[11].shortname newestedition
+indexingdoc[0].context[11].datatype 0
+indexingdoc[0].context[12].fullname year.year
+indexingdoc[0].context[12].shortname year
+indexingdoc[0].context[12].datatype 0
+indexingdoc[0].context[13].fullname did.did
+indexingdoc[0].context[13].shortname did
+indexingdoc[0].context[13].datatype 0
+indexingdoc[0].context[14].fullname scorekey.scorekey
+indexingdoc[0].context[14].shortname scorekey
+indexingdoc[0].context[14].datatype 0
+indexingdoc[0].context[15].fullname cbid.cbid
+indexingdoc[0].context[15].shortname cbid
+indexingdoc[0].context[15].datatype 0
+indexingdoc[0].summary[32]
+indexingdoc[0].summary[0].name sddocname
+indexingdoc[0].summary[0].datatype 2
+indexingdoc[0].summary[1].name bgndata
+indexingdoc[0].summary[1].datatype 2
+indexingdoc[0].summary[2].name sales
+indexingdoc[0].summary[2].datatype 0
+indexingdoc[0].summary[3].name pto
+indexingdoc[0].summary[3].datatype 0
+indexingdoc[0].summary[4].name mid
+indexingdoc[0].summary[4].datatype 0
+indexingdoc[0].summary[5].name ew
+indexingdoc[0].summary[5].datatype 2
+indexingdoc[0].summary[6].name surl
+indexingdoc[0].summary[6].datatype 2
+indexingdoc[0].summary[7].name userrate
+indexingdoc[0].summary[7].datatype 0
+indexingdoc[0].summary[8].name pid
+indexingdoc[0].summary[8].datatype 2
+indexingdoc[0].summary[9].name weight
+indexingdoc[0].summary[9].datatype 1
+indexingdoc[0].summary[10].name url
+indexingdoc[0].summary[10].datatype 2
+indexingdoc[0].summary[11].name isbn
+indexingdoc[0].summary[11].datatype 2
+indexingdoc[0].summary[12].name fmt
+indexingdoc[0].summary[12].datatype 2
+indexingdoc[0].summary[13].name albumid
+indexingdoc[0].summary[13].datatype 2
+indexingdoc[0].summary[14].name disp_song
+indexingdoc[0].summary[14].datatype 2
+indexingdoc[0].summary[15].name song
+indexingdoc[0].summary[15].datatype 2
+indexingdoc[0].summary[16].name pfrom
+indexingdoc[0].summary[16].datatype 0
+indexingdoc[0].summary[17].name bgnpfrom
+indexingdoc[0].summary[17].datatype 1
+indexingdoc[0].summary[18].name categories
+indexingdoc[0].summary[18].datatype 2
+indexingdoc[0].summary[19].name data
+indexingdoc[0].summary[19].datatype 2
+indexingdoc[0].summary[20].name numreview
+indexingdoc[0].summary[20].datatype 0
+indexingdoc[0].summary[21].name bgnsellers
+indexingdoc[0].summary[21].datatype 0
+indexingdoc[0].summary[22].name image
+indexingdoc[0].summary[22].datatype 2
+indexingdoc[0].summary[23].name artist
+indexingdoc[0].summary[23].datatype 2
+indexingdoc[0].summary[24].name title
+indexingdoc[0].summary[24].datatype 2
+indexingdoc[0].summary[25].name newestedition
+indexingdoc[0].summary[25].datatype 0
+indexingdoc[0].summary[26].name bgnpto
+indexingdoc[0].summary[26].datatype 2
+indexingdoc[0].summary[27].name year
+indexingdoc[0].summary[27].datatype 0
+indexingdoc[0].summary[28].name did
+indexingdoc[0].summary[28].datatype 0
+indexingdoc[0].summary[29].name scorekey
+indexingdoc[0].summary[29].datatype 0
+indexingdoc[0].summary[30].name cbid
+indexingdoc[0].summary[30].datatype 0
+indexingdoc[0].summary[31].name ranklog
+indexingdoc[0].summary[31].datatype 2
+indexingdoc[0].attribute[13]
+indexingdoc[0].attribute[0].name sales
+indexingdoc[0].attribute[0].datatype 0
+indexingdoc[0].attribute[0].containertype SINGLE
+indexingdoc[0].attribute[1].name pto
+indexingdoc[0].attribute[1].datatype 0
+indexingdoc[0].attribute[1].containertype SINGLE
+indexingdoc[0].attribute[2].name mid
+indexingdoc[0].attribute[2].datatype 0
+indexingdoc[0].attribute[2].containertype ARRAY
+indexingdoc[0].attribute[3].name weight
+indexingdoc[0].attribute[3].datatype 1
+indexingdoc[0].attribute[3].containertype SINGLE
+indexingdoc[0].attribute[4].name bgnpfrom
+indexingdoc[0].attribute[4].datatype 1
+indexingdoc[0].attribute[4].containertype SINGLE
+indexingdoc[0].attribute[5].name artistspid
+indexingdoc[0].attribute[5].datatype 2
+indexingdoc[0].attribute[5].containertype WEIGHTEDSET
+indexingdoc[0].attribute[5].containerproperty[0]
+indexingdoc[0].attribute[6].name newestedition
+indexingdoc[0].attribute[6].datatype 0
+indexingdoc[0].attribute[6].containertype SINGLE
+indexingdoc[0].attribute[7].name year
+indexingdoc[0].attribute[7].datatype 0
+indexingdoc[0].attribute[7].containertype ARRAY
+indexingdoc[0].attribute[8].name did
+indexingdoc[0].attribute[8].datatype 0
+indexingdoc[0].attribute[8].containertype SINGLE
+indexingdoc[0].attribute[9].name cbid
+indexingdoc[0].attribute[9].datatype 0
+indexingdoc[0].attribute[9].containertype SINGLE
+indexingdoc[0].attribute[10].name artistspid_c
+indexingdoc[0].attribute[10].datatype 2
+indexingdoc[0].attribute[10].containertype WEIGHTEDSET
+indexingdoc[0].attribute[10].containerproperty[1]
+indexingdoc[0].attribute[10].containerproperty[0].propertyname "createifnonexistant"
+indexingdoc[0].attribute[11].name artistspid_r
+indexingdoc[0].attribute[11].datatype 2
+indexingdoc[0].attribute[11].containertype WEIGHTEDSET
+indexingdoc[0].attribute[11].containerproperty[1]
+indexingdoc[0].attribute[11].containerproperty[0].propertyname "removeifzero"
+indexingdoc[0].attribute[12].name artistspid_c_r
+indexingdoc[0].attribute[12].datatype 2
+indexingdoc[0].attribute[12].containertype WEIGHTEDSET
+indexingdoc[0].attribute[12].containerproperty[2]
+indexingdoc[0].attribute[12].containerproperty[0].propertyname "createifnonexistant"
+indexingdoc[0].attribute[12].containerproperty[1].propertyname "removeifzero"
diff --git a/indexinglanguage/src/test/cfg/indexingdocument.cfg b/indexinglanguage/src/test/cfg/indexingdocument.cfg
new file mode 100644
index 00000000000..5bebe6e9d22
--- /dev/null
+++ b/indexinglanguage/src/test/cfg/indexingdocument.cfg
@@ -0,0 +1,210 @@
+indexingdoc[1]
+indexingdoc[0].name music
+indexingdoc[0].source[1]
+indexingdoc[0].source[0] music
+indexingdoc[0].context[19]
+indexingdoc[0].context[0].fullname sddocname.sddocname
+indexingdoc[0].context[0].shortname sddocname
+indexingdoc[0].context[0].datatype 2
+indexingdoc[0].context[1].fullname sales.sales
+indexingdoc[0].context[1].shortname sales
+indexingdoc[0].context[1].datatype 0
+indexingdoc[0].context[2].fullname pto.pto
+indexingdoc[0].context[2].shortname pto
+indexingdoc[0].context[2].datatype 0
+indexingdoc[0].context[3].fullname default.keys
+indexingdoc[0].context[3].shortname keys
+indexingdoc[0].context[3].datatype 2
+indexingdoc[0].context[4].fullname mid.mid
+indexingdoc[0].context[4].shortname mid
+indexingdoc[0].context[4].datatype 0
+indexingdoc[0].context[5].fullname default.ew
+indexingdoc[0].context[5].shortname ew
+indexingdoc[0].context[5].datatype 2
+indexingdoc[0].context[6].fullname fmt.fmt
+indexingdoc[0].context[6].shortname fmt
+indexingdoc[0].context[6].datatype 2
+indexingdoc[0].context[7].fullname default.song
+indexingdoc[0].context[7].shortname song
+indexingdoc[0].context[7].datatype 2
+indexingdoc[0].context[8].fullname categories.categories
+indexingdoc[0].context[8].shortname categories
+indexingdoc[0].context[8].datatype 2
+indexingdoc[0].context[9].fullname default.artist
+indexingdoc[0].context[9].shortname artist
+indexingdoc[0].context[9].datatype 2
+indexingdoc[0].context[10].fullname artistspid3.artistspid3
+indexingdoc[0].context[10].shortname artistspid3
+indexingdoc[0].context[10].datatype 2
+indexingdoc[0].context[11].fullname default.title
+indexingdoc[0].context[11].shortname title
+indexingdoc[0].context[11].datatype 2
+indexingdoc[0].context[12].fullname newestedition.newestedition
+indexingdoc[0].context[12].shortname newestedition
+indexingdoc[0].context[12].datatype 0
+indexingdoc[0].context[13].fullname year.year
+indexingdoc[0].context[13].shortname year
+indexingdoc[0].context[13].datatype 0
+indexingdoc[0].context[14].fullname endyear.endyear
+indexingdoc[0].context[14].shortname endyear
+indexingdoc[0].context[14].datatype 0
+indexingdoc[0].context[15].fullname did.did
+indexingdoc[0].context[15].shortname did
+indexingdoc[0].context[15].datatype 0
+indexingdoc[0].context[16].fullname scorekey.scorekey
+indexingdoc[0].context[16].shortname scorekey
+indexingdoc[0].context[16].datatype 0
+indexingdoc[0].context[17].fullname cbid.cbid
+indexingdoc[0].context[17].shortname cbid
+indexingdoc[0].context[17].datatype 0
+indexingdoc[0].context[18].fullname titles.titles
+indexingdoc[0].context[18].shortname titles
+indexingdoc[0].context[18].datatype 2
+indexingdoc[0].summary[40]
+indexingdoc[0].summary[0].name distance
+indexingdoc[0].summary[0].datatype 0
+indexingdoc[0].summary[1].name sddocname
+indexingdoc[0].summary[1].datatype 2
+indexingdoc[0].summary[2].name bgndata
+indexingdoc[0].summary[2].datatype 2
+indexingdoc[0].summary[3].name sales
+indexingdoc[0].summary[3].datatype 0
+indexingdoc[0].summary[4].name pto
+indexingdoc[0].summary[4].datatype 0
+indexingdoc[0].summary[5].name mid
+indexingdoc[0].summary[5].datatype 2
+indexingdoc[0].summary[6].name ew
+indexingdoc[0].summary[6].datatype 2
+indexingdoc[0].summary[7].name surl
+indexingdoc[0].summary[7].datatype 2
+indexingdoc[0].summary[8].name userrate
+indexingdoc[0].summary[8].datatype 0
+indexingdoc[0].summary[9].name pid
+indexingdoc[0].summary[9].datatype 2
+indexingdoc[0].summary[10].name weight
+indexingdoc[0].summary[10].datatype 1
+indexingdoc[0].summary[11].name url
+indexingdoc[0].summary[11].datatype 2
+indexingdoc[0].summary[12].name isbn
+indexingdoc[0].summary[12].datatype 2
+indexingdoc[0].summary[13].name fmt
+indexingdoc[0].summary[13].datatype 2
+indexingdoc[0].summary[14].name albumid
+indexingdoc[0].summary[14].datatype 2
+indexingdoc[0].summary[15].name disp_song
+indexingdoc[0].summary[15].datatype 2
+indexingdoc[0].summary[16].name song
+indexingdoc[0].summary[16].datatype 2
+indexingdoc[0].summary[17].name pfrom
+indexingdoc[0].summary[17].datatype 0
+indexingdoc[0].summary[18].name bgnpfrom
+indexingdoc[0].summary[18].datatype 1
+indexingdoc[0].summary[19].name categories
+indexingdoc[0].summary[19].datatype 2
+indexingdoc[0].summary[20].name data
+indexingdoc[0].summary[20].datatype 2
+indexingdoc[0].summary[21].name numreview
+indexingdoc[0].summary[21].datatype 0
+indexingdoc[0].summary[22].name bgnsellers
+indexingdoc[0].summary[22].datatype 0
+indexingdoc[0].summary[23].name image
+indexingdoc[0].summary[23].datatype 2
+indexingdoc[0].summary[24].name artist
+indexingdoc[0].summary[24].datatype 2
+indexingdoc[0].summary[25].name artistspid
+indexingdoc[0].summary[25].datatype 2
+indexingdoc[0].summary[26].name artistspid3
+indexingdoc[0].summary[26].datatype 2
+indexingdoc[0].summary[27].name title
+indexingdoc[0].summary[27].datatype 2
+indexingdoc[0].summary[28].name newestedition
+indexingdoc[0].summary[28].datatype 0
+indexingdoc[0].summary[29].name bgnpto
+indexingdoc[0].summary[29].datatype 2
+indexingdoc[0].summary[30].name year
+indexingdoc[0].summary[30].datatype 2
+indexingdoc[0].summary[31].name endyear
+indexingdoc[0].summary[31].datatype 2
+indexingdoc[0].summary[32].name did
+indexingdoc[0].summary[32].datatype 0
+indexingdoc[0].summary[33].name scorekey
+indexingdoc[0].summary[33].datatype 0
+indexingdoc[0].summary[34].name cbid
+indexingdoc[0].summary[34].datatype 0
+indexingdoc[0].summary[35].name titles
+indexingdoc[0].summary[35].datatype 2
+indexingdoc[0].summary[36].name ranklog
+indexingdoc[0].summary[36].datatype 2
+indexingdoc[0].summary[37].name rankfeatures
+indexingdoc[0].summary[37].datatype 2
+indexingdoc[0].summary[38].name summaryfeatures
+indexingdoc[0].summary[38].datatype 2
+indexingdoc[0].summary[39].name documentid
+indexingdoc[0].summary[39].datatype 2
+indexingdoc[0].attribute[22]
+indexingdoc[0].attribute[0].name sales
+indexingdoc[0].attribute[0].datatype 0
+indexingdoc[0].attribute[0].containertype SINGLE
+indexingdoc[0].attribute[1].name pto
+indexingdoc[0].attribute[1].datatype 0
+indexingdoc[0].attribute[1].containertype SINGLE
+indexingdoc[0].attribute[2].name mid
+indexingdoc[0].attribute[2].datatype 0
+indexingdoc[0].attribute[2].containertype ARRAY
+indexingdoc[0].attribute[3].name ew
+indexingdoc[0].attribute[3].datatype 2
+indexingdoc[0].attribute[3].containertype SINGLE
+indexingdoc[0].attribute[4].name weight
+indexingdoc[0].attribute[4].datatype 1
+indexingdoc[0].attribute[4].containertype SINGLE
+indexingdoc[0].attribute[5].name bgnpfrom
+indexingdoc[0].attribute[5].datatype 1
+indexingdoc[0].attribute[5].containertype SINGLE
+indexingdoc[0].attribute[6].name artist
+indexingdoc[0].attribute[6].datatype 2
+indexingdoc[0].attribute[6].containertype SINGLE
+indexingdoc[0].attribute[7].name artistspid
+indexingdoc[0].attribute[7].datatype 2
+indexingdoc[0].attribute[7].containertype WEIGHTEDSET
+indexingdoc[0].attribute[8].name artistspid2
+indexingdoc[0].attribute[8].datatype 1
+indexingdoc[0].attribute[8].containertype WEIGHTEDSET
+indexingdoc[0].attribute[9].name title
+indexingdoc[0].attribute[9].datatype 2
+indexingdoc[0].attribute[9].containertype SINGLE
+indexingdoc[0].attribute[10].name newestedition
+indexingdoc[0].attribute[10].datatype 0
+indexingdoc[0].attribute[10].containertype SINGLE
+indexingdoc[0].attribute[11].name year
+indexingdoc[0].attribute[11].datatype 0
+indexingdoc[0].attribute[11].containertype ARRAY
+indexingdoc[0].attribute[12].name endyear
+indexingdoc[0].attribute[12].datatype 0
+indexingdoc[0].attribute[12].containertype ARRAY
+indexingdoc[0].attribute[13].name did
+indexingdoc[0].attribute[13].datatype 0
+indexingdoc[0].attribute[13].containertype SINGLE
+indexingdoc[0].attribute[14].name cbid
+indexingdoc[0].attribute[14].datatype 0
+indexingdoc[0].attribute[14].containertype SINGLE
+indexingdoc[0].attribute[15].name noupdate
+indexingdoc[0].attribute[15].datatype 2
+indexingdoc[0].attribute[15].containertype SINGLE
+indexingdoc[0].attribute[16].name noupdate2
+indexingdoc[0].attribute[16].datatype 2
+indexingdoc[0].attribute[16].containertype SINGLE
+indexingdoc[0].attribute[17].name multiposition2d_position
+indexingdoc[0].attribute[17].datatype 4
+indexingdoc[0].attribute[17].containertype ARRAY
+indexingdoc[0].attribute[18].name extracategories
+indexingdoc[0].attribute[18].datatype 2
+indexingdoc[0].attribute[18].containertype ARRAY
+indexingdoc[0].attribute[19].name default_fieldlength
+indexingdoc[0].attribute[19].datatype 0
+indexingdoc[0].attribute[19].containertype ARRAY
+indexingdoc[0].attribute[20].name fmt_fieldlength
+indexingdoc[0].attribute[20].datatype 0
+indexingdoc[0].attribute[20].containertype SINGLE
+indexingdoc[0].attribute[21].name categories_fieldlength
+indexingdoc[0].attribute[21].datatype 0
+indexingdoc[0].attribute[21].containertype SINGLE
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentTestCase.java
new file mode 100644
index 00000000000..188dfb3f8f5
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentTestCase.java
@@ -0,0 +1,141 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@SuppressWarnings({ "unchecked" })
+public class DocumentTestCase {
+
+    @Test
+    public void requireThatArrayOfStructIsProcessedCorrectly() throws ParseException {
+        DocumentType docType = new DocumentType("my_input");
+        docType.addField(new Field("my_str", DataType.getArray(DataType.STRING)));
+        docType.addField(new Field("my_pos", DataType.getArray(PositionDataType.INSTANCE)));
+
+        Document doc = new Document(docType, "doc:scheme:");
+        Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING));
+        arr.add(new StringFieldValue("6;9"));
+        doc.setFieldValue("my_str", arr);
+
+        assertNotNull(doc = Expression.execute(Expression.fromString("input my_str | for_each { to_pos } | index my_pos"), doc));
+        assertNotNull(doc.getFieldValue("my_str"));
+        FieldValue val = doc.getFieldValue("my_pos");
+        assertNotNull(val);
+        assertEquals(DataType.getArray(PositionDataType.INSTANCE), val.getDataType());
+        assertTrue(val instanceof Array);
+        arr = (Array)val;
+        assertEquals(1, arr.size());
+        assertNotNull(val = arr.getFieldValue(0));
+        assertEquals(PositionDataType.INSTANCE, val.getDataType());
+        assertTrue(val instanceof Struct);
+        Struct pos = (Struct)val;
+        assertEquals(new IntegerFieldValue(6), PositionDataType.getXValue(pos));
+        assertEquals(new IntegerFieldValue(9), PositionDataType.getYValue(pos));
+    }
+
+    @Test
+    public void requireThatConcatenationWorks() throws ParseException {
+        DocumentType docType = new DocumentType("my_input");
+        docType.addField(new Field("arr_a", DataType.getArray(DataType.STRING)));
+        docType.addField(new Field("arr_b", DataType.getArray(DataType.STRING)));
+        docType.addField(new Field("out", DataType.getArray(DataType.STRING)));
+
+        Expression exp = Expression.fromString("input arr_a . input arr_b | index out");
+        {
+            Document doc = new Document(docType, "doc:scheme:");
+            assertNotNull(doc = Expression.execute(exp, doc));
+            FieldValue val = doc.getFieldValue("out");
+            assertNotNull(val);
+            assertEquals(DataType.getArray(DataType.STRING), val.getDataType());
+            assertTrue(val instanceof Array);
+            Array arr = (Array)val;
+            assertEquals(0, arr.size());
+        }
+        {
+            Document doc = new Document(docType, "doc:scheme:");
+            Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING));
+            arr.add(new StringFieldValue("a1"));
+            doc.setFieldValue("arr_a", arr);
+
+            assertNotNull(doc = Expression.execute(exp, doc));
+            FieldValue val = doc.getFieldValue("out");
+            assertNotNull(val);
+            assertEquals(DataType.getArray(DataType.STRING), val.getDataType());
+            assertTrue(val instanceof Array);
+            arr = (Array)val;
+            assertEquals(1, arr.size());
+        }
+        {
+            Document doc = new Document(docType, "doc:scheme:");
+            Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING));
+            arr.add(new StringFieldValue("a1"));
+            arr.add(new StringFieldValue("a2"));
+            doc.setFieldValue("arr_a", arr);
+            arr = new Array<StringFieldValue>(DataType.getArray(DataType.STRING));
+            arr.add(new StringFieldValue("b1"));
+            doc.setFieldValue("arr_b", arr);
+
+            assertNotNull(doc = Expression.execute(exp, doc));
+            FieldValue val = doc.getFieldValue("out");
+            assertNotNull(val);
+            assertEquals(DataType.getArray(DataType.STRING), val.getDataType());
+            assertTrue(val instanceof Array);
+            arr = (Array)val;
+            assertEquals(3, arr.size());
+        }
+    }
+
+    @Test
+    public void requireThatConcatenationOfEmbracedStatementsWorks() throws ParseException {
+        DocumentType docType = new DocumentType("my_input");
+        docType.addField(new Field("str_a", DataType.STRING));
+        docType.addField(new Field("str_b", DataType.STRING));
+        docType.addField(new Field("out", DataType.getArray(DataType.STRING)));
+
+        Expression exp = Expression.fromString("(input str_a | split ',') . (input str_b | split ',') | index out");
+        {
+            Document doc = new Document(docType, "doc:scheme:");
+            assertNotNull(doc = Expression.execute(exp, doc));
+            FieldValue val = doc.getFieldValue("out");
+            assertNotNull(val);
+            assertEquals(DataType.getArray(DataType.STRING), val.getDataType());
+            assertTrue(val instanceof Array);
+            Array arr = (Array)val;
+            assertEquals(0, arr.size());
+        }
+        {
+            Document doc = new Document(docType, "doc:scheme:");
+            doc.setFieldValue("str_a", new StringFieldValue("a1"));
+
+            assertNotNull(doc = Expression.execute(exp, doc));
+            FieldValue val = doc.getFieldValue("out");
+            assertNotNull(val);
+            assertEquals(DataType.getArray(DataType.STRING), val.getDataType());
+            assertTrue(val instanceof Array);
+            Array arr = (Array)val;
+            assertEquals(1, arr.size());
+        }
+        {
+            Document doc = new Document(docType, "doc:scheme:");
+            doc.setFieldValue("str_a", new StringFieldValue("a1,a2"));
+            doc.setFieldValue("str_b", new StringFieldValue("b1"));
+
+            assertNotNull(doc = Expression.execute(exp, doc));
+            FieldValue val = doc.getFieldValue("out");
+            assertNotNull(val);
+            assertEquals(DataType.getArray(DataType.STRING), val.getDataType());
+            assertTrue(val instanceof Array);
+            Array arr = (Array)val;
+            assertEquals(3, arr.size());
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToPathUpdateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToPathUpdateTestCase.java
new file mode 100644
index 00000000000..feef07a46bd
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToPathUpdateTestCase.java
@@ -0,0 +1,115 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DocumentToPathUpdateTestCase {
+
+    @Test
+    public void requireThatIntegerAssignIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField(new Field("my_int", DataType.INT));
+
+        FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "my_int", "", new IntegerFieldValue(69));
+        Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd);
+        assertNotNull(doc);
+        doc.setFieldValue("my_int", new IntegerFieldValue(96));
+
+        DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldPathUpdates().size());
+        assertNotNull(upd = docUpd.getFieldPathUpdates().get(0));
+
+        assertTrue(upd instanceof AssignFieldPathUpdate);
+        assertEquals("my_int", upd.getOriginalFieldPath());
+        assertEquals(new IntegerFieldValue(96), ((AssignFieldPathUpdate)upd).getNewValue());
+    }
+
+    @Test
+    public void requireThatStringAssignIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField(new Field("my_str", DataType.STRING));
+
+        FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "my_str", "", new StringFieldValue("69"));
+        Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd);
+        assertNotNull(doc);
+        doc.setFieldValue("my_str", new StringFieldValue("96"));
+
+        DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldPathUpdates().size());
+        assertNotNull(upd = docUpd.getFieldPathUpdates().get(0));
+
+        assertTrue(upd instanceof AssignFieldPathUpdate);
+        assertEquals("my_str", upd.getOriginalFieldPath());
+        assertEquals(new StringFieldValue("96"), ((AssignFieldPathUpdate)upd).getNewValue());
+    }
+
+    @Test
+    public void requireThatStructAssignIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        StructDataType structType = new StructDataType("my_struct");
+        structType.addField(new Field("b", DataType.INT));
+        docType.addField(new Field("a", structType));
+
+        Struct struct = structType.createFieldValue();
+        struct.setFieldValue("b", new IntegerFieldValue(69));
+        FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "a", "", struct);
+        Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("a");
+        assertTrue(obj instanceof Struct);
+        struct = (Struct)obj;
+        struct.setFieldValue("b", new IntegerFieldValue(96));
+
+        DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldPathUpdates().size());
+        assertNotNull(upd = docUpd.getFieldPathUpdates().get(0));
+
+        assertTrue(upd instanceof AssignFieldPathUpdate);
+        assertEquals("a", upd.getOriginalFieldPath());
+        assertEquals(struct, ((AssignFieldPathUpdate)upd).getNewValue());
+    }
+
+    @Test
+    public void requireThatStructElementAssignIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        StructDataType structType = new StructDataType("my_struct");
+        structType.addField(new Field("b", DataType.INT));
+        docType.addField(new Field("a", structType));
+
+        FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "a.b", "", new IntegerFieldValue(69));
+        Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("a");
+        assertTrue(obj instanceof Struct);
+        Struct struct = (Struct)obj;
+        struct.setFieldValue("b", new IntegerFieldValue(96));
+
+        DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldPathUpdates().size());
+        assertNotNull(upd = docUpd.getFieldPathUpdates().get(0));
+
+        assertTrue(upd instanceof AssignFieldPathUpdate);
+        assertEquals("a.b", upd.getOriginalFieldPath());
+        obj = ((AssignFieldPathUpdate)upd).getNewValue();
+        assertTrue(obj instanceof IntegerFieldValue);
+        assertEquals(96, ((IntegerFieldValue)obj).getInteger());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToValueUpdateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToValueUpdateTestCase.java
new file mode 100644
index 00000000000..ecf43080b4b
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToValueUpdateTestCase.java
@@ -0,0 +1,340 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.update.*;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@SuppressWarnings({ "unchecked" })
+public class DocumentToValueUpdateTestCase {
+
+    @Test
+    public void requireThatSddocnameFieldIsIgnored() {
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField(new Field("sddocname", DataType.STRING));
+
+        ValueUpdate valueUpd = ValueUpdate.createAssign(new StringFieldValue("69"));
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("sddocname"), valueUpd);
+        doc.setFieldValue("sddocname", new StringFieldValue("96"));
+
+        UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd);
+        assertNull(adapter.getOutput());
+    }
+
+    @Test
+    public void requireThatEmptyFieldUpdatesAreIgnored() {
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField(new Field("my_int", DataType.INT));
+        docType.addField(new Field("my_str", DataType.STRING));
+
+        ValueUpdate valueUpd = ValueUpdate.createAssign(new IntegerFieldValue(69));
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_int"), valueUpd);
+        doc.setFieldValue("my_int", new IntegerFieldValue(96));
+
+        UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd);
+        DocumentUpdate docUpd = adapter.getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldUpdates().size());
+        assertEquals("my_int", docUpd.getFieldUpdate(0).getField().getName());
+    }
+
+    @Test
+    public void requireThatIntegerAssignIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField(new Field("my_int", DataType.INT));
+
+        ValueUpdate valueUpd = ValueUpdate.createAssign(new IntegerFieldValue(69));
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_int"), valueUpd);
+        doc.setFieldValue("my_int", new IntegerFieldValue(96));
+
+        UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd);
+        DocumentUpdate docUpd = adapter.getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldUpdates().size());
+
+        FieldUpdate fieldUpd = docUpd.getFieldUpdate(0);
+        assertNotNull(fieldUpd);
+
+        assertEquals(docType.getField("my_int"), fieldUpd.getField());
+        assertEquals(1, fieldUpd.getValueUpdates().size());
+        assertNotNull(valueUpd = fieldUpd.getValueUpdate(0));
+        assertTrue(valueUpd instanceof AssignValueUpdate);
+        assertEquals(new IntegerFieldValue(96), valueUpd.getValue());
+    }
+
+    @Test
+    public void requireThatClearFieldIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField(new Field("my_int", DataType.INT));
+
+        ValueUpdate valueUpd = ValueUpdate.createClear();
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_int"), valueUpd);
+        assertNotNull(doc.getFieldValue("my_int"));
+
+        UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd);
+        DocumentUpdate docUpd = adapter.getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldUpdates().size());
+
+        FieldUpdate fieldUpd = docUpd.getFieldUpdate(0);
+        assertNotNull(fieldUpd);
+
+        assertEquals(docType.getField("my_int"), fieldUpd.getField());
+        assertEquals(1, fieldUpd.getValueUpdates().size());
+        assertNotNull(valueUpd = fieldUpd.getValueUpdate(0));
+        assertTrue(valueUpd instanceof ClearValueUpdate);
+    }
+
+    @Test
+    public void requireThatStringAssignIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField(new Field("my_str", DataType.STRING));
+
+        ValueUpdate valueUpd = ValueUpdate.createAssign(new StringFieldValue("69"));
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_str"), valueUpd);
+        doc.setFieldValue("my_str", new StringFieldValue("96"));
+
+        UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd);
+        DocumentUpdate docUpd = adapter.getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldUpdates().size());
+
+        FieldUpdate fieldUpd = docUpd.getFieldUpdate(0);
+        assertNotNull(fieldUpd);
+
+        assertEquals(docType.getField("my_str"), fieldUpd.getField());
+        assertEquals(1, fieldUpd.getValueUpdates().size());
+        assertNotNull(valueUpd = fieldUpd.getValueUpdate(0));
+        assertTrue(valueUpd instanceof AssignValueUpdate);
+        assertEquals(new StringFieldValue("96"), valueUpd.getValue());
+    }
+
+    @Test
+    public void requireThatStructAssignIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        StructDataType structType = new StructDataType("my_struct");
+        structType.addField(new Field("b", DataType.INT));
+        docType.addField(new Field("a", structType));
+
+        Struct struct = structType.createFieldValue();
+        struct.setFieldValue("b", new IntegerFieldValue(69));
+        ValueUpdate valueUpd = ValueUpdate.createAssign(struct);
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("a"), valueUpd);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("a");
+        assertTrue(obj instanceof Struct);
+        struct = (Struct)obj;
+        struct.setFieldValue("b", new IntegerFieldValue(96));
+
+        UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd);
+        DocumentUpdate docUpd = adapter.getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldUpdates().size());
+
+        FieldUpdate fieldUpd = docUpd.getFieldUpdate(0);
+        assertNotNull(fieldUpd);
+
+        assertEquals(docType.getField("a"), fieldUpd.getField());
+        assertEquals(1, fieldUpd.getValueUpdates().size());
+        assertNotNull(valueUpd = fieldUpd.getValueUpdate(0));
+        assertTrue(valueUpd instanceof AssignValueUpdate);
+        assertEquals(struct, valueUpd.getValue());
+    }
+
+    @Test
+    public void requireThatArrayElementAddIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        ArrayDataType arrType = DataType.getArray(DataType.STRING);
+        docType.addField(new Field("my_arr", arrType));
+
+        ValueUpdate valueUpd = ValueUpdate.createAdd(new StringFieldValue("foo"));
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_arr"), valueUpd);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("my_arr");
+        assertTrue(obj instanceof Array);
+        Array<StringFieldValue> arr = (Array<StringFieldValue>)obj;
+        arr.set(0, new StringFieldValue("bar"));
+
+        UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd);
+        DocumentUpdate docUpd = adapter.getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldUpdates().size());
+
+        FieldUpdate fieldUpd = docUpd.getFieldUpdate(0);
+        assertNotNull(fieldUpd);
+
+        assertEquals(docType.getField("my_arr"), fieldUpd.getField());
+        assertEquals(1, fieldUpd.getValueUpdates().size());
+        assertNotNull(valueUpd = fieldUpd.getValueUpdate(0));
+        assertTrue(valueUpd instanceof AddValueUpdate);
+        assertEquals(new StringFieldValue("bar"), valueUpd.getValue());
+    }
+
+    @Test
+    public void requireThatArrayElementRemoveIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        ArrayDataType arrType = DataType.getArray(DataType.STRING);
+        docType.addField(new Field("my_arr", arrType));
+
+        ValueUpdate valueUpd = ValueUpdate.createRemove(new StringFieldValue("foo"));
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_arr"), valueUpd);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("my_arr");
+        assertTrue(obj instanceof Array);
+        Array<StringFieldValue> arr = (Array<StringFieldValue>)obj;
+        arr.set(0, new StringFieldValue("bar"));
+
+        UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd);
+        DocumentUpdate docUpd = adapter.getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldUpdates().size());
+
+        FieldUpdate fieldUpd = docUpd.getFieldUpdate(0);
+        assertNotNull(fieldUpd);
+
+        assertEquals(docType.getField("my_arr"), fieldUpd.getField());
+        assertEquals(1, fieldUpd.getValueUpdates().size());
+        assertNotNull(valueUpd = fieldUpd.getValueUpdate(0));
+        assertTrue(valueUpd instanceof RemoveValueUpdate);
+        assertEquals(new StringFieldValue("bar"), valueUpd.getValue());
+    }
+
+    @Test
+    public void requireThatArrayOfStructElementAddIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        StructDataType structType = new StructDataType("my_struct");
+        structType.addField(new Field("b", DataType.INT));
+        docType.addField(new Field("a", DataType.getArray(structType)));
+
+        Struct struct = structType.createFieldValue();
+        struct.setFieldValue("b", new IntegerFieldValue(69));
+        ValueUpdate valueUpd = ValueUpdate.createAdd(struct);
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("a"), valueUpd);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("a");
+        assertTrue(obj instanceof Array);
+        Array<Struct> arr = (Array<Struct>)obj;
+        struct = structType.createFieldValue();
+        struct.setFieldValue("b", new IntegerFieldValue(96));
+        arr.set(0, struct);
+
+        UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd);
+        DocumentUpdate docUpd = adapter.getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldUpdates().size());
+
+        FieldUpdate fieldUpd = docUpd.getFieldUpdate(0);
+        assertNotNull(fieldUpd);
+
+        assertEquals(docType.getField("a"), fieldUpd.getField());
+        assertEquals(1, fieldUpd.getValueUpdates().size());
+        assertNotNull(valueUpd = fieldUpd.getValueUpdate(0));
+        assertTrue(valueUpd instanceof AddValueUpdate);
+        assertEquals(struct, valueUpd.getValue());
+    }
+
+    @Test
+    public void requireThatWsetElementAssignIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        WeightedSetDataType wsetType = DataType.getWeightedSet(DataType.STRING);
+        docType.addField(new Field("my_wset", wsetType));
+
+        ValueUpdate valueUpd = ValueUpdate.createMap(new StringFieldValue("foo"), ValueUpdate.createAssign(new IntegerFieldValue(69)));
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_wset"), valueUpd);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("my_wset");
+        assertTrue(obj instanceof WeightedSet);
+        WeightedSet<StringFieldValue> wset = (WeightedSet<StringFieldValue>)obj;
+        wset.put(new StringFieldValue("foo"), 96);
+
+        UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd);
+        DocumentUpdate docUpd = adapter.getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldUpdates().size());
+
+        FieldUpdate fieldUpd = docUpd.getFieldUpdate(0);
+        assertNotNull(fieldUpd);
+
+        assertEquals(docType.getField("my_wset"), fieldUpd.getField());
+        assertEquals(1, fieldUpd.getValueUpdates().size());
+        assertNotNull(valueUpd = fieldUpd.getValueUpdate(0));
+        assertTrue(valueUpd instanceof MapValueUpdate);
+        assertEquals(new StringFieldValue("foo"), valueUpd.getValue());
+        assertNotNull(valueUpd = ((MapValueUpdate)valueUpd).getUpdate());
+        assertTrue(valueUpd instanceof AssignValueUpdate);
+        assertEquals(new IntegerFieldValue(96), valueUpd.getValue());
+    }
+
+    @Test
+    public void requireThatWsetElementAddIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        WeightedSetDataType wsetType = DataType.getWeightedSet(DataType.STRING);
+        docType.addField(new Field("my_wset", wsetType));
+
+        ValueUpdate valueUpd = ValueUpdate.createAdd(new StringFieldValue("foo"), 69);
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_wset"), valueUpd);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("my_wset");
+        assertTrue(obj instanceof WeightedSet);
+        WeightedSet<StringFieldValue> wset = (WeightedSet<StringFieldValue>)obj;
+        wset.put(new StringFieldValue("foo"), 96);
+
+        UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd);
+        DocumentUpdate docUpd = adapter.getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldUpdates().size());
+
+        FieldUpdate fieldUpd = docUpd.getFieldUpdate(0);
+        assertNotNull(fieldUpd);
+
+        assertEquals(docType.getField("my_wset"), fieldUpd.getField());
+        assertEquals(1, fieldUpd.getValueUpdates().size());
+        assertNotNull(valueUpd = fieldUpd.getValueUpdate(0));
+        assertTrue(valueUpd instanceof AddValueUpdate);
+        assertEquals(new StringFieldValue("foo"), valueUpd.getValue());
+        assertEquals(96, ((AddValueUpdate)valueUpd).getWeight());
+    }
+
+    @Test
+    public void requireThatWsetElementRemoveIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        WeightedSetDataType wsetType = DataType.getWeightedSet(DataType.STRING);
+        docType.addField(new Field("my_wset", wsetType));
+
+        ValueUpdate valueUpd = ValueUpdate.createRemove(new StringFieldValue("foo"));
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_wset"), valueUpd);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("my_wset");
+        assertTrue(obj instanceof WeightedSet);
+        WeightedSet<StringFieldValue> wset = (WeightedSet<StringFieldValue>)obj;
+        wset.remove(new StringFieldValue("foo"));
+        wset.add(new StringFieldValue("bar"));
+
+        UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd);
+        DocumentUpdate docUpd = adapter.getOutput();
+        assertNotNull(docUpd);
+        assertEquals(1, docUpd.getFieldUpdates().size());
+
+        FieldUpdate fieldUpd = docUpd.getFieldUpdate(0);
+        assertNotNull(fieldUpd);
+
+        assertEquals(docType.getField("my_wset"), fieldUpd.getField());
+        assertEquals(1, fieldUpd.getValueUpdates().size());
+        assertNotNull(valueUpd = fieldUpd.getValueUpdate(0));
+        assertTrue(valueUpd instanceof RemoveValueUpdate);
+        assertEquals(new StringFieldValue("bar"), valueUpd.getValue());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentUpdateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentUpdateTestCase.java
new file mode 100644
index 00000000000..aec8d0332c2
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentUpdateTestCase.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.document.update.AddValueUpdate;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.ValueUpdate;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DocumentUpdateTestCase {
+
+    @Test
+    public void requireThatArrayOfStructElementAddIsProcessedCorrectly() throws ParseException {
+        DocumentType docType = new DocumentType("my_input");
+        docType.addField(new Field("my_str", DataType.getArray(DataType.STRING)));
+        docType.addField(new Field("my_pos", DataType.getArray(PositionDataType.INSTANCE)));
+
+        DocumentUpdate docUpdate = new DocumentUpdate(docType, "doc:scheme:");
+        docUpdate.addFieldUpdate(FieldUpdate.createAdd(docType.getField("my_str"), new StringFieldValue("6;9")));
+        docUpdate = Expression.execute(Expression.fromString("input my_str | for_each { to_pos } | index my_pos"), docUpdate);
+
+        assertNotNull(docUpdate);
+        assertEquals(0, docUpdate.getFieldPathUpdates().size());
+        assertEquals(1, docUpdate.getFieldUpdates().size());
+
+        FieldUpdate fieldUpd = docUpdate.getFieldUpdate(0);
+        assertNotNull(fieldUpd);
+        assertEquals(docType.getField("my_pos"), fieldUpd.getField());
+        assertEquals(1, fieldUpd.getValueUpdates().size());
+
+        ValueUpdate valueUpd = fieldUpd.getValueUpdate(0);
+        assertNotNull(valueUpd);
+        assertTrue(valueUpd instanceof AddValueUpdate);
+
+        Object val = valueUpd.getValue();
+        assertNotNull(val);
+        assertTrue(val instanceof Struct);
+        Struct pos = (Struct)val;
+        assertEquals(PositionDataType.INSTANCE, pos.getDataType());
+        assertEquals(new IntegerFieldValue(6), PositionDataType.getXValue(pos));
+        assertEquals(new IntegerFieldValue(9), PositionDataType.getYValue(pos));
+    }
+
+    @Test
+    public void requireThatCreateIfNonExistentFlagIsPropagated() throws ParseException {
+        DocumentType docType = new DocumentType("my_input");
+        docType.addField(new Field("my_str", DataType.getArray(DataType.STRING)));
+        DocumentUpdate upd = new DocumentUpdate(docType, "doc:scheme:");
+        upd.addFieldUpdate(FieldUpdate.createAdd(docType.getField("my_str"), new StringFieldValue("foo")));
+        upd.setCreateIfNonExistent(true);
+
+        upd = Expression.execute(Expression.fromString("input my_str | index my_str"), upd);
+        assertTrue(upd.getCreateIfNonExistent());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java
new file mode 100644
index 00000000000..cd5f17c3b98
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java
@@ -0,0 +1,281 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.Base64DecodeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.Base64EncodeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.CatExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.CompositeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.EchoExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ExecutionContext;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.GetFieldExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.GetVarExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.GuardExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.HexDecodeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.HexEncodeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.HostNameExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.IfThenExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.InputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.JoinExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.LowerCaseExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.NormalizeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.NowExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.OptimizePredicateExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ParenthesisExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.RandomExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SelectInputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SplitExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SubstringExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SwitchExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ThisExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToArrayExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToByteExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToDoubleExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToFloatExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToIntegerExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToLongExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToStringExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToWsetExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.TrimExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.VerificationContext;
+import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression;
+import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ExpressionConverterTestCase {
+
+    @SuppressWarnings("unchecked")
+    @Test
+    public void requireThatAllExpressionTypesCanBeTraversed() {
+        assertConvertable(new ArithmeticExpression(new InputExpression("foo"), ArithmeticExpression.Operator.ADD,
+                                                   new InputExpression("bar")));
+        assertConvertable(new AttributeExpression("foo"));
+        assertConvertable(new Base64DecodeExpression());
+        assertConvertable(new Base64EncodeExpression());
+        assertConvertable(new CatExpression(new InputExpression("foo"), new IndexExpression("bar")));
+        assertConvertable(new ClearStateExpression());
+        assertConvertable(new EchoExpression());
+        assertConvertable(new ForEachExpression(new IndexExpression("foo")));
+        assertConvertable(new GetFieldExpression("foo"));
+        assertConvertable(new GetVarExpression("foo"));
+        assertConvertable(new GuardExpression(new IndexExpression("foo")));
+        assertConvertable(new HexDecodeExpression());
+        assertConvertable(new HexEncodeExpression());
+        assertConvertable(new HostNameExpression());
+        assertConvertable(new IfThenExpression(new InputExpression("foo"), IfThenExpression.Comparator.EQ,
+                                               new InputExpression("bar"),
+                                               new IndexExpression("baz"),
+                                               new IndexExpression("cox")));
+        assertConvertable(new IndexExpression("foo"));
+        assertConvertable(new InputExpression("foo"));
+        assertConvertable(new JoinExpression("foo"));
+        assertConvertable(new LowerCaseExpression());
+        assertConvertable(new NormalizeExpression(new SimpleLinguistics()));
+        assertConvertable(new NowExpression());
+        assertConvertable(new OptimizePredicateExpression());
+        assertConvertable(new ParenthesisExpression(new InputExpression("foo")));
+        assertConvertable(new RandomExpression(69));
+        assertConvertable(new ScriptExpression(new StatementExpression(new InputExpression("foo"))));
+        assertConvertable(new SelectInputExpression(new Pair<String, Expression>("foo", new IndexExpression("bar")),
+                                                    new Pair<String, Expression>("bar", new IndexExpression("foo"))));
+        assertConvertable(new SetLanguageExpression());
+        assertConvertable(new SetValueExpression(new IntegerFieldValue(69)));
+        assertConvertable(new SetVarExpression("foo"));
+        assertConvertable(new SplitExpression("foo"));
+        assertConvertable(new StatementExpression(new InputExpression("foo")));
+        assertConvertable(new SubstringExpression(6, 9));
+        assertConvertable(new SummaryExpression("foo"));
+        assertConvertable(new SwitchExpression(Collections.singletonMap("foo", (Expression)new IndexExpression("bar")),
+                                               new InputExpression("baz")));
+        assertConvertable(new ThisExpression());
+        assertConvertable(new ToArrayExpression());
+        assertConvertable(new ToByteExpression());
+        assertConvertable(new ToDoubleExpression());
+        assertConvertable(new ToFloatExpression());
+        assertConvertable(new ToIntegerExpression());
+        assertConvertable(new TokenizeExpression(new SimpleLinguistics(), new AnnotatorConfig()));
+        assertConvertable(new ToLongExpression());
+        assertConvertable(new ToPositionExpression());
+        assertConvertable(new ToStringExpression());
+        assertConvertable(new ToWsetExpression(false, false));
+        assertConvertable(new TrimExpression());
+        assertConvertable(new ZCurveExpression());
+    }
+
+    @Test
+    public void requireThatScriptElementsCanBeRemoved() {
+        StatementExpression foo = new StatementExpression(new AttributeExpression("foo"));
+        StatementExpression bar = new StatementExpression(new AttributeExpression("bar"));
+        ScriptExpression before = new ScriptExpression(foo, bar);
+
+        Expression after = new SearchReplace(foo, null).convert(before);
+        assertTrue(after instanceof ScriptExpression);
+        assertEquals(1, ((ScriptExpression)after).size());
+        assertEquals(bar, ((ScriptExpression)after).get(0));
+
+        after = new SearchReplace(bar, null).convert(before);
+        assertTrue(after instanceof ScriptExpression);
+        assertEquals(1, ((ScriptExpression)after).size());
+        assertEquals(foo, ((ScriptExpression)after).get(0));
+    }
+
+    @Test
+    public void requireThatSwitchElementsCanBeRemoved() {
+        Map<String, Expression> cases = new HashMap<>();
+        Expression foo = new AttributeExpression("foo");
+        Expression bar = new AttributeExpression("bar");
+        cases.put("foo", foo);
+        cases.put("bar", bar);
+        SwitchExpression before = new SwitchExpression(cases);
+
+        Expression after = new SearchReplace(foo, null).convert(before);
+        assertTrue(after instanceof SwitchExpression);
+        assertEquals(1, ((SwitchExpression)after).getCases().size());
+
+        after = new SearchReplace(bar, null).convert(before);
+        assertTrue(after instanceof SwitchExpression);
+        assertEquals(1, ((SwitchExpression)after).getCases().size());
+    }
+
+    @Test
+    public void requireThatUnknownCompositeThrows() {
+        try {
+            new MyTraverser().convert(new MyComposite());
+            fail();
+        } catch (UnsupportedOperationException e) {
+            assertEquals(NoSuchMethodException.class, e.getCause().getClass());
+        }
+    }
+
+    @Test
+    public void requireThatConversionExceptionCanBeThrown() {
+        final RuntimeException expectedCause = new RuntimeException();
+        try {
+            new ExpressionConverter() {
+
+                @Override
+                protected boolean shouldConvert(Expression exp) {
+                    return exp instanceof AttributeExpression;
+                }
+
+                @Override
+                protected Expression doConvert(Expression exp) {
+                    throw expectedCause;
+                }
+            }.convert(new StatementExpression(new AttributeExpression("foo")));
+            fail();
+        } catch (RuntimeException e) {
+            assertSame(expectedCause, e);
+        }
+    }
+
+    @Test
+    public void requireThatCatConversionIgnoresNull() {
+        Expression exp = new ExpressionConverter() {
+
+            @Override
+            protected boolean shouldConvert(Expression exp) {
+                return exp instanceof AttributeExpression;
+            }
+
+            @Override
+            protected Expression doConvert(Expression exp) {
+                return null;
+            }
+        }.convert(new CatExpression(new AttributeExpression("foo"), new IndexExpression("bar")));
+        assertEquals(new CatExpression(new IndexExpression("bar")), exp);
+    }
+
+    private static void assertConvertable(Expression exp) {
+        MyTraverser traverser = new MyTraverser();
+        assertEquals(traverser.convert(exp), exp);
+    }
+
+    private static class SearchReplace extends ExpressionConverter {
+
+        final Expression searchFor;
+        final Expression replaceWith;
+
+        private SearchReplace(Expression searchFor, Expression replaceWith) {
+            this.searchFor = searchFor;
+            this.replaceWith = replaceWith;
+        }
+
+        @Override
+        protected boolean shouldConvert(Expression exp) {
+            return exp == searchFor;
+        }
+
+        @Override
+        protected Expression doConvert(Expression exp) {
+            return replaceWith;
+        }
+    }
+
+    private static class MyTraverser extends ExpressionConverter {
+
+        @Override
+        protected boolean shouldConvert(Expression exp) {
+            return false;
+        }
+
+        @Override
+        protected Expression doConvert(Expression exp) {
+            return exp;
+        }
+    }
+
+    private static class MyComposite extends CompositeExpression {
+
+        @Override
+        protected void doExecute(ExecutionContext ctx) {
+
+        }
+
+        @Override
+        protected void doVerify(VerificationContext ctx) {
+
+        }
+
+        @Override
+        public DataType requiredInputType() {
+            return null;
+        }
+
+        @Override
+        public DataType createdOutputType() {
+            return null;
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizerTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizerTestCase.java
new file mode 100644
index 00000000000..2258c7d5a13
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizerTestCase.java
@@ -0,0 +1,84 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:magnarn@yahoo-inc.com">Magnar Nedland</a>
+ */
+public class ExpressionOptimizerTestCase {
+    @Test
+    public void requireThatSimpleExpressionsAreNotChanged() {
+        assertNotOptimized("input foo");
+    }
+
+    @Test
+    public void requireThatStatementsBeforeOneThatIgnoresInputAreRemoved() {
+        checkStatementThatIgnoresInput(new InputExpression("foo"));
+        checkStatementThatIgnoresInput(new NowExpression());
+        checkStatementThatIgnoresInput(new SetValueExpression(new IntegerFieldValue(42)));
+        checkStatementThatIgnoresInput(new HostNameExpression());
+        checkStatementThatIgnoresInput(new RandomExpression(42));
+        checkStatementThatIgnoresInput(new ArithmeticExpression(
+                new NowExpression(), ArithmeticExpression.Operator.ADD, new NowExpression()));
+        checkStatementThatIgnoresInput(new CatExpression(new NowExpression(), new NowExpression()));
+        checkStatementThatIgnoresInput(new ParenthesisExpression(new NowExpression()));
+        assertOptimized("input foo | (trim | now)", "(now)");
+        assertOptimized("input foo | get_var bar", "get_var bar");
+        assertOptimized("1 | index foo | now | 0", "1 | index foo | 0");
+    }
+
+    @Test
+    public void requireThatStatementsBeforeOneThatConsidersInputAreKept() {
+        assertNotOptimized("input foo | random");
+        assertNotOptimized("input foo | trim + now");
+        assertNotOptimized("input foo | (trim + now) - now");
+        assertNotOptimized("input foo | (now . trim)");
+        assertNotOptimized("input foo | (trim)");
+        assertNotOptimized("input foo | (trim | trim)");
+        assertNotOptimized("input foo | switch {case 'bar': now; case 'foo': now; }");
+        assertNotOptimized("{ input test | { summary test; }; }");
+        assertNotOptimized("1 | set_var foo | 0 | set_var bar");
+        assertNotOptimized("1 | index foo | 0 | index bar");
+        assertNotOptimized("1 | echo | 0 | echo");
+        assertNotOptimized("'foo' | if (1 < 2) { now | summary } else { 42 | summary } | summary");
+        assertNotOptimized("{ 0 | set_var tmp; " +
+                           "  input foo | split ';' | for_each { to_int + get_var tmp | set_var tmp };" +
+                           "  get_var tmp | attribute bar; }");
+        assertNotOptimized("0 | set_var tmp | " +
+                           "input foo | split ';' | for_each { to_int + get_var tmp | set_var tmp } | " +
+                           "get_var tmp | attribute bar");
+    }
+
+    private void checkStatementThatIgnoresInput(Expression exp) {
+        assertOptimized(new StatementExpression(new InputExpression("xyzzy"), exp), exp);
+        assertOptimized(new StatementExpression(new InputExpression("xyzzy"), exp, new LowerCaseExpression()),
+                        new StatementExpression(exp, new LowerCaseExpression()));
+    }
+
+    private void assertOptimized(Expression input, Expression expected) {
+        assertEquals(expected.toString(), new ExpressionOptimizer().convert(input).toString());
+    }
+
+    private void assertOptimized(String input, String expected) {
+        try {
+            assertOptimized(Expression.fromString(input), Expression.fromString(expected));
+        } catch (ParseException e) {
+            fail(e.getMessage());
+        }
+    }
+
+    private void assertNotOptimized(String script) {
+        assertOptimized(script, script);
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcherTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcherTestCase.java
new file mode 100644
index 00000000000..2e47ccb7ea3
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcherTestCase.java
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.CatExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.GuardExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.IfThenExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ParenthesisExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SelectInputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SwitchExpression;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ExpressionSearcherTestCase {
+
+    private static final ExpressionSearcher searcher = new ExpressionSearcher<>(IndexExpression.class);
+
+    @Test
+    public void requireThatExpressionsCanBeFound() {
+        IndexExpression exp = new IndexExpression("foo");
+        assertFound(exp, new ArithmeticExpression(exp, ArithmeticExpression.Operator.ADD,
+                                                  new AttributeExpression("foo")));
+        assertFound(exp, new ArithmeticExpression(new AttributeExpression("foo"),
+                                                  ArithmeticExpression.Operator.ADD, exp));
+        assertFound(exp, new CatExpression(exp));
+        assertFound(exp, new CatExpression(new AttributeExpression("foo"), exp));
+        assertFound(exp, new ForEachExpression(exp));
+        assertFound(exp, new GuardExpression(exp));
+        assertFound(exp, new IfThenExpression(exp,
+                                              IfThenExpression.Comparator.EQ,
+                                              new AttributeExpression("foo"),
+                                              new AttributeExpression("bar"),
+                                              new AttributeExpression("baz")));
+        assertFound(exp, new IfThenExpression(new AttributeExpression("foo"),
+                                              IfThenExpression.Comparator.EQ,
+                                              exp,
+                                              new AttributeExpression("bar"),
+                                              new AttributeExpression("baz")));
+        assertFound(exp, new IfThenExpression(new AttributeExpression("foo"),
+                                              IfThenExpression.Comparator.EQ,
+                                              new AttributeExpression("bar"),
+                                              exp,
+                                              new AttributeExpression("baz")));
+        assertFound(exp, new IfThenExpression(new AttributeExpression("foo"),
+                                              IfThenExpression.Comparator.EQ,
+                                              new AttributeExpression("bar"),
+                                              new AttributeExpression("baz"),
+                                              exp));
+        assertFound(exp, new ParenthesisExpression(exp));
+        assertFound(exp, new ScriptExpression(new StatementExpression(exp)));
+        assertFound(exp, new ScriptExpression(new StatementExpression(new AttributeExpression("foo")),
+                                              new StatementExpression(exp)));
+        assertFound(exp, new SelectInputExpression(
+                Arrays.asList(new Pair<String, Expression>("foo", exp),
+                              new Pair<String, Expression>("bar", new AttributeExpression("bar")))));
+        assertFound(exp, new SelectInputExpression(
+                Arrays.asList(new Pair<String, Expression>("foo", new AttributeExpression("bar")),
+                              new Pair<String, Expression>("bar", exp))));
+        assertFound(exp, new StatementExpression(exp));
+        assertFound(exp, new StatementExpression(new AttributeExpression("foo"), exp));
+        assertFound(exp, new SwitchExpression(
+                Collections.singletonMap("foo", exp),
+                new AttributeExpression("bar")));
+        assertFound(exp, new SwitchExpression(
+                Collections.singletonMap("foo", new AttributeExpression("bar")),
+                exp));
+    }
+
+    private static void assertFound(IndexExpression searchFor, Expression searchIn) {
+        assertTrue(searcher.containedIn(searchIn));
+        assertSame(searchFor, searcher.searchIn(searchIn));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitorTestCase.java
new file mode 100644
index 00000000000..95a916cd0ad
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitorTestCase.java
@@ -0,0 +1,141 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.Base64DecodeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.Base64EncodeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.CatExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.EchoExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.GetFieldExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.GetVarExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.GuardExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.HexDecodeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.HexEncodeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.HostNameExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.IfThenExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.InputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.JoinExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.LowerCaseExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.NormalizeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.NowExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.OptimizePredicateExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ParenthesisExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.RandomExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SelectInputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SplitExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SubstringExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SwitchExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ThisExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToArrayExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToByteExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToDoubleExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToFloatExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToIntegerExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToLongExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToStringExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ToWsetExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.TrimExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression;
+import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
+import org.junit.Test;
+
+import java.util.Collections;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ExpressionVisitorTestCase {
+
+    @SuppressWarnings("unchecked")
+    @Test
+    public void requireThatAllExpressionsAreVisited() {
+        assertCount(3, new ArithmeticExpression(new InputExpression("foo"), ArithmeticExpression.Operator.ADD,
+                                                new InputExpression("bar")));
+        assertCount(1, new AttributeExpression("foo"));
+        assertCount(1, new Base64DecodeExpression());
+        assertCount(1, new Base64EncodeExpression());
+        assertCount(3, new CatExpression(new InputExpression("foo"), new IndexExpression("bar")));
+        assertCount(1, new ClearStateExpression());
+        assertCount(1, new EchoExpression());
+        assertCount(2, new ForEachExpression(new IndexExpression("foo")));
+        assertCount(1, new GetFieldExpression("foo"));
+        assertCount(1, new GetVarExpression("foo"));
+        assertCount(2, new GuardExpression(new IndexExpression("foo")));
+        assertCount(1, new HexDecodeExpression());
+        assertCount(1, new HexEncodeExpression());
+        assertCount(1, new HostNameExpression());
+        assertCount(5, new IfThenExpression(new InputExpression("foo"), IfThenExpression.Comparator.EQ,
+                                            new InputExpression("bar"),
+                                            new IndexExpression("baz"),
+                                            new IndexExpression("cox")));
+        assertCount(1, new IndexExpression("foo"));
+        assertCount(1, new InputExpression("foo"));
+        assertCount(1, new JoinExpression("foo"));
+        assertCount(1, new LowerCaseExpression());
+        assertCount(1, new NormalizeExpression(new SimpleLinguistics()));
+        assertCount(1, new NowExpression());
+        assertCount(1, new OptimizePredicateExpression());
+        assertCount(2, new ParenthesisExpression(new InputExpression("foo")));
+        assertCount(1, new RandomExpression(69));
+        assertCount(3, new ScriptExpression(new StatementExpression(new InputExpression("foo"))));
+        assertCount(3, new SelectInputExpression(new Pair<String, Expression>("foo", new IndexExpression("bar")),
+                                                 new Pair<String, Expression>("bar", new IndexExpression("foo"))));
+        assertCount(1, new SetLanguageExpression());
+        assertCount(1, new SetValueExpression(new IntegerFieldValue(69)));
+        assertCount(1, new SetVarExpression("foo"));
+        assertCount(1, new SplitExpression("foo"));
+        assertCount(2, new StatementExpression(new InputExpression("foo")));
+        assertCount(1, new SummaryExpression("foo"));
+        assertCount(1, new SubstringExpression(6, 9));
+        assertCount(3, new SwitchExpression(Collections.singletonMap("foo", (Expression)new IndexExpression("bar")),
+                                            new InputExpression("baz")));
+        assertCount(1, new ThisExpression());
+        assertCount(1, new ToArrayExpression());
+        assertCount(1, new ToByteExpression());
+        assertCount(1, new ToDoubleExpression());
+        assertCount(1, new ToFloatExpression());
+        assertCount(1, new ToIntegerExpression());
+        assertCount(1, new TokenizeExpression(new SimpleLinguistics(), new AnnotatorConfig()));
+        assertCount(1, new ToLongExpression());
+        assertCount(1, new ToPositionExpression());
+        assertCount(1, new ToStringExpression());
+        assertCount(1, new ToWsetExpression(false, false));
+        assertCount(1, new TrimExpression());
+        assertCount(1, new ZCurveExpression());
+    }
+
+    private static void assertCount(int expectedCount, Expression exp) {
+        MyVisitor visitor = new MyVisitor();
+        visitor.visit(exp);
+        assertEquals(expectedCount, visitor.count);
+    }
+
+    private static class MyVisitor extends ExpressionVisitor {
+
+        int count = 0;
+
+        @Override
+        protected void doVisit(Expression exp) {
+            ++count;
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/FieldValueConverterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/FieldValueConverterTestCase.java
new file mode 100644
index 00000000000..96140be7500
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/FieldValueConverterTestCase.java
@@ -0,0 +1,423 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlStream;
+import org.junit.Test;
+
+import java.util.Iterator;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class FieldValueConverterTestCase {
+
+    @Test
+    public void requireThatNullIsIgnored() {
+        assertNull(new FieldValueConverter() {
+
+            @Override
+            protected boolean shouldConvert(FieldValue value) {
+                throw new AssertionError();
+            }
+
+            @Override
+            protected FieldValue doConvert(FieldValue value) {
+                throw new AssertionError();
+            }
+        }.convert(null));
+    }
+
+    @Test
+    public void requireThatUnknownTypesFallThrough() {
+        FieldValue val = new MyFieldValue();
+        assertSame(val, new FieldValueConverter() {
+
+            @Override
+            protected boolean shouldConvert(FieldValue value) {
+                return false;
+            }
+
+            @Override
+            protected FieldValue doConvert(FieldValue value) {
+                throw new AssertionError();
+            }
+        }.convert(val));
+    }
+
+    @Test
+    public void requireThatSingleValueIsConverted() {
+        StringFieldValue before = new StringFieldValue("69");
+        FieldValue after = new StringMarker().doConvert(before);
+
+        assertTrue(after instanceof StringFieldValue);
+        assertEquals(new StringFieldValue("69'"), after);
+    }
+
+    @Test
+    public void requireThatArrayElementsAreConverted() {
+        Array<StringFieldValue> before = new Array<>(DataType.getArray(DataType.STRING));
+        before.add(new StringFieldValue("6"));
+        before.add(new StringFieldValue("9"));
+        FieldValue after = new StringMarker().convert(before);
+
+        assertTrue(after instanceof Array);
+        assertEquals(new StringFieldValue("6'"), ((Array)after).get(0));
+        assertEquals(new StringFieldValue("9'"), ((Array)after).get(1));
+    }
+
+    @Test
+    public void requireThatConvertedArrayElementCompatibilityIsEnforced() {
+        Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING));
+        StringFieldValue foo = new StringFieldValue("foo");
+        StringFieldValue bar = new StringFieldValue("bar");
+        arr.add(foo);
+        arr.add(bar);
+        try {
+            new SearchReplace(foo, new IntegerFieldValue(69)).convert(arr);
+            fail();
+        } catch (IllegalArgumentException e) {
+
+        }
+        try {
+            new SearchReplace(bar, new IntegerFieldValue(69)).convert(arr);
+            fail();
+        } catch (IllegalArgumentException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatMapEntriesAreConverted() {
+        MapFieldValue<StringFieldValue, StringFieldValue> before =
+                new MapFieldValue<>(DataType.getMap(DataType.STRING, DataType.STRING));
+        before.put(new StringFieldValue("6"), new StringFieldValue("9"));
+        before.put(new StringFieldValue("9"), new StringFieldValue("6"));
+        FieldValue after = new StringMarker().convert(before);
+
+        assertTrue(after instanceof MapFieldValue);
+        assertEquals(new StringFieldValue("6'"), ((MapFieldValue)after).get(new StringFieldValue("9'")));
+        assertEquals(new StringFieldValue("9'"), ((MapFieldValue)after).get(new StringFieldValue("6'")));
+    }
+
+    @Test
+    public void requireThatMapElementsCanBeRemoved() {
+        MapFieldValue<StringFieldValue, StringFieldValue> before =
+                new MapFieldValue<>(DataType.getMap(DataType.STRING, DataType.STRING));
+        StringFieldValue foo = new StringFieldValue("foo");
+        StringFieldValue bar = new StringFieldValue("bar");
+        StringFieldValue baz = new StringFieldValue("baz");
+        StringFieldValue cox = new StringFieldValue("cox");
+        before.put(foo, bar);
+        before.put(baz, cox);
+
+        FieldValue after = new SearchReplace(foo, null).convert(before);
+        assertTrue(after instanceof MapFieldValue);
+        assertEquals(1, ((MapFieldValue)after).size());
+
+        after = new SearchReplace(bar, null).convert(before);
+        assertTrue(after instanceof MapFieldValue);
+        assertEquals(1, ((MapFieldValue)after).size());
+
+        after = new SearchReplace(baz, null).convert(before);
+        assertTrue(after instanceof MapFieldValue);
+        assertEquals(1, ((MapFieldValue)after).size());
+
+        after = new SearchReplace(cox, null).convert(before);
+        assertTrue(after instanceof MapFieldValue);
+        assertEquals(1, ((MapFieldValue)after).size());
+    }
+
+    @Test
+    public void requireThatConvertedMapElementCompatibilityIsEnforced() {
+        MapFieldValue<StringFieldValue, StringFieldValue> before =
+                new MapFieldValue<>(DataType.getMap(DataType.STRING, DataType.STRING));
+        StringFieldValue foo = new StringFieldValue("foo");
+        StringFieldValue bar = new StringFieldValue("bar");
+        StringFieldValue baz = new StringFieldValue("baz");
+        StringFieldValue cox = new StringFieldValue("cox");
+        before.put(foo, bar);
+        before.put(baz, cox);
+
+        try {
+            new SearchReplace(foo, new IntegerFieldValue(69)).convert(before);
+            fail();
+        } catch (IllegalArgumentException e) {
+
+        }
+        try {
+            new SearchReplace(bar, new IntegerFieldValue(69)).convert(before);
+            fail();
+        } catch (IllegalArgumentException e) {
+
+        }
+        try {
+            new SearchReplace(baz, new IntegerFieldValue(69)).convert(before);
+            fail();
+        } catch (IllegalArgumentException e) {
+
+        }
+        try {
+            new SearchReplace(cox, new IntegerFieldValue(69)).convert(before);
+            fail();
+        } catch (IllegalArgumentException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatWsetElementsAreConverted() {
+        WeightedSet<StringFieldValue> before = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+        before.put(new StringFieldValue("6"), 69);
+        before.put(new StringFieldValue("9"), 96);
+        FieldValue after = new StringMarker().convert(before);
+
+        assertTrue(after instanceof WeightedSet);
+        @SuppressWarnings("unchecked")
+        WeightedSet<StringFieldValue> w = (WeightedSet<StringFieldValue>) after;
+        assertEquals(2, w.size());
+        assertTrue(w.contains(new StringFieldValue("9'")));
+        assertEquals(96, w.get(new StringFieldValue("9'")).intValue());
+        assertTrue(w.contains(new StringFieldValue("6'")));
+        assertEquals(69, w.get(new StringFieldValue("6'")).intValue());
+    }
+
+    @Test
+    public void requireThatWsetElementsCanBeRemoved() {
+        WeightedSet<StringFieldValue> before = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+        StringFieldValue foo = new StringFieldValue("foo");
+        StringFieldValue bar = new StringFieldValue("bar");
+        before.put(foo, 6);
+        before.put(bar, 9);
+
+        FieldValue after = new SearchReplace(foo, null).convert(before);
+        assertTrue(after instanceof WeightedSet);
+        assertEquals(1, ((WeightedSet)after).size());
+
+        after = new SearchReplace(bar, null).convert(before);
+        assertTrue(after instanceof WeightedSet);
+        assertEquals(1, ((WeightedSet)after).size());
+    }
+
+    @Test
+    public void requireThatConvertedWsetElementCompatibilityIsEnforced() {
+        WeightedSet<StringFieldValue> before = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+        StringFieldValue foo = new StringFieldValue("foo");
+        StringFieldValue bar = new StringFieldValue("bar");
+        before.put(foo, 6);
+        before.put(bar, 9);
+
+        try {
+            new SearchReplace(foo, new IntegerFieldValue(69)).convert(before);
+            fail();
+        } catch (IllegalArgumentException e) {
+
+        }
+        try {
+            new SearchReplace(bar, new IntegerFieldValue(69)).convert(before);
+            fail();
+        } catch (IllegalArgumentException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatEmptyCollectionsAreConvertedToNull() {
+        FieldValueConverter converter = new FieldValueConverter() {
+
+            @Override
+            protected boolean shouldConvert(FieldValue value) {
+                return false;
+            }
+
+            @Override
+            protected FieldValue doConvert(FieldValue value) {
+                throw new AssertionError();
+            }
+        };
+        assertNull(converter.convert(new Array(DataType.getArray(DataType.STRING))));
+        assertNull(converter.convert(new MapFieldValue(DataType.getMap(DataType.STRING, DataType.STRING))));
+        assertNull(converter.convert(new WeightedSet<StringFieldValue>(DataType.getWeightedSet(DataType.STRING))));
+    }
+
+    @Test
+    public void requireThatStructElementsAreConverted() {
+        StructDataType type = new StructDataType("foo");
+        type.addField(new Field("bar", DataType.STRING));
+        type.addField(new Field("baz", DataType.STRING));
+        Struct before = type.createFieldValue();
+        before.setFieldValue("bar", new StringFieldValue("6"));
+        before.setFieldValue("baz", new StringFieldValue("9"));
+        FieldValue after = new StringMarker().convert(before);
+
+        assertTrue(after instanceof Struct);
+        assertEquals(new StringFieldValue("6'"), ((Struct)after).getFieldValue("bar"));
+        assertEquals(new StringFieldValue("9'"), ((Struct)after).getFieldValue("baz"));
+    }
+
+    @Test
+    public void requireThatStructElementsCanBeRemoved() {
+        StructDataType type = new StructDataType("foo");
+        type.addField(new Field("bar", DataType.STRING));
+        type.addField(new Field("baz", DataType.STRING));
+        Struct before = type.createFieldValue();
+        StringFieldValue barVal = new StringFieldValue("6");
+        StringFieldValue bazVal = new StringFieldValue("9");
+        before.setFieldValue("bar", barVal);
+        before.setFieldValue("baz", bazVal);
+
+        FieldValue after = new SearchReplace(barVal, null).convert(before);
+        assertTrue(after instanceof Struct);
+        assertNull(((Struct)after).getFieldValue("bar"));
+        assertEquals(bazVal, ((Struct)after).getFieldValue("baz"));
+
+        after = new SearchReplace(bazVal, null).convert(before);
+        assertTrue(after instanceof Struct);
+        assertEquals(barVal, ((Struct)after).getFieldValue("bar"));
+        assertNull(((Struct)after).getFieldValue("baz"));
+    }
+
+    @Test
+    public void requireThatStructArrayElementsAreConverted() {
+        StructDataType type = new StructDataType("foo");
+        type.addField(new Field("bar", DataType.STRING));
+        type.addField(new Field("baz", DataType.STRING));
+        Array<Struct> before = new Array<>(DataType.getArray(type));
+        Struct elem = type.createFieldValue();
+        elem.setFieldValue("bar", new StringFieldValue("6"));
+        elem.setFieldValue("baz", new StringFieldValue("9"));
+        before.add(elem);
+        elem = type.createFieldValue();
+        elem.setFieldValue("bar", new StringFieldValue("9"));
+        elem.setFieldValue("baz", new StringFieldValue("6"));
+        before.add(elem);
+        FieldValue after = new StringMarker().convert(before);
+
+        assertTrue(after instanceof Array);
+        FieldValue val = ((Array)after).getFieldValue(0);
+        assertTrue(val instanceof Struct);
+        assertEquals(new StringFieldValue("6'"), ((Struct)val).getFieldValue("bar"));
+        assertEquals(new StringFieldValue("9'"), ((Struct)val).getFieldValue("baz"));
+        val = ((Array)after).getFieldValue(1);
+        assertTrue(val instanceof Struct);
+        assertEquals(new StringFieldValue("9'"), ((Struct)val).getFieldValue("bar"));
+        assertEquals(new StringFieldValue("6'"), ((Struct)val).getFieldValue("baz"));
+    }
+
+    @Test
+    public void requireThatArrayChangeNestedType() {
+        Array<StringFieldValue> before = new Array<>(DataType.getArray(DataType.STRING));
+        before.add(new StringFieldValue("69"));
+        FieldValue after = new IntegerParser().convert(before);
+
+        assertTrue(after instanceof Array);
+        assertEquals(DataType.getArray(DataType.INT), after.getDataType());
+    }
+
+    @Test
+    public void requireThatMapChangeNestedType() {
+        MapFieldValue<StringFieldValue, StringFieldValue> before =
+                new MapFieldValue<>(
+                        DataType.getMap(DataType.STRING, DataType.STRING));
+        before.put(new StringFieldValue("6"), new StringFieldValue("9"));
+        FieldValue after = new IntegerParser().convert(before);
+
+        assertTrue(after instanceof MapFieldValue);
+        assertEquals(DataType.getMap(DataType.INT, DataType.INT), after.getDataType());
+    }
+
+    @Test
+    public void requireThatWsetChangeNestedType() {
+        assertWsetChangeNestedType(false, false);
+        assertWsetChangeNestedType(false, true);
+        assertWsetChangeNestedType(true, false);
+        assertWsetChangeNestedType(true, true);
+    }
+
+    private static void assertWsetChangeNestedType(boolean createIfNonExistent, boolean removeIfZero) {
+        WeightedSet<StringFieldValue> before =
+                new WeightedSet<>(DataType.getWeightedSet(DataType.STRING, createIfNonExistent, removeIfZero));
+        before.put(new StringFieldValue("6"), 9);
+        FieldValue after = new IntegerParser().convert(before);
+
+        assertTrue(after instanceof WeightedSet);
+        assertEquals(DataType.getWeightedSet(DataType.INT, createIfNonExistent, removeIfZero), after.getDataType());
+    }
+
+    private static class StringMarker extends StringFieldConverter {
+
+        @Override
+        protected FieldValue doConvert(StringFieldValue value) {
+            return new StringFieldValue(value.toString() + "'");
+        }
+    }
+
+    private static class IntegerParser extends StringFieldConverter {
+
+        @Override
+        protected FieldValue doConvert(StringFieldValue value) {
+            return new IntegerFieldValue(Integer.valueOf(value.toString()));
+        }
+    }
+
+    private static class SearchReplace extends FieldValueConverter {
+
+        final FieldValue searchFor;
+        final FieldValue replaceWith;
+
+        private SearchReplace(FieldValue searchFor, FieldValue replaceWith) {
+            this.searchFor = searchFor;
+            this.replaceWith = replaceWith;
+        }
+
+        @Override
+        protected boolean shouldConvert(FieldValue value) {
+            return value == searchFor;
+        }
+
+        @Override
+        protected FieldValue doConvert(FieldValue value) {
+            return replaceWith;
+        }
+    }
+
+    private static class MyFieldValue extends FieldValue {
+
+        @Override
+        public DataType getDataType() {
+            return null;
+        }
+
+        @Override
+        public void printXml(XmlStream xml) {
+
+        }
+
+        @Override
+        public void clear() {
+
+        }
+
+        @Override
+        public void assign(Object obj) {
+
+        }
+
+        @Override
+        public void serialize(Field field, FieldWriter writer) {
+
+        }
+
+        @Override
+        public void deserialize(Field field, FieldReader reader) {
+
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/PathUpdateToDocumentTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/PathUpdateToDocumentTestCase.java
new file mode 100644
index 00000000000..3472920dcdf
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/PathUpdateToDocumentTestCase.java
@@ -0,0 +1,141 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class PathUpdateToDocumentTestCase {
+
+    @Test
+    public void requireThatIntegerFieldsAreConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField(new Field("my_int", DataType.INT));
+
+        FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "my_int", "", new IntegerFieldValue(69));
+        Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd);
+        assertNotNull(doc);
+
+        assertEquals(69, ((IntegerFieldValue)doc.getFieldValue("my_int")).getInteger());
+    }
+
+    @Test
+    public void requireThatStringFieldsAreConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        docType.addField(new Field("my_str", DataType.STRING));
+
+        FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "my_str", "", new StringFieldValue("69"));
+        Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd);
+        assertNotNull(doc);
+
+        assertEquals("69", ((StringFieldValue)doc.getFieldValue("my_str")).getString());
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    @Test
+    public void requireThatArrayFieldsAreConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        ArrayDataType arrType = DataType.getArray(DataType.INT);
+        docType.addField(new Field("my_arr", arrType));
+
+        Array<IntegerFieldValue> arrVal = arrType.createFieldValue();
+        arrVal.add(new IntegerFieldValue(6));
+        arrVal.add(new IntegerFieldValue(9));
+        FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "my_arr", "", arrVal);
+
+        Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("my_arr");
+        assertTrue(obj instanceof Array);
+        Array arr = (Array)obj;
+        assertEquals(2, arr.size());
+        assertEquals(new IntegerFieldValue(6), arr.get(0));
+        assertEquals(new IntegerFieldValue(9), arr.get(1));
+    }
+
+    @Test
+    public void requireThatWsetKeysAreConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        WeightedSetDataType wsetType = DataType.getWeightedSet(DataType.STRING);
+        docType.addField(new Field("my_wset", wsetType));
+
+        FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "my_wset{69}", "", new IntegerFieldValue(96));
+
+        Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("my_wset");
+        assertTrue(obj instanceof WeightedSet);
+        WeightedSet wset = (WeightedSet)obj;
+        assertEquals(1, wset.size());
+        assertEquals(96, wset.get(new StringFieldValue("69")).intValue());
+    }
+
+    @Test
+    public void requireThatNestedStructsAreConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        StructDataType structType = new StructDataType("my_struct");
+        structType.addField(new Field("b", DataType.INT));
+        docType.addField(new Field("a", structType));
+
+        FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "a.b", "", new IntegerFieldValue(69));
+
+        Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("a");
+        assertTrue(obj instanceof Struct);
+        Struct struct = (Struct)obj;
+        assertEquals(new IntegerFieldValue(69), struct.getFieldValue("b"));
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    @Test
+    public void requireThatAddIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        ArrayDataType arrType = DataType.getArray(DataType.INT);
+        docType.addField(new Field("my_arr", arrType));
+
+        Array<IntegerFieldValue> arrVal = arrType.createFieldValue();
+        arrVal.add(new IntegerFieldValue(6));
+        arrVal.add(new IntegerFieldValue(9));
+        FieldPathUpdate upd = new AddFieldPathUpdate(docType, "my_arr", "", arrVal);
+
+        Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("my_arr");
+        assertTrue(obj instanceof Array);
+        Array arr = (Array)obj;
+        assertEquals(2, arr.size());
+        assertEquals(new IntegerFieldValue(6), arr.get(0));
+        assertEquals(new IntegerFieldValue(9), arr.get(1));
+    }
+
+    @Test
+    public void requireThatRemoveIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        ArrayDataType arrType = DataType.getArray(DataType.INT);
+        docType.addField(new Field("my_arr", arrType));
+
+        FieldPathUpdate upd = new RemoveFieldPathUpdate(docType, "my_arr", "");
+
+        Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("my_arr");
+        assertTrue(obj instanceof Array);
+        Array arr = (Array)obj;
+        assertEquals(0, arr.size());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptParserTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptParserTestCase.java
new file mode 100644
index 00000000000..5b034ec90b6
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptParserTestCase.java
@@ -0,0 +1,100 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.expressions.EchoExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.InputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
+import com.yahoo.vespa.indexinglanguage.parser.IndexingInput;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class ScriptParserTestCase {
+
+    @Test
+    public void requireThatExpressionParserCanBeInvoked() throws ParseException {
+        try {
+            ScriptParser.parseExpression(newContext("foo"));
+        } catch (ParseException e) {
+            assertException(e, "Encountered \" <IDENTIFIER> \"foo \"\" at line 1, column 1.");
+        }
+        assertEquals(new InputExpression("foo"),
+                     ScriptParser.parseExpression(newContext("input foo")));
+        assertEquals(new StatementExpression(new InputExpression("foo"), new EchoExpression()),
+                     ScriptParser.parseExpression(newContext("input foo | echo")));
+        assertEquals(new ScriptExpression(new StatementExpression(new InputExpression("foo")),
+                                          new StatementExpression(new EchoExpression())),
+                     ScriptParser.parseExpression(newContext("{ input foo; echo }")));
+    }
+
+    @Test
+    public void requireThatStatementParserCanBeInvoked() throws ParseException {
+        try {
+            ScriptParser.parseStatement(newContext("foo"));
+        } catch (ParseException e) {
+            assertException(e, "Encountered \" <IDENTIFIER> \"foo \"\" at line 1, column 1.");
+        }
+        assertEquals(new StatementExpression(new InputExpression("foo")),
+                     ScriptParser.parseStatement(newContext("input foo")));
+        assertEquals(new StatementExpression(new InputExpression("foo"), new EchoExpression()),
+                     ScriptParser.parseStatement(newContext("input foo | echo")));
+        assertEquals(new StatementExpression(new ScriptExpression(new StatementExpression(new InputExpression("foo")),
+                                                                  new StatementExpression(new EchoExpression()))),
+                     ScriptParser.parseStatement(newContext("{ input foo; echo }")));
+    }
+
+    @Test
+    public void requireThatScriptParserCanBeInvoked() throws ParseException {
+        try {
+            ScriptParser.parseScript(newContext("foo"));
+        } catch (ParseException e) {
+            assertException(e, "Encountered \" <IDENTIFIER> \"foo \"\" at line 1, column 1.");
+        }
+        try {
+            ScriptParser.parseScript(newContext("input foo"));
+        } catch (ParseException e) {
+            assertException(e, "Encountered \" \"input\" \"input \"\" at line 1, column 1.");
+        }
+        try {
+            ScriptParser.parseScript(newContext("input foo | echo"));
+        } catch (ParseException e) {
+            assertException(e, "Encountered \" \"input\" \"input \"\" at line 1, column 1.");
+        }
+        assertEquals(new ScriptExpression(new StatementExpression(new InputExpression("foo")),
+                                          new StatementExpression(new EchoExpression())),
+                     ScriptParser.parseScript(newContext("{ input foo; echo }")));
+    }
+
+    @Test
+    public void requireThatStatementParserBacksUpStream() throws ParseException {
+        ScriptParserContext config = newContext("input foo input bar");
+        assertEquals(new StatementExpression(new InputExpression("foo")), ScriptParser.parseStatement(config));
+        assertEquals(new StatementExpression(new InputExpression("bar")), ScriptParser.parseStatement(config));
+    }
+
+    @Test
+    public void requireThatScriptParserBacksUpStream() throws ParseException {
+        ScriptParserContext config = newContext("{ input foo }{ input bar }");
+        assertEquals(new ScriptExpression(new StatementExpression(new InputExpression("foo"))),
+                     ScriptParser.parseScript(config));
+        assertEquals(new ScriptExpression(new StatementExpression(new InputExpression("bar"))),
+                     ScriptParser.parseScript(config));
+    }
+
+    private static void assertException(ParseException e, String expectedMessage) throws ParseException {
+        if (!e.getMessage().startsWith(expectedMessage)) {
+            throw e;
+        }
+    }
+
+    private static ScriptParserContext newContext(String input) {
+        return new ScriptParserContext(new SimpleLinguistics()).setInputStream(new IndexingInput(input));
+    }
+
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java
new file mode 100644
index 00000000000..41a020e1384
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ScriptTestCase {
+
+    private final DocumentType type;
+
+    public ScriptTestCase() {
+        type = new DocumentType("mytype");
+        type.addField("in-1", DataType.STRING);
+        type.addField("in-2", DataType.STRING);
+        type.addField("out-1", DataType.STRING);
+        type.addField("out-2", DataType.STRING);
+    }
+
+    @Test
+    public void requireThatScriptExecutesStatements() {
+        Document input = new Document(type, "doc:scheme:");
+        input.setFieldValue("in-1", new StringFieldValue("6"));
+        input.setFieldValue("in-2", new StringFieldValue("9"));
+
+        Expression exp = new ScriptExpression(
+                new StatementExpression(new InputExpression("in-1"), new AttributeExpression("out-1")),
+                new StatementExpression(new InputExpression("in-2"), new AttributeExpression("out-2")));
+        Document output = Expression.execute(exp, input);
+        assertNotNull(output);
+        assertEquals(new StringFieldValue("6"), output.getFieldValue("out-1"));
+        assertEquals(new StringFieldValue("9"), output.getFieldValue("out-2"));
+    }
+
+    @Test
+    public void requireThatEachStatementHasEmptyInput() {
+        Document input = new Document(type, "doc:scheme:");
+        input.setFieldValue(input.getField("in-1"), new StringFieldValue("69"));
+
+        Expression exp = new ScriptExpression(
+                new StatementExpression(new InputExpression("in-1"), new AttributeExpression("out-1")),
+                new StatementExpression(new AttributeExpression("out-2")));
+        try {
+            exp.verify(input);
+            fail();
+        } catch (VerificationException e) {
+            assertTrue(e.getExpression() instanceof ScriptExpression);
+            assertEquals("Expected any input, got null.", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatFactoryMethodWorks() throws ParseException {
+        Document input = new Document(type, "doc:scheme:");
+        input.setFieldValue("in-1", new StringFieldValue("FOO"));
+
+        Document output = Expression.execute(Expression.fromString("input 'in-1' | { index 'out-1'; lowercase | index 'out-2' }"), input);
+        assertNotNull(output);
+        assertEquals(new StringFieldValue("FOO"), output.getFieldValue("out-1"));
+        assertEquals(new StringFieldValue("foo"), output.getFieldValue("out-2"));
+    }
+
+    @Test
+    public void requireThatIfExpressionPassesOriginalInputAlong() throws ParseException {
+        Document input = new Document(type, "doc:scheme:");
+        Document output = Expression.execute(Expression.fromString("'foo' | if (1 < 2) { 'bar' | index 'out-1' } else { 'baz' | index 'out-1' } | index 'out-1'"), input);
+        assertNotNull(output);
+        assertEquals(new StringFieldValue("foo"), output.getFieldValue("out-1"));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactoryTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactoryTestCase.java
new file mode 100644
index 00000000000..13b2d1c3249
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactoryTestCase.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.ValueUpdate;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * @author <a href="mailto:magnarn@yahoo-inc.com">Magnar Nedland</a>
+ */
+public class SimpleAdapterFactoryTestCase {
+
+    @Test
+    public void requireThatCompleteUpdatesAreCombined() {
+        DocumentType docType = new DocumentType("my_type");
+        DocumentUpdate update = new DocumentUpdate(docType, "doc:foo:1");
+        Field field1 = new Field("int1", DataType.INT);
+        Field field2 = new Field("int2", DataType.INT);
+        Field field3 = new Field("int3", DataType.INT);
+        docType.addField(field1);
+        docType.addField(field2);
+        docType.addField(field3);
+        update.addFieldUpdate(FieldUpdate.createAssign(field1, new IntegerFieldValue(10)));
+        update.addFieldUpdate(FieldUpdate.createAssign(field2, new IntegerFieldValue(20)));
+        update.addFieldUpdate(FieldUpdate.createIncrement(field3, 30));
+
+        SimpleAdapterFactory factory = new SimpleAdapterFactory();
+        List<UpdateAdapter> adapters = factory.newUpdateAdapterList(update);
+        assertEquals(2, adapters.size());
+    }
+
+    @Test
+    public void requireThatFieldUpdateCanHaveManyPartialUpdatesForOneField() {
+        DocumentType docType = new DocumentType("my_type");
+        DocumentUpdate docUpdate = new DocumentUpdate(docType, "doc:foo:1");
+        Field field = new Field("my_int", DataType.INT);
+        docType.addField(field);
+        FieldUpdate fieldUpdate = FieldUpdate.create(field);
+        fieldUpdate.addValueUpdate(ValueUpdate.createIncrement(1));
+        fieldUpdate.addValueUpdate(ValueUpdate.createIncrement(2));
+        fieldUpdate.addValueUpdate(ValueUpdate.createIncrement(4));
+        docUpdate.addFieldUpdate(fieldUpdate);
+
+        SimpleAdapterFactory factory = new SimpleAdapterFactory();
+        List<UpdateAdapter> adapters = factory.newUpdateAdapterList(docUpdate);
+        assertEquals(4, adapters.size());
+
+        UpdateAdapter adapter = adapters.get(0);
+        assertNotNull(adapter);
+        assertEquals(new IntegerFieldValue(1), adapter.getInputValue("my_int"));
+        assertNotNull(adapter = adapters.get(1));
+        assertEquals(new IntegerFieldValue(2), adapter.getInputValue("my_int"));
+        assertNotNull(adapter = adapters.get(2));
+        assertEquals(new IntegerFieldValue(4), adapter.getInputValue("my_int"));
+        assertNotNull(adapter = adapters.get(3));
+        assertNull(adapter.getInputValue("my_int")); // always add an adapter for complete updates
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapterTestCase.java
new file mode 100644
index 00000000000..ec9505e6ca8
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapterTestCase.java
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.vespa.indexinglanguage.expressions.VerificationException;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SimpleDocumentAdapterTestCase {
+
+    @Test
+    public void requireThatStructFieldsCanBeAccessed() {
+        DataType barType = DataType.STRING;
+        FieldValue bar = barType.createFieldValue("bar");
+
+        StructDataType fooType = new StructDataType("my_struct");
+        fooType.addField(new Field("bar", barType));
+        Struct foo = new Struct(fooType);
+        foo.setFieldValue("bar", bar);
+
+        DocumentType docType = new DocumentType("my_doc");
+        docType.addField("foo", fooType);
+        Document doc = new Document(docType, "doc:scheme:");
+        doc.setFieldValue("foo", foo);
+
+        DocumentAdapter adapter = new SimpleDocumentAdapter(doc);
+        assertEquals(fooType, adapter.getInputType(null, "foo"));
+        assertEquals(foo, adapter.getInputValue("foo"));
+        assertEquals(barType, adapter.getInputType(null, "foo.bar"));
+        assertEquals(bar, adapter.getInputValue("foo.bar"));
+    }
+
+    @Test
+    public void requireThatUnknownFieldsReturnNull() {
+        DocumentType docType = new DocumentType("my_doc");
+        Document doc = new Document(docType, "doc:scheme:");
+
+        DocumentAdapter adapter = new SimpleDocumentAdapter(doc);
+        try {
+            adapter.getInputType(null, "foo");
+            fail();
+        } catch (VerificationException e) {
+            assertEquals("Input field 'foo' not found.", e.getMessage());
+        }
+        assertNull(adapter.getInputValue("foo"));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleTestAdapter.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleTestAdapter.java
new file mode 100644
index 00000000000..9f25f376a2a
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleTestAdapter.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.FieldPath;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.FieldValueAdapter;
+import com.yahoo.vespa.indexinglanguage.expressions.VerificationException;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SimpleTestAdapter implements FieldValueAdapter {
+
+    private final Map<String, DataType> types = new HashMap<String, DataType>();
+    private final Map<String, FieldValue> values = new HashMap<String, FieldValue>();
+
+    public SimpleTestAdapter(Field... fields) {
+        for (Field field : fields) {
+            types.put(field.getName(), field.getDataType());
+        }
+    }
+
+    public SimpleTestAdapter createField(Field field) {
+        types.put(field.getName(), field.getDataType());
+        return this;
+    }
+
+    @Override
+    public DataType getInputType(Expression exp, String fieldName) {
+        return types.get(fieldName);
+    }
+
+    @Override
+    public FieldValue getInputValue(String fieldName) {
+        return values.get(fieldName);
+    }
+
+    @Override
+    public FieldValue getInputValue(FieldPath fieldPath) {
+        return values.get(fieldPath.toString());
+    }
+
+    @Override
+    public void tryOutputType(Expression exp, String fieldName, DataType valueType) {
+        DataType fieldType = types.get(fieldName);
+        if (fieldType == null) {
+            throw new VerificationException(exp, "Field '" + fieldName + "' not found.");
+        }
+        if (!fieldType.isAssignableFrom(valueType)) {
+            throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to field '" +
+                                                 fieldName + "' which is " + fieldType.getName() + ".");
+        }
+    }
+
+    @Override
+    public SimpleTestAdapter setOutputValue(Expression exp, String fieldName, FieldValue fieldValue) {
+        values.put(fieldName, fieldValue);
+        return this;
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/TypedExpressionConverterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/TypedExpressionConverterTestCase.java
new file mode 100644
index 00000000000..0e1e42ff391
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/TypedExpressionConverterTestCase.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class TypedExpressionConverterTestCase {
+
+    @Test
+    public void requireThatOnlyExpressionsOfGivenTypeIsConverted() {
+        assertConvert(new AttributeExpression("foo"),
+                      new IndexExpression("foo"));
+        assertConvert(new StatementExpression(new AttributeExpression("foo")),
+                      new StatementExpression(new IndexExpression("foo")));
+        assertConvert(new SummaryExpression("foo"),
+                      new SummaryExpression("foo"));
+        assertConvert(new StatementExpression(new SummaryExpression("foo")),
+                      new StatementExpression(new SummaryExpression("foo")));
+    }
+
+    private static void assertConvert(Expression before, Expression expectedAfter) {
+        assertEquals(expectedAfter, new MyConverter().convert(before));
+    }
+
+    private static class MyConverter extends TypedExpressionConverter<AttributeExpression> {
+
+        public MyConverter() {
+            super(AttributeExpression.class);
+        }
+
+        @Override
+        protected Expression typedConvert(AttributeExpression exp) {
+            return new IndexExpression(exp.getFieldName());
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueTransformProviderTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueTransformProviderTestCase.java
new file mode 100644
index 00000000000..fdcfd05736e
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueTransformProviderTestCase.java
@@ -0,0 +1,173 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import org.junit.Test;
+
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ValueTransformProviderTestCase {
+
+    @Test
+    public void requireThatInsertWorks() {
+        assertProvided(new StatementExpression(new IndexExpression("foo")),
+                       new StatementExpression(new LowerCaseExpression(),
+                                               new IndexExpression("foo")));
+    }
+
+    @Test
+    public void requireThatInsertionIsJustInTime() {
+        assertProvided(new StatementExpression(new TrimExpression(),
+                                               new IndexExpression("foo")),
+                       new StatementExpression(new TrimExpression(),
+                                               new LowerCaseExpression(),
+                                               new IndexExpression("foo")));
+    }
+
+    @Test
+    public void requireThatExistingTransformIsDetected() {
+        assertProvided(new StatementExpression(new LowerCaseExpression(),
+                                               new IndexExpression("foo")),
+                       new StatementExpression(new LowerCaseExpression(),
+                                               new IndexExpression("foo")));
+    }
+
+    @Test
+    public void requireThatExistingRedundantTransformIsRemoved() {
+        assertProvided(new StatementExpression(new IndexExpression("foo"),
+                                               new LowerCaseExpression(),
+                                               new IndexExpression("bar")),
+                       new StatementExpression(new LowerCaseExpression(),
+                                               new IndexExpression("foo"),
+                                               new IndexExpression("bar")));
+    }
+
+    @Test
+    public void requireThatNoRedundantTransformIsInserted() {
+        assertProvided(new StatementExpression(new IndexExpression("foo"),
+                                               new IndexExpression("bar")),
+                       new StatementExpression(new LowerCaseExpression(),
+                                               new IndexExpression("foo"),
+                                               new IndexExpression("bar")));
+    }
+
+    @Test
+    public void requireThatExistingTransformIsPreserved() {
+        assertProvided(new StatementExpression(new LowerCaseExpression(),
+                                               new IndexExpression("foo"),
+                                               new IndexExpression("bar")),
+                       new StatementExpression(new LowerCaseExpression(),
+                                               new IndexExpression("foo"),
+                                               new IndexExpression("bar")));
+    }
+
+    @Test
+    public void requireThatCompositeInsertWorks() {
+        assertProvided(new StatementExpression(new ForEachExpression(new IndexExpression("foo"))),
+                       new StatementExpression(new ForEachExpression(new StatementExpression(
+                               new LowerCaseExpression(),
+                               new IndexExpression("foo")))));
+    }
+
+    @Test
+    public void requireThatStatementsAreManagedSeparately() {
+        assertProvided(new ScriptExpression(new StatementExpression(new IndexExpression("foo")),
+                                            new StatementExpression(new IndexExpression("bar"))),
+                       new ScriptExpression(new StatementExpression(new LowerCaseExpression(),
+                                                                    new IndexExpression("foo")),
+                                            new StatementExpression(new LowerCaseExpression(),
+                                                                    new IndexExpression("bar"))));
+    }
+
+    @Test
+    public void requireThatIfThenBranchesAreManagedSeparately() {
+        assertProvided(new StatementExpression(new IfThenExpression(new IndexExpression("a"),
+                                                                    IfThenExpression.Comparator.EQ,
+                                                                    new IndexExpression("b"),
+                                                                    new IndexExpression("c"),
+                                                                    new IndexExpression("d")),
+                                               new IndexExpression("e")),
+                       new StatementExpression(new IfThenExpression(new StatementExpression(new LowerCaseExpression(),
+                                                                                            new IndexExpression("a")),
+                                                                    IfThenExpression.Comparator.EQ,
+                                                                    new StatementExpression(new LowerCaseExpression(),
+                                                                                            new IndexExpression("b")),
+                                                                    new StatementExpression(new LowerCaseExpression(),
+                                                                                            new IndexExpression("c")),
+                                                                    new StatementExpression(new LowerCaseExpression(),
+                                                                                            new IndexExpression("d"))),
+                                               new LowerCaseExpression(),
+                                               new IndexExpression("e")));
+    }
+
+    @Test
+    public void requireThatSelectInputBranchesAreManagedSeparately() {
+        List<Pair<String, Expression>> before = new LinkedList<Pair<String, Expression>>();
+        before.add(new Pair<String, Expression>("a", new IndexExpression("b")));
+        before.add(new Pair<String, Expression>("c", new IndexExpression("d")));
+
+        List<Pair<String, Expression>> after = new LinkedList<Pair<String, Expression>>();
+        after.add(new Pair<String, Expression>("a", new StatementExpression(new LowerCaseExpression(),
+                                                                            new IndexExpression("b"))));
+        after.add(new Pair<String, Expression>("c", new StatementExpression(new LowerCaseExpression(),
+                                                                            new IndexExpression("d"))));
+
+        assertProvided(new StatementExpression(new SelectInputExpression(before),
+                                               new IndexExpression("e")),
+                       new StatementExpression(new SelectInputExpression(after),
+                                               new LowerCaseExpression(),
+                                               new IndexExpression("e")));
+    }
+
+    @Test
+    public void requireThatSwitchBranchesAreManagedSeparately() {
+        Map<String, Expression> before = new LinkedHashMap<String, Expression>();
+        before.put("a", new IndexExpression("b"));
+        before.put("c", new IndexExpression("d"));
+
+        Map<String, Expression> after = new LinkedHashMap<String, Expression>();
+        after.put("a", new StatementExpression(new LowerCaseExpression(),
+                                               new IndexExpression("b")));
+        after.put("c", new StatementExpression(new LowerCaseExpression(),
+                                               new IndexExpression("d")));
+
+        assertProvided(new StatementExpression(new SwitchExpression(before,
+                                                                    new IndexExpression("e")),
+                                               new IndexExpression("f")),
+                       new StatementExpression(new SwitchExpression(after,
+                                                                    new StatementExpression(new LowerCaseExpression(),
+                                                                                            new IndexExpression("e"))),
+                                               new LowerCaseExpression(),
+                                               new IndexExpression("f")));
+    }
+
+    private void assertProvided(Expression before, Expression after) {
+        assertEquals(after, new MyProvider().convert(before));
+    }
+
+    private static class MyProvider extends ValueTransformProvider {
+
+        MyProvider() {
+            super(LowerCaseExpression.class);
+        }
+
+        @Override
+        protected boolean requiresTransform(Expression exp) {
+            return exp instanceof IndexExpression;
+        }
+
+        @Override
+        protected Expression newTransform() {
+            return new LowerCaseExpression();
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueUpdateToDocumentTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueUpdateToDocumentTestCase.java
new file mode 100644
index 00000000000..333c07b9e2d
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueUpdateToDocumentTestCase.java
@@ -0,0 +1,156 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.update.ValueUpdate;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ValueUpdateToDocumentTestCase {
+
+    @Test
+    public void requireThatIntegerFieldsAreConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        Field field = new Field("my_int", DataType.INT);
+        docType.addField(field);
+
+        ValueUpdate update = ValueUpdate.createAssign(new IntegerFieldValue(42));
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update);
+        assertNotNull(doc);
+
+        assertEquals(42, ((IntegerFieldValue)doc.getFieldValue("my_int")).getInteger());
+    }
+
+
+    @Test
+    public void requireThatClearValueUpdatesAreConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        Field field = new Field("my_int", DataType.INT);
+        docType.addField(field);
+
+        ValueUpdate update = ValueUpdate.createClear();
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update);
+        assertNotNull(doc);
+
+        assertNotNull(doc.getFieldValue("my_int"));
+        assertEquals(new IntegerFieldValue(), doc.getFieldValue("my_int"));
+    }
+
+
+    @Test
+    public void requireThatStringFieldsAreConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        Field field = new Field("my_str", DataType.STRING);
+        docType.addField(field);
+
+        ValueUpdate update = ValueUpdate.createAssign(new StringFieldValue("42"));
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update);
+        assertNotNull(doc);
+
+        assertEquals("42", ((StringFieldValue)doc.getFieldValue("my_str")).getString());
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    @Test
+    public void requireThatArrayFieldsAreConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        ArrayDataType arrType = DataType.getArray(DataType.INT);
+        Field field = new Field("my_arr", arrType);
+        docType.addField(field);
+
+        Array<IntegerFieldValue> arrVal = arrType.createFieldValue();
+        arrVal.add(new IntegerFieldValue(6));
+        arrVal.add(new IntegerFieldValue(9));
+        ValueUpdate update = ValueUpdate.createAssign(arrVal);
+
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("my_arr");
+        assertTrue(obj instanceof Array);
+        Array arr = (Array)obj;
+        assertEquals(2, arr.size());
+        assertEquals(new IntegerFieldValue(6), arr.get(0));
+        assertEquals(new IntegerFieldValue(9), arr.get(1));
+    }
+
+    @Test
+    public void requireThatWsetKeysAreConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        WeightedSetDataType wsetType = DataType.getWeightedSet(DataType.STRING);
+        Field field = new Field("my_wset", wsetType);
+        docType.addField(field);
+
+        ValueUpdate update = ValueUpdate.createMap(new StringFieldValue("69"), ValueUpdate.createAssign(new IntegerFieldValue(96)));
+
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("my_wset");
+        assertTrue(obj instanceof WeightedSet);
+        WeightedSet wset = (WeightedSet)obj;
+        assertEquals(1, wset.size());
+        assertEquals(96, wset.get(new StringFieldValue("69")).intValue());
+    }
+
+    @Test
+    public void requireThatNestedStructsAreConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        StructDataType structType = new StructDataType("my_struct");
+        structType.addField(new Field("b", DataType.INT));
+        Field field = new Field("a", structType);
+        docType.addField(field);
+
+        ValueUpdate update = ValueUpdate.createMap(new StringFieldValue("b"), ValueUpdate.createAssign(new IntegerFieldValue(42)));
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("a");
+        assertTrue(obj instanceof Struct);
+        Struct struct = (Struct)obj;
+        assertEquals(new IntegerFieldValue(42), struct.getFieldValue("b"));
+    }
+
+    @SuppressWarnings({ "unchecked" })
+    @Test
+    public void requireThatAddIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        ArrayDataType arrType = DataType.getArray(DataType.INT);
+        Field field = new Field("my_arr", arrType);
+        docType.addField(field);
+
+        ValueUpdate update = ValueUpdate.createMap(new IntegerFieldValue(0), ValueUpdate.createAdd(new IntegerFieldValue(6)));
+
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("my_arr");
+        assertTrue(obj instanceof Array);
+        Array arr = (Array)obj;
+        assertEquals(1, arr.size());
+        assertEquals(new IntegerFieldValue(6), arr.get(0));
+    }
+
+    @Test
+    public void requireThatRemoveIsConverted() {
+        DocumentType docType = new DocumentType("my_type");
+        ArrayDataType arrType = DataType.getArray(DataType.INT);
+        Field field = new Field("my_arr", arrType);
+        docType.addField(field);
+
+        ValueUpdate update = ValueUpdate.createClear();
+
+        Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update);
+        assertNotNull(doc);
+
+        FieldValue obj = doc.getFieldValue("my_arr");
+        assertTrue(obj instanceof Array);
+        Array arr = (Array)obj;
+        assertEquals(0, arr.size());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticTestCase.java
new file mode 100644
index 00000000000..51d2a3717a9
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticTestCase.java
@@ -0,0 +1,205 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression.Operator;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ArithmeticTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        ArithmeticExpression exp = newArithmetic(6, Operator.ADD, 9);
+        assertEquals(newLong(6), exp.getLeftHandSide());
+        assertEquals(Operator.ADD, exp.getOperator());
+        assertEquals(newLong(9), exp.getRightHandSide());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = newArithmetic(6, Operator.ADD, 9);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(newArithmetic(1, Operator.DIV, 1)));
+        assertFalse(exp.equals(newArithmetic(6, Operator.DIV, 1)));
+        assertFalse(exp.equals(newArithmetic(6, Operator.ADD, 1)));
+        assertEquals(exp, newArithmetic(6, Operator.ADD, 9));
+        assertEquals(exp.hashCode(), newArithmetic(6, Operator.ADD, 9).hashCode());
+    }
+
+    @Test
+    public void requireThatConstructorDoesNotAcceptNull() {
+        try {
+            newArithmetic(null, Operator.ADD, new SimpleExpression());
+            fail();
+        } catch (NullPointerException e) {
+
+        }
+        try {
+            newArithmetic(new SimpleExpression(), null, new SimpleExpression());
+            fail();
+        } catch (NullPointerException e) {
+
+        }
+        try {
+            newArithmetic(new SimpleExpression(), Operator.ADD, null);
+            fail();
+        } catch (NullPointerException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatVerifyCallsAreForwarded() {
+        assertVerify(SimpleExpression.newOutput(DataType.INT), Operator.ADD,
+                     SimpleExpression.newOutput(DataType.INT), null);
+        assertVerifyThrows(SimpleExpression.newOutput(null), Operator.ADD,
+                           SimpleExpression.newOutput(DataType.INT), null,
+                           "Attempting to perform arithmetic on a null value.");
+        assertVerifyThrows(SimpleExpression.newOutput(DataType.INT), Operator.ADD,
+                           SimpleExpression.newOutput(null), null,
+                           "Attempting to perform arithmetic on a null value.");
+        assertVerifyThrows(SimpleExpression.newOutput(null), Operator.ADD,
+                           SimpleExpression.newOutput(null), null,
+                           "Attempting to perform arithmetic on a null value.");
+        assertVerifyThrows(SimpleExpression.newOutput(DataType.INT), Operator.ADD,
+                           SimpleExpression.newOutput(DataType.STRING), null,
+                           "Attempting to perform unsupported arithmetic: [int] + [string]");
+        assertVerifyThrows(SimpleExpression.newOutput(DataType.STRING), Operator.ADD,
+                           SimpleExpression.newOutput(DataType.STRING), null,
+                           "Attempting to perform unsupported arithmetic: [string] + [string]");
+    }
+
+    @Test
+    public void requireThatOperandInputCanBeNull() {
+        SimpleExpression reqNull = new SimpleExpression();
+        SimpleExpression reqInt = new SimpleExpression().setRequiredInput(DataType.INT);
+        assertNull(newArithmetic(reqNull, Operator.ADD, reqNull).requiredInputType());
+        assertEquals(DataType.INT, newArithmetic(reqInt, Operator.ADD, reqNull).requiredInputType());
+        assertEquals(DataType.INT, newArithmetic(reqInt, Operator.ADD, reqInt).requiredInputType());
+        assertEquals(DataType.INT, newArithmetic(reqNull, Operator.ADD, reqInt).requiredInputType());
+    }
+
+    @Test
+    public void requireThatOperandsAreInputCompatible() {
+        assertVerify(new SimpleExpression().setRequiredInput(DataType.INT), Operator.ADD,
+                     new SimpleExpression().setRequiredInput(DataType.INT), DataType.INT);
+        assertVerifyThrows(new SimpleExpression().setRequiredInput(DataType.INT), Operator.ADD,
+                           new SimpleExpression().setRequiredInput(DataType.STRING), null,
+                           "Operands require conflicting input types, int vs string.");
+    }
+
+    @Test
+    public void requireThatResultIsCalculated() {
+        for (int i = 0; i < 50; ++i) {
+            LongFieldValue lhs = new LongFieldValue(i);
+            LongFieldValue rhs = new LongFieldValue(100 - i);
+            assertResult(lhs, Operator.ADD, rhs, new LongFieldValue(lhs.getLong() + rhs.getLong()));
+            assertResult(lhs, Operator.SUB, rhs, new LongFieldValue(lhs.getLong() - rhs.getLong()));
+            assertResult(lhs, Operator.DIV, rhs, new LongFieldValue(lhs.getLong() / rhs.getLong()));
+            assertResult(lhs, Operator.MOD, rhs, new LongFieldValue(lhs.getLong() % rhs.getLong()));
+            assertResult(lhs, Operator.MUL, rhs, new LongFieldValue(lhs.getLong() * rhs.getLong()));
+        }
+    }
+
+    @Test
+    public void requireThatArithmeticWithNullEvaluatesToNull() {
+        assertNull(newArithmetic(new SimpleExpression(), Operator.ADD,
+                                 new SetValueExpression(new LongFieldValue(69))).execute());
+        assertNull(newArithmetic(new SetValueExpression(new LongFieldValue(69)), Operator.ADD,
+                                 new SimpleExpression()).execute());
+    }
+
+    @Test
+    public void requireThatNonNumericOperandThrows() {
+        try {
+            newArithmetic(new SetValueExpression(new IntegerFieldValue(6)), Operator.ADD,
+                          new SetValueExpression(new StringFieldValue("9"))).execute();
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Unsupported operation: [int] + [string]", e.getMessage());
+        }
+        try {
+            newArithmetic(new SetValueExpression(new StringFieldValue("6")), Operator.ADD,
+                          new SetValueExpression(new IntegerFieldValue(9))).execute();
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Unsupported operation: [string] + [int]", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatProperNumericalTypeIsUsed() {
+        for (Operator op : Operator.values()) {
+            assertType(DataType.INT, op, DataType.INT, DataType.INT);
+            assertType(DataType.LONG, op, DataType.INT, DataType.LONG);
+            assertType(DataType.LONG, op, DataType.LONG, DataType.LONG);
+            assertType(DataType.INT, op, DataType.LONG, DataType.LONG);
+
+            assertType(DataType.FLOAT, op, DataType.FLOAT, DataType.FLOAT);
+            assertType(DataType.DOUBLE, op, DataType.FLOAT, DataType.DOUBLE);
+            assertType(DataType.DOUBLE, op, DataType.DOUBLE, DataType.DOUBLE);
+            assertType(DataType.FLOAT, op, DataType.DOUBLE, DataType.DOUBLE);
+
+            assertType(DataType.INT, op, DataType.FLOAT, DataType.FLOAT);
+            assertType(DataType.INT, op, DataType.DOUBLE, DataType.DOUBLE);
+        }
+    }
+
+    private void assertResult(FieldValue lhs, Operator op, FieldValue rhs, FieldValue expected) {
+        assertEquals(expected, evaluate(new SetValueExpression(lhs), op,
+                                        new SetValueExpression(rhs)));
+    }
+
+    private void assertType(DataType lhs, Operator op, DataType rhs, DataType expected) {
+        assertEquals(expected, newArithmetic(SimpleExpression.newOutput(lhs), op,
+                                             SimpleExpression.newOutput(rhs)).verify());
+        assertEquals(expected, newArithmetic(lhs.createFieldValue(6), op,
+                                             rhs.createFieldValue(9)).execute().getDataType());
+    }
+
+    private static FieldValue evaluate(Expression lhs, Operator op, Expression rhs) {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        new ArithmeticExpression(lhs, op, rhs).execute(ctx);
+        return ctx.getValue();
+    }
+
+    private static ArithmeticExpression newArithmetic(long lhs, Operator op, long rhs) {
+        return newArithmetic(new LongFieldValue(lhs), op, new LongFieldValue(rhs));
+    }
+
+    private static ArithmeticExpression newArithmetic(FieldValue lhs, Operator op, FieldValue rhs) {
+        return newArithmetic(new SetValueExpression(lhs), op, new SetValueExpression(rhs));
+    }
+
+    private static ArithmeticExpression newArithmetic(Expression lhs, Operator op, Expression rhs) {
+        return new ArithmeticExpression(lhs, op, rhs);
+    }
+
+    private static SetValueExpression newLong(long val) {
+        return new SetValueExpression(new LongFieldValue(val));
+    }
+
+    private static void assertVerify(Expression lhs, Operator op, Expression rhs, DataType val) {
+        new ArithmeticExpression(lhs, op, rhs).verify(val);
+    }
+
+    private static void assertVerifyThrows(Expression lhs, Operator op, Expression rhs, DataType val,
+                                           String expectedException) {
+        try {
+            new ArithmeticExpression(lhs, op, rhs).verify(val);
+            fail();
+        } catch (VerificationException e) {
+            assertEquals(expectedException, e.getMessage());
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpressionTestCase.java
new file mode 100644
index 00000000000..097e0f21bc1
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpressionTestCase.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertExecute;
+import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertVerify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class AttributeExpressionTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        AttributeExpression exp = new AttributeExpression("foo");
+        assertEquals("foo", exp.getFieldName());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new AttributeExpression("foo");
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new AttributeExpression("bar")));
+        assertFalse(exp.equals(new IndexExpression("foo")));
+        assertEquals(exp, new AttributeExpression("foo"));
+        assertEquals(exp.hashCode(), new AttributeExpression("foo").hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        assertVerify(new AttributeExpression("foo"));
+    }
+
+    @Test
+    public void requireThatExpressionCanBeExecuted() {
+        assertExecute(new AttributeExpression("foo"));
+    }
+}
\ No newline at end of file
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java
new file mode 100644
index 00000000000..3ed01e464be
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Base64DecodeTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new Base64DecodeExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new Base64DecodeExpression());
+        assertEquals(exp.hashCode(), new Base64DecodeExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatInputIsDecoded() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("zcIoHQ"));
+        new Base64DecodeExpression().execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof LongFieldValue);
+        assertEquals(489210573L, ((LongFieldValue)val).getLong());
+    }
+
+    @Test
+    public void requireThatEmptyStringDecodesToLongMinValue() {
+        assertEquals(new LongFieldValue(Long.MIN_VALUE),
+                     new Base64DecodeExpression().execute(new StringFieldValue("")));
+    }
+
+    @Test
+    public void requireThatInputDoesNotExceedMaxLength() {
+        try {
+            new Base64DecodeExpression().execute(new StringFieldValue("abcdefghijlkm"));
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Base64 value 'abcdefghijlkm' is out of range.", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatIllegalInputThrows() {
+        try {
+            new Base64DecodeExpression().execute(new StringFieldValue("???"));
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Illegal base64 value '???'.", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new Base64DecodeExpression();
+        assertVerify(DataType.STRING, exp, DataType.LONG);
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.LONG, exp, "Expected string input, got long.");
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeTestCase.java
new file mode 100644
index 00000000000..3f7b92cba5c
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeTestCase.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Base64EncodeTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new Base64EncodeExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new Base64EncodeExpression());
+        assertEquals(exp.hashCode(), new Base64EncodeExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatInputIsEncoded() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new LongFieldValue(489210573L));
+        new Base64EncodeExpression().execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof StringFieldValue);
+        assertEquals("zcIoHQAAAAA=", ((StringFieldValue)val).getString());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new Base64EncodeExpression();
+        assertVerify(DataType.LONG, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected long input, got null.");
+        assertVerifyThrows(DataType.STRING, exp, "Expected long input, got string.");
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java
new file mode 100644
index 00000000000..89b8c89a48f
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java
@@ -0,0 +1,247 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class CatTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        Expression foo = new AttributeExpression("foo");
+        Expression bar = new AttributeExpression("bar");
+        CatExpression exp = new CatExpression(foo, bar);
+        assertEquals(2, exp.size());
+        assertSame(foo, exp.get(0));
+        assertSame(bar, exp.get(1));
+        assertEquals(Arrays.asList(foo, bar), exp.asList());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression foo = new AttributeExpression("foo");
+        Expression bar = new AttributeExpression("bar");
+        Expression exp = new CatExpression(foo, bar);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new StatementExpression(foo, bar)));
+        assertFalse(exp.equals(new CatExpression()));
+        assertFalse(exp.equals(new CatExpression(foo)));
+        assertFalse(exp.equals(new CatExpression(bar, foo)));
+        assertEquals(exp, new CatExpression(foo, bar));
+        assertEquals(exp.hashCode(), new CatExpression(foo, bar).hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        assertVerify(new SetValueExpression(new StringFieldValue("foo")),
+                     new SetValueExpression(new StringFieldValue("bar")), null);
+        assertVerify(new SimpleExpression().setRequiredInput(DataType.STRING),
+                     new SimpleExpression().setRequiredInput(DataType.STRING), DataType.STRING);
+        assertVerifyThrows(new SimpleExpression().setCreatedOutput(null),
+                           new SimpleExpression().setCreatedOutput(DataType.STRING), null,
+                           "Attempting to concatenate a null value ");
+        assertVerifyThrows(new SimpleExpression().setRequiredInput(DataType.STRING),
+                           new SimpleExpression().setRequiredInput(DataType.INT), null,
+                           "Operands require conflicting input types, string vs int.");
+        assertVerifyThrows(new SimpleExpression().setRequiredInput(DataType.STRING),
+                           new SimpleExpression().setRequiredInput(DataType.STRING), null,
+                           "Expected string input, got null.");
+        assertVerifyThrows(new SimpleExpression().setRequiredInput(DataType.STRING),
+                           new SimpleExpression().setRequiredInput(DataType.STRING), DataType.INT,
+                           "Expected string input, got int.");
+    }
+
+    @Test
+    public void requireThatPrimitivesAreConcatenated() {
+        assertEquals(new StringFieldValue("69"), evaluate(new StringFieldValue("6"), new StringFieldValue("9")));
+        assertEquals(new StringFieldValue("69"), evaluate(new StringFieldValue("6"), new IntegerFieldValue(9)));
+        assertEquals(new StringFieldValue("69"), evaluate(new IntegerFieldValue(6), new IntegerFieldValue(9)));
+        assertEquals(new StringFieldValue("69"), evaluate(new IntegerFieldValue(6), new StringFieldValue("9")));
+    }
+
+    @Test
+    public void requireThatPrimitivesCanNotBeNull() {
+        assertNull(evaluate(DataType.STRING, new StringFieldValue("69"), DataType.STRING, null));
+        assertNull(evaluate(DataType.STRING, null, DataType.STRING, new StringFieldValue("69")));
+        assertNull(evaluate(DataType.INT, new IntegerFieldValue(69), DataType.INT, null));
+        assertNull(evaluate(DataType.INT, null, DataType.INT, new IntegerFieldValue(69)));
+    }
+
+    @Test
+    public void requireThatPrimitiveVerificationWorks() {
+        assertEquals(DataType.STRING, evaluate(DataType.getArray(DataType.STRING), DataType.STRING));
+        assertEquals(DataType.STRING, evaluate(DataType.STRING, DataType.getArray(DataType.STRING)));
+        assertEquals(DataType.STRING, evaluate(DataType.getWeightedSet(DataType.STRING), DataType.STRING));
+        assertEquals(DataType.STRING, evaluate(DataType.STRING, DataType.getWeightedSet(DataType.STRING)));
+    }
+
+    @Test
+    public void requireThatInputValueIsAvailableToAllInnerExpressions() {
+        assertEquals(new StringFieldValue("foobarfoo"),
+                     new StatementExpression(new SetValueExpression(new StringFieldValue("foo")),
+                                             new CatExpression(new ThisExpression(),
+                                                               new SetValueExpression(new StringFieldValue("bar")),
+                                                               new ThisExpression())).execute());
+    }
+
+    @Test
+    public void requiredThatRequiredInputTypeAllowsNull() {
+         assertVerify(new SetValueExpression(new StringFieldValue("foo")), new TrimExpression(), DataType.STRING);
+         assertVerify(new TrimExpression(), new SetValueExpression(new StringFieldValue("foo")), DataType.STRING);
+    }
+
+    @Test
+    public void requireThatArraysAreConcatenated() {
+        Array<StringFieldValue> lhs = new Array<>(DataType.getArray(DataType.STRING));
+        lhs.add(new StringFieldValue("6"));
+        Array<StringFieldValue> rhs = new Array<>(DataType.getArray(DataType.STRING));
+        rhs.add(new StringFieldValue("9"));
+
+        FieldValue val = evaluate(lhs, rhs);
+        assertTrue(val instanceof Array);
+
+        Array arr = (Array)val;
+        assertEquals(2, arr.size());
+        assertEquals(new StringFieldValue("6"), arr.get(0));
+        assertEquals(new StringFieldValue("9"), arr.get(1));
+    }
+
+    @Test
+    public void requireThatArraysCanBeNull() {
+        DataType type = DataType.getArray(DataType.STRING);
+        Array<StringFieldValue> arr = new Array<>(type);
+        arr.add(new StringFieldValue("9"));
+
+        FieldValue val = evaluate(type, null, type, arr);
+        assertEquals(type, val.getDataType());
+        assertEquals(1, ((Array)val).size());
+        assertEquals(new StringFieldValue("9"), ((Array)val).get(0));
+
+        val = evaluate(type, arr, type, null);
+        assertEquals(type, val.getDataType());
+        assertEquals(1, ((Array)val).size());
+        assertEquals(new StringFieldValue("9"), ((Array)val).get(0));
+    }
+
+    @Test
+    public void requireThatWsetsAreConcatenated() {
+        WeightedSet<StringFieldValue> lhs = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+        lhs.put(new StringFieldValue("6"), 9);
+        WeightedSet<StringFieldValue> rhs = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+        rhs.put(new StringFieldValue("9"), 6);
+
+        FieldValue val = evaluate(lhs, rhs);
+        assertTrue(val instanceof WeightedSet);
+
+        WeightedSet wset = (WeightedSet)val;
+        assertEquals(2, wset.size());
+        assertEquals(Integer.valueOf(9), wset.get(new StringFieldValue("6")));
+        assertEquals(Integer.valueOf(6), wset.get(new StringFieldValue("9")));
+    }
+
+    @Test
+    public void requireThatWsetsCanBeNull() {
+        DataType type = DataType.getWeightedSet(DataType.STRING);
+        WeightedSet<StringFieldValue> wset = new WeightedSet<>(type);
+        wset.put(new StringFieldValue("6"), 9);
+
+        FieldValue val = evaluate(type, null, type, wset);
+        assertEquals(type, val.getDataType());
+        assertEquals(1, ((WeightedSet)val).size());
+        assertEquals(Integer.valueOf(9), ((WeightedSet)val).get(new StringFieldValue("6")));
+
+        val = evaluate(type, wset, type, null);
+        assertEquals(type, val.getDataType());
+        assertEquals(1, ((WeightedSet)val).size());
+        assertEquals(Integer.valueOf(9), ((WeightedSet)val).get(new StringFieldValue("6")));
+    }
+
+    @Test
+    public void requireThatCollectionTypesMustBeCompatible() {
+        assertEquals(DataType.getArray(DataType.STRING), evaluate(DataType.getArray(DataType.STRING),
+                                                                  DataType.getArray(DataType.STRING)));
+        assertEquals(DataType.STRING, evaluate(DataType.getArray(DataType.STRING), DataType.getArray(DataType.INT)));
+        assertEquals(DataType.STRING,
+                     evaluate(DataType.getArray(DataType.STRING), DataType.getWeightedSet(DataType.STRING)));
+
+        assertEquals(DataType.getWeightedSet(DataType.STRING), evaluate(DataType.getWeightedSet(DataType.STRING),
+                                                                        DataType.getWeightedSet(DataType.STRING)));
+        assertEquals(DataType.STRING,
+                     evaluate(DataType.getWeightedSet(DataType.STRING), DataType.getWeightedSet(DataType.INT)));
+        assertEquals(DataType.STRING,
+                     evaluate(DataType.getWeightedSet(DataType.STRING), DataType.getArray(DataType.STRING)));
+    }
+
+    @Test
+    public void requireThatCollectionValuesMustBeCompatible() {
+        {
+            Array<StringFieldValue> arrA = new Array<>(DataType.getArray(DataType.STRING));
+            arrA.add(new StringFieldValue("6"));
+            Array<IntegerFieldValue> arrB = new Array<>(DataType.getArray(DataType.INT));
+            arrB.add(new IntegerFieldValue(9));
+            assertEquals(new StringFieldValue(arrA.toString() + arrB.toString()), evaluate(arrA, arrB));
+            assertEquals(new StringFieldValue(arrB.toString() + arrA.toString()), evaluate(arrB, arrA));
+        }
+        {
+            Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING));
+            arr.add(new StringFieldValue("6"));
+            WeightedSet<StringFieldValue> wset = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+            wset.add(new StringFieldValue("9"));
+            assertEquals(new StringFieldValue(arr.toString() + wset.toString()), evaluate(arr, wset));
+            assertEquals(new StringFieldValue(wset.toString() + arr.toString()), evaluate(wset, arr));
+        }
+        {
+            WeightedSet<StringFieldValue> wsetA = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+            wsetA.add(new StringFieldValue("6"));
+            WeightedSet<IntegerFieldValue> wsetB = new WeightedSet<>(DataType.getWeightedSet(DataType.INT));
+            wsetB.add(new IntegerFieldValue(9));
+            assertEquals(new StringFieldValue(wsetA.toString() + wsetB.toString()), evaluate(wsetA, wsetB));
+            assertEquals(new StringFieldValue(wsetB.toString() + wsetA.toString()), evaluate(wsetB, wsetA));
+        }
+    }
+
+    private static void assertVerify(Expression expA, Expression expB, DataType val) {
+        new CatExpression(expA, expB).verify(val);
+    }
+
+    private static void assertVerifyThrows(Expression expA, Expression expB, DataType val, String expectedException) {
+        try {
+            new CatExpression(expA, expB).verify(val);
+            fail();
+        } catch (VerificationException e) {
+            if (!e.getMessage().startsWith(expectedException)) {
+                assertEquals(expectedException, e.getMessage());
+            }
+        }
+    }
+
+    private static FieldValue evaluate(FieldValue valA, FieldValue valB) {
+        return evaluate(valA.getDataType(), valA, valB.getDataType(), valB);
+    }
+
+    private static FieldValue evaluate(DataType typeA, FieldValue valA, DataType typeB, FieldValue valB) {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter(new Field("a", typeA),
+                                                                          new Field("b", typeB)));
+        ctx.setOutputValue(null, "a", valA);
+        ctx.setOutputValue(null, "b", valB);
+        new CatExpression(new InputExpression("a"), new InputExpression("b")).execute(ctx);
+        return ctx.getValue();
+    }
+
+    private static DataType evaluate(DataType typeA, DataType typeB) {
+        SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("a", typeA), new Field("b", typeB));
+        VerificationContext ctx = new VerificationContext(adapter);
+        new CatExpression(new InputExpression("a"), new InputExpression("b")).verify(ctx);
+        return ctx.getValue();
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateTestCase.java
new file mode 100644
index 00000000000..4fdb63a934e
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateTestCase.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ClearStateTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new ClearStateExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new ClearStateExpression());
+        assertEquals(exp.hashCode(), new ClearStateExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExecutionContextIsCleared() {
+        MyExecution ctx = new MyExecution();
+        ctx.execute(new ClearStateExpression());
+        assertTrue(ctx.cleared);
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new ClearStateExpression();
+        assertVerify(null, exp, null);
+        assertVerify(DataType.INT, exp, null);
+        assertVerify(DataType.STRING, exp, null);
+    }
+
+    @Test
+    public void requireThatVerificationContextIsCleared() {
+        MyVerification ctx = new MyVerification();
+        ctx.execute(new ClearStateExpression());
+        assertTrue(ctx.cleared);
+    }
+
+    private static class MyExecution extends ExecutionContext {
+
+        boolean cleared = false;
+
+        @Override
+        public ExecutionContext clear() {
+            cleared = true;
+            return this;
+        }
+    }
+
+    private static class MyVerification extends VerificationContext {
+
+        boolean cleared = false;
+
+        @Override
+        public VerificationContext clear() {
+            cleared = true;
+            return this;
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpressionTestCase.java
new file mode 100644
index 00000000000..0ca70441b65
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpressionTestCase.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class CompositeExpressionTestCase {
+
+    @Test
+    public void requireThatToScriptBlockOutputIsParsable() throws ParseException {
+        Expression exp = new SetValueExpression(new IntegerFieldValue(69));
+        assertScript("{ 69; }", exp);
+        assertScript("{ 69; }", new StatementExpression(exp));
+        assertScript("{ 69; }", new ScriptExpression(new StatementExpression(exp)));
+    }
+
+    private static void assertScript(String expectedScript, Expression exp) throws ParseException {
+        String str = CompositeExpression.toScriptBlock(exp);
+        assertEquals(expectedScript, str);
+        assertNotNull(ScriptExpression.fromString(str));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/EchoTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/EchoTestCase.java
new file mode 100644
index 00000000000..9e1ae2e5350
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/EchoTestCase.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class EchoTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        assertSame(System.out, new EchoExpression().getOutputStream());
+
+        PrintStream out = new PrintStream(System.out);
+        assertSame(out, new EchoExpression(out).getOutputStream());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        PrintStream out = new PrintStream(System.out);
+        Expression exp = new EchoExpression(out);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new EchoExpression()));
+        assertFalse(exp.equals(new EchoExpression(new PrintStream(System.err))));
+        assertEquals(exp, new EchoExpression(out));
+        assertEquals(exp.hashCode(), new EchoExpression(out).hashCode());
+    }
+
+    @Test
+    public void requireThatValueIsEchoed() {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("69"));
+        new EchoExpression(new PrintStream(out)).execute(ctx);
+
+        assertEquals("69" + System.getProperty("line.separator"), out.toString());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new EchoExpression();
+        assertVerify(DataType.INT, exp, DataType.INT);
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected any input, got null.");
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java
new file mode 100644
index 00000000000..63447145613
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.annotation.*;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import java.util.Iterator;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ExactTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new ExactExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new ExactExpression());
+        assertEquals(exp.hashCode(), new ExactExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatValueIsNotChanged() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("FOO"));
+        new ExactExpression().execute(ctx);
+
+        assertEquals("FOO", String.valueOf(ctx.getValue()));
+    }
+
+    @Test
+    public void requireThatValueIsAnnotated() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("FOO"));
+        new ExactExpression().execute(ctx);
+
+        assertAnnotation(0, 3, new StringFieldValue("foo"), (StringFieldValue)ctx.getValue());
+    }
+
+    @Test
+    public void requireThatThereIsNoSegmentation() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("FOO BAR"));
+        new ExactExpression().execute(ctx);
+
+        assertAnnotation(0, 7, new StringFieldValue("foo bar"), (StringFieldValue)ctx.getValue());
+    }
+
+    @Test
+    public void requireThatRedundantAnnotationValueIsIgnored() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("foo"));
+        new ExactExpression().execute(ctx);
+
+        assertAnnotation(0, 3, null, (StringFieldValue)ctx.getValue());
+    }
+
+    @Test
+    public void requireThatEmptyStringsAreNotAnnotated() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue(""));
+        new ExactExpression().execute(ctx);
+
+        assertNull(((StringFieldValue)ctx.getValue()).getSpanTree(SpanTrees.LINGUISTICS));
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new ExactExpression();
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected string input, got int.");
+    }
+
+    private static void assertAnnotation(int expectedFrom, int expectedLen, StringFieldValue expectedVal,
+                                         StringFieldValue actualVal)
+    {
+        SpanTree tree = actualVal.getSpanTree(SpanTrees.LINGUISTICS);
+        assertNotNull(tree);
+        SpanList root = (SpanList)tree.getRoot();
+        assertNotNull(root);
+
+        Iterator<SpanNode> nodeIt = root.childIterator();
+        assertTrue(nodeIt.hasNext());
+        SpanNode node = nodeIt.next();
+        assertNotNull(node);
+        assertEquals(expectedFrom, node.getFrom());
+        assertEquals(expectedLen, node.getLength());
+
+        Iterator<Annotation> annoIt = tree.iterator(node);
+        assertTrue(annoIt.hasNext());
+        Annotation anno = annoIt.next();
+        assertNotNull(anno);
+        assertEquals(expectedVal, anno.getFieldValue());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContextTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContextTestCase.java
new file mode 100644
index 00000000000..770f2f5662c
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContextTestCase.java
@@ -0,0 +1,106 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.language.Language;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ExecutionContextTestCase {
+
+    @Test
+    public void requireThatValueCanBeSet() {
+        ExecutionContext ctx = new ExecutionContext();
+        FieldValue val = new StringFieldValue("foo");
+        ctx.setValue(val);
+        assertSame(val, ctx.getValue());
+    }
+
+    @Test
+    public void requireThatVariablesCanBeSet() {
+        ExecutionContext ctx = new ExecutionContext();
+        FieldValue val = new StringFieldValue("foo");
+        ctx.setVariable("foo", val);
+        assertSame(val, ctx.getVariable("foo"));
+    }
+
+    @Test
+    public void requireThatLanguageCanBeSet() {
+        ExecutionContext ctx = new ExecutionContext();
+        ctx.setLanguage(Language.ARABIC);
+        assertEquals(Language.ARABIC, ctx.getLanguage());
+    }
+
+    @Test
+    public void requireThatNullLanguageThrowsException() {
+        ExecutionContext ctx = new ExecutionContext();
+        try {
+            ctx.setLanguage(null);
+            fail();
+        } catch (NullPointerException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatClearRemovesValue() {
+        ExecutionContext ctx = new ExecutionContext();
+        ctx.setValue(new StringFieldValue("foo"));
+        ctx.clear();
+        assertNull(ctx.getValue());
+    }
+
+    @Test
+    public void requireThatClearRemovesVariables() {
+        ExecutionContext ctx = new ExecutionContext();
+        ctx.setVariable("foo", new StringFieldValue("foo"));
+        ctx.clear();
+        assertNull(ctx.getVariable("foo"));
+    }
+
+    @Test
+    public void requireThatClearDoesNotClearLanguage() {
+        ExecutionContext ctx = new ExecutionContext();
+        ctx.setLanguage(Language.ARABIC);
+        ctx.clear();
+        assertEquals(Language.ARABIC, ctx.getLanguage());
+    }
+
+    @Test
+    public void requireThatResolveLanguageDefaultsToEnglishWithoutLinguistics() {
+        ExecutionContext ctx = new ExecutionContext();
+        assertEquals(Language.ENGLISH, ctx.resolveLanguage(null));
+    }
+
+    @Test
+    public void requireThatResolveLanguageDefaultsToEnglishWithoutValue() {
+        ExecutionContext ctx = new ExecutionContext();
+        assertEquals(Language.ENGLISH, ctx.resolveLanguage(new SimpleLinguistics()));
+    }
+
+    @Test
+    public void requireThatLanguageCanBeResolved() {
+        ExecutionContext ctx = new ExecutionContext();
+        ctx.setValue(new StringFieldValue("\u3072\u3089\u304c\u306a"));
+        assertEquals(Language.JAPANESE, ctx.resolveLanguage(new SimpleLinguistics()));
+        ctx.setValue(new StringFieldValue("\ud55c\uae00\uacfc"));
+        assertEquals(Language.KOREAN, ctx.resolveLanguage(new SimpleLinguistics()));
+    }
+
+    @Test
+    public void requireThatExplicitLanguagePreventsDetection() {
+        ExecutionContext ctx = new ExecutionContext();
+        ctx.setLanguage(Language.ARABIC);
+        ctx.setValue(new StringFieldValue("\u3072\u3089\u304c\u306a"));
+        assertEquals(Language.ARABIC, ctx.resolveLanguage(new SimpleLinguistics()));
+        ctx.setValue(new StringFieldValue("\ud55c\uae00\uacfc"));
+        assertEquals(Language.ARABIC, ctx.resolveLanguage(new SimpleLinguistics()));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssert.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssert.java
new file mode 100644
index 00000000000..62173461744
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssert.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+
+import java.util.regex.Pattern;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+class ExpressionAssert {
+
+    public static void assertVerifyCtx(VerificationContext ctx, Expression exp, DataType expectedValueAfter) {
+        assertEquals(expectedValueAfter, exp.verify(ctx));
+    }
+
+    public static void assertVerify(DataType valueBefore, Expression exp, DataType expectedValueAfter) {
+        assertVerifyCtx(new VerificationContext().setValue(valueBefore), exp, expectedValueAfter);
+    }
+
+    public static void assertVerifyThrows(DataType valueBefore, Expression exp, String expectedException) {
+        assertVerifyCtxThrows(new VerificationContext().setValue(valueBefore), exp, expectedException);
+    }
+
+    public static void assertVerifyCtxThrows(VerificationContext ctx, Expression exp, String expectedException) {
+        try {
+            exp.verify(ctx);
+            fail();
+        } catch (VerificationException e) {
+            if (!Pattern.matches(expectedException, e.getMessage())) {
+                assertEquals(expectedException, e.getMessage());
+            }
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssertTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssertTestCase.java
new file mode 100644
index 00000000000..c4f7872cfcc
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssertTestCase.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class ExpressionAssertTestCase {
+
+    @Test
+    public void requireThatAssertVerifyMethodThrowsWhenAppropriate() {
+        Throwable thrown = null;
+        try {
+            assertVerify(DataType.INT, new SimpleExpression(), DataType.STRING);
+        } catch (Throwable t) {
+            thrown = t;
+        }
+        assertNotNull(thrown);
+
+        thrown = null;
+        try {
+            assertVerifyThrows(DataType.INT, new SimpleExpression(),
+                               "unchecked expected exception message");
+        } catch (Throwable t) {
+            thrown = t;
+        }
+        assertNotNull(thrown);
+
+        thrown = null;
+        try {
+            assertVerifyThrows(DataType.INT, SimpleExpression.newRequired(DataType.STRING),
+                               "wrong expected exception message");
+        } catch (Throwable t) {
+            thrown = t;
+        }
+        assertNotNull(thrown);
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionTestCase.java
new file mode 100644
index 00000000000..172bed83862
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionTestCase.java
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class ExpressionTestCase {
+
+    @Test
+    public void requireThatInputTypeIsCheckedBeforeExecute() {
+        assertExecute(newRequiredInput(DataType.INT), null);
+        assertExecute(newRequiredInput(DataType.INT), new IntegerFieldValue(69));
+        assertExecuteThrows(newRequiredInput(DataType.INT), new StringFieldValue("foo"),
+                            new IllegalArgumentException("expected int input, got string"));
+    }
+
+    @Test
+    public void requireThatOutputTypeIsCheckedAfterExecute() {
+        assertExecute(newCreatedOutput(DataType.INT, (FieldValue)null), null);
+        assertExecute(newCreatedOutput(DataType.INT, new IntegerFieldValue(69)), null);
+        assertExecuteThrows(newCreatedOutput(DataType.INT, new StringFieldValue("foo")), null,
+                            new IllegalStateException("expected int output, got string"));
+    }
+
+    @Test
+    public void requireThatInputTypeIsCheckedBeforeVerify() {
+        assertVerify(newRequiredInput(DataType.INT), DataType.INT);
+        assertVerifyThrows(newRequiredInput(DataType.INT), null,
+                           "Expected int input, got null.");
+        assertVerifyThrows(newRequiredInput(DataType.INT), UnresolvedDataType.INSTANCE,
+                           "Failed to resolve input type.");
+        assertVerifyThrows(newRequiredInput(DataType.INT), DataType.STRING,
+                           "Expected int input, got string.");
+    }
+
+    @Test
+    public void requireThatOutputTypeIsCheckedAfterVerify() {
+        assertVerify(newCreatedOutput(DataType.INT, DataType.INT), null);
+        assertVerifyThrows(newCreatedOutput(DataType.INT, (DataType)null), null,
+                           "Expected int output, got null.");
+        assertVerifyThrows(newCreatedOutput(DataType.INT, UnresolvedDataType.INSTANCE), null,
+                           "Failed to resolve output type.");
+        assertVerifyThrows(newCreatedOutput(DataType.INT, DataType.STRING), null,
+                           "Expected int output, got string.");
+    }
+
+    @Test
+    public void requireThatEqualsMethodWorks() {
+        assertTrue(Expression.equals(null, null));
+        assertTrue(Expression.equals(1, 1));
+        assertFalse(Expression.equals(1, 2));
+        assertFalse(Expression.equals(1, null));
+        assertFalse(Expression.equals(null, 2));
+    }
+
+    private static Expression newRequiredInput(DataType requiredInput) {
+        return new SimpleExpression().setRequiredInput(requiredInput);
+    }
+
+    private static Expression newCreatedOutput(DataType createdOutput, FieldValue actualOutput) {
+        return new SimpleExpression().setCreatedOutput(createdOutput).setExecuteValue(actualOutput);
+    }
+
+    private static Expression newCreatedOutput(DataType createdOutput, DataType actualOutput) {
+        return new SimpleExpression().setCreatedOutput(createdOutput).setVerifyValue(actualOutput);
+    }
+
+    private static void assertExecute(Expression exp, FieldValue val) {
+        exp.execute(val);
+    }
+
+    private static void assertExecuteThrows(Expression exp, FieldValue val, Exception expectedException) {
+        try {
+            exp.execute(val);
+            fail();
+        } catch (RuntimeException e) {
+            assertEquals(expectedException.getClass(), e.getClass());
+            assertTrue(e.getMessage().contains(expectedException.getMessage()));
+        }
+    }
+
+    private static void assertVerify(Expression exp, DataType val) {
+        exp.verify(val);
+    }
+
+    private static void assertVerifyThrows(Expression exp, DataType val, String expectedException) {
+        try {
+            exp.verify(val);
+            fail();
+        } catch (VerificationException e) {
+            assertEquals(expectedException, e.getMessage());
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenTestCase.java
new file mode 100644
index 00000000000..6e6ca016aa4
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenTestCase.java
@@ -0,0 +1,93 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.annotation.*;
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class FlattenTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new FlattenExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new FlattenExpression());
+        assertEquals(exp.hashCode(), new FlattenExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatAnnotationsAreFlattened() {
+        SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS);
+        tree.annotate(new Span(0, 3), new Annotation(AnnotationTypes.TERM, new StringFieldValue("oof")));
+        tree.annotate(new Span(4, 3), new Annotation(AnnotationTypes.TERM, new StringFieldValue("rab")));
+        tree.annotate(new Span(8, 3), new Annotation(AnnotationTypes.TERM, new StringFieldValue("zab")));
+
+        StringFieldValue val = new StringFieldValue("foo bar baz");
+        val.setSpanTree(tree);
+
+        assertEquals(new StringFieldValue("foo[oof] bar[rab] baz[zab]"), new FlattenExpression().execute(val));
+    }
+
+    @Test
+    public void requireThatNonTermAnnotationsAreIgnored() {
+        SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS);
+        tree.annotate(new Span(0, 3), new Annotation(AnnotationTypes.STEM, new StringFieldValue("oof")));
+
+        StringFieldValue val = new StringFieldValue("foo");
+        val.setSpanTree(tree);
+
+        assertEquals(new StringFieldValue("foo"), new FlattenExpression().execute(val));
+    }
+
+    @Test
+    public void requireThatNonSpanAnnotationsAreIgnored() {
+        SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS);
+        tree.annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("oof")));
+
+        StringFieldValue val = new StringFieldValue("foo");
+        val.setSpanTree(tree);
+
+        assertEquals(new StringFieldValue("foo"), new FlattenExpression().execute(val));
+    }
+
+    @Test
+    public void requireThatAnnotationsAreSorted() {
+        SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS);
+        tree.annotate(new Span(0, 3), new Annotation(AnnotationTypes.TERM, new StringFieldValue("cox")));
+        tree.annotate(new Span(0, 3), new Annotation(AnnotationTypes.TERM, new StringFieldValue("baz")));
+        tree.annotate(new Span(0, 3), new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));
+
+        StringFieldValue val = new StringFieldValue("foo");
+        val.setSpanTree(tree);
+
+        assertEquals(new StringFieldValue("foo[bar, baz, cox]"), new FlattenExpression().execute(val));
+    }
+
+    @Test
+    public void requireThatAnnotationsWithoutFieldValueUseOriginalSpan() {
+        SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS);
+        tree.annotate(new Span(0, 3), new Annotation(AnnotationTypes.TERM));
+
+        StringFieldValue val = new StringFieldValue("foo");
+        val.setSpanTree(tree);
+
+        assertEquals(new StringFieldValue("foo[foo]"), new FlattenExpression().execute(val));
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new FlattenExpression();
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected string input, got int.");
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java
new file mode 100644
index 00000000000..2bf00c98490
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java
@@ -0,0 +1,254 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ForEachTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        Expression innerExp = new AttributeExpression("foo");
+        ForEachExpression exp = new ForEachExpression(innerExp);
+        assertSame(innerExp, exp.getInnerExpression());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression innerExp = new AttributeExpression("foo");
+        Expression exp = new ForEachExpression(innerExp);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new ForEachExpression(new AttributeExpression("bar"))));
+        assertEquals(exp, new ForEachExpression(innerExp));
+        assertEquals(exp.hashCode(), new ForEachExpression(innerExp).hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new ForEachExpression(SimpleExpression.newConversion(DataType.INT, DataType.STRING));
+        assertVerify(DataType.getArray(DataType.INT), exp, DataType.getArray(DataType.STRING));
+        assertVerifyThrows(null, exp, "Expected any input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected Array, Struct or WeightedSet input, got int.");
+        assertVerifyThrows(DataType.getArray(DataType.STRING), exp, "Expected int input, got string.");
+    }
+
+    @Test
+    public void requireThatStructFieldCompatibilityIsVerified() {
+        StructDataType type = new StructDataType("my_struct");
+        type.addField(new Field("foo", DataType.INT));
+        assertVerify(type, new ForEachExpression(new SimpleExpression()), type);
+        assertVerifyThrows(type, new ForEachExpression(SimpleExpression.newConversion(DataType.STRING, DataType.INT)),
+                           "Expected string input, got int.");
+        assertVerifyThrows(type, new ForEachExpression(SimpleExpression.newConversion(DataType.INT, DataType.STRING)),
+                           "Expected int output, got string.");
+    }
+
+    @Test
+    public void requireThatEachTokenIsExecutedSeparately() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING));
+        arr.add(new StringFieldValue("6"));
+        arr.add(new StringFieldValue("9"));
+        ctx.setValue(arr);
+
+        MyCollector exp = new MyCollector();
+        new ForEachExpression(exp).execute(ctx);
+
+        assertEquals(2, exp.lst.size());
+
+        FieldValue val = exp.lst.get(0);
+        assertTrue(val instanceof StringFieldValue);
+        assertEquals("6", ((StringFieldValue)val).getString());
+
+        val = exp.lst.get(1);
+        assertTrue(val instanceof StringFieldValue);
+        assertEquals("9", ((StringFieldValue)val).getString());
+    }
+
+    @Test
+    public void requireThatCreatedOutputTypeDependsOnInnerExpression() {
+        assertNull(new ForEachExpression(new SimpleExpression()).createdOutputType());
+        assertNotNull(new ForEachExpression(new SetValueExpression(new IntegerFieldValue(69))).createdOutputType());
+    }
+
+    @Test
+    public void requireThatArrayCanBeConverted() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        Array<StringFieldValue> before = new Array<>(DataType.getArray(DataType.STRING));
+        before.add(new StringFieldValue("6"));
+        before.add(new StringFieldValue("9"));
+        ctx.setValue(before);
+
+        new ForEachExpression(new ToIntegerExpression()).execute(ctx);
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof Array);
+
+        Array after = (Array)val;
+        assertEquals(2, after.size());
+        assertEquals(new IntegerFieldValue(6), after.get(0));
+        assertEquals(new IntegerFieldValue(9), after.get(1));
+    }
+
+    @Test
+    public void requireThatEmptyArrayCanBeConverted() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new Array<StringFieldValue>(DataType.getArray(DataType.STRING)));
+
+        new ForEachExpression(new ToIntegerExpression()).execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof Array);
+        assertEquals(DataType.INT, ((Array)val).getDataType().getNestedType());
+        assertTrue(((Array)val).isEmpty());
+    }
+
+    @Test
+    public void requireThatIllegalInputValueThrows() {
+        try {
+            new ForEachExpression(new SimpleExpression()).execute(new StringFieldValue("foo"));
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Expected Array, Struct or WeightedSet input, got string.", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatArrayWithNullCanBeConverted() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING));
+        arr.add(new StringFieldValue("foo"));
+        ctx.setValue(arr);
+
+        new ForEachExpression(SimpleExpression.newConversion(DataType.STRING, DataType.INT)
+                                              .setExecuteValue(null)).execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof Array);
+        assertEquals(DataType.INT, ((Array)val).getDataType().getNestedType());
+        assertTrue(((Array)val).isEmpty());
+    }
+
+    @Test
+    public void requireThatWsetCanBeConverted() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        WeightedSet<StringFieldValue> before = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING));
+        before.put(new StringFieldValue("6"), 9);
+        before.put(new StringFieldValue("9"), 6);
+        ctx.setValue(before);
+
+        new ForEachExpression(new ToIntegerExpression()).execute(ctx);
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof WeightedSet);
+
+        WeightedSet after = (WeightedSet)val;
+        assertEquals(2, after.size());
+        assertEquals(new Integer(9), after.get(new IntegerFieldValue(6)));
+        assertEquals(new Integer(6), after.get(new IntegerFieldValue(9)));
+    }
+
+    @Test
+    public void requireThatEmptyWsetCanBeConverted() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new WeightedSet<StringFieldValue>(DataType.getWeightedSet(DataType.STRING)));
+
+        new ForEachExpression(new ToIntegerExpression()).execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof WeightedSet);
+        assertEquals(DataType.INT, ((WeightedSet)val).getDataType().getNestedType());
+        assertTrue(((WeightedSet)val).isEmpty());
+    }
+
+    @Test
+    public void requireThatStructContentCanBeConverted() {
+        StructDataType type = new StructDataType("my_type");
+        type.addField(new Field("my_str", DataType.STRING));
+        Struct struct = new Struct(type);
+        struct.setFieldValue("my_str", new StringFieldValue("  foo  "));
+
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(struct);
+
+        new ForEachExpression(new TrimExpression()).execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof Struct);
+        assertEquals(type, val.getDataType());
+        assertEquals(new StringFieldValue("foo"), ((Struct)val).getFieldValue("my_str"));
+    }
+
+    @Test
+    public void requireThatIncompatibleStructFieldsFailToValidate() {
+        StructDataType type = new StructDataType("my_type");
+        type.addField(new Field("my_int", DataType.INT));
+
+        VerificationContext ctx = new VerificationContext(new SimpleTestAdapter());
+        ctx.setValue(type);
+
+        try {
+            new ForEachExpression(new ToArrayExpression()).verify(ctx);
+            fail();
+        } catch (VerificationException e) {
+            assertEquals("Expected int output, got Array<int>.", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatIncompatibleStructFieldsFailToExecute() {
+        StructDataType type = new StructDataType("my_type");
+        type.addField(new Field("my_int", DataType.INT));
+        Struct struct = new Struct(type);
+        struct.setFieldValue("my_int", new IntegerFieldValue(69));
+
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(struct);
+
+        try {
+            new ForEachExpression(new ToArrayExpression()).execute(ctx);
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Class class com.yahoo.document.datatypes.Array not applicable to an class " +
+                         "com.yahoo.document.datatypes.IntegerFieldValue instance.", e.getMessage());
+        }
+    }
+
+    private static class MyCollector extends Expression {
+
+        List<FieldValue> lst = new LinkedList<>();
+
+        @Override
+        protected void doExecute(ExecutionContext ctx) {
+            lst.add(ctx.getValue());
+        }
+
+        @Override
+        protected void doVerify(VerificationContext ctx) {
+
+        }
+
+        @Override
+        public DataType requiredInputType() {
+            return null;
+        }
+
+        @Override
+        public DataType createdOutputType() {
+            return null;
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldTestCase.java
new file mode 100644
index 00000000000..1cb55e288da
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldTestCase.java
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.vespa.indexinglanguage.SimpleDocumentAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class GetFieldTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        GetFieldExpression exp = new GetFieldExpression("foo");
+        assertEquals("foo", exp.getFieldName());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new GetFieldExpression("foo");
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new GetFieldExpression("bar")));
+        assertEquals(exp, new GetFieldExpression("foo"));
+        assertEquals(exp.hashCode(), new GetFieldExpression("foo").hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        StructDataType type = new StructDataType("my_struct");
+        type.addField(new Field("foo", DataType.STRING));
+        Expression exp = new GetFieldExpression("foo");
+        assertVerify(type, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected any input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected structured input, got int.");
+        assertVerifyThrows(type, new GetFieldExpression("bar"), "Field 'bar' not found.");
+    }
+
+    @Test
+    public void requireThatStructFieldsCanBeRead() {
+        DataType barType = DataType.STRING;
+        FieldValue bar = barType.createFieldValue("bar");
+
+        StructDataType fooType = new StructDataType("my_struct");
+        fooType.addField(new Field("bar", barType));
+        Struct foo = new Struct(fooType);
+        foo.setFieldValue("bar", bar);
+
+        DocumentType docType = new DocumentType("my_doc");
+        docType.addField("foo", fooType);
+        Document doc = new Document(docType, "doc:scheme:");
+        doc.setFieldValue("foo", foo);
+
+        ExecutionContext ctx = new ExecutionContext(new SimpleDocumentAdapter(doc));
+        assertEquals(bar, new StatementExpression(new InputExpression("foo"),
+                                                  new GetFieldExpression("bar")).execute(ctx));
+    }
+
+    @Test
+    public void requireThatIllegalInputThrows() {
+        try {
+            new GetFieldExpression("foo").execute(new StringFieldValue("bar"));
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Expected structured input, got string.", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatUnknownFieldThrows() {
+        try {
+            new GetFieldExpression("foo").execute(new StructDataType("my_struct").createFieldValue());
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Field 'foo' not found.", e.getMessage());
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarTestCase.java
new file mode 100644
index 00000000000..8455a47d781
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarTestCase.java
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class GetVarTestCase {
+
+    @Test
+    public void requireThatAccessorsWorks() {
+        GetVarExpression exp = new GetVarExpression("foo");
+        assertEquals("foo", exp.getVariableName());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new GetVarExpression("foo");
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new GetVarExpression("bar")));
+        assertEquals(exp, new GetVarExpression("foo"));
+        assertEquals(exp.hashCode(), new GetVarExpression("foo").hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        VerificationContext ctx = new VerificationContext();
+        ctx.setVariable("foo", DataType.STRING);
+
+        assertEquals(DataType.STRING, new GetVarExpression("foo").verify(ctx));
+        try {
+            new GetVarExpression("bar").verify(ctx);
+            fail();
+        } catch (VerificationException e) {
+            assertEquals("Variable 'bar' not found.", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatSymbolIsRead() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setVariable("in", new IntegerFieldValue(69));
+        new GetVarExpression("in").execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof IntegerFieldValue);
+        assertEquals(69, ((IntegerFieldValue)val).getInteger());
+    }
+
+    @Test
+    public void requireThatGetVarCanBeUsedToImplementSum() throws ParseException {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setOutputValue(null, "in", new StringFieldValue("0;1;2;3;4;5;6;7;8;9"));
+        ScriptExpression.fromString("{ 0 | set_var tmp; " +
+                                    "  input in | split ';' | for_each { to_int + get_var tmp | set_var tmp };" +
+                                    "  get_var tmp | attribute out; }").execute(ctx);
+
+        FieldValue val = ctx.getInputValue("out");
+        assertTrue(val instanceof IntegerFieldValue);
+        assertEquals(45, ((IntegerFieldValue)val).getInteger());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java
new file mode 100644
index 00000000000..b6e7b1c7b41
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java
@@ -0,0 +1,126 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.update.AssignValueUpdate;
+import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.ValueUpdate;
+import com.yahoo.language.Language;
+import com.yahoo.vespa.indexinglanguage.SimpleAdapterFactory;
+import com.yahoo.vespa.indexinglanguage.UpdateAdapter;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+import org.junit.Test;
+
+import java.util.List;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class GuardTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        Expression innerExp = new AttributeExpression("foo");
+        GuardExpression exp = new GuardExpression(innerExp);
+        assertSame(innerExp, exp.getInnerExpression());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression innerExp = new AttributeExpression("foo");
+        Expression exp = new GuardExpression(innerExp);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new GuardExpression(new AttributeExpression("bar"))));
+        assertEquals(exp, new GuardExpression(innerExp));
+        assertEquals(exp.hashCode(), new GuardExpression(innerExp).hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new GuardExpression(SimpleExpression.newConversion(DataType.INT, DataType.STRING));
+        assertVerify(DataType.INT, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected int input, got null.");
+        assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string.");
+    }
+
+    @Test
+    public void requireThatInputFieldsAreIncludedByDocument() throws ParseException {
+        DocumentType docType = new DocumentType("my_input");
+        docType.addField(new Field("my_lng", DataType.LONG));
+        docType.addField(new Field("my_str", DataType.STRING));
+
+        Document doc = new Document(docType, "doc:scheme:");
+        doc.setFieldValue("my_str", new StringFieldValue("69"));
+        assertNotNull(doc = Expression.execute(Expression.fromString("guard { input my_str | to_int | attribute my_lng }"), doc));
+        assertEquals(new LongFieldValue(69), doc.getFieldValue("my_lng"));
+    }
+
+    @Test
+    public void requireThatInputFieldsAreIncludedByUpdate() throws ParseException {
+        DocumentType docType = new DocumentType("my_input");
+        docType.addField(new Field("my_lng", DataType.LONG));
+        docType.addField(new Field("my_str", DataType.STRING));
+
+        DocumentUpdate docUpdate = new DocumentUpdate(docType, "doc:scheme:");
+        docUpdate.addFieldUpdate(FieldUpdate.createAssign(docType.getField("my_str"), new StringFieldValue("69")));
+        assertNotNull(docUpdate = Expression.execute(Expression.fromString("guard { input my_str | to_int | attribute my_lng }"), docUpdate));
+
+        assertEquals(0, docUpdate.getFieldPathUpdates().size());
+        assertEquals(1, docUpdate.getFieldUpdates().size());
+
+        FieldUpdate fieldUpd = docUpdate.getFieldUpdate(0);
+        assertNotNull(fieldUpd);
+        assertEquals(docType.getField("my_lng"), fieldUpd.getField());
+        assertEquals(1, fieldUpd.getValueUpdates().size());
+
+        ValueUpdate valueUpd = fieldUpd.getValueUpdate(0);
+        assertNotNull(valueUpd);
+        assertTrue(valueUpd instanceof AssignValueUpdate);
+        assertEquals(new LongFieldValue(69), valueUpd.getValue());
+    }
+
+    @Test
+    public void requireThatConstFieldsAreIncludedByDocument() throws ParseException {
+        DocumentType docType = new DocumentType("my_input");
+        docType.addField(new Field("my_lng", DataType.LONG));
+        docType.addField(new Field("my_str", DataType.STRING));
+
+        Document doc = new Document(docType, "doc:scheme:");
+        doc.setFieldValue("my_str", new StringFieldValue("foo"));
+        assertNotNull(doc = Expression.execute(Expression.fromString("guard { now | attribute my_lng }"), doc));
+        assertTrue(doc.getFieldValue("my_lng") instanceof LongFieldValue);
+    }
+
+    @Test
+    public void requireThatConstFieldsAreSkippedByUpdate() throws ParseException {
+        DocumentType docType = new DocumentType("my_input");
+        docType.addField(new Field("my_int", DataType.INT));
+        docType.addField(new Field("my_str", DataType.STRING));
+
+        DocumentUpdate docUpdate = new DocumentUpdate(docType, "doc:scheme:");
+        docUpdate.addFieldUpdate(FieldUpdate.createAssign(docType.getField("my_str"), new StringFieldValue("foo")));
+        assertNull(Expression.execute(Expression.fromString("guard { now | attribute my_int }"), docUpdate));
+    }
+
+    @Test
+    public void requireThatLanguageCanBeSetByUpdate() throws ParseException {
+        DocumentType docType = new DocumentType("my_input");
+        docType.addField(new Field("my_str", DataType.STRING));
+        DocumentUpdate docUpdate = new DocumentUpdate(docType, "doc:scheme:");
+        docUpdate.addFieldUpdate(FieldUpdate.createAssign(docType.getField("my_str"), new StringFieldValue("foo")));
+
+        SimpleAdapterFactory factory = new SimpleAdapterFactory();
+        List<UpdateAdapter> lst = factory.newUpdateAdapterList(docUpdate);
+        assertEquals(1, lst.size());
+
+        ExecutionContext ctx = new ExecutionContext(lst.get(0));
+        Expression.fromString("guard { 'en' | set_language }").execute(ctx);
+        assertEquals(Language.ENGLISH, ctx.getLanguage());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeTestCase.java
new file mode 100644
index 00000000000..69083d48101
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeTestCase.java
@@ -0,0 +1,83 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class HexDecodeTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new HexDecodeExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new HexDecodeExpression());
+        assertEquals(exp.hashCode(), new HexDecodeExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new HexDecodeExpression();
+        assertVerify(DataType.STRING, exp, DataType.LONG);
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.LONG, exp, "Expected string input, got long.");
+    }
+
+    @Test
+    public void requireInputIsDecoded() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("1d28c2cd"));
+        new HexDecodeExpression().execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof LongFieldValue);
+        assertEquals(489210573L, ((LongFieldValue)val).getLong());
+    }
+
+    @Test
+    public void requireThatLargeInputIsDecoded() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("ff7a3c87fd74abff"));
+        new HexDecodeExpression().execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof LongFieldValue);
+        assertEquals(-37651092108694529L, ((LongFieldValue)val).getLong());
+    }
+
+    @Test
+    public void requireThatEmptyStringDecodesToLongMinValue() {
+        assertEquals(new LongFieldValue(Long.MIN_VALUE),
+                     new HexDecodeExpression().execute(new StringFieldValue("")));
+    }
+
+    @Test
+    public void requireThatInputDoesNotExceedMaxLength() {
+        try {
+            new HexDecodeExpression().execute(new StringFieldValue("1ffffffffffffffff"));
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Hex value '1ffffffffffffffff' is out of range.", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatIllegalInputThrows() {
+        try {
+            new HexDecodeExpression().execute(new StringFieldValue("???"));
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Illegal hex value '???'.", e.getMessage());
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeTestCase.java
new file mode 100644
index 00000000000..4873e428de8
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeTestCase.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class HexEncodeTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new HexEncodeExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new HexEncodeExpression());
+        assertEquals(exp.hashCode(), new HexEncodeExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new HexEncodeExpression();
+        assertVerify(DataType.LONG, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected long input, got null.");
+        assertVerifyThrows(DataType.STRING, exp, "Expected long input, got string.");
+    }
+
+    @Test
+    public void requireThatInputIsEncoded() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new LongFieldValue(489210573L));
+        new HexEncodeExpression().execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof StringFieldValue);
+        assertEquals("1d28c2cd", ((StringFieldValue)val).getString());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameTestCase.java
new file mode 100644
index 00000000000..713a8d7bcb0
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameTestCase.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class HostNameTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new HostNameExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new HostNameExpression());
+        assertEquals(exp.hashCode(), new HostNameExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new HostNameExpression();
+        assertVerify(null, exp, DataType.STRING);
+        assertVerify(DataType.INT, exp, DataType.STRING);
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+    }
+
+    @Test
+    public void requireThatHostnameIsSet() throws UnknownHostException {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        new HostNameExpression().execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof StringFieldValue);
+        assertEquals(HostNameExpression.normalizeHostName(InetAddress.getLocalHost().getHostName()),
+                     ((StringFieldValue)val).getString());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java
new file mode 100644
index 00000000000..b59564e9600
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java
@@ -0,0 +1,319 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlStream;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static com.yahoo.vespa.indexinglanguage.expressions.IfThenExpression.Comparator;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class IfThenTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        Expression lhs = new AttributeExpression("lhs");
+        Expression rhs = new AttributeExpression("rhs");
+        Expression ifTrue = new AttributeExpression("ifTrue");
+        Expression ifFalse = new AttributeExpression("ifFalse");
+        IfThenExpression exp = new IfThenExpression(lhs, Comparator.EQ, rhs, ifTrue, ifFalse);
+        assertSame(lhs, exp.getLeftHandSide());
+        assertSame(rhs, exp.getRightHandSide());
+        assertEquals(Comparator.EQ, exp.getComparator());
+        assertSame(ifTrue, exp.getIfTrueExpression());
+        assertSame(ifFalse, exp.getIfFalseExpression());
+    }
+
+    @Test
+    public void requireThatRequiredInputTypeCompatibilityIsVerified() {
+        Expression exp = newRequiredInput(DataType.STRING, Comparator.EQ, DataType.STRING,
+                                          DataType.STRING, DataType.STRING);
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected string input, got int.");
+        assertVerifyThrows(null, newRequiredInput(DataType.INT, Comparator.EQ, DataType.STRING,
+                                                  DataType.STRING, DataType.STRING),
+                           "Operands require conflicting input types, int vs string.");
+        assertVerifyThrows(null, newRequiredInput(DataType.STRING, Comparator.EQ, DataType.INT,
+                                                  DataType.STRING, DataType.STRING),
+                           "Operands require conflicting input types, string vs int.");
+        assertVerifyThrows(null, newRequiredInput(DataType.STRING, Comparator.EQ, DataType.STRING,
+                                                  DataType.INT, DataType.STRING),
+                           "Operands require conflicting input types, string vs int.");
+        assertVerifyThrows(null, newRequiredInput(DataType.STRING, Comparator.EQ, DataType.STRING,
+                                                  DataType.STRING, DataType.INT),
+                           "Operands require conflicting input types, string vs int.");
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        assertVerify(DataType.STRING, new FlattenExpression(), DataType.STRING);
+        assertVerifyThrows(null, new FlattenExpression(),
+                           "Expected string input, got null.");
+        assertVerifyThrows(DataType.INT, new FlattenExpression(),
+                           "Expected string input, got int.");
+    }
+
+    @Test
+    public void requireThatIfFalseDefaultsToNull() {
+        IfThenExpression exp = new IfThenExpression(new AttributeExpression("lhs"), Comparator.EQ,
+                                                    new AttributeExpression("rhs"), new AttributeExpression("ifTrue"));
+        assertNull(exp.getIfFalseExpression());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression lhs = new AttributeExpression("lhs");
+        Expression rhs = new AttributeExpression("rhs");
+        Expression ifTrue = new AttributeExpression("ifTrue");
+        Expression ifFalse = new AttributeExpression("ifFalse");
+        Expression exp = new IfThenExpression(lhs, Comparator.EQ, rhs, ifTrue, ifFalse);
+
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new IfThenExpression(new IndexExpression("lhs"), Comparator.NE,
+                                                    new IndexExpression("rhs"),
+                                                    new IndexExpression("ifTrue"),
+                                                    new IndexExpression("ifFalse"))));
+        assertFalse(exp.equals(new IfThenExpression(new AttributeExpression("lhs"), Comparator.NE,
+                                                    new IndexExpression("rhs"),
+                                                    new IndexExpression("ifTrue"),
+                                                    new IndexExpression("ifFalse"))));
+        assertFalse(exp.equals(new IfThenExpression(new AttributeExpression("lhs"), Comparator.EQ,
+                                                    new IndexExpression("rhs"),
+                                                    new IndexExpression("ifTrue"),
+                                                    new IndexExpression("ifFalse"))));
+        assertFalse(exp.equals(new IfThenExpression(new AttributeExpression("lhs"), Comparator.EQ,
+                                                    new AttributeExpression("rhs"),
+                                                    new IndexExpression("ifTrue"),
+                                                    new IndexExpression("ifFalse"))));
+        assertFalse(exp.equals(new IfThenExpression(new AttributeExpression("lhs"), Comparator.EQ,
+                                                    new AttributeExpression("rhs"),
+                                                    new AttributeExpression("ifTrue"),
+                                                    new IndexExpression("ifFalse"))));
+        assertEquals(exp, new IfThenExpression(new AttributeExpression("lhs"), Comparator.EQ,
+                                               new AttributeExpression("rhs"),
+                                               new AttributeExpression("ifTrue"),
+                                               new AttributeExpression("ifFalse")));
+        assertEquals(exp.hashCode(), new IfThenExpression(new AttributeExpression("lhs"), Comparator.EQ,
+                                                          new AttributeExpression("rhs"),
+                                                          new AttributeExpression("ifTrue"),
+                                                          new AttributeExpression("ifFalse")).hashCode());
+    }
+
+    @Test
+    public void requireThatAllChildrenSeeInputValue() {
+        FieldValueAdapter adapter = createTestAdapter();
+        new StatementExpression(new SetValueExpression(new IntegerFieldValue(69)),
+                                new IfThenExpression(new AttributeExpression("lhs"),
+                                                     Comparator.EQ,
+                                                     new AttributeExpression("rhs"),
+                                                     new AttributeExpression("ifTrue"),
+                                                     new AttributeExpression("ifFalse"))).execute(adapter);
+        assertEquals(new IntegerFieldValue(69), adapter.getInputValue("lhs"));
+        assertEquals(new IntegerFieldValue(69), adapter.getInputValue("rhs"));
+        assertEquals(new IntegerFieldValue(69), adapter.getInputValue("ifTrue"));
+        assertNull(null, adapter.getInputValue("ifFalse"));
+
+        adapter = createTestAdapter();
+        new StatementExpression(new SetValueExpression(new IntegerFieldValue(69)),
+                                new IfThenExpression(new AttributeExpression("lhs"),
+                                                     Comparator.NE,
+                                                     new AttributeExpression("rhs"),
+                                                     new AttributeExpression("ifTrue"),
+                                                     new AttributeExpression("ifFalse"))).execute(adapter);
+        assertEquals(new IntegerFieldValue(69), adapter.getInputValue("lhs"));
+        assertEquals(new IntegerFieldValue(69), adapter.getInputValue("rhs"));
+        assertNull(null, adapter.getInputValue("ifTrue"));
+        assertEquals(new IntegerFieldValue(69), adapter.getInputValue("ifFalse"));
+    }
+
+    @Test
+    public void requireThatElseExpIsOptional() {
+        ExecutionContext ctx = new ExecutionContext();
+        Expression exp = new IfThenExpression(new SetValueExpression(new IntegerFieldValue(6)),
+                                              Comparator.GT,
+                                              new SetValueExpression(new IntegerFieldValue(9)),
+                                              new SetValueExpression(new StringFieldValue("69")));
+        FieldValue val = ctx.setValue(new IntegerFieldValue(96)).execute(exp).getValue();
+        assertTrue(val instanceof IntegerFieldValue);
+        assertEquals(96, ((IntegerFieldValue)val).getInteger());
+    }
+
+    @Test
+    public void requireThatNonNumericValuesUseFieldValueCompareTo() {
+        FieldValue small = new MyFieldValue(6);
+        FieldValue large = new MyFieldValue(9);
+
+        assertCmpTrue(small, Comparator.EQ, small);
+        assertCmpFalse(small, Comparator.NE, small);
+        assertCmpFalse(small, Comparator.GT, small);
+        assertCmpTrue(small, Comparator.GE, small);
+        assertCmpFalse(small, Comparator.LT, small);
+        assertCmpTrue(small, Comparator.LE, small);
+
+        assertCmpFalse(small, Comparator.EQ, large);
+        assertCmpTrue(small, Comparator.NE, large);
+        assertCmpFalse(small, Comparator.GT, large);
+        assertCmpFalse(small, Comparator.GE, large);
+        assertCmpTrue(small, Comparator.LT, large);
+        assertCmpTrue(small, Comparator.LE, large);
+    }
+
+    @Test
+    public void requireThatNumericValuesSupportNumericCompareTo() {
+        List<NumericFieldValue> sixes = Arrays.asList(new ByteFieldValue((byte)6),
+                                                      new DoubleFieldValue(6.0),
+                                                      new FloatFieldValue(6.0f),
+                                                      new IntegerFieldValue(6),
+                                                      new LongFieldValue(6L));
+        List<NumericFieldValue> nines = Arrays.asList(new ByteFieldValue((byte)9),
+                                                      new DoubleFieldValue(9.0),
+                                                      new FloatFieldValue(9.0f),
+                                                      new IntegerFieldValue(9),
+                                                      new LongFieldValue(9L));
+        for (NumericFieldValue lhs : sixes) {
+            for (NumericFieldValue rhs : sixes) {
+                assertCmpTrue(lhs, Comparator.EQ, rhs);
+                assertCmpFalse(lhs, Comparator.NE, rhs);
+                assertCmpFalse(lhs, Comparator.GT, rhs);
+                assertCmpTrue(lhs, Comparator.GE, rhs);
+                assertCmpFalse(lhs, Comparator.LT, rhs);
+                assertCmpTrue(lhs, Comparator.LE, rhs);
+            }
+            for (NumericFieldValue rhs : nines) {
+                assertCmpFalse(lhs, Comparator.EQ, rhs);
+                assertCmpTrue(lhs, Comparator.NE, rhs);
+                assertCmpFalse(lhs, Comparator.GT, rhs);
+                assertCmpFalse(lhs, Comparator.GE, rhs);
+                assertCmpTrue(lhs, Comparator.LT, rhs);
+                assertCmpTrue(lhs, Comparator.LE, rhs);
+            }
+        }
+        for (NumericFieldValue lhs : nines) {
+            for (NumericFieldValue rhs : nines) {
+                assertCmpTrue(lhs, Comparator.EQ, rhs);
+                assertCmpFalse(lhs, Comparator.NE, rhs);
+                assertCmpFalse(lhs, Comparator.GT, rhs);
+                assertCmpTrue(lhs, Comparator.GE, rhs);
+                assertCmpFalse(lhs, Comparator.LT, rhs);
+                assertCmpTrue(lhs, Comparator.LE, rhs);
+            }
+            for (NumericFieldValue rhs : sixes) {
+                assertCmpFalse(lhs, Comparator.EQ, rhs);
+                assertCmpTrue(lhs, Comparator.NE, rhs);
+                assertCmpTrue(lhs, Comparator.GT, rhs);
+                assertCmpTrue(lhs, Comparator.GE, rhs);
+                assertCmpFalse(lhs, Comparator.LT, rhs);
+                assertCmpFalse(lhs, Comparator.LE, rhs);
+            }
+        }
+    }
+
+    @Test
+    public void requireThatNullLeftOrRightHandSideEvaluatesToNull() {
+        Expression exp = new IfThenExpression(new GetVarExpression("lhs"), Comparator.EQ, new GetVarExpression("rhs"),
+                                              new SetValueExpression(new StringFieldValue("true")),
+                                              new SetValueExpression(new StringFieldValue("false")));
+        assertEquals(new StringFieldValue("true"),
+                     exp.execute(new ExecutionContext().setVariable("lhs", new IntegerFieldValue(69))
+                                                       .setVariable("rhs", new IntegerFieldValue(69))));
+        assertEquals(new StringFieldValue("false"),
+                     exp.execute(new ExecutionContext().setVariable("lhs", new IntegerFieldValue(6))
+                                                       .setVariable("rhs", new IntegerFieldValue(9))));
+        assertNull(exp.execute(new ExecutionContext().setVariable("lhs", new IntegerFieldValue(69))));
+        assertNull(exp.execute(new ExecutionContext().setVariable("rhs", new IntegerFieldValue(69))));
+    }
+
+    private static void assertCmpTrue(FieldValue lhs, Comparator cmp, FieldValue rhs) {
+        assertTrue(evaluateIfThen(lhs, cmp, rhs));
+    }
+
+    private static void assertCmpFalse(FieldValue lhs, Comparator cmp, FieldValue rhs) {
+        assertFalse(evaluateIfThen(lhs, cmp, rhs));
+    }
+
+    private static boolean evaluateIfThen(FieldValue lhs, Comparator cmp, FieldValue rhs) {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        new StatementExpression(
+                new SetValueExpression(new IntegerFieldValue(1)),
+                new IfThenExpression(new SetValueExpression(lhs), cmp, new SetValueExpression(rhs),
+                                     new SetVarExpression("true"),
+                                     new SetVarExpression("false"))).execute(ctx);
+        return ctx.getVariable("true") != null;
+    }
+
+    private static FieldValueAdapter createTestAdapter() {
+        return new SimpleTestAdapter(new Field("lhs", DataType.INT),
+                                     new Field("rhs", DataType.INT),
+                                     new Field("ifTrue", DataType.INT),
+                                     new Field("ifFalse", DataType.INT));
+    }
+
+    private static Expression newRequiredInput(DataType lhs, Comparator cmp, DataType rhs, DataType ifTrue,
+                                               DataType ifFalse) {
+        return new IfThenExpression(new SimpleExpression().setRequiredInput(lhs), cmp,
+                                    new SimpleExpression().setRequiredInput(rhs),
+                                    new SimpleExpression().setRequiredInput(ifTrue),
+                                    ifFalse != null ? new SimpleExpression().setRequiredInput(ifFalse) : null);
+    }
+
+    private static class MyFieldValue extends FieldValue {
+
+        final Integer val;
+
+        MyFieldValue(int val) {
+            this.val = val;
+        }
+
+        @Override
+        public DataType getDataType() {
+            return null;
+        }
+
+        @Override
+        public void printXml(XmlStream xml) {
+
+        }
+
+        @Override
+        public void clear() {
+
+        }
+
+        @Override
+        public void assign(Object o) {
+
+        }
+
+        @Override
+        public void serialize(Field field, FieldWriter writer) {
+
+        }
+
+        @Override
+        public void deserialize(Field field, FieldReader reader) {
+
+        }
+
+        @Override
+        public int compareTo(FieldValue rhs) {
+            if (!(rhs instanceof MyFieldValue)) {
+                throw new AssertionError();
+            }
+            return val.compareTo(((MyFieldValue)rhs).val);
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IndexExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IndexExpressionTestCase.java
new file mode 100644
index 00000000000..5097a14b870
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IndexExpressionTestCase.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertExecute;
+import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertVerify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class IndexExpressionTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        IndexExpression exp = new IndexExpression("foo");
+        assertEquals("foo", exp.getFieldName());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression lhs = new IndexExpression("foo");
+        assertFalse(lhs.equals(new Object()));
+        assertFalse(lhs.equals(new AttributeExpression("foo")));
+        assertFalse(lhs.equals(new IndexExpression("bar")));
+        assertEquals(lhs, new IndexExpression("foo"));
+        assertEquals(lhs.hashCode(), new IndexExpression("foo").hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        assertVerify(new IndexExpression("foo"));
+    }
+
+    @Test
+    public void requireThatExpressionCanBeExecuted() {
+        assertExecute(new IndexExpression("foo"));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/InputTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/InputTestCase.java
new file mode 100644
index 00000000000..ed53f50444c
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/InputTestCase.java
@@ -0,0 +1,81 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.Struct;
+import com.yahoo.vespa.indexinglanguage.SimpleDocumentAdapter;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class InputTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        InputExpression exp = new InputExpression("foo");
+        assertEquals("foo", exp.getFieldName());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new InputExpression(null);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new InputExpression("foo")));
+        assertEquals(exp, new InputExpression(null));
+        assertEquals(exp.hashCode(), new InputExpression(null).hashCode());
+
+        exp = new InputExpression("foo");
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new InputExpression("bar")));
+        assertEquals(exp, new InputExpression("foo"));
+        assertEquals(exp.hashCode(), new InputExpression("foo").hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("foo", DataType.STRING));
+        adapter.setOutputValue(null, "foo", new StringFieldValue("69"));
+        assertEquals(DataType.STRING, new InputExpression("foo").verify(adapter));
+        try {
+            new InputExpression("bar").verify(adapter);
+            fail();
+        } catch (VerificationException e) {
+            assertEquals("Field 'bar' not found.", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatFieldIsRead() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter(new Field("in", DataType.STRING)));
+        ctx.setOutputValue(null, "in", new StringFieldValue("69"));
+        new InputExpression("in").execute(ctx);
+
+        assertEquals(new StringFieldValue("69"), ctx.getValue());
+    }
+
+    @Test
+    public void requireThatStructFieldsCanBeRead() {
+        DataType barType = DataType.STRING;
+        FieldValue bar = barType.createFieldValue("bar");
+
+        StructDataType fooType = new StructDataType("my_struct");
+        fooType.addField(new Field("bar", barType));
+        Struct foo = new Struct(fooType);
+        foo.setFieldValue("bar", bar);
+
+        DocumentType docType = new DocumentType("my_doc");
+        docType.addField("foo", fooType);
+        Document doc = new Document(docType, "doc:scheme:");
+        doc.setFieldValue("foo", foo);
+
+        ExecutionContext ctx = new ExecutionContext(new SimpleDocumentAdapter(doc));
+        assertEquals(foo, new InputExpression("foo").execute(ctx));
+        assertEquals(bar, new InputExpression("foo.bar").execute(ctx));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/JoinTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/JoinTestCase.java
new file mode 100644
index 00000000000..09abbf95688
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/JoinTestCase.java
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class JoinTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        JoinExpression exp = new JoinExpression("foo");
+        assertEquals("foo", exp.getDelimiter());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new JoinExpression("foo");
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new JoinExpression("bar")));
+        assertEquals(exp, new JoinExpression("foo"));
+        assertEquals(exp.hashCode(), new JoinExpression("foo").hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new JoinExpression(";");
+        assertVerify(DataType.getArray(DataType.INT), exp, DataType.STRING);
+        assertVerify(DataType.getArray(DataType.STRING), exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected any input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected Array input, got int.");
+    }
+
+    @Test
+    public void requireThatValueIsJoined() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING));
+        arr.add(new StringFieldValue("6"));
+        arr.add(new StringFieldValue("9"));
+        ctx.setValue(arr);
+
+        new JoinExpression(";").execute(ctx);
+        assertEquals(new StringFieldValue("6;9"), ctx.getValue());
+    }
+
+    @Test
+    public void requireThatNonArrayInputThrows() {
+        try {
+            new JoinExpression(";").execute(new StringFieldValue("foo"));
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Expected Array input, got string.", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatAccessorWorks() {
+        JoinExpression exp = new JoinExpression(";");
+        assertEquals(";", exp.getDelimiter());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseTestCase.java
new file mode 100644
index 00000000000..96d83847fa8
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseTestCase.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LowerCaseTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new LowerCaseExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new LowerCaseExpression());
+        assertEquals(exp.hashCode(), new LowerCaseExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new LowerCaseExpression();
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected string input, got int.");
+    }
+
+    @Test
+    public void requireThatStringIsLowerCased() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("FOO"));
+        new LowerCaseExpression().execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof StringFieldValue);
+        assertEquals("foo", ((StringFieldValue)val).getString());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolverTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolverTestCase.java
new file mode 100644
index 00000000000..b4f03dacd64
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolverTestCase.java
@@ -0,0 +1,122 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class MathResolverTestCase {
+
+    // --------------------------------------------------------------------------------
+    //
+    // Tests
+    //
+    // --------------------------------------------------------------------------------
+
+    @Test
+    public void requireThatOperatorsWork() {
+        assertEquals(3, evaluate(1, ArithmeticExpression.Operator.ADD, 2));
+        assertEquals(1, evaluate(3, ArithmeticExpression.Operator.SUB, 2));
+        assertEquals(2, evaluate(4, ArithmeticExpression.Operator.DIV, 2));
+        assertEquals(1, evaluate(3, ArithmeticExpression.Operator.MOD, 2));
+        assertEquals(4, evaluate(2, ArithmeticExpression.Operator.MUL, 2));
+    }
+
+    @Test
+    public void requireThatOperatorPrecedenceIsCorrect() {
+        assertEquals(3, evaluate(1, ArithmeticExpression.Operator.ADD, 1, ArithmeticExpression.Operator.ADD, 1));
+        assertEquals(1, evaluate(1, ArithmeticExpression.Operator.ADD, 1, ArithmeticExpression.Operator.SUB, 1));
+        assertEquals(4, evaluate(2, ArithmeticExpression.Operator.ADD, 4, ArithmeticExpression.Operator.DIV, 2));
+        assertEquals(2, evaluate(1, ArithmeticExpression.Operator.ADD, 3, ArithmeticExpression.Operator.MOD, 2));
+        assertEquals(3, evaluate(1, ArithmeticExpression.Operator.ADD, 1, ArithmeticExpression.Operator.MUL, 2));
+
+        assertEquals(1, evaluate(1, ArithmeticExpression.Operator.SUB, 1, ArithmeticExpression.Operator.ADD, 1));
+        assertEquals(-1, evaluate(1, ArithmeticExpression.Operator.SUB, 1, ArithmeticExpression.Operator.SUB, 1));
+        assertEquals(-1, evaluate(1, ArithmeticExpression.Operator.SUB, 4, ArithmeticExpression.Operator.DIV, 2));
+        assertEquals(1, evaluate(2, ArithmeticExpression.Operator.SUB, 3, ArithmeticExpression.Operator.MOD, 2));
+        assertEquals(-1, evaluate(1, ArithmeticExpression.Operator.SUB, 1, ArithmeticExpression.Operator.MUL, 2));
+
+        assertEquals(3, evaluate(4, ArithmeticExpression.Operator.DIV, 2, ArithmeticExpression.Operator.ADD, 1));
+        assertEquals(1, evaluate(4, ArithmeticExpression.Operator.DIV, 2, ArithmeticExpression.Operator.SUB, 1));
+        assertEquals(2, evaluate(4, ArithmeticExpression.Operator.DIV, 2, ArithmeticExpression.Operator.DIV, 1));
+        assertEquals(2, evaluate(4, ArithmeticExpression.Operator.DIV, 2, ArithmeticExpression.Operator.MOD, 3));
+        assertEquals(2, evaluate(4, ArithmeticExpression.Operator.DIV, 2, ArithmeticExpression.Operator.MUL, 1));
+
+        assertEquals(2, evaluate(3, ArithmeticExpression.Operator.MOD, 2, ArithmeticExpression.Operator.ADD, 1));
+        assertEquals(0, evaluate(3, ArithmeticExpression.Operator.MOD, 2, ArithmeticExpression.Operator.SUB, 1));
+        assertEquals(1, evaluate(3, ArithmeticExpression.Operator.MOD, 2, ArithmeticExpression.Operator.DIV, 1));
+        assertEquals(1, evaluate(3, ArithmeticExpression.Operator.MOD, 2, ArithmeticExpression.Operator.MOD, 2));
+        assertEquals(1, evaluate(3, ArithmeticExpression.Operator.MOD, 2, ArithmeticExpression.Operator.MUL, 1));
+
+        assertEquals(3, evaluate(1, ArithmeticExpression.Operator.MUL, 2, ArithmeticExpression.Operator.ADD, 1));
+        assertEquals(1, evaluate(1, ArithmeticExpression.Operator.MUL, 2, ArithmeticExpression.Operator.SUB, 1));
+        assertEquals(2, evaluate(2, ArithmeticExpression.Operator.MUL, 2, ArithmeticExpression.Operator.DIV, 2));
+        assertEquals(0, evaluate(1, ArithmeticExpression.Operator.MUL, 2, ArithmeticExpression.Operator.MOD, 2));
+        assertEquals(4, evaluate(1, ArithmeticExpression.Operator.MUL, 2, ArithmeticExpression.Operator.MUL, 2));
+    }
+
+    @Test
+    public void requireThatFirstOperatorIsAdd() {
+        MathResolver resolver = new MathResolver();
+        for (ArithmeticExpression.Operator type : ArithmeticExpression.Operator.values()) {
+            if (type == ArithmeticExpression.Operator.ADD) {
+                continue;
+            }
+            try {
+                resolver.push(type, newInteger(69));
+            } catch (IllegalArgumentException e) {
+                assertEquals("First item in an arithmetic operation must be an addition.", e.getMessage());
+            }
+        }
+    }
+
+    @Test
+    public void requireThatNullOperatorThrowsException() {
+        try {
+            new MathResolver().push(null, newInteger(69));
+            fail();
+        } catch (NullPointerException e) {
+
+        }
+    }
+
+    // --------------------------------------------------------------------------------
+    //
+    // Utilities
+    //
+    // --------------------------------------------------------------------------------
+
+    private static Expression newInteger(int val) {
+        return new SetValueExpression(new IntegerFieldValue(val));
+    }
+
+    private static int evaluate(Expression exp) {
+        FieldValue val = new ExecutionContext(new SimpleTestAdapter()).execute(exp).getValue();
+        assertTrue(val instanceof IntegerFieldValue);
+        return ((IntegerFieldValue)val).getInteger();
+    }
+
+    private static int evaluate(int lhs, ArithmeticExpression.Operator op, int rhs) {
+        MathResolver resolver = new MathResolver();
+        resolver.push(ArithmeticExpression.Operator.ADD, newInteger(lhs));
+        resolver.push(op, newInteger(rhs));
+        return evaluate(resolver.resolve());
+    }
+
+    private static int evaluate(int valA,
+                                ArithmeticExpression.Operator opB, int valC,
+                                ArithmeticExpression.Operator opD, int valE)
+    {
+        MathResolver resolver = new MathResolver();
+        resolver.push(ArithmeticExpression.Operator.ADD, newInteger(valA));
+        resolver.push(opB, newInteger(valC));
+        resolver.push(opD, newInteger(valE));
+        return evaluate(resolver.resolve());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java
new file mode 100644
index 00000000000..c4cd1716940
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java
@@ -0,0 +1,109 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.annotation.*;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.process.TokenType;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.Iterator;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author bratseth
+ */
+public class NGramTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        Linguistics linguistics = new SimpleLinguistics();
+        NGramExpression exp = new NGramExpression(linguistics, 69);
+        assertSame(linguistics, exp.getLinguistics());
+        assertEquals(69, exp.getGramSize());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Linguistics linguistics = new SimpleLinguistics();
+        NGramExpression exp = new NGramExpression(linguistics, 69);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new NGramExpression(Mockito.mock(Linguistics.class), 96)));
+        assertFalse(exp.equals(new NGramExpression(linguistics, 96)));
+        assertEquals(exp, new NGramExpression(linguistics, 69));
+        assertEquals(exp.hashCode(), new NGramExpression(new SimpleLinguistics(), 69).hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new NGramExpression(new SimpleLinguistics(), 69);
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected string input, got int.");
+    }
+
+    @Test
+    public void testNGrams() {
+        ExecutionContext context = new ExecutionContext(new SimpleTestAdapter());
+        context.setValue(new StringFieldValue("en gul Bille sang... "));
+        new NGramExpression(new SimpleLinguistics(), 3).execute(context);
+
+        StringFieldValue value = (StringFieldValue)context.getValue();
+        assertEquals("Grams are pure annotations - field value is unchanged", "en gul Bille sang... ",
+                     value.getString());
+        SpanTree gramTree = value.getSpanTree(SpanTrees.LINGUISTICS);
+        assertNotNull(gramTree);
+        SpanList grams = (SpanList)gramTree.getRoot();
+        Iterator<SpanNode> i = grams.childIterator();
+        assertSpan(0, 2, true, i, gramTree);  // en
+        assertSpan(2, 1, false, i, gramTree); // <space>
+        assertSpan(3, 3, true, i, gramTree);  // gul
+        assertSpan(6, 1, false, i, gramTree); // <space>
+        assertSpan(7, 3, true, i, gramTree, "bil");  // Bil
+        assertSpan(8, 3, true, i, gramTree);
+        assertSpan(9, 3, true, i, gramTree);
+        assertSpan(12, 1, false, i, gramTree); // <space>
+        assertSpan(13, 3, true, i, gramTree);
+        assertSpan(14, 3, true, i, gramTree);
+        assertSpan(17, 4, false, i, gramTree); // <...space>
+        assertFalse(i.hasNext());
+    }
+
+    private void assertSpan(int from, int length, boolean gram, Iterator<SpanNode> i, SpanTree tree) {
+        assertSpan(from, length, gram, i, tree, null);
+    }
+
+    private void assertSpan(int from, int length, boolean gram, Iterator<SpanNode> i, SpanTree tree, String termValue) {
+        if (!i.hasNext()) {
+            fail("No more spans");
+        }
+        SpanNode gramSpan = i.next();
+        assertEquals("gram start", from, gramSpan.getFrom());
+        assertEquals("gram length", length, gramSpan.getLength());
+        assertTrue(gramSpan.isLeafNode());
+        Iterator<Annotation> annotations = tree.iterator(gramSpan);
+        Annotation typeAnnotation = annotations.next();
+        assertEquals(AnnotationTypes.TOKEN_TYPE, typeAnnotation.getType());
+        int typeInt = ((IntegerFieldValue)typeAnnotation.getFieldValue()).getInteger();
+        if (gram) {
+            assertEquals(TokenType.ALPHABETIC.getValue(), typeInt);
+            Annotation termAnnotation = annotations.next();
+            assertEquals(AnnotationTypes.TERM, termAnnotation.getType());
+            if (termValue == null) {
+                assertNull(termAnnotation.getFieldValue());
+            } else {
+                assertEquals(termValue, ((StringFieldValue)termAnnotation.getFieldValue()).getString());
+            }
+        } else { // gap between grams
+            assertEquals(TokenType.PUNCTUATION.getValue(), typeInt);
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java
new file mode 100644
index 00000000000..65f966f9691
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.language.Language;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class NormalizeTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        Linguistics linguistics = new SimpleLinguistics();
+        NormalizeExpression exp = new NormalizeExpression(linguistics);
+        assertSame(linguistics, exp.getLinguistics());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Linguistics linguistics = new SimpleLinguistics();
+        NormalizeExpression exp = new NormalizeExpression(linguistics);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new NormalizeExpression(Mockito.mock(Linguistics.class))));
+        assertEquals(exp, new NormalizeExpression(linguistics));
+        assertEquals(exp.hashCode(), new NormalizeExpression(linguistics).hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new NormalizeExpression(new SimpleLinguistics());
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected string input, got int.");
+    }
+
+    @Test
+    public void requireThatInputIsNormalized() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setLanguage(Language.ENGLISH);
+        ctx.setValue(new StringFieldValue("b\u00e9yonc\u00e8"));
+        new NormalizeExpression(new SimpleLinguistics()).execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof StringFieldValue);
+        assertEquals("beyonce", ((StringFieldValue)val).getString());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NowTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NowTestCase.java
new file mode 100644
index 00000000000..dce37a1ef4a
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NowTestCase.java
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class NowTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        MyTimer timer = new MyTimer();
+        NowExpression exp = new NowExpression(timer);
+        assertSame(timer, exp.getTimer());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        MyTimer timer = new MyTimer();
+        NowExpression exp = new NowExpression(timer);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new NowExpression(new MyTimer())));
+        assertEquals(exp, new NowExpression(timer));
+        assertEquals(exp.hashCode(), new NowExpression(timer).hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new NowExpression();
+        assertVerify(null, exp, DataType.LONG);
+        assertVerify(DataType.INT, exp, DataType.LONG);
+        assertVerify(DataType.STRING, exp, DataType.LONG);
+    }
+
+    @Test
+    public void requireThatCurrentTimeIsSet() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        new NowExpression(new MyTimer()).execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof LongFieldValue);
+        assertEquals(69L, ((LongFieldValue)val).getLong());
+    }
+
+    private class MyTimer implements NowExpression.Timer {
+
+        @Override
+        public long currentTimeSeconds() {
+            return 69L;
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateTestCase.java
new file mode 100644
index 00000000000..c835eca4057
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateTestCase.java
@@ -0,0 +1,97 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.PredicateFieldValue;
+import com.yahoo.document.predicate.Predicate;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyCtx;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyCtxThrows;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class OptimizePredicateTestCase {
+
+    @Test
+    public void requireThatOptimizerIsCalledWithCloneOfInput() {
+        final Predicate predicateA = Mockito.mock(Predicate.class);
+        final Predicate predicateB = Mockito.mock(Predicate.class);
+        final PredicateFieldValue input = new PredicateFieldValue(predicateA);
+        ExecutionContext ctx = new ExecutionContext()
+                .setValue(input)
+                .setVariable("arity", new IntegerFieldValue(10));
+        FieldValue output = new OptimizePredicateExpression(
+                (predicate, options) -> {
+                    assertNotSame(predicateA, predicate);
+                    return predicateB;
+                }
+        ).execute(ctx);
+        assertNotSame(output, input);
+        assertTrue(output instanceof PredicateFieldValue);
+        assertSame(predicateB, ((PredicateFieldValue)output).getPredicate());
+    }
+
+    @Test
+    public void requireThatPredicateOptionsAreSet() {
+        final Predicate predicate = Mockito.mock(Predicate.class);
+        final PredicateFieldValue input = new PredicateFieldValue(predicate);
+        ExecutionContext ctx = new ExecutionContext()
+                .setValue(input)
+                .setVariable("arity", new IntegerFieldValue(10));
+        new OptimizePredicateExpression((predicate1, options) -> {
+            assertEquals(10, options.getArity());
+            assertEquals(0x8000000000000000L, options.getLowerBound());
+            assertEquals(0x7fffffffffffffffL, options.getUpperBound());
+            return predicate1;
+        }).execute(ctx);
+        ctx.setVariable("upper_bound", new LongFieldValue(1000));
+        ctx.setVariable("lower_bound", new LongFieldValue(0));
+        new OptimizePredicateExpression(
+                (value, options) -> {
+                    assertEquals(10, options.getArity());
+                    assertEquals(0, options.getLowerBound());
+                    assertEquals(1000, options.getUpperBound());
+                    return value;
+                }
+        ).execute(ctx);
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new OptimizePredicateExpression();
+        assertTrue(exp.equals(exp));
+        assertFalse(exp.equals(new Object()));
+        assertTrue(exp.equals(new OptimizePredicateExpression()));
+        assertEquals(exp.hashCode(), new OptimizePredicateExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new OptimizePredicateExpression();
+        assertVerifyThrows(null, exp, "Expected predicate input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected predicate input, got int.");
+        assertVerifyThrows(DataType.PREDICATE, exp, "Variable 'arity' must be set.");
+
+        VerificationContext context = new VerificationContext().setValue(DataType.PREDICATE);
+        context.setVariable("arity", DataType.STRING);
+        assertVerifyCtxThrows(context, exp, "Variable 'arity' must have type int.");
+        context.setVariable("arity", DataType.INT);
+        assertVerifyCtx(context, exp, DataType.PREDICATE);
+        context.setVariable("lower_bound", DataType.INT);
+        assertVerifyCtxThrows(context, exp, "Variable 'lower_bound' must have type long.");
+        context.setVariable("lower_bound", DataType.LONG);
+        assertVerifyCtx(context, exp, DataType.PREDICATE);
+        context.setVariable("upper_bound", DataType.INT);
+        assertVerifyCtxThrows(context, exp,  "Variable 'upper_bound' must have type long.");
+        context.setVariable("upper_bound", DataType.LONG);
+        assertVerifyCtx(context, exp, DataType.PREDICATE);
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssert.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssert.java
new file mode 100644
index 00000000000..2e6befea363
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssert.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+class OutputAssert {
+
+    public static void assertExecute(OutputExpression exp) {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter(new Field(exp.getFieldName(), DataType.STRING)));
+        ctx.setValue(new StringFieldValue("69"));
+        ctx.execute(exp);
+
+        FieldValue out = ctx.getInputValue(exp.getFieldName());
+        assertTrue(out instanceof StringFieldValue);
+        assertEquals("69", ((StringFieldValue)out).getString());
+    }
+
+    public static void assertVerify(OutputExpression exp) {
+        assertVerify(new MyAdapter(null), DataType.INT, exp);
+        assertVerify(new MyAdapter(null), DataType.STRING, exp);
+        assertVerifyThrows(new MyAdapter(null), null, exp, "Expected any input, got null.");
+        assertVerifyThrows(new MyAdapter(new VerificationException(null, "foo")), DataType.INT, exp, "foo");
+    }
+
+    public static void assertVerify(FieldTypeAdapter adapter, DataType value, Expression exp) {
+        assertEquals(value, new VerificationContext(adapter).setValue(value).execute(exp).getValue());
+    }
+
+    public static void assertVerifyThrows(FieldTypeAdapter adapter, DataType value, Expression exp,
+                                          String expectedException)
+    {
+        try {
+            new VerificationContext(adapter).setValue(value).execute(exp);
+            fail();
+        } catch (VerificationException e) {
+            assertEquals(expectedException, e.getMessage());
+        }
+    }
+
+    private static class MyAdapter implements FieldTypeAdapter {
+
+        final RuntimeException e;
+
+        MyAdapter(RuntimeException e) {
+            this.e = e;
+        }
+
+        @Override
+        public DataType getInputType(Expression exp, String fieldName) {
+            throw new AssertionError();
+        }
+
+        @Override
+        public void tryOutputType(Expression exp, String fieldName, DataType valueType) {
+            if (e != null) {
+                throw e;
+            }
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssertTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssertTestCase.java
new file mode 100644
index 00000000000..dd1a0bdaa5d
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssertTestCase.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertVerifyThrows;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class OutputAssertTestCase {
+
+    @Test
+    public void requireThatAssertVerifyMethodThrowsWhenAppropriate() {
+        Throwable thrown = null;
+        try {
+            assertVerify(null, DataType.INT, SimpleExpression.newRequired(DataType.STRING));
+        } catch (Throwable t) {
+            thrown = t;
+        }
+        assertNotNull(thrown);
+
+        thrown = null;
+        try {
+            assertVerifyThrows(null, DataType.INT, SimpleExpression.newRequired(DataType.INT),
+                               "unchecked expected exception message");
+        } catch (Throwable t) {
+            thrown = t;
+        }
+        assertNotNull(thrown);
+
+        thrown = null;
+        try {
+            assertVerifyThrows(null, DataType.INT, SimpleExpression.newRequired(DataType.STRING),
+                               "wrong expected exception message");
+        } catch (Throwable t) {
+            thrown = t;
+        }
+        assertNotNull(thrown);
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisTestCase.java
new file mode 100644
index 00000000000..2e7bf148d60
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisTestCase.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ParenthesisTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        Expression innerExp = new AttributeExpression("foo");
+        ParenthesisExpression exp = new ParenthesisExpression(innerExp);
+        assertSame(innerExp, exp.getInnerExpression());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression innerExp = new AttributeExpression("foo");
+        Expression exp = new ParenthesisExpression(innerExp);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new ParenthesisExpression(new AttributeExpression("bar"))));
+        assertEquals(exp, new ParenthesisExpression(innerExp));
+        assertEquals(exp.hashCode(), new ParenthesisExpression(innerExp).hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new ParenthesisExpression(SimpleExpression.newConversion(DataType.INT, DataType.STRING));
+        assertVerify(DataType.INT, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected int input, got null.");
+        assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string.");
+    }
+
+    @Test
+    public void requireThatNestedExpressionIsRun() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter(new Field("in", DataType.STRING)));
+        ctx.setOutputValue(null, "in", new StringFieldValue("69"));
+        new ParenthesisExpression(new InputExpression("in")).execute(ctx);
+
+        assertTrue(ctx.getValue() instanceof StringFieldValue);
+        assertEquals("69", ((StringFieldValue)ctx.getValue()).getString());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/RandomTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/RandomTestCase.java
new file mode 100644
index 00000000000..7ccda30d957
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/RandomTestCase.java
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class RandomTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        RandomExpression exp = new RandomExpression(69);
+        assertEquals(Integer.valueOf(69), exp.getMaxValue());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new RandomExpression(69);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new RandomExpression(96)));
+        assertEquals(exp, new RandomExpression(69));
+        assertEquals(exp.hashCode(), new RandomExpression(69).hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new RandomExpression();
+        assertVerify(null, exp, DataType.INT);
+        assertVerify(DataType.INT, exp, DataType.INT);
+        assertVerify(DataType.STRING, exp, DataType.INT);
+    }
+
+    @Test
+    public void requireThatRandomValueIsSet() {
+        for (int i = 0; i < 666; ++i) {
+            ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+            new RandomExpression(69).execute(ctx);
+
+            FieldValue val = ctx.getValue();
+            assertTrue(val instanceof IntegerFieldValue);
+            assertTrue(((IntegerFieldValue)val).getInteger() < 69);
+        }
+    }
+
+    @Test
+    public void requireThatInputValueIsParsedAsMaxIfNoneIsConfigured() {
+        for (int i = 0; i < 666; ++i) {
+            ExecutionContext ctx = new ExecutionContext().setValue(new IntegerFieldValue(69));
+            new RandomExpression().execute(ctx);
+
+            FieldValue val = ctx.getValue();
+            assertTrue(val instanceof IntegerFieldValue);
+            assertTrue(((IntegerFieldValue)val).getInteger() < 69);
+        }
+    }
+
+    @Test
+    public void requireThatIllegalInputThrowsNumberFormatException() {
+        try {
+            new RandomExpression().execute(new ExecutionContext());
+            fail();
+        } catch (NumberFormatException e) {
+
+        }
+        try {
+            new RandomExpression().execute(new ExecutionContext().setValue(new StringFieldValue("foo")));
+            fail();
+        } catch (NumberFormatException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatDefaultMaxIsNull() {
+        assertNull(new RandomExpression().getMaxValue());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java
new file mode 100644
index 00000000000..e204596bbcf
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java
@@ -0,0 +1,124 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ScriptTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        ScriptExpression exp = newScript();
+        assertTrue(exp.isEmpty());
+        assertEquals(0, exp.size());
+
+        StatementExpression foo = newStatement(new AttributeExpression("foo"));
+        StatementExpression bar = newStatement(new AttributeExpression("bar"));
+        exp = newScript(foo, bar);
+        assertFalse(exp.isEmpty());
+        assertEquals(2, exp.size());
+        assertSame(foo, exp.get(0));
+        assertSame(bar, exp.get(1));
+        assertEquals(Arrays.asList(foo, bar), exp.asList());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        StatementExpression foo = newStatement(new AttributeExpression("foo"));
+        StatementExpression bar = newStatement(new AttributeExpression("bar"));
+        Expression exp = newScript(foo, bar);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new CatExpression(foo, bar)));
+        assertFalse(exp.equals(newScript(newStatement(new IndexExpression("foo")))));
+        assertFalse(exp.equals(newScript(newStatement(new IndexExpression("foo")),
+                                         newStatement(new IndexExpression("bar")))));
+        assertFalse(exp.equals(newScript(newStatement(foo),
+                                         newStatement(new IndexExpression("bar")))));
+        assertEquals(exp, newScript(foo, bar));
+        assertEquals(exp.hashCode(), newScript(foo, bar).hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = newScript(newStatement(SimpleExpression.newConversion(DataType.INT, DataType.STRING)));
+        assertVerify(DataType.INT, exp, DataType.INT); // does not touch output
+        assertVerifyThrows(null, exp, "Expected int input, got null.");
+        assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string.");
+
+        assertVerifyThrows(null, newScript(newStatement(SimpleExpression.newConversion(DataType.INT, DataType.STRING)),
+                                           newStatement(SimpleExpression.newConversion(DataType.STRING, DataType.INT))),
+                           "Statements require conflicting input types, int vs string.");
+    }
+
+    @Test
+    public void requireThatInputValueIsAvailableToAllStatements() {
+        SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out-1", DataType.INT),
+                                                          new Field("out-2", DataType.INT));
+        newStatement(new SetValueExpression(new IntegerFieldValue(69)),
+                     newScript(newStatement(new AttributeExpression("out-1"),
+                                            new AttributeExpression("out-2")))).execute(adapter);
+        assertEquals(new IntegerFieldValue(69), adapter.getInputValue("out-1"));
+        assertEquals(new IntegerFieldValue(69), adapter.getInputValue("out-2"));
+    }
+
+    @Test
+    public void requireThatScriptEvaluatesToInputValue() {
+        SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out", DataType.INT));
+        newStatement(new SetValueExpression(new IntegerFieldValue(6)),
+                     newScript(newStatement(new SetValueExpression(new IntegerFieldValue(9)))),
+                     new AttributeExpression("out")).execute(adapter);
+        assertEquals(new IntegerFieldValue(6), adapter.getInputValue("out"));
+    }
+
+    @Test
+    public void requireThatVariablesAreAvailableInScript() {
+        SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out", DataType.INT));
+        newScript(newStatement(new SetValueExpression(new IntegerFieldValue(69)),
+                               new SetVarExpression("tmp")),
+                  newStatement(new GetVarExpression("tmp"),
+                               new AttributeExpression("out"))).execute(adapter);
+        assertEquals(new IntegerFieldValue(69), adapter.getInputValue("out"));
+    }
+
+    @Test
+    public void requireThatVariablesAreAvailableOutsideScript() {
+        SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out", DataType.INT));
+        newStatement(newScript(newStatement(new SetValueExpression(new IntegerFieldValue(69)),
+                                            new SetVarExpression("tmp"))),
+                     new GetVarExpression("tmp"),
+                     new AttributeExpression("out")).execute(adapter);
+        assertEquals(new IntegerFieldValue(69), adapter.getInputValue("out"));
+    }
+
+    @Test
+    public void requireThatVariablesReplaceOthersOutsideScript() {
+        SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out", DataType.INT));
+        newStatement(new SetValueExpression(new IntegerFieldValue(6)),
+                     new SetVarExpression("tmp"),
+                     newScript(newStatement(new SetValueExpression(new IntegerFieldValue(9)),
+                                            new SetVarExpression("tmp"))),
+                     new GetVarExpression("tmp"),
+                     new AttributeExpression("out")).execute(adapter);
+        assertEquals(new IntegerFieldValue(9), adapter.getInputValue("out"));
+    }
+
+    private static ScriptExpression newScript(StatementExpression... args) {
+        return new ScriptExpression(args);
+    }
+
+    private static StatementExpression newStatement(Expression... args) {
+        return new StatementExpression(args);
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java
new file mode 100644
index 00000000000..a19b1a8f349
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java
@@ -0,0 +1,111 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SelectInputTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        List<Pair<String, Expression>> cases = new LinkedList<>();
+        Pair<String, Expression> foo = new Pair<String, Expression>("foo", new AttributeExpression("foo"));
+        Pair<String, Expression> bar = new Pair<String, Expression>("bar", new AttributeExpression("bar"));
+        cases.add(foo);
+        cases.add(bar);
+        SelectInputExpression exp = new SelectInputExpression(cases);
+
+        assertEquals(2, exp.getCases().size());
+        assertSame(foo, exp.getCases().get(0));
+        assertSame(bar, exp.getCases().get(1));
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression foo = new AttributeExpression("foo");
+        Expression exp = newSelectInput(foo, "bar", "baz");
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(newSelectInput(new IndexExpression("foo"))));
+        assertFalse(exp.equals(newSelectInput(new IndexExpression("foo"), "bar")));
+        assertFalse(exp.equals(newSelectInput(new IndexExpression("foo"), "bar", "baz")));
+        assertFalse(exp.equals(newSelectInput(foo)));
+        assertFalse(exp.equals(newSelectInput(foo, "bar")));
+        assertEquals(exp, newSelectInput(foo, "bar", "baz"));
+        assertEquals(exp.hashCode(), newSelectInput(foo, "bar", "baz").hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        SimpleTestAdapter adapter = new SimpleTestAdapter();
+        adapter.createField(new Field("my_int", DataType.INT));
+        adapter.createField(new Field("my_str", DataType.STRING));
+
+        Expression exp = newSelectInput(new AttributeExpression("my_int"), "my_int");
+        assertVerify(adapter, null, exp);
+        assertVerify(adapter, DataType.INT, exp);
+        assertVerify(adapter, DataType.STRING, exp);
+
+        assertVerifyThrows(adapter, newSelectInput(new AttributeExpression("my_int"), "my_str"),
+                           "Can not assign string to field 'my_int' which is int.");
+        assertVerifyThrows(adapter, newSelectInput(new AttributeExpression("my_int"), "my_unknown"),
+                           "Field 'my_unknown' not found.");
+    }
+
+    @Test
+    public void requireThatSelectedExpressionIsRun() {
+        assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("foo"), "foo");
+        assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("bar"), "bar");
+        assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("foo", "bar"), "foo");
+        assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("bar", "baz"), "bar");
+        assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("baz", "cox"), null);
+    }
+
+    private static void assertVerify(FieldTypeAdapter adapter, DataType value, Expression exp) {
+        assertEquals(value, exp.verify(new VerificationContext(adapter).setValue(value)));
+    }
+
+    private static void assertVerifyThrows(FieldTypeAdapter adapter, Expression exp, String expectedException) {
+        try {
+            exp.verify(new VerificationContext(adapter));
+            fail();
+        } catch (VerificationException e) {
+            assertEquals(expectedException, e.getMessage());
+        }
+    }
+
+    private static SelectInputExpression newSelectInput(Expression exp, String... fieldNames) {
+        List<Pair<String, Expression>> cases = new LinkedList<>();
+        for (String fieldName : fieldNames) {
+            cases.add(new Pair<>(fieldName, exp));
+        }
+        return new SelectInputExpression(cases);
+    }
+
+    private static void assertSelect(List<String> inputField, List<String> availableFields, String expected) {
+        SimpleTestAdapter adapter = new SimpleTestAdapter();
+        ExecutionContext ctx = new ExecutionContext(adapter);
+        for (String fieldName : availableFields) {
+            adapter.createField(new Field(fieldName, DataType.STRING));
+            ctx.setOutputValue(null, fieldName, new StringFieldValue(fieldName));
+        }
+        List<Pair<String, Expression>> cases = new LinkedList<>();
+        for (String fieldName : inputField) {
+            cases.add(new Pair<String, Expression>(fieldName, new SetVarExpression("out")));
+        }
+        new SelectInputExpression(cases).execute(ctx);
+        assertEquals(expected != null ? new StringFieldValue(expected) : null, ctx.getVariable("out"));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageTestCase.java
new file mode 100644
index 00000000000..782cedcc571
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageTestCase.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.language.Language;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SetLanguageTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new SetLanguageExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new SetLanguageExpression());
+        assertEquals(exp.hashCode(), new SetLanguageExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new SetLanguageExpression();
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected string input, got int.");
+    }
+
+    @Test
+    public void requireThatLanguageIsSet() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("en"));
+        new SetLanguageExpression().execute(ctx);
+        assertEquals(Language.ENGLISH, ctx.getLanguage());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueTestCase.java
new file mode 100644
index 00000000000..a6e55076286
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueTestCase.java
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SetValueTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        FieldValue foo = new StringFieldValue("foo");
+        SetValueExpression exp = new SetValueExpression(foo);
+        assertSame(foo, exp.getValue());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        FieldValue foo = new StringFieldValue("foo");
+        Expression exp = new SetValueExpression(foo);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new SetValueExpression(new StringFieldValue("bar"))));
+        assertEquals(exp, new SetValueExpression(foo));
+        assertEquals(exp.hashCode(), new SetValueExpression(foo).hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new SetValueExpression(new StringFieldValue("foo"));
+        assertVerify(null, exp, DataType.STRING);
+        assertVerify(DataType.INT, exp, DataType.STRING);
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+    }
+
+    @Test
+    public void requireThatNullValueThrowsException() {
+        try {
+            new SetValueExpression(null);
+            fail();
+        } catch (NullPointerException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatValueIsSet() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        new SetValueExpression(new StringFieldValue("69")).execute(ctx);
+        assertEquals(new StringFieldValue("69"), ctx.getValue());
+    }
+
+    @Test
+    public void requireThatLongFieldValueGetsATrailingL() {
+        assertEquals("69L", new SetValueExpression(new LongFieldValue(69)).toString());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarTestCase.java
new file mode 100644
index 00000000000..2a3d913733b
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarTestCase.java
@@ -0,0 +1,81 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SetVarTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        SetVarExpression exp = new SetVarExpression("foo");
+        assertEquals("foo", exp.getVariableName());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new SetVarExpression("foo");
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new SetVarExpression("bar")));
+        assertEquals(exp, new SetVarExpression("foo"));
+        assertEquals(exp.hashCode(), new SetVarExpression("foo").hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new SetVarExpression("foo");
+        assertVerify(DataType.INT, exp, DataType.INT);
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected any input, got null.");
+
+        try {
+            new VerificationContext().setVariable("foo", DataType.INT).setValue(DataType.STRING).execute(exp);
+            fail();
+        } catch (VerificationException e) {
+            assertEquals("Attempting to assign conflicting types to variable 'foo', int vs string.", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatSymbolIsWritten() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new IntegerFieldValue(69));
+        new SetVarExpression("out").execute(ctx);
+
+        FieldValue val = ctx.getVariable("out");
+        assertTrue(val instanceof IntegerFieldValue);
+        assertEquals(69, ((IntegerFieldValue)val).getInteger());
+    }
+
+    @Test
+    public void requireThatVariableTypeCanNotChange() {
+        VerificationContext ctx = new VerificationContext(new SimpleTestAdapter());
+        ctx.setValue(DataType.INT);
+        new SetVarExpression("out").verify(ctx);
+
+        try {
+            ctx.setValue(DataType.STRING);
+            new SetVarExpression("out").verify(ctx);
+            fail();
+        } catch (VerificationException e) {
+            assertTrue(e.getExpression() instanceof SetVarExpression);
+            assertEquals("Attempting to assign conflicting types to variable 'out', int vs string.", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatAccessorWorks() {
+        SetVarExpression exp = new SetVarExpression("69");
+        assertEquals("69", exp.getVariableName());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java
new file mode 100644
index 00000000000..11f269b4d01
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java
@@ -0,0 +1,117 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.datatypes.FieldValue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+class SimpleExpression extends Expression {
+
+    private boolean hasExecuteValue = false;
+    private boolean hasVerifyValue = false;
+    private FieldValue executeValue;
+    private DataType verifyValue;
+    private DataType requiredInput;
+    private DataType createdOutput;
+
+    public SimpleExpression setExecuteValue(FieldValue executeValue) {
+        this.hasExecuteValue = true;
+        this.executeValue = executeValue;
+        return this;
+    }
+
+    public SimpleExpression setVerifyValue(DataType verifyValue) {
+        this.hasVerifyValue = true;
+        this.verifyValue = verifyValue;
+        return this;
+    }
+
+    public SimpleExpression setRequiredInput(DataType requiredInput) {
+        this.requiredInput = requiredInput;
+        return this;
+    }
+
+    public SimpleExpression setCreatedOutput(DataType createdOutput) {
+        this.createdOutput = createdOutput;
+        return this;
+    }
+
+    @Override
+    protected void doExecute(ExecutionContext ctx) {
+        if (hasExecuteValue) {
+            ctx.setValue(executeValue);
+        }
+    }
+
+    @Override
+    protected void doVerify(VerificationContext ctx) {
+        if (hasVerifyValue) {
+            ctx.setValue(verifyValue);
+        }
+    }
+
+    @Override
+    public DataType requiredInputType() {
+        return requiredInput;
+    }
+
+    @Override
+    public DataType createdOutputType() {
+        return createdOutput;
+    }
+
+    @Override
+    public int hashCode() {
+        return hashCode(executeValue) + hashCode(verifyValue) + hashCode(requiredInput) + hashCode(createdOutput);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof SimpleExpression)) {
+            return false;
+        }
+        SimpleExpression rhs = (SimpleExpression)obj;
+        if (hasExecuteValue != rhs.hasExecuteValue) {
+            return false;
+        }
+        if (!equals(executeValue, rhs.executeValue)) {
+            return false;
+        }
+        if (hasVerifyValue != rhs.hasVerifyValue) {
+            return false;
+        }
+        if (!equals(verifyValue, rhs.verifyValue)) {
+            return false;
+        }
+        if (!equals(requiredInput, rhs.requiredInput)) {
+            return false;
+        }
+        if (!equals(createdOutput, rhs.createdOutput)) {
+            return false;
+        }
+        return true;
+    }
+
+    public static SimpleExpression newOutput(DataType createdOutput) {
+        return new SimpleExpression().setCreatedOutput(createdOutput)
+                                     .setVerifyValue(createdOutput);
+    }
+
+    public static SimpleExpression newRequired(DataType requiredInput) {
+        return new SimpleExpression().setRequiredInput(requiredInput);
+    }
+
+    public static SimpleExpression newConversion(DataType requiredInput, DataType createdOutput) {
+        return new SimpleExpression().setRequiredInput(requiredInput)
+                                     .setCreatedOutput(createdOutput)
+                                     .setExecuteValue(createdOutput.createFieldValue())
+                                     .setVerifyValue(createdOutput);
+    }
+
+    private static int hashCode(Object obj) {
+        return obj != null ? obj.hashCode() : 0;
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpressionTestCase.java
new file mode 100644
index 00000000000..a44891cc55f
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpressionTestCase.java
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class SimpleExpressionTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        SimpleExpression exp = new SimpleExpression();
+        assertNull(exp.requiredInputType());
+        assertNull(exp.createdOutputType());
+        assertNull(exp.execute());
+        assertNull(exp.verify());
+
+        assertEquals(DataType.INT, new SimpleExpression().setRequiredInput(DataType.INT).requiredInputType());
+        assertEquals(DataType.INT, new SimpleExpression().setCreatedOutput(DataType.INT).createdOutputType());
+        assertEquals(DataType.INT, new SimpleExpression().setVerifyValue(DataType.INT).verify());
+        assertEquals(new IntegerFieldValue(69),
+                     new SimpleExpression().setExecuteValue(new IntegerFieldValue(69)).execute());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new SimpleExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new SimpleExpression());
+        assertEquals(exp.hashCode(), new SimpleExpression().hashCode());
+
+        exp = new SimpleExpression().setExecuteValue(null);
+        assertFalse(exp.equals(new SimpleExpression()));
+        assertEquals(exp, new SimpleExpression().setExecuteValue(null));
+
+        exp = new SimpleExpression().setExecuteValue(new IntegerFieldValue(69));
+        assertFalse(exp.equals(new SimpleExpression().setExecuteValue(new IntegerFieldValue(96))));
+        assertEquals(exp, new SimpleExpression().setExecuteValue(new IntegerFieldValue(69)));
+
+        exp = new SimpleExpression().setVerifyValue(null);
+        assertFalse(exp.equals(new SimpleExpression()));
+        assertEquals(exp, new SimpleExpression().setVerifyValue(null));
+
+        exp = new SimpleExpression().setVerifyValue(DataType.INT);
+        assertFalse(exp.equals(new SimpleExpression().setVerifyValue(DataType.STRING)));
+        assertEquals(exp, new SimpleExpression().setVerifyValue(DataType.INT));
+
+        exp = new SimpleExpression().setRequiredInput(DataType.INT);
+        assertFalse(exp.equals(new SimpleExpression().setRequiredInput(DataType.STRING)));
+        assertEquals(exp, new SimpleExpression().setRequiredInput(DataType.INT));
+
+        exp = new SimpleExpression().setCreatedOutput(DataType.INT);
+        assertFalse(exp.equals(new SimpleExpression().setCreatedOutput(DataType.STRING)));
+        assertEquals(exp, new SimpleExpression().setCreatedOutput(DataType.INT));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SplitTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SplitTestCase.java
new file mode 100644
index 00000000000..4abab5c7c7d
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SplitTestCase.java
@@ -0,0 +1,77 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SplitTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        SplitExpression exp = new SplitExpression("foo");
+        assertEquals("foo", exp.getSplitPattern().toString());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new SplitExpression("foo");
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new SplitExpression("bar")));
+        assertEquals(exp, new SplitExpression("foo"));
+        assertEquals(exp.hashCode(), new SplitExpression("foo").hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new SplitExpression(";");
+        assertVerify(DataType.STRING, exp, DataType.getArray(DataType.STRING));
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected string input, got int.");
+    }
+
+    @Test
+    public void requireThatValueIsSplit() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("6;9"));
+        new SplitExpression(";").execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val.getDataType().equals(DataType.getArray(DataType.STRING)));
+        assertTrue(val instanceof Array);
+
+        Array arr = (Array)val;
+        assertEquals(new StringFieldValue("6"), arr.get(0));
+        assertEquals(new StringFieldValue("9"), arr.get(1));
+    }
+
+    @Test
+    public void requireThatNullInputProducesNullOutput() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        new SplitExpression(";").execute(ctx);
+
+        assertNull(ctx.getValue());
+    }
+
+    @Test
+    public void requireThatEmptyInputProducesEmptyArray() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue(""));
+        new SplitExpression(";").execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val.getDataType().equals(DataType.getArray(DataType.STRING)));
+        assertTrue(val instanceof Array);
+        assertEquals(0, ((Array)val).size());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java
new file mode 100644
index 00000000000..cc496126c01
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java
@@ -0,0 +1,120 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class StatementTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        StatementExpression exp = newStatement();
+        assertTrue(exp.isEmpty());
+        assertEquals(0, exp.size());
+
+        Expression foo = new AttributeExpression("foo");
+        Expression bar = new AttributeExpression("bar");
+        exp = newStatement(foo, bar);
+        assertFalse(exp.isEmpty());
+        assertEquals(2, exp.size());
+        assertSame(foo, exp.get(0));
+        assertSame(bar, exp.get(1));
+        assertEquals(Arrays.asList(foo, bar), exp.asList());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression foo = new AttributeExpression("foo");
+        Expression bar = new AttributeExpression("bar");
+        Expression exp = newStatement(foo, bar);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new CatExpression(foo, bar)));
+        assertFalse(exp.equals(newStatement(new IndexExpression("foo"))));
+        assertFalse(exp.equals(newStatement(new IndexExpression("foo"),
+                                            new IndexExpression("bar"))));
+        assertFalse(exp.equals(newStatement(foo,
+                                            new IndexExpression("bar"))));
+        assertEquals(exp, newStatement(foo, bar));
+        assertEquals(exp.hashCode(), newStatement(foo, bar).hashCode());
+    }
+
+    @Test
+    public void requireThatStatementIsFlattened() {
+        Expression foo = SimpleExpression.newConversion(DataType.INT, DataType.STRING);
+        Expression bar = SimpleExpression.newConversion(DataType.STRING, DataType.INT);
+        assertEquals(newStatement(foo, bar), newStatement(foo, bar));
+        assertEquals(newStatement(foo, bar), newStatement(null, foo, bar));
+        assertEquals(newStatement(foo, bar), newStatement(foo, null, bar));
+        assertEquals(newStatement(foo, bar), newStatement(foo, bar, null));
+        assertEquals(newStatement(foo, bar), newStatement(newStatement(foo), bar));
+        assertEquals(newStatement(foo, bar), newStatement(newStatement(foo), newStatement(bar)));
+        assertEquals(newStatement(foo, bar), newStatement(foo, newStatement(bar)));
+        assertEquals(newStatement(foo, bar), newStatement(newStatement(foo, bar)));
+    }
+
+    @Test
+    public void requireThatRequiredInputIsDeterminedByFirstNonNullRequiredInput() {
+        assertEquals(DataType.INT, newStatement(SimpleExpression.newRequired(DataType.INT)).requiredInputType());
+        assertEquals(DataType.INT, newStatement(new SimpleExpression(),
+                                                SimpleExpression.newRequired(DataType.INT)).requiredInputType());
+        assertEquals(DataType.INT, newStatement(SimpleExpression.newRequired(DataType.INT),
+                                                SimpleExpression.newRequired(DataType.INT)).requiredInputType());
+    }
+
+    @Test
+    public void requireThatRequiredInputIsNullIfAnyOutputIsCreatedFirst() {
+        assertNull(newStatement(new SimpleExpression().setCreatedOutput(DataType.INT),
+                                new SimpleExpression().setRequiredInput(DataType.INT)).requiredInputType());
+    }
+
+    @Test
+    public void requireThatCreatedOutputIsDeterminedByLastNonNullCreatedOutput() {
+        assertEquals(DataType.STRING, newStatement(SimpleExpression.newOutput(DataType.STRING)).createdOutputType());
+        assertEquals(DataType.STRING, newStatement(SimpleExpression.newOutput(DataType.INT),
+                                                   SimpleExpression.newOutput(DataType.STRING)).createdOutputType());
+        assertEquals(DataType.STRING, newStatement(SimpleExpression.newOutput(DataType.STRING),
+                                                   new SimpleExpression()).createdOutputType());
+    }
+
+    @Test
+    public void requireThatInternalVerificationIsPerformed() {
+        Expression exp = newStatement(SimpleExpression.newOutput(DataType.STRING),
+                                      SimpleExpression.newConversion(DataType.INT, DataType.STRING));
+        assertVerifyThrows(null, exp, "Expected int input, got string.");
+        assertVerifyThrows(DataType.INT, exp, "Expected int input, got string.");
+        assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string.");
+
+        exp = newStatement(SimpleExpression.newOutput(DataType.INT),
+                           SimpleExpression.newConversion(DataType.INT, DataType.STRING));
+        assertVerify(null, exp, DataType.STRING);
+        assertVerify(DataType.INT, exp, DataType.STRING);
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+    }
+
+    @Test
+    public void requireThatStatementIsExecuted() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        StatementExpression statement = newStatement(new SetValueExpression(new IntegerFieldValue(69)));
+        newStatement(statement).execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof IntegerFieldValue);
+        assertEquals(69, ((IntegerFieldValue)val).getInteger());
+    }
+
+    private static StatementExpression newStatement(Expression... args) {
+        return new StatementExpression(args);
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringTestCase.java
new file mode 100644
index 00000000000..04cf2ba5ef6
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringTestCase.java
@@ -0,0 +1,77 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SubstringTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        SubstringExpression exp = new SubstringExpression(6, 9);
+        assertEquals(6, exp.getFrom());
+        assertEquals(9, exp.getTo());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new SubstringExpression(6, 9);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new SubstringExpression(66, 99)));
+        assertFalse(exp.equals(new SubstringExpression(6, 99)));
+        assertEquals(exp, new SubstringExpression(6, 9));
+        assertEquals(exp.hashCode(), new SubstringExpression(6, 9).hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new SubstringExpression(6, 9);
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected string input, got int.");
+    }
+
+    @Test
+    public void requireThatRangeIsCappedToInput() {
+        assertEquals(new StringFieldValue(""), new SubstringExpression(6, 9).execute(new StringFieldValue("012345")));
+        assertEquals(new StringFieldValue("345"), new SubstringExpression(3, 9).execute(new StringFieldValue("012345")));
+    }
+
+    @Test
+    public void requireThatIllegalRangeThrowsException() {
+        assertIndexOutOfBounds(-1, 69);
+        assertIndexOutOfBounds(69, -1);
+        assertIndexOutOfBounds(-9, -6);
+        assertIndexOutOfBounds(9, 6);
+    }
+
+    private static void assertIndexOutOfBounds(int from, int to) {
+        try {
+            new SubstringExpression(from, to);
+            fail();
+        } catch (IndexOutOfBoundsException e) {
+
+        }
+    }
+
+    @Test
+    public void requireThatStringIsSliced() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("666999"));
+        new SubstringExpression(2, 4).execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof StringFieldValue);
+        assertEquals("69", ((StringFieldValue)val).getString());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SummaryExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SummaryExpressionTestCase.java
new file mode 100644
index 00000000000..8256f3ed730
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SummaryExpressionTestCase.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertExecute;
+import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertVerify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class SummaryExpressionTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        SummaryExpression exp = new SummaryExpression("foo");
+        assertEquals("foo", exp.getFieldName());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new SummaryExpression("foo");
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new AttributeExpression("foo")));
+        assertFalse(exp.equals(new SummaryExpression("bar")));
+        assertEquals(exp, new SummaryExpression("foo"));
+        assertEquals(exp.hashCode(), new SummaryExpression("foo").hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        assertVerify(new SummaryExpression("foo"));
+    }
+
+    @Test
+    public void requireThatExpressionCanBeExecuted() {
+        assertExecute(new SummaryExpression("foo"));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java
new file mode 100644
index 00000000000..ef1198b238e
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java
@@ -0,0 +1,133 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SwitchTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        Map<String, Expression> cases = new HashMap<>();
+        Expression foo = new AttributeExpression("foo");
+        Expression bar = new AttributeExpression("bar");
+        Expression baz = new AttributeExpression("baz");
+        cases.put("foo", foo);
+        cases.put("bar", bar);
+        SwitchExpression exp = new SwitchExpression(cases, baz);
+        assertEquals(2, exp.getCases().size());
+        assertSame(foo, exp.getCases().get("foo"));
+        assertSame(bar, exp.getCases().get("bar"));
+        assertSame(baz, exp.getDefaultExpression());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Map<String, Expression> cases = new HashMap<>();
+        Expression foo = new AttributeExpression("foo");
+        Expression bar = new AttributeExpression("bar");
+        Expression baz = new AttributeExpression("baz");
+        cases.put("foo", foo);
+        cases.put("bar", bar);
+        SwitchExpression exp = new SwitchExpression(cases, baz);
+
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new SwitchExpression(Collections.singletonMap("foo", foo))));
+        assertFalse(exp.equals(new SwitchExpression(Collections.singletonMap("foo", foo), baz)));
+        assertFalse(exp.equals(new SwitchExpression(cases)));
+        assertEquals(exp, new SwitchExpression(cases, baz));
+        assertEquals(exp.hashCode(), new SwitchExpression(cases, baz).hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression foo = SimpleExpression.newConversion(DataType.STRING, DataType.INT);
+        Expression exp = new SwitchExpression(Collections.singletonMap("foo", foo));
+        assertVerify(DataType.STRING, exp, DataType.STRING); // does not touch output
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected string input, got int.");
+    }
+
+    @Test
+    public void requireThatCasesAreVerified() {
+        Map<String, Expression> cases = new HashMap<>();
+        cases.put("foo", SimpleExpression.newRequired(DataType.INT));
+        assertVerifyThrows(DataType.STRING, new SwitchExpression(cases),
+                           "Expected int input, got string.");
+        assertVerifyThrows(DataType.STRING, new SwitchExpression(Collections.<String, Expression>emptyMap(),
+                                                                 SimpleExpression.newRequired(DataType.INT)),
+                           "Expected int input, got string.");
+    }
+
+    @Test
+    public void requireThatIllegalArgumentThrows() {
+        try {
+            new SwitchExpression(Collections.<String, Expression>emptyMap()).execute(new IntegerFieldValue(69));
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Expected string input, got int.", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatDefaultExpressionIsNullIfNotGiven() {
+        assertNull(new SwitchExpression(Collections.<String, Expression>emptyMap()).getDefaultExpression());
+    }
+
+    @Test
+    public void requireThatIsEmptyReflectsOnBothCasesAndDefault() {
+        assertTrue(new SwitchExpression(Collections.<String, Expression>emptyMap()).isEmpty());
+        assertTrue(new SwitchExpression(Collections.<String, Expression>emptyMap(), null).isEmpty());
+        assertFalse(new SwitchExpression(Collections.<String, Expression>emptyMap(),
+                                         new AttributeExpression("foo")).isEmpty());
+        assertFalse(new SwitchExpression(Collections.singletonMap("foo", new AttributeExpression("foo")),
+                                         null).isEmpty());
+        assertFalse(new SwitchExpression(Collections.singletonMap("foo", new AttributeExpression("foo")),
+                                         new AttributeExpression("foo")).isEmpty());
+
+    }
+
+    @Test
+    public void requireThatCorrectExpressionIsExecuted() {
+        Map<String, Expression> cases = new HashMap<>();
+        cases.put("foo", new StatementExpression(new SetValueExpression(new StringFieldValue("bar")),
+                                                 new SetVarExpression("out")));
+        cases.put("baz", new StatementExpression(new SetValueExpression(new StringFieldValue("cox")),
+                                                 new SetVarExpression("out")));
+        Expression exp = new SwitchExpression(cases);
+        assertEvaluate(new StringFieldValue("foo"), exp, new StringFieldValue("bar"));
+        assertEvaluate(new StringFieldValue("baz"), exp, new StringFieldValue("cox"));
+        assertEvaluate(new StringFieldValue("???"), exp, null);
+    }
+
+    @Test
+    public void requireThatDefaultExpressionIsExecuted() {
+        Map<String, Expression> cases = new HashMap<>();
+        cases.put("foo", new StatementExpression(new SetValueExpression(new StringFieldValue("bar")),
+                                                 new SetVarExpression("out")));
+        Expression defaultExp = new StatementExpression(new SetValueExpression(new StringFieldValue("cox")),
+                                                        new SetVarExpression("out"));
+        Expression exp = new SwitchExpression(cases, defaultExp);
+        assertEvaluate(new StringFieldValue("foo"), exp, new StringFieldValue("bar"));
+        assertEvaluate(new StringFieldValue("baz"), exp, new StringFieldValue("cox"));
+        assertEvaluate(null, exp, new StringFieldValue("cox"));
+    }
+
+    private static void assertEvaluate(FieldValue input, Expression exp, FieldValue expectedOutVar) {
+        assertEquals(expectedOutVar, new ExecutionContext().setValue(input).execute(exp).getVariable("out"));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ThisTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ThisTestCase.java
new file mode 100644
index 00000000000..344a1b0b414
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ThisTestCase.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ThisTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new ThisExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new ThisExpression());
+        assertEquals(exp.hashCode(), new ThisExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new ThisExpression();
+        assertVerify(DataType.INT, exp, DataType.INT);
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected any input, got null.");
+    }
+
+    @Test
+    public void requireThatValueIsPreserved() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("69"));
+        new ThisExpression().execute(ctx);
+
+        assertEquals(new StringFieldValue("69"), ctx.getValue());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayTestCase.java
new file mode 100644
index 00000000000..193b8748d49
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayTestCase.java
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.Array;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToArrayTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new ToArrayExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new ToArrayExpression());
+        assertEquals(exp.hashCode(), new ToArrayExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new ToArrayExpression();
+        assertVerify(DataType.INT, exp, DataType.getArray(DataType.INT));
+        assertVerify(DataType.STRING, exp, DataType.getArray(DataType.STRING));
+        assertVerifyThrows(null, exp, "Expected any input, got null.");
+    }
+
+    @Test
+    public void requireThatValueIsConverted() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("69")).execute(new ToArrayExpression());
+
+        FieldValue val = ctx.getValue();
+        assertEquals(Array.class, val.getClass());
+
+        Array arr = (Array)val;
+        ArrayDataType type = arr.getDataType();
+        assertEquals(DataType.STRING, type.getNestedType());
+        assertEquals(1, arr.size());
+        assertEquals(new StringFieldValue("69"), arr.get(0));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteTestCase.java
new file mode 100644
index 00000000000..baa5ed5be93
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteTestCase.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.ByteFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToByteTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new ToByteExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new ToByteExpression());
+        assertEquals(exp.hashCode(), new ToByteExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new ToByteExpression();
+        assertVerify(DataType.INT, exp, DataType.BYTE);
+        assertVerify(DataType.STRING, exp, DataType.BYTE);
+        assertVerifyThrows(null, exp, "Expected any input, got null.");
+    }
+
+    @Test
+    public void requireThatValueIsConverted() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("69")).execute(new ToByteExpression());
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof ByteFieldValue);
+        assertEquals(69, ((ByteFieldValue)val).getByte());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleTestCase.java
new file mode 100644
index 00000000000..abd4f397b3b
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleTestCase.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.DoubleFieldValue;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToDoubleTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new ToDoubleExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new ToDoubleExpression());
+        assertEquals(exp.hashCode(), new ToDoubleExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new ToDoubleExpression();
+        assertVerify(DataType.INT, exp, DataType.DOUBLE);
+        assertVerify(DataType.STRING, exp, DataType.DOUBLE);
+        assertVerifyThrows(null, exp, "Expected any input, got null.");
+    }
+
+    @Test
+    public void requireThatValueIsConverted() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("6.9")).execute(new ToDoubleExpression());
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof DoubleFieldValue);
+        assertEquals(6.9, ((DoubleFieldValue)val).getDouble(), 1e-6);
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatTestCase.java
new file mode 100644
index 00000000000..b8b3f4ee51e
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatTestCase.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.FloatFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToFloatTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new ToFloatExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new ToFloatExpression());
+        assertEquals(exp.hashCode(), new ToFloatExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new ToFloatExpression();
+        assertVerify(DataType.INT, exp, DataType.FLOAT);
+        assertVerify(DataType.STRING, exp, DataType.FLOAT);
+        assertVerifyThrows(null, exp, "Expected any input, got null.");
+    }
+
+    @Test
+    public void requireThatValueIsConverted() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("6.9f")).execute(new ToFloatExpression());
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof FloatFieldValue);
+        assertEquals(6.9f, ((FloatFieldValue)val).getFloat(), 1e-6);
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerTestCase.java
new file mode 100644
index 00000000000..c3d863de645
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerTestCase.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToIntegerTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new ToIntegerExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new ToIntegerExpression());
+        assertEquals(exp.hashCode(), new ToIntegerExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new ToIntegerExpression();
+        assertVerify(DataType.INT, exp, DataType.INT);
+        assertVerify(DataType.STRING, exp, DataType.INT);
+        assertVerifyThrows(null, exp, "Expected any input, got null.");
+    }
+
+    @Test
+    public void requireThatValueIsConverted() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("69")).execute(new ToIntegerExpression());
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof IntegerFieldValue);
+        assertEquals(69, ((IntegerFieldValue)val).getInteger());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongTestCase.java
new file mode 100644
index 00000000000..b4fc310db48
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongTestCase.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToLongTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new ToLongExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new ToLongExpression());
+        assertEquals(exp.hashCode(), new ToLongExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new ToLongExpression();
+        assertVerify(DataType.INT, exp, DataType.LONG);
+        assertVerify(DataType.STRING, exp, DataType.LONG);
+        assertVerifyThrows(null, exp, "Expected any input, got null.");
+    }
+
+    @Test
+    public void requireThatValueIsConverted() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("69")).execute(new ToLongExpression());
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof LongFieldValue);
+        assertEquals(69L, ((LongFieldValue)val).getLong());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionTestCase.java
new file mode 100644
index 00000000000..fb370405643
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionTestCase.java
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.StructuredFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToPositionTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new ToPositionExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new ToPositionExpression());
+        assertEquals(exp.hashCode(), new ToPositionExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new ToPositionExpression();
+        assertVerify(DataType.STRING, exp, PositionDataType.INSTANCE);
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected string input, got int.");
+    }
+
+    @Test
+    public void requireThatPositionIsParsed() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("6;9")).execute(new ToPositionExpression());
+
+        FieldValue out = ctx.getValue();
+        assertTrue(out instanceof StructuredFieldValue);
+        assertEquals(PositionDataType.INSTANCE, out.getDataType());
+
+        FieldValue val = ((StructuredFieldValue)out).getFieldValue("x");
+        assertTrue(val instanceof IntegerFieldValue);
+        assertEquals(6, ((IntegerFieldValue)val).getInteger());
+
+        val = ((StructuredFieldValue)out).getFieldValue("y");
+        assertTrue(val instanceof IntegerFieldValue);
+        assertEquals(9, ((IntegerFieldValue)val).getInteger());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringTestCase.java
new file mode 100644
index 00000000000..d20d75ee5dd
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringTestCase.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToStringTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new ToStringExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new ToStringExpression());
+        assertEquals(exp.hashCode(), new ToStringExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new ToStringExpression();
+        assertVerify(DataType.INT, exp, DataType.STRING);
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected any input, got null.");
+    }
+
+    @Test
+    public void requireThatValueIsConverted() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new IntegerFieldValue(69)).execute(new ToStringExpression());
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof StringFieldValue);
+        assertEquals("69", ((StringFieldValue)val).getString());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetTestCase.java
new file mode 100644
index 00000000000..e761e4332dc
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetTestCase.java
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.datatypes.WeightedSet;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ToWsetTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        assertAccessors(false, false);
+        assertAccessors(false, true);
+        assertAccessors(true, false);
+        assertAccessors(true, true);
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new ToWsetExpression(true, true);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new ToWsetExpression(false, false)));
+        assertFalse(exp.equals(new ToWsetExpression(true, false)));
+        assertFalse(exp.equals(new ToWsetExpression(false, true)));
+        assertEquals(exp, new ToWsetExpression(true, true));
+        assertEquals(exp.hashCode(), new ToWsetExpression(true, true).hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        assertVerify(false, false);
+        assertVerify(false, true);
+        assertVerify(true, false);
+        assertVerify(true, true);
+    }
+
+    @Test
+    public void requireThatValueIsConverted() {
+        assertConvert(false, false);
+        assertConvert(false, true);
+        assertConvert(true, false);
+        assertConvert(true, true);
+    }
+
+    private static void assertVerify(boolean createIfNonExistent, boolean removeIfZero) {
+        Expression exp = new ToWsetExpression(createIfNonExistent, removeIfZero);
+        ExpressionAssert.assertVerify(DataType.INT, exp,
+                                      DataType.getWeightedSet(DataType.INT, createIfNonExistent, removeIfZero));
+        ExpressionAssert.assertVerify(DataType.STRING, exp,
+                                      DataType.getWeightedSet(DataType.STRING, createIfNonExistent, removeIfZero));
+        assertVerifyThrows(null, exp, "Expected any input, got null.");
+    }
+
+    private static void assertConvert(boolean createIfNonExistent, boolean removeIfZero) {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("69")).execute(new ToWsetExpression(createIfNonExistent, removeIfZero));
+
+        FieldValue val = ctx.getValue();
+        assertEquals(WeightedSet.class, val.getClass());
+
+        WeightedSet wset = (WeightedSet)val;
+        WeightedSetDataType type = wset.getDataType();
+        assertEquals(DataType.STRING, type.getNestedType());
+        assertEquals(createIfNonExistent, type.createIfNonExistent());
+        assertEquals(removeIfZero, type.removeIfZero());
+
+        assertEquals(1, wset.size());
+        assertEquals(Integer.valueOf(1), wset.get(new StringFieldValue("69")));
+    }
+
+    private static void assertAccessors(boolean createIfNonExistent, boolean removeIfZero) {
+        ToWsetExpression exp = new ToWsetExpression(createIfNonExistent, removeIfZero);
+        assertEquals(createIfNonExistent, exp.getCreateIfNonExistent());
+        assertEquals(removeIfZero, exp.getRemoveIfZero());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java
new file mode 100644
index 00000000000..9fdab3b978f
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.annotation.SpanTrees;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.language.Language;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class TokenizeTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        Linguistics linguistics = new SimpleLinguistics();
+        AnnotatorConfig config = new AnnotatorConfig();
+        TokenizeExpression exp = new TokenizeExpression(linguistics, config);
+        assertSame(linguistics, exp.getLinguistics());
+        assertSame(config, exp.getConfig());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        AnnotatorConfig config = new AnnotatorConfig().setLanguage(Language.ARABIC);
+        Expression exp = new TokenizeExpression(new SimpleLinguistics(), config);
+        assertFalse(exp.equals(new Object()));
+        assertFalse(exp.equals(new TokenizeExpression(Mockito.mock(Linguistics.class),
+                                                      new AnnotatorConfig().setLanguage(Language.SPANISH))));
+        assertFalse(exp.equals(new TokenizeExpression(new SimpleLinguistics(),
+                                                      new AnnotatorConfig().setLanguage(Language.SPANISH))));
+        assertEquals(exp, new TokenizeExpression(new SimpleLinguistics(), config));
+        assertEquals(exp.hashCode(), new TokenizeExpression(new SimpleLinguistics(), config).hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new TokenizeExpression(new SimpleLinguistics(), new AnnotatorConfig());
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected string input, got int.");
+    }
+
+    @Test
+    public void requireThatValueIsAnnotated() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("foo"));
+        new TokenizeExpression(new SimpleLinguistics(), new AnnotatorConfig()).execute(ctx);
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof StringFieldValue);
+        assertNotNull(((StringFieldValue)val).getSpanTree(SpanTrees.LINGUISTICS));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TrimTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TrimTestCase.java
new file mode 100644
index 00000000000..6057cf2f211
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TrimTestCase.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class TrimTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new TrimExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new TrimExpression());
+        assertEquals(exp.hashCode(), new TrimExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new TrimExpression();
+        assertVerify(DataType.STRING, exp, DataType.STRING);
+        assertVerifyThrows(null, exp, "Expected string input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected string input, got int.");
+    }
+
+    @Test
+    public void requireThatStringIsTrimmed() {
+        ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
+        ctx.setValue(new StringFieldValue("  69  ")).execute(new TrimExpression());
+
+        FieldValue val = ctx.getValue();
+        assertTrue(val instanceof StringFieldValue);
+        assertEquals("69", ((StringFieldValue)val).getString());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedDataTypeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedDataTypeTestCase.java
new file mode 100644
index 00000000000..a1e549c3830
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedDataTypeTestCase.java
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.serialization.FieldReader;
+import com.yahoo.document.serialization.FieldWriter;
+import com.yahoo.document.serialization.XmlStream;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class UnresolvedDataTypeTestCase {
+
+    @Test
+    public void requireThatFieldValueIsUnresolved() {
+        assertEquals(UnresolvedFieldValue.class, UnresolvedDataType.INSTANCE.createFieldValue().getClass());
+    }
+
+    @Test
+    public void requireThatTypeIsCompatibleWithAnything() {
+        assertFalse(UnresolvedDataType.INSTANCE.isValueCompatible(null));
+        assertTrue(UnresolvedDataType.INSTANCE.isValueCompatible(new MyFieldValue()));
+    }
+
+    private static class MyFieldValue extends FieldValue {
+
+        @Override
+        public DataType getDataType() {
+            return null;
+        }
+
+        @Override
+        public void printXml(XmlStream xml) {
+
+        }
+
+        @Override
+        public void clear() {
+
+        }
+
+        @Override
+        public void assign(Object o) {
+
+        }
+
+        @Override
+        public void serialize(Field field, FieldWriter writer) {
+
+        }
+
+        @Override
+        public void deserialize(Field field, FieldReader reader) {
+
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValueTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValueTestCase.java
new file mode 100644
index 00000000000..74ea3b35954
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValueTestCase.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class UnresolvedFieldValueTestCase {
+
+    @Test
+    public void requireThatDataTypeIsUnresolved() {
+        assertEquals(UnresolvedDataType.INSTANCE, new UnresolvedFieldValue().getDataType());
+    }
+
+    @Test
+    public void requireThatNoMethodsAreSupported() {
+        UnresolvedFieldValue val = new UnresolvedFieldValue();
+        try {
+            val.printXml(null);
+            fail();
+        } catch (UnsupportedOperationException e) {
+
+        }
+        try {
+            val.clear();
+            fail();
+        } catch (UnsupportedOperationException e) {
+
+        }
+        try {
+            val.assign(new UnresolvedFieldValue());
+            fail();
+        } catch (UnsupportedOperationException e) {
+
+        }
+        try {
+            val.serialize(null, null);
+            fail();
+        } catch (UnsupportedOperationException e) {
+
+        }
+        try {
+            val.deserialize(null, null);
+            fail();
+        } catch (UnsupportedOperationException e) {
+
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContextTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContextTestCase.java
new file mode 100644
index 00000000000..10edcb546fe
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContextTestCase.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import org.junit.Test;
+
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class VerificationContextTestCase {
+
+    @Test
+    public void requireThatValueCanBeSet() {
+        VerificationContext ctx = new VerificationContext();
+        DataType val = DataType.STRING;
+        ctx.setValue(val);
+        assertSame(val, ctx.getValue());
+    }
+
+    @Test
+    public void requireThatVariablesCanBeSet() {
+        VerificationContext ctx = new VerificationContext();
+        DataType val = DataType.STRING;
+        ctx.setVariable("foo", val);
+        assertSame(val, ctx.getVariable("foo"));
+    }
+
+    @Test
+    public void requireThatClearRemovesValue() {
+        VerificationContext ctx = new VerificationContext();
+        ctx.setValue(DataType.STRING);
+        ctx.clear();
+        assertNull(ctx.getValue());
+    }
+
+    @Test
+    public void requireThatClearRemovesVariables() {
+        VerificationContext ctx = new VerificationContext();
+        ctx.setVariable("foo", DataType.STRING);
+        ctx.clear();
+        assertNull(ctx.getVariable("foo"));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationExceptionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationExceptionTestCase.java
new file mode 100644
index 00000000000..412da319016
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationExceptionTestCase.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class VerificationExceptionTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        Expression exp = new SimpleExpression();
+        VerificationException e = new VerificationException(exp, "foo");
+        assertSame(exp, e.getExpression());
+        assertEquals("foo", e.getMessage());
+        assertTrue(e.toString().contains(exp.toString()));
+        assertTrue(e.toString().contains(e.getMessage()));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveTestCase.java
new file mode 100644
index 00000000000..503aacdd551
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveTestCase.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.geo.ZCurve;
+import org.junit.Test;
+
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify;
+import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ZCurveTestCase {
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        Expression exp = new ZCurveExpression();
+        assertFalse(exp.equals(new Object()));
+        assertEquals(exp, new ZCurveExpression());
+        assertEquals(exp.hashCode(), new ZCurveExpression().hashCode());
+    }
+
+    @Test
+    public void requireThatExpressionCanBeVerified() {
+        Expression exp = new ZCurveExpression();
+        assertVerify(PositionDataType.INSTANCE, exp, DataType.LONG);
+        assertVerifyThrows(null, exp, "Expected position input, got null.");
+        assertVerifyThrows(DataType.INT, exp, "Expected position input, got int.");
+    }
+
+    @Test
+    public void requireThatInputIsEncoded() {
+        assertEquals(new LongFieldValue(ZCurve.encode(6, 9)),
+                     new ZCurveExpression().execute(PositionDataType.valueOf(6, 9)));
+    }
+
+    @Test
+    public void requireThatMissingFieldEvaluatesToNull() {
+        assertNull(new ZCurveExpression().execute(PositionDataType.valueOf(null, 9)));
+        assertNull(new ZCurveExpression().execute(PositionDataType.valueOf(6, null)));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfigTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfigTestCase.java
new file mode 100644
index 00000000000..163e4bff4a8
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfigTestCase.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.linguistics;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.process.StemMode;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class AnnotatorConfigTestCase {
+
+    @Test
+    public void requireThatAccessorsWork() {
+        AnnotatorConfig config = new AnnotatorConfig();
+        for (Language language : Language.values()) {
+            config.setLanguage(language);
+            assertEquals(language, config.getLanguage());
+        }
+        for (StemMode mode : StemMode.values()) {
+            config.setStemMode(mode);
+            assertEquals(mode, config.getStemMode());
+        }
+        config.setRemoveAccents(true);
+        assertTrue(config.getRemoveAccents());
+        config.setRemoveAccents(false);
+        assertFalse(config.getRemoveAccents());
+    }
+
+    @Test
+    public void requireThatCopyConstructorIsImplemented() {
+        AnnotatorConfig config = new AnnotatorConfig();
+        config.setLanguage(Language.ARABIC);
+        config.setStemMode(StemMode.SHORTEST);
+        config.setRemoveAccents(!config.getRemoveAccents());
+
+        AnnotatorConfig other = new AnnotatorConfig(config);
+        assertEquals(config.getLanguage(), other.getLanguage());
+        assertEquals(config.getStemMode(), other.getStemMode());
+        assertEquals(config.getRemoveAccents(), other.getRemoveAccents());
+    }
+
+    @Test
+    public void requireThatHashCodeAndEqualsAreImplemented() {
+        AnnotatorConfig config = newConfig(Language.DUTCH, StemMode.NONE, true);
+        assertFalse(config.equals(new Object()));
+        assertFalse(config.equals(newConfig(Language.SPANISH, StemMode.SHORTEST, false)));
+        assertFalse(config.equals(newConfig(Language.DUTCH, StemMode.SHORTEST, false)));
+        assertFalse(config.equals(newConfig(Language.DUTCH, StemMode.NONE, false)));
+        assertEquals(config, newConfig(Language.DUTCH, StemMode.NONE, true));
+        assertEquals(config.hashCode(), newConfig(Language.DUTCH, StemMode.NONE, true).hashCode());
+    }
+
+    private static AnnotatorConfig newConfig(Language language, StemMode stemMode,
+                                             boolean removeAccents) {
+        AnnotatorConfig config = new AnnotatorConfig();
+        config.setLanguage(language);
+        config.setStemMode(stemMode);
+        config.setRemoveAccents(removeAccents);
+        return config;
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java
new file mode 100644
index 00000000000..805bdc96904
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java
@@ -0,0 +1,250 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.linguistics;

+

+import com.yahoo.document.annotation.Annotation;

+import com.yahoo.document.annotation.AnnotationTypes;

+import com.yahoo.document.annotation.SpanTree;

+import com.yahoo.document.annotation.SpanTrees;

+import com.yahoo.document.datatypes.StringFieldValue;

+import com.yahoo.language.Language;

+import com.yahoo.language.Linguistics;

+import com.yahoo.language.process.StemMode;

+import com.yahoo.language.process.Token;

+import com.yahoo.language.process.TokenType;

+import com.yahoo.language.process.Tokenizer;

+import com.yahoo.language.simple.SimpleToken;

+

+import org.junit.Test;

+import org.mockito.Mockito;

+

+import java.util.*;

+

+import static org.junit.Assert.assertEquals;

+import static org.junit.Assert.assertTrue;

+

+/**

+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>

+ */

+public class LinguisticsAnnotatorTestCase {

+

+    private static final AnnotatorConfig CONFIG = new AnnotatorConfig();

+

+    // --------------------------------------------------------------------------------

+    //

+    // Tests

+    //

+    // --------------------------------------------------------------------------------

+

+    @Test

+    public void requireThatAnnotateFailsWithZeroTokens() {

+        assertAnnotations(null, "foo");

+    }

+

+    @Test

+    public void requireThatAnnotateFailsWithoutIndexableTokenString() {

+        for (TokenType type : TokenType.values()) {

+            if (type.isIndexable()) {

+                continue;

+            }

+            assertAnnotations(null, "foo", newToken("foo", "bar", type));

+        }

+    }

+

+    @Test

+    public void requireThatIndexableTokenStringsAreAnnotated() {

+        SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);

+        expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));

+        for (TokenType type : TokenType.values()) {

+            if (!type.isIndexable()) {

+                continue;

+            }

+            assertAnnotations(expected, "foo", newToken("foo", "bar", type));

+        }

+    }

+

+    @Test

+    public void requireThatSpecialTokenStringsAreAnnotatedRegardlessOfType() {

+        SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);

+        expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));

+        for (TokenType type : TokenType.values()) {

+            assertAnnotations(expected, "foo", newToken("foo", "bar", type, true));

+        }

+    }

+

+    @Test

+    public void requireThatTermAnnotationsAreEmptyIfOrigIsLowerCase() {

+        SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);

+        expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM));

+        for (boolean specialToken : Arrays.asList(true, false)) {

+            for (TokenType type : TokenType.values()) {

+                if (!specialToken && !type.isIndexable()) {

+                    continue;

+                }

+                assertAnnotations(expected, "foo", newToken("foo", "foo", type, specialToken));

+            }

+        }

+    }

+

+    @Test

+    public void requireThatTermAnnotationsAreLowerCased() {

+        SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);

+        expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));

+        for (boolean specialToken : Arrays.asList(true, false)) {

+            for (TokenType type : TokenType.values()) {

+                if (!specialToken && !type.isIndexable()) {

+                    continue;

+                }

+                assertAnnotations(expected, "foo", newToken("foo", "BAR", type, specialToken));

+            }

+        }

+    }

+

+    @Test

+    public void requireThatCompositeTokensAreFlattened() {

+        SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);

+        expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("foo")));

+        expected.spanList().span(3, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));

+        expected.spanList().span(6, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("baz")));

+

+        SimpleToken token = newToken("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC)

+                .addComponent(newToken("FOO", "foo", TokenType.ALPHABETIC).setOffset(0))

+                .addComponent(newToken("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3)

+                                      .addComponent(newToken("BAR", "bar", TokenType.ALPHABETIC).setOffset(3))

+                                      .addComponent(newToken("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6)));

+        assertAnnotations(expected, "foobarbaz", token);

+    }

+

+    @Test

+    public void requireThatCompositeSpecialTokensAreNotFlattened() {

+        SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);

+        expected.spanList().span(0, 9).annotate(new Annotation(AnnotationTypes.TERM,

+                                                               new StringFieldValue("foobarbaz")));

+

+        SimpleToken token = newToken("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC).setSpecialToken(true)

+                .addComponent(newToken("FOO", "foo", TokenType.ALPHABETIC).setOffset(0))

+                .addComponent(newToken("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3)

+                                      .addComponent(newToken("BAR", "bar", TokenType.ALPHABETIC).setOffset(3))

+                                      .addComponent(newToken("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6)));

+        assertAnnotations(expected, "foobarbaz", token);

+    }

+

+    @Test

+    public void requireThatErrorTokensAreSkipped() {

+        assertAnnotations(null, "foo", new SimpleToken("foo").setType(TokenType.ALPHABETIC)

+                                                             .setOffset(-1));

+    }

+

+    @Test

+    public void requireThatTermReplacementsAreApplied() {

+        SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);

+        expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));

+        for (boolean specialToken : Arrays.asList(true, false)) {

+            for (TokenType type : TokenType.values()) {

+                if (!specialToken && !type.isIndexable()) {

+                    continue;

+                }

+                assertAnnotations(expected, "foo",

+                                  newLinguistics(Arrays.asList(newToken("foo", "foo", type, specialToken)),

+                                                 Collections.singletonMap("foo", "bar")));

+            }

+        }

+    }

+

+    @Test

+    public void requireThatExistingAnnotationsAreKept() {

+        SpanTree spanTree = new SpanTree(SpanTrees.LINGUISTICS);

+        spanTree.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("baz")));

+

+        StringFieldValue val = new StringFieldValue("foo");

+        val.setSpanTree(spanTree);

+

+        Linguistics linguistics = newLinguistics(Arrays.asList(newToken("foo", "bar", TokenType.ALPHABETIC, false)),

+                                                 Collections.<String, String>emptyMap());

+        new LinguisticsAnnotator(linguistics, CONFIG).annotate(val);

+

+        assertTrue(new LinguisticsAnnotator(linguistics, CONFIG).annotate(val));

+        assertEquals(spanTree, val.getSpanTree(SpanTrees.LINGUISTICS));

+    }

+

+    @Test

+    public void requireThatMaxTermOccurencesIsHonored() {

+        final String inputTerm = "foo";

+        final String stemmedInputTerm = "bar"; // completely different from

+                                               // inputTerm for safer test

+        final String paddedInputTerm = inputTerm + " ";

+        final SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);

+        final int inputTermOccurence = AnnotatorConfig.DEFAULT_MAX_TERM_OCCURRENCES * 2;

+        for (int i = 0; i < AnnotatorConfig.DEFAULT_MAX_TERM_OCCURRENCES; ++i) {

+            expected.spanList().span(i * paddedInputTerm.length(), inputTerm.length())

+                    .annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue(stemmedInputTerm)));

+        }

+        for (TokenType type : TokenType.values()) {

+            if (!type.isIndexable()) {

+                continue;

+            }

+            StringBuilder input = new StringBuilder();

+            Token[] tokens = new Token[inputTermOccurence];

+            for (int i = 0; i < inputTermOccurence; ++i) {

+                SimpleToken t = newToken(inputTerm, stemmedInputTerm, type);

+                t.setOffset(i * paddedInputTerm.length());

+                tokens[i] = t;

+                input.append(paddedInputTerm);

+            }

+            assertAnnotations(expected, input.toString(), tokens);

+        }

+    }

+

+    // --------------------------------------------------------------------------------

+    //

+    // Utilities

+    //

+    // --------------------------------------------------------------------------------

+

+    private static SimpleToken newToken(String orig, String stem, TokenType type) {

+        return newToken(orig, stem, type, false);

+    }

+

+    private static SimpleToken newToken(String orig, String stem, TokenType type, boolean specialToken) {

+        return new SimpleToken(orig).setTokenString(stem)

+                                    .setType(type)

+                                    .setSpecialToken(specialToken);

+    }

+

+    private static void assertAnnotations(SpanTree expected, String value, Token... tokens) {

+        assertAnnotations(expected, value, newLinguistics(Arrays.asList(tokens), Collections.<String, String>emptyMap()));

+    }

+

+    private static void assertAnnotations(SpanTree expected, String str, Linguistics linguistics) {

+        StringFieldValue val = new StringFieldValue(str);

+        assertEquals(expected != null, new LinguisticsAnnotator(linguistics, CONFIG).annotate(val));

+        assertEquals(expected, val.getSpanTree(SpanTrees.LINGUISTICS));

+    }

+

+    private static Linguistics newLinguistics(List<? extends Token> tokens, Map<String, String> replacementTerms) {

+        Linguistics linguistics = Mockito.mock(Linguistics.class);

+        Mockito.when(linguistics.getTokenizer()).thenReturn(new MyTokenizer(tokens, replacementTerms));

+        return linguistics;

+    }

+

+    private static class MyTokenizer implements Tokenizer {

+

+        final List<Token> tokens;

+        final Map<String, String> replacementTerms;

+

+        public MyTokenizer(List<? extends Token> tokens, Map<String, String> replacementTerms) {

+            this.tokens = new ArrayList<>(tokens);

+            this.replacementTerms = replacementTerms;

+        }

+

+        @Override

+        public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) {

+            return tokens;

+        }

+

+        @Override

+        public String getReplacementTerm(String term) {

+            String replacement = replacementTerms.get(term);

+            return replacement != null ? replacement : term;

+        }

+    }

+}

diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/DefaultFieldNameTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/DefaultFieldNameTestCase.java
new file mode 100644
index 00000000000..8d279f14b5b
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/DefaultFieldNameTestCase.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.parser;
+
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.ScriptParserContext;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.InputExpression;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class DefaultFieldNameTestCase {
+
+    @Test
+    public void requireThatDefaultFieldNameIsAppliedWhenArgumentIsMissing() throws ParseException {
+        IndexingInput input = new IndexingInput("input");
+        InputExpression exp = (InputExpression)Expression.newInstance(new ScriptParserContext(new SimpleLinguistics())
+                                                                              .setInputStream(input)
+                                                                              .setDefaultFieldName("foo"));
+        assertEquals("foo", exp.getFieldName());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java
new file mode 100644
index 00000000000..a70de62c1a7
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java
@@ -0,0 +1,91 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.parser;
+
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ExpressionTestCase {
+
+    @Test
+    public void requireThatAllExpressionTypesAreParsed() throws ParseException {
+        assertExpression(ArithmeticExpression.class, "1 + 2");
+        assertExpression(AttributeExpression.class, "attribute");
+        assertExpression(Base64DecodeExpression.class, "base64decode");
+        assertExpression(Base64EncodeExpression.class, "base64encode");
+        assertExpression(CatExpression.class, "1 . 2");
+        assertExpression(ClearStateExpression.class, "clear_state");
+        assertExpression(EchoExpression.class, "echo");
+        assertExpression(ExactExpression.class, "exact");
+        assertExpression(FlattenExpression.class, "flatten");
+        assertExpression(ForEachExpression.class, "for_each { 1 }");
+        assertExpression(GetFieldExpression.class, "get_field field1");
+        assertExpression(GetVarExpression.class, "get_var myvar1");
+        assertExpression(GuardExpression.class, "guard { 1 }");
+        assertExpression(GuardExpression.class, "guard { 1; 2 }");
+        assertExpression(HexDecodeExpression.class, "hexdecode");
+        assertExpression(HexEncodeExpression.class, "hexencode");
+        assertExpression(HostNameExpression.class, "hostname");
+        assertExpression(IfThenExpression.class, "if (1 < 2) { 3 }");
+        assertExpression(IfThenExpression.class, "if (1 < 2) { 3 } else { 4 }");
+        assertExpression(IndexExpression.class, "index");
+        assertExpression(InputExpression.class, "input");
+        assertExpression(InputExpression.class, "input field1");
+        assertExpression(JoinExpression.class, "join '1'");
+        assertExpression(LowerCaseExpression.class, "lowercase");
+        assertExpression(NGramExpression.class, "ngram 1");
+        assertExpression(NormalizeExpression.class, "normalize");
+        assertExpression(NowExpression.class, "now");
+        assertExpression(OptimizePredicateExpression.class, "optimize_predicate");
+        assertExpression(ParenthesisExpression.class, "(1)");
+        assertExpression(RandomExpression.class, "random");
+        assertExpression(RandomExpression.class, "random 1");
+        assertExpression(ScriptExpression.class, "{ 1; 2 }");
+        assertExpression(SelectInputExpression.class, "select_input { field1: 2; }");
+        assertExpression(SetLanguageExpression.class, "set_language");
+        assertExpression(SetValueExpression.class, "1");
+        assertExpression(SetVarExpression.class, "set_var myvar1");
+        assertExpression(SplitExpression.class, "split '1'");
+        assertExpression(StatementExpression.class, "1 | 2");
+        assertExpression(SubstringExpression.class, "substring 1 2");
+        assertExpression(SummaryExpression.class, "summary");
+        assertExpression(SwitchExpression.class, "switch { case '1': 2; }");
+        assertExpression(ThisExpression.class, "this");
+        assertExpression(ToArrayExpression.class, "to_array");
+        assertExpression(ToByteExpression.class, "to_byte");
+        assertExpression(ToDoubleExpression.class, "to_double");
+        assertExpression(ToFloatExpression.class, "to_float");
+        assertExpression(ToIntegerExpression.class, "to_int");
+        assertExpression(TokenizeExpression.class, "tokenize");
+        assertExpression(TokenizeExpression.class, "tokenize stem");
+        assertExpression(TokenizeExpression.class, "tokenize stem normalize");
+        assertExpression(TokenizeExpression.class, "tokenize stem:\"ALL\" normalize");
+        assertExpression(TokenizeExpression.class, "tokenize stem:\"ALL\"");
+        assertExpression(TokenizeExpression.class, "tokenize normalize");
+        assertExpression(ToLongExpression.class, "to_long");
+        assertExpression(ToPositionExpression.class, "to_pos");
+        assertExpression(ToStringExpression.class, "to_string");
+        assertExpression(ToWsetExpression.class, "to_wset");
+        assertExpression(ToWsetExpression.class, "to_wset create_if_non_existent");
+        assertExpression(ToWsetExpression.class, "to_wset remove_if_zero");
+        assertExpression(ToWsetExpression.class, "to_wset create_if_non_existent remove_if_zero");
+        assertExpression(ToWsetExpression.class, "to_wset remove_if_zero create_if_non_existent");
+        assertExpression(TrimExpression.class, "trim");
+        assertExpression(ZCurveExpression.class, "zcurve");
+    }
+
+    private static void assertExpression(Class expectedClass, String str) throws ParseException {
+        Linguistics linguistics = new SimpleLinguistics();
+        Expression foo = Expression.fromString(str, linguistics);
+        assertEquals(expectedClass, foo.getClass());
+        Expression bar = Expression.fromString(foo.toString(), linguistics);
+        assertEquals(foo.hashCode(), bar.hashCode());
+        assertEquals(foo, bar);
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/FieldNameTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/FieldNameTestCase.java
new file mode 100644
index 00000000000..9cce117bec3
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/FieldNameTestCase.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.parser;
+
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class FieldNameTestCase {
+
+    @Test
+    public void requireThatDotCanBeUsedInFieldName() throws ParseException {
+        assertEquals(new AttributeExpression("foo.bar"), Expression.fromString("attribute foo . bar"));
+        assertEquals(new IndexExpression("foo.bar"), Expression.fromString("index foo . bar"));
+        assertEquals(new SummaryExpression("foo.bar"), Expression.fromString("summary foo . bar"));
+    }
+
+    @Test
+    public void requireThatCatDotIsNotConfusedWithFieldName() throws ParseException {
+        assertEquals(new CatExpression(new InputExpression("foo"), new InputExpression("bar")),
+                     Expression.fromString("input foo . input bar"));
+        assertEquals(new CatExpression(new InputExpression("foo"), new SetValueExpression(new StringFieldValue("bar"))),
+                     Expression.fromString("input foo . 'bar'"));
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/IdentifierTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/IdentifierTestCase.java
new file mode 100644
index 00000000000..69840bff19e
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/IdentifierTestCase.java
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.parser;
+
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class IdentifierTestCase {
+
+    @Test
+    public void requireThatThereAreNoReservedWords() throws ParseException {
+        List<String> tokens = Arrays.asList("attribute",
+                                            "base64_decode",
+                                            "base64_encode",
+                                            "clear_state",
+                                            "compact_phrase",
+                                            "create_if_non_existent",
+                                            "echo",
+                                            "exact",
+                                            "flatten",
+                                            "for_each",
+                                            "get_field",
+                                            "get_var",
+                                            "guard",
+                                            "hex_decode",
+                                            "hex_encode",
+                                            "host_name",
+                                            "if",
+                                            "index",
+                                            "join",
+                                            "linguistics",
+                                            "lower_case",
+                                            "ngram",
+                                            "normalize",
+                                            "now",
+                                            "optimize_predicate",
+                                            "predicate_to_raw",
+                                            "put_symbol",
+                                            "random",
+                                            "raw_to_predicate",
+                                            "remove_ctrl_chars",
+                                            "remove_if_zero",
+                                            "remove_so_si",
+                                            "select_field",
+                                            "set_language",
+                                            "set_var",
+                                            "split",
+                                            "substring",
+                                            "summary",
+                                            "switch",
+                                            "this",
+                                            "tokenize",
+                                            "to_array",
+                                            "to_double",
+                                            "to_float",
+                                            "to_int",
+                                            "to_long",
+                                            "to_pos",
+                                            "to_string",
+                                            "to_wset",
+                                            "trim",
+                                            "zcurve");
+        for (String str : tokens) {
+            IndexingParser parser = new IndexingParser(new IndexingInput(str));
+            assertEquals(str, parser.identifier());
+        }
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/MathTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/MathTestCase.java
new file mode 100644
index 00000000000..fee4d6ed1a5
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/MathTestCase.java
@@ -0,0 +1,92 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.parser;
+
+import com.yahoo.document.datatypes.*;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class MathTestCase {
+
+    private static final double EPS = 1e-6;
+
+    @Test
+    public void requireThatNumericTypeIsPreserved() throws ParseException {
+        assertEquals(1, evaluateDouble("1.0"), EPS);
+        assertEquals(3, evaluateDouble("1.0 + 2"), EPS);
+        assertEquals(-1, evaluateDouble("1.0 - 2"), EPS);
+        assertEquals(2, evaluateDouble("1.0 * 2"), EPS);
+        assertEquals(0.5, evaluateDouble("1.0 / 2"), EPS);
+        assertEquals(1, evaluateDouble("1.0 % 2"), EPS);
+
+        assertEquals(1, evaluateFloat("1.0f"), EPS);
+        assertEquals(3, evaluateFloat("1.0f + 2"), EPS);
+        assertEquals(-1, evaluateFloat("1.0f - 2"), EPS);
+        assertEquals(2, evaluateFloat("1.0f * 2"), EPS);
+        assertEquals(0.5, evaluateFloat("1.0f / 2"), EPS);
+        assertEquals(1, evaluateFloat("1.0f % 2"), EPS);
+
+        assertEquals(1, evaluateInteger("1"));
+        assertEquals(3, evaluateInteger("1 + 2"));
+        assertEquals(-1, evaluateInteger("1 - 2"));
+        assertEquals(2, evaluateInteger("1 * 2"));
+        assertEquals(0, evaluateInteger("1 / 2"));
+        assertEquals(1, evaluateInteger("1 % 2"));
+
+        assertEquals(1, evaluateLong("1L"));
+        assertEquals(3, evaluateLong("1L + 2"));
+        assertEquals(-1, evaluateLong("1L - 2"));
+        assertEquals(2, evaluateLong("1L * 2"));
+        assertEquals(0, evaluateLong("1L / 2"));
+        assertEquals(1, evaluateLong("1L % 2"));
+
+        assertEquals(3, evaluateDouble("1.0 + 2"), EPS);
+        assertEquals(3, evaluateDouble("1.0 + 2L"), EPS);
+        assertEquals(3, evaluateDouble("1.0 + 2.0f"), EPS);
+        assertEquals(3, evaluateFloat("1.0f + 2"), EPS);
+        assertEquals(3, evaluateFloat("1.0f + 2L"), EPS);
+        assertEquals(3, evaluateLong("1L + 2"));
+    }
+
+    @Test
+    public void requireThatParenthesisControlsPrecedence() throws ParseException {
+        assertEquals(2, evaluateInteger("1 - 2 + 3"));
+        assertEquals(2, evaluateInteger("(1 - 2) + 3"));
+        assertEquals(-4, evaluateInteger("1 - (2 + 3)"));
+        assertEquals(2, evaluateInteger("(1 - 2 + 3)"));
+    }
+
+    private static double evaluateDouble(String script) throws ParseException {
+        FieldValue val = evaluateMath(script);
+        assertTrue(val instanceof DoubleFieldValue);
+        return ((DoubleFieldValue)val).getDouble();
+    }
+
+    private static double evaluateFloat(String script) throws ParseException {
+        FieldValue val = evaluateMath(script);
+        assertTrue(val instanceof FloatFieldValue);
+        return ((FloatFieldValue)val).getFloat();
+    }
+
+    private static long evaluateInteger(String script) throws ParseException {
+        FieldValue val = evaluateMath(script);
+        assertTrue(val instanceof IntegerFieldValue);
+        return ((IntegerFieldValue)val).getInteger();
+    }
+
+    private static long evaluateLong(String script) throws ParseException {
+        FieldValue val = evaluateMath(script);
+        assertTrue(val instanceof LongFieldValue);
+        return ((LongFieldValue)val).getLong();
+    }
+
+    private static FieldValue evaluateMath(String script) throws ParseException {
+        return Expression.fromString(script).execute();
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/NumberTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/NumberTestCase.java
new file mode 100644
index 00000000000..3ff4e957fc6
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/NumberTestCase.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.parser;
+
+import com.yahoo.document.datatypes.*;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class NumberTestCase {
+
+    @Test
+    public void requireThatCorrectNumberTypeIsParsed() throws ParseException {
+        assertTrue(Expression.fromString("6.9").execute() instanceof DoubleFieldValue);
+        assertTrue(Expression.fromString("6.9f").execute() instanceof FloatFieldValue);
+        assertTrue(Expression.fromString("6.9F").execute() instanceof FloatFieldValue);
+        assertTrue(Expression.fromString("69").execute() instanceof IntegerFieldValue);
+        assertTrue(Expression.fromString("69l").execute() instanceof LongFieldValue);
+        assertTrue(Expression.fromString("69L").execute() instanceof LongFieldValue);
+        assertTrue(Expression.fromString("0x69").execute() instanceof IntegerFieldValue);
+        assertTrue(Expression.fromString("0x69l").execute() instanceof LongFieldValue);
+        assertTrue(Expression.fromString("0x69L").execute() instanceof LongFieldValue);
+        assertTrue(Expression.fromString("'69'").execute() instanceof StringFieldValue);
+        assertTrue(Expression.fromString("\"69\"").execute() instanceof StringFieldValue);
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/PrecedenceTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/PrecedenceTestCase.java
new file mode 100644
index 00000000000..418e307c539
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/PrecedenceTestCase.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.parser;
+
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class PrecedenceTestCase {
+
+    @Test
+    public void requireThatMathPrecedesConcat() throws ParseException {
+        assertEquals("15", evaluate("1 . 2 + 3"));
+        assertEquals("33", evaluate("1 + 2 . 3"));
+    }
+
+    private static String evaluate(String script) throws ParseException {
+        return String.valueOf(Expression.fromString(script).execute());
+    }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ScriptTestCase.java
new file mode 100644
index 00000000000..ff2def03c09
--- /dev/null
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ScriptTestCase.java
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.parser;
+
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ScriptTestCase {
+
+    @Test
+    public void requireThatRootProductionIsFlexible() throws ParseException {
+        assertRoot(SetValueExpression.class, "1");
+        assertRoot(StatementExpression.class, "1 | echo");
+        assertRoot(StatementExpression.class, "{ 1 | echo }");
+        assertRoot(StatementExpression.class, "{ 1 | echo; }");
+        assertRoot(ScriptExpression.class, "{ 1 | echo; 2 | echo }");
+    }
+
+    @Test
+    public void requireThatNewlineIsAllowedAfterStatement() throws ParseException {
+        assertScript("{ 1 }");
+        assertScript("{\n 1 }");
+        assertScript("{ 1\n }");
+        assertScript("{\n 1\n }");
+
+        assertScript("{ 1; }");
+        assertScript("{\n 1; }");
+        assertScript("{ 1;\n }");
+        assertScript("{\n 1;\n }");
+
+        assertScript("{ 1; 2 }");
+        assertScript("{\n 1; 2 }");
+        assertScript("{\n 1;\n 2 }");
+        assertScript("{\n 1; 2\n }");
+        assertScript("{ 1;\n 2\n }");
+        assertScript("{\n 1;\n 2\n }");
+    }
+
+    @Test
+    public void requireThatNewlineIsAllowedWithinStatement() throws ParseException {
+        assertStatement("1 |\n 2");
+    }
+
+    private void assertRoot(Class expectedClass, String input) throws ParseException {
+        assertEquals(expectedClass, new IndexingParser(input).root().getClass());
+    }
+
+    private static void assertScript(String input) throws ParseException {
+        assertNotNull(new IndexingParser(input).script());
+    }
+
+    private static void assertStatement(String input) throws ParseException {
+        assertNotNull(new IndexingParser(input).statement());
+    }
+}