Publish

26 files changed, 938 insertions, 0 deletions

diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/TextSerialize.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/TextSerialize.java
new file mode 100644
index 00000000000..bac9f2af237
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/TextSerialize.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.search.query.textserialize.item.ItemContext;
+import com.yahoo.search.query.textserialize.item.ItemFormHandler;
+import com.yahoo.search.query.textserialize.parser.ParseException;
+import com.yahoo.search.query.textserialize.parser.Parser;
+import com.yahoo.search.query.textserialize.parser.TokenMgrError;
+import com.yahoo.search.query.textserialize.serializer.QueryTreeSerializer;
+
+import java.io.StringReader;
+
+/**
+ * @author tonytv
+ * Facade
+ * Allows serializing/deserializing  a query to the programmatic format.
+ */
+public class TextSerialize {
+    public static Item parse(String serializedQuery) {
+        try {
+            ItemContext context = new ItemContext();
+            Object result = new Parser(new StringReader(serializedQuery.replace("'", "\"")), new ItemFormHandler(), context).start();
+            context.connectItems();
+
+            if (!(result instanceof Item)) {
+                throw new RuntimeException("The serialized query '" + serializedQuery + "' did not evaluate to an Item" +
+                        "(type = " + result.getClass() + ")");
+            }
+            return (Item) result;
+        } catch (ParseException e) {
+            throw new RuntimeException(e);
+        } catch (TokenMgrError e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static String serialize(Item item) {
+        return new QueryTreeSerializer().serialize(item);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/AndNotRestConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/AndNotRestConverter.java
new file mode 100644
index 00000000000..c4e54ca748d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/AndNotRestConverter.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.NotItem;
+
+import java.util.List;
+
+import static com.yahoo.search.query.textserialize.item.ListUtil.butFirst;
+import static com.yahoo.search.query.textserialize.item.ListUtil.first;
+
+/**
+ * @author tonytv
+ */
+public class AndNotRestConverter extends CompositeConverter<NotItem> {
+    static final String andNotRest = "AND-NOT-REST";
+
+    public AndNotRestConverter() {
+        super(NotItem.class);
+    }
+
+    @Override
+    protected void addChildren(NotItem item, ItemArguments arguments, ItemContext context) {
+        if (firstIsNull(arguments.children)) {
+            addNegativeItems(item, arguments.children);
+        } else {
+            addItems(item, arguments.children);
+        }
+    }
+
+    private void addNegativeItems(NotItem notItem, List<Object> children) {
+        for (Object child: butFirst(children)) {
+            TypeCheck.ensureInstanceOf(child, Item.class);
+            notItem.addNegativeItem((Item) child);
+        }
+    }
+
+    private void addItems(NotItem notItem, List<Object> children) {
+        for (Object child : children) {
+            TypeCheck.ensureInstanceOf(child, Item.class);
+            notItem.addItem((Item) child);
+        }
+    }
+
+
+    private boolean firstIsNull(List<Object> children) {
+        return !children.isEmpty() && first(children) == null;
+    }
+
+    @Override
+    protected String getFormName(Item item) {
+        return andNotRest;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/CompositeConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/CompositeConverter.java
new file mode 100644
index 00000000000..7f7c5e48d0a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/CompositeConverter.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.search.query.textserialize.serializer.DispatchForm;
+import com.yahoo.search.query.textserialize.serializer.ItemIdMapper;
+
+import java.util.ListIterator;
+
+/**
+ * @author tonytv
+ */
+public class CompositeConverter<T extends CompositeItem> implements ItemFormConverter {
+    private final Class<T> itemClass;
+
+    public CompositeConverter(Class<T> itemClass) {
+        this.itemClass = itemClass;
+    }
+
+    @Override
+    public Object formToItem(String name, ItemArguments arguments, ItemContext itemContext) {
+        T item = newInstance();
+        addChildren(item, arguments, itemContext);
+        return item;
+    }
+
+    protected void addChildren(T item, ItemArguments arguments, ItemContext itemContext) {
+        for (Object child : arguments.children) {
+            item.addItem(asItem(child));
+        }
+        ItemInitializer.initialize(item, arguments, itemContext);
+    }
+
+    private static Item asItem(Object child) {
+        if (!(child instanceof Item) && child != null) {
+            throw new RuntimeException("Expected query item, but got '" + child.toString() +
+                    "' [" + child.getClass().getName() + "]");
+        }
+        return (Item) child;
+    }
+
+    private T newInstance() {
+        try {
+            return itemClass.newInstance();
+        } catch (InstantiationException | IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public DispatchForm itemToForm(Item item, ItemIdMapper itemIdMapper) {
+        CompositeItem compositeItem = (CompositeItem) item;
+
+        DispatchForm form = new DispatchForm(getFormName(item));
+        for (ListIterator<Item> i = compositeItem.getItemIterator(); i.hasNext() ;) {
+            form.addChild(i.next());
+        }
+        ItemInitializer.initializeForm(form, item, itemIdMapper);
+        return form;
+    }
+
+    protected String getFormName(Item item) {
+        return item.getItemType().name();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ExactStringConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ExactStringConverter.java
new file mode 100644
index 00000000000..4b68ecfe5a9
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ExactStringConverter.java
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.ExactstringItem;
+
+/**
+ * @author balder
+ */
+// TODO: balder to fix javadoc
+public class ExactStringConverter extends WordConverter {
+    @Override
+    ExactstringItem newTermItem(String word) {
+        return new ExactstringItem(word);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/IntConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/IntConverter.java
new file mode 100644
index 00000000000..43b96d17773
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/IntConverter.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.IntItem;
+import com.yahoo.prelude.query.TermItem;
+
+/**
+ * @author tonytv
+ */
+public class IntConverter extends TermConverter {
+    @Override
+    IntItem newTermItem(String word) {
+        return new IntItem(word);
+    }
+
+    @Override
+    protected String getValue(TermItem item) {
+        return ((IntItem)item).getNumber();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemArguments.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemArguments.java
new file mode 100644
index 00000000000..50cc9c42773
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemArguments.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static com.yahoo.search.query.textserialize.item.ListUtil.firstInstanceOf;
+
+/**
+ * @author tonytv
+ */
+public class ItemArguments {
+    public final Map<?, ?> properties;
+    public final List<Object> children;
+
+    public ItemArguments(List<Object> arguments) {
+        if (firstInstanceOf(arguments, Map.class)) {
+            properties = (Map<?, ?>) ListUtil.first(arguments);
+            children = ListUtil.rest(arguments);
+        } else {
+            properties = Collections.emptyMap();
+            children = arguments;
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemContext.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemContext.java
new file mode 100644
index 00000000000..fd21b4e02e1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemContext.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.TaggableItem;
+
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.Map;
+
+/**
+ * @author tonytv
+ */
+public class ItemContext {
+    private class Connectivity {
+        final String id;
+        final double strength;
+
+        public Connectivity(String id, double strength) {
+            this.id = id;
+            this.strength = strength;
+        }
+    }
+
+    private final Map<String, Item> itemById = new HashMap<>();
+    private final Map<TaggableItem, Connectivity> connectivityByItem = new IdentityHashMap<>();
+
+
+    public void setItemId(String id, Item item) {
+        itemById.put(id, item);
+    }
+
+    public void setConnectivity(TaggableItem item, String id, Double strength) {
+        connectivityByItem.put(item, new Connectivity(id, strength));
+    }
+
+    public void connectItems() {
+        for (Map.Entry<TaggableItem, Connectivity> entry : connectivityByItem.entrySet()) {
+            entry.getKey().setConnectivity(getItem(entry.getValue().id), entry.getValue().strength);
+        }
+    }
+
+    private Item getItem(String id) {
+        Item item = itemById.get(id);
+        if (item == null)
+            throw new IllegalArgumentException("No item with id '" + id + "'.");
+        return item;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemExecutorRegistry.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemExecutorRegistry.java
new file mode 100644
index 00000000000..20ef9f4e5cc
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemExecutorRegistry.java
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.EquivItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.NearItem;
+import com.yahoo.prelude.query.ONearItem;
+import com.yahoo.prelude.query.OrItem;
+import com.yahoo.prelude.query.PhraseItem;
+import com.yahoo.prelude.query.RankItem;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author tonytv
+ */
+public class ItemExecutorRegistry {
+
+    private static final Map<String, ItemFormConverter> executorsByName = new HashMap<>();
+    static {
+        register(Item.ItemType.AND, createCompositeConverter(AndItem.class));
+        register(Item.ItemType.OR, createCompositeConverter(OrItem.class));
+        register(Item.ItemType.RANK, createCompositeConverter(RankItem.class));
+        register(Item.ItemType.PHRASE, createCompositeConverter(PhraseItem.class));
+        register(Item.ItemType.EQUIV, createCompositeConverter(EquivItem.class));
+
+        register(AndNotRestConverter.andNotRest, new AndNotRestConverter());
+
+        register(Item.ItemType.NEAR, new NearConverter(NearItem.class));
+        register(Item.ItemType.ONEAR, new NearConverter(ONearItem.class));
+
+        register(Item.ItemType.WORD, new WordConverter());
+        register(Item.ItemType.INT, new IntConverter());
+        register(Item.ItemType.PREFIX, new PrefixConverter());
+        register(Item.ItemType.SUBSTRING, new SubStringConverter());
+        register(Item.ItemType.EXACT, new ExactStringConverter());
+        register(Item.ItemType.SUFFIX, new SuffixConverter());
+    }
+
+    private static <T extends CompositeItem> ItemFormConverter createCompositeConverter(Class<T> itemClass) {
+        return new CompositeConverter<>(itemClass);
+    }
+
+    private static void register(Item.ItemType type, ItemFormConverter executor) {
+        register(type.toString(), executor);
+    }
+
+    private static void register(String type, ItemFormConverter executor) {
+        executorsByName.put(type, executor);
+    }
+
+    public static ItemFormConverter getByName(String name) {
+        ItemFormConverter executor = executorsByName.get(name);
+        ensureNotNull(executor, name);
+        return executor;
+    }
+
+    private static void ensureNotNull(ItemFormConverter executor, String name) {
+        if (executor == null) {
+            throw new RuntimeException("No item type named '" + name + "'.");
+        }
+    }
+
+    public static ItemFormConverter getByType(Item.ItemType itemType) {
+        String name = (itemType == Item.ItemType.NOT) ? AndNotRestConverter.andNotRest : itemType.name();
+        return getByName(name);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemFormConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemFormConverter.java
new file mode 100644
index 00000000000..256ad569686
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemFormConverter.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.search.query.textserialize.serializer.DispatchForm;
+import com.yahoo.search.query.textserialize.serializer.ItemIdMapper;
+
+/**
+ * @author tonytv
+ */
+public interface ItemFormConverter {
+    Object formToItem(String name, ItemArguments arguments, ItemContext context);
+    DispatchForm itemToForm(Item item, ItemIdMapper itemIdMapper);
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemFormHandler.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemFormHandler.java
new file mode 100644
index 00000000000..81b13a107c8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemFormHandler.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.search.query.textserialize.parser.DispatchFormHandler;
+
+import java.util.List;
+
+/**
+ * @author tonytv
+ */
+public class ItemFormHandler implements DispatchFormHandler{
+    @Override
+    public Object dispatch(String name, List<Object> arguments, Object dispatchContext) {
+        ItemFormConverter executor = ItemExecutorRegistry.getByName(name);
+        return executor.formToItem(name, new ItemArguments(arguments), (ItemContext)dispatchContext);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemInitializer.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemInitializer.java
new file mode 100644
index 00000000000..ae54165abef
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemInitializer.java
@@ -0,0 +1,137 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.IndexedItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.TaggableItem;
+import com.yahoo.search.query.textserialize.serializer.DispatchForm;
+import com.yahoo.search.query.textserialize.serializer.ItemIdMapper;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author tonytv
+ */
+public class ItemInitializer {
+    private static final String indexProperty = "index";
+    private static final String idProperty = "id";
+    private static final String significanceProperty = "significance";
+    private static final String uniqueIdProperty = "uniqueId";
+    private static final String weightProperty = "weight";
+
+    public static void initialize(Item item, ItemArguments arguments, ItemContext itemContext) {
+        storeIdInContext(item, arguments.properties, itemContext);
+
+        Object weight = arguments.properties.get(weightProperty);
+        if (weight != null) {
+            TypeCheck.ensureInstanceOf(weight, Number.class);
+            item.setWeight(((Number)weight).intValue());
+        }
+
+        if (item instanceof TaggableItem) {
+            initializeTaggableItem((TaggableItem)item, arguments, itemContext);
+        }
+
+        if (item instanceof IndexedItem) {
+            initializeIndexedItem((IndexedItem)item, arguments, itemContext);
+        }
+    }
+
+    private static void storeIdInContext(Item item, Map<?, ?> properties, ItemContext itemContext) {
+        Object id = properties.get("id");
+        if (id != null) {
+            TypeCheck.ensureInstanceOf(id, String.class);
+            itemContext.setItemId((String) id, item);
+        }
+    }
+
+    private static void initializeTaggableItem(TaggableItem item, ItemArguments arguments, ItemContext itemContext) {
+        Object connectivity = arguments.properties.get("connectivity");
+        if (connectivity != null) {
+            storeConnectivityInContext(item, connectivity, itemContext);
+        }
+
+        Object significance = arguments.properties.get(significanceProperty);
+        if (significance != null) {
+            TypeCheck.ensureInstanceOf(significance, Number.class);
+            item.setSignificance(((Number)significance).doubleValue());
+        }
+
+        Object uniqueId = arguments.properties.get(uniqueIdProperty);
+        if (uniqueId != null) {
+            TypeCheck.ensureInstanceOf(uniqueId, Number.class);
+            item.setUniqueID(((Number)uniqueId).intValue());
+        }
+    }
+
+    private static void initializeIndexedItem(IndexedItem indexedItem, ItemArguments arguments, ItemContext itemContext) {
+        Object index = arguments.properties.get(indexProperty);
+        if (index != null) {
+            TypeCheck.ensureInstanceOf(index, String.class);
+            indexedItem.setIndexName((String) index);
+        }
+    }
+
+    private static void storeConnectivityInContext(TaggableItem item, Object connectivity, ItemContext itemContext) {
+        TypeCheck.ensureInstanceOf(connectivity, List.class);
+        List<?> connectivityList = (List<?>) connectivity;
+        if (connectivityList.size() != 2) {
+            throw new IllegalArgumentException("Expected two elements for connectivity, got " + connectivityList.size());
+        }
+
+        Object id = connectivityList.get(0);
+        Object strength = connectivityList.get(1);
+
+        TypeCheck.ensureInstanceOf(id, String.class);
+        TypeCheck.ensureInstanceOf(strength, Number.class);
+
+        itemContext.setConnectivity(item, (String)id, ((Number)strength).doubleValue());
+    }
+
+    public static void initializeForm(DispatchForm form, Item item, ItemIdMapper itemIdMapper) {
+        if (item.getWeight() != Item.DEFAULT_WEIGHT) {
+            form.setProperty(weightProperty, item.getWeight());
+        }
+
+        if (item instanceof IndexedItem) {
+            initializeIndexedForm(form, (IndexedItem) item);
+        }
+        if (item instanceof TaggableItem) {
+            initializeTaggableForm(form, (TaggableItem) item, itemIdMapper);
+        }
+        initializeFormWithIdIfConnected(form, item, itemIdMapper);
+    }
+
+    private static void initializeFormWithIdIfConnected(DispatchForm form, Item item, ItemIdMapper itemIdMapper) {
+        if (item.hasConnectivityBackLink()) {
+            form.setProperty(idProperty, itemIdMapper.getId(item));
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void initializeTaggableForm(DispatchForm form, TaggableItem taggableItem, ItemIdMapper itemIdMapper) {
+        Item connectedItem = taggableItem.getConnectedItem();
+        if (connectedItem != null) {
+            form.setProperty("connectivity",
+                    Arrays.asList(itemIdMapper.getId(connectedItem), taggableItem.getConnectivity()));
+        }
+
+        if (taggableItem.hasExplicitSignificance()) {
+            form.setProperty(significanceProperty, taggableItem.getSignificance());
+        }
+
+        if (taggableItem.hasUniqueID()) {
+            form.setProperty(uniqueIdProperty, taggableItem.getUniqueID());
+        }
+    }
+
+    private static void initializeIndexedForm(DispatchForm form, IndexedItem item) {
+        String index = item.getIndexName();
+        if (!index.isEmpty()) {
+            form.setProperty(indexProperty, index);
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ListUtil.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ListUtil.java
new file mode 100644
index 00000000000..9349b01a3bc
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ListUtil.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import java.util.*;
+
+/**
+ * @author tonytv
+ */
+public class ListUtil {
+    public static <T> List<T> rest(List<T> list) {
+        return list.subList(1, list.size());
+    }
+
+    public static <T> T first(Collection<T> collection) {
+        return collection.iterator().next();
+    }
+
+    public static boolean firstInstanceOf(Collection<?> collection, @SuppressWarnings("rawtypes") Class c) {
+        return !collection.isEmpty() && c.isInstance(first(collection));
+    }
+
+    public static <T> List<T> butFirst(List<T> list) {
+        return list.subList(1, list.size());
+    }
+
+    public static <T> Iterable<T> butFirst(final Collection<T> collection) {
+        return () -> {
+            Iterator<T> i = collection.iterator();
+            i.next();
+            return i;
+        };
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/NearConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/NearConverter.java
new file mode 100644
index 00000000000..3be8d3d1c65
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/NearConverter.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.NearItem;
+import com.yahoo.search.query.textserialize.serializer.DispatchForm;
+import com.yahoo.search.query.textserialize.serializer.ItemIdMapper;
+
+/**
+ * @author tonytv
+ */
+@SuppressWarnings("rawtypes")
+public class NearConverter extends CompositeConverter {
+    final private String distanceProperty = "distance";;
+
+    @SuppressWarnings("unchecked")
+    public NearConverter(Class<? extends NearItem> nearItemClass) {
+        super(nearItemClass);
+    }
+
+    @Override
+    public Object formToItem(String name, ItemArguments arguments, ItemContext itemContext) {
+        NearItem nearItem = (NearItem) super.formToItem(name, arguments, itemContext);
+        setDistance(nearItem, arguments);
+        return nearItem;
+    }
+
+    private void setDistance(NearItem nearItem, ItemArguments arguments) {
+        Object distance = arguments.properties.get(distanceProperty);
+        if (distance != null) {
+            TypeCheck.ensureInteger(distance);
+            nearItem.setDistance(((Number)distance).intValue());
+        }
+    }
+
+    @Override
+    public DispatchForm itemToForm(Item item, ItemIdMapper itemIdMapper) {
+        DispatchForm dispatchForm = super.itemToForm(item, itemIdMapper);
+
+        NearItem nearItem = (NearItem)item;
+        dispatchForm.setProperty(distanceProperty, nearItem.getDistance());
+        return dispatchForm;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/PrefixConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/PrefixConverter.java
new file mode 100644
index 00000000000..cb3a6c1943c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/PrefixConverter.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.PrefixItem;
+
+/**
+ * @author tonytv
+ */
+public class PrefixConverter extends WordConverter {
+    @Override
+    PrefixItem newTermItem(String word) {
+        return new PrefixItem(word);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/SubStringConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/SubStringConverter.java
new file mode 100644
index 00000000000..e61a189684f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/SubStringConverter.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.SubstringItem;
+
+/**
+ * @author tonytv
+ */
+public class SubStringConverter extends WordConverter {
+    @Override
+    SubstringItem newTermItem(String word) {
+        return new SubstringItem(word);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/SuffixConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/SuffixConverter.java
new file mode 100644
index 00000000000..4390e3464d2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/SuffixConverter.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.SuffixItem;
+
+/**
+ * @author tonytv
+ */
+public class SuffixConverter extends WordConverter {
+    @Override
+    SuffixItem newTermItem(String word) {
+        return new SuffixItem(word);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TermConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TermConverter.java
new file mode 100644
index 00000000000..8bc6cba7f67
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TermConverter.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.TermItem;
+import com.yahoo.search.query.textserialize.serializer.DispatchForm;
+import com.yahoo.search.query.textserialize.serializer.ItemIdMapper;
+
+/**
+ * @author tonytv
+ */
+public abstract class TermConverter implements ItemFormConverter {
+    @Override
+    public Object formToItem(String name, ItemArguments arguments, ItemContext context) {
+        ensureOnlyOneChild(arguments);
+        String word = getWord(arguments);
+
+        TermItem item = newTermItem(word);
+        ItemInitializer.initialize(item, arguments, context);
+        return item;
+    }
+
+    abstract TermItem newTermItem(String word);
+
+
+    private void ensureOnlyOneChild(ItemArguments arguments) {
+        if (arguments.children.size() != 1) {
+            throw new IllegalArgumentException("Expected exactly one argument, got '" +
+                    arguments.children.toString() + "'");
+        }
+    }
+
+    private String getWord(ItemArguments arguments) {
+        Object word = arguments.children.get(0);
+
+        if (!(word instanceof String)) {
+            throw new RuntimeException("Expected string, got '" + word + "' [" + word.getClass().getName() + "].");
+        }
+        return (String)word;
+    }
+
+    @Override
+    public DispatchForm itemToForm(Item item, ItemIdMapper itemIdMapper) {
+        TermItem termItem = (TermItem)item;
+
+        DispatchForm form = new DispatchForm(termItem.getItemType().name());
+        ItemInitializer.initializeForm(form, item, itemIdMapper);
+        form.addChild(getValue(termItem));
+        return form;
+    }
+
+    protected abstract String getValue(TermItem item);
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TypeCheck.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TypeCheck.java
new file mode 100644
index 00000000000..a6e38d288a4
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TypeCheck.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.protect.Validator;
+
+/**
+ * @author tonytv
+ */
+public class TypeCheck {
+    public static void ensureInstanceOf(Object object, Class<?> c) {
+        Validator.ensureInstanceOf(expectationString(c.getName(), object.getClass().getSimpleName()),
+                object, c);
+    }
+
+    public static void ensureInteger(Object value) {
+        ensureInstanceOf(value, Number.class);
+        Number number = (Number)value;
+
+        int intValue = number.intValue();
+        if (intValue != number.doubleValue())
+            throw new IllegalArgumentException("Invalid integer '" + number + "'");
+    }
+
+    private static String expectationString(String expected, String got) {
+        return "Expected " + expected + ", but got " + got;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/WordConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/WordConverter.java
new file mode 100644
index 00000000000..dce33e392ae
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/WordConverter.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.TermItem;
+import com.yahoo.prelude.query.WordItem;
+
+/**
+ * @author tonytv
+ */
+public class WordConverter extends TermConverter {
+    @Override
+    WordItem newTermItem(String word) {
+        return new WordItem(word);
+    }
+
+    @Override
+    protected String getValue(TermItem item) {
+        return ((WordItem)item).getWord();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/package-info.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/package-info.java
new file mode 100644
index 00000000000..1e1d3052731
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.query.textserialize;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/.gitignore b/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/.gitignore
new file mode 100644
index 00000000000..add88bd6807
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/.gitignore
@@ -0,0 +1,7 @@
+/TokenMgrError.java
+/Token.java
+/SimpleCharStream.java
+/ParserTokenManager.java
+/ParserConstants.java
+/ParseException.java
+/Parser.java
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/DispatchFormHandler.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/DispatchFormHandler.java
new file mode 100644
index 00000000000..33c8e36bd57
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/DispatchFormHandler.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.parser;
+
+import java.util.List;
+
+/**
+ * @author tonytv
+ */
+public interface DispatchFormHandler {
+    Object dispatch(String name, List<Object> arguments, Object dispatchContext);
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/DispatchForm.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/DispatchForm.java
new file mode 100644
index 00000000000..091efa0a01b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/DispatchForm.java
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.serializer;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author tonytv
+ */
+public class DispatchForm {
+    private final String name;
+    public final Map<Object, Object> properties = new LinkedHashMap<>();
+    public final List<Object> children = new ArrayList<>();
+
+    public DispatchForm(String name) {
+        this.name = name;
+    }
+
+    public void addChild(Object child) {
+        children.add(child);
+    }
+
+    /**
+     * Only public for the purpose of testing.
+     */
+    public String serialize(ItemIdMapper itemIdMapper) {
+        StringBuilder builder = new StringBuilder();
+        builder.append('(').append(name);
+
+        serializeProperties(builder, itemIdMapper);
+        serializeChildren(builder, itemIdMapper);
+
+        builder.append(')');
+        return builder.toString();
+    }
+
+    private void serializeProperties(StringBuilder builder, ItemIdMapper itemIdMapper) {
+        if (properties.isEmpty())
+            return;
+
+        builder.append(' ').append(Serializer.serializeMap(properties, itemIdMapper));
+    }
+
+
+    private void serializeChildren(StringBuilder builder, ItemIdMapper itemIdMapper) {
+        for (Object child : children) {
+            builder.append(' ').append(Serializer.serialize(child, itemIdMapper));
+        }
+    }
+
+    public void setProperty(Object key, Object value) {
+        properties.put(key, value);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/ItemIdMapper.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/ItemIdMapper.java
new file mode 100644
index 00000000000..c32a7f52c0a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/ItemIdMapper.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.serializer;
+
+import com.yahoo.prelude.query.Item;
+
+import java.util.IdentityHashMap;
+import java.util.Map;
+
+/**
+ * @author tonytv
+ */
+public class ItemIdMapper {
+    private final Map<Item, String> idByItem = new IdentityHashMap<>();
+    private int idCounter = 0;
+
+    public String getId(Item item) {
+        String id = idByItem.get(item);
+        if (id != null) {
+            return id;
+        } else {
+            idByItem.put(item, generateId(item));
+            return getId(item);
+        }
+    }
+
+    private String generateId(Item item) {
+        return item.getName() + "_" + nextCount();
+    }
+
+    private int nextCount() {
+        return idCounter++;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/QueryTreeSerializer.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/QueryTreeSerializer.java
new file mode 100644
index 00000000000..e3090930369
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/QueryTreeSerializer.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.serializer;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.search.query.textserialize.item.ItemExecutorRegistry;
+
+
+/**
+ * @author tonytv
+ */
+public class QueryTreeSerializer {
+    public String serialize(Item root) {
+        ItemIdMapper itemIdMapper = new ItemIdMapper();
+        return ItemExecutorRegistry.getByType(root.getItemType()).itemToForm(root, itemIdMapper).serialize(itemIdMapper);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/Serializer.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/Serializer.java
new file mode 100644
index 00000000000..e8352254551
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/Serializer.java
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.serializer;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.search.query.textserialize.item.ItemExecutorRegistry;
+
+import java.util.List;
+import java.util.Map;
+
+import static com.yahoo.search.query.textserialize.item.ListUtil.butFirst;
+import static com.yahoo.search.query.textserialize.item.ListUtil.first;
+
+/**
+ * @author tonytv
+ */
+class Serializer {
+    static String serialize(Object child, ItemIdMapper itemIdMapper) {
+        if (child instanceof DispatchForm) {
+            return ((DispatchForm) child).serialize(itemIdMapper);
+        } else if (child instanceof Item) {
+            return serializeItem((Item) child, itemIdMapper);
+        } else if (child instanceof String) {
+            return serializeString((String) child);
+        } else if (child instanceof Number) {
+            return child.toString();
+        } else if (child instanceof Map) {
+            return serializeMap((Map<?, ?>)child, itemIdMapper);
+        } else if (child instanceof List) {
+            return serializeList((List<?>)child, itemIdMapper);
+        } else {
+            throw new IllegalArgumentException("Can't serialize type " + child.getClass());
+        }
+    }
+
+    private static String serializeString(String string) {
+        return '"' + string.replace("\\", "\\\\").replace("\"", "\\\"") + '"';
+    }
+
+    static String serializeList(List<?> list, ItemIdMapper itemIdMapper) {
+        StringBuilder builder = new StringBuilder();
+        builder.append('[');
+
+        if (!list.isEmpty()) {
+            builder.append(serialize(first(list), itemIdMapper));
+
+            for (Object element : butFirst(list)) {
+                builder.append(", ").append(serialize(element, itemIdMapper));
+            }
+        }
+
+        builder.append(']');
+        return builder.toString();
+    }
+
+    static String serializeMap(Map<?, ?> map, ItemIdMapper itemIdMapper) {
+        StringBuilder builder = new StringBuilder();
+        builder.append("{");
+
+        if (!map.isEmpty()) {
+            serializeEntry(builder, first(map.entrySet()), itemIdMapper);
+            for (Map.Entry<?, ?> entry : butFirst(map.entrySet())) {
+                builder.append(", ");
+                serializeEntry(builder, entry, itemIdMapper);
+            }
+        }
+
+        builder.append('}');
+        return builder.toString();
+    }
+
+    static void serializeEntry(StringBuilder builder, Map.Entry<?, ?> entry, ItemIdMapper itemIdMapper) {
+        builder.append(serialize(entry.getKey(), itemIdMapper)).append(' ').
+                append(serialize(entry.getValue(), itemIdMapper));
+    }
+
+    static String serializeItem(Item item, ItemIdMapper itemIdMapper) {
+        return ItemExecutorRegistry.getByType(item.getItemType()).itemToForm(item, itemIdMapper).serialize(itemIdMapper);
+    }
+}