diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /indexinglanguage |
Publish
Diffstat (limited to 'indexinglanguage')
195 files changed, 17644 insertions, 0 deletions
diff --git a/indexinglanguage/.gitignore b/indexinglanguage/.gitignore new file mode 100644 index 00000000000..b19abb01892 --- /dev/null +++ b/indexinglanguage/.gitignore @@ -0,0 +1,14 @@ +*.iml +*.ipr +*.iws +.classpath +.project +archive +build +classes +lib/* +libexec +log +target +testLogs +/pom.xml.build diff --git a/indexinglanguage/OWNERS b/indexinglanguage/OWNERS new file mode 100644 index 00000000000..7ae1acb1be9 --- /dev/null +++ b/indexinglanguage/OWNERS @@ -0,0 +1 @@ +geirst diff --git a/indexinglanguage/README b/indexinglanguage/README new file mode 100644 index 00000000000..e56a0c1f1dd --- /dev/null +++ b/indexinglanguage/README @@ -0,0 +1 @@ +Indexing Language interpreter diff --git a/indexinglanguage/pom.xml b/indexinglanguage/pom.xml new file mode 100644 index 00000000000..a4328091d35 --- /dev/null +++ b/indexinglanguage/pom.xml @@ -0,0 +1,112 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>indexinglanguage</artifactId> + <packaging>jar</packaging> + <version>6-SNAPSHOT</version> + <name>indexinglanguage</name> + <description>Interpreter for the Indexing Language</description> + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>document</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>linguistics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>predicate-search-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <version>1.4</version> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:unchecked</arg> + <arg>-Xlint:deprecation</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <!-- append to the packaging phase. --> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>javacc-maven-plugin</artifactId> + <executions> + <execution> + <id>javacc</id> + <goals> + <goal>javacc</goal> + </goals> + <configuration> + <lookAhead>1</lookAhead> + <isStatic>false</isStatic> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <configuration> + <updateReleaseInfo>true</updateReleaseInfo> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/AdapterFactory.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/AdapterFactory.java new file mode 100644 index 00000000000..349555ee3fc --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/AdapterFactory.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentUpdate; + +import java.util.List; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface AdapterFactory { + + public DocumentAdapter newDocumentAdapter(Document doc); + + public List<UpdateAdapter> newUpdateAdapterList(DocumentUpdate upd); +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/DocumentAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/DocumentAdapter.java new file mode 100644 index 00000000000..93d848fe0ca --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/DocumentAdapter.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.Document; +import com.yahoo.vespa.indexinglanguage.expressions.FieldValueAdapter; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface DocumentAdapter extends FieldValueAdapter { + + public Document getFullOutput(); + + public Document getUpdatableOutput(); +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java new file mode 100644 index 00000000000..0653fba6e86 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java @@ -0,0 +1,136 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.collections.Pair; +import com.yahoo.vespa.indexinglanguage.expressions.*; + +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +@SuppressWarnings({ "UnusedDeclaration" }) +public abstract class ExpressionConverter implements Cloneable { + + public final Expression convert(Expression exp) { + if (exp == null) { + return null; + } + if (shouldConvert(exp)) { + return doConvert(exp); + } + if (!(exp instanceof CompositeExpression)) { + return exp; + } + try { + // The class.getMethod here takes 8% of the cpu time in reading the SSBE application package + // TODO: Implement double dispatch through visitor instead? + return (Expression)ExpressionConverter.class.getMethod("innerConvert", exp.getClass()).invoke(this, exp); + } catch (IllegalAccessException | NoSuchMethodException e) { + throw new UnsupportedOperationException(exp.getClass().getName(), e); + } catch (InvocationTargetException e) { + Throwable t = e.getTargetException(); + throw t instanceof RuntimeException ? (RuntimeException)t : new RuntimeException(t); + } + } + + public Expression innerConvert(ArithmeticExpression exp) { + return new ArithmeticExpression(convert(exp.getLeftHandSide()), + exp.getOperator(), + convert(exp.getRightHandSide())); + } + + public Expression innerConvert(CatExpression exp) { + List<Expression> lst = new LinkedList<>(); + for (Expression innerExp : exp) { + Expression next = convert(innerExp); + if (next != null) { + lst.add(next); + } + } + return new CatExpression(lst); + } + + public Expression innerConvert(ForEachExpression exp) { + return new ForEachExpression(convert(exp.getInnerExpression())); + } + + public Expression innerConvert(GuardExpression exp) { + return new GuardExpression(convert(exp.getInnerExpression())); + } + + public Expression innerConvert(IfThenExpression exp) { + return new IfThenExpression(branch().convert(exp.getLeftHandSide()), + exp.getComparator(), + branch().convert(exp.getRightHandSide()), + branch().convert(exp.getIfTrueExpression()), + branch().convert(exp.getIfFalseExpression())); + } + + public Expression innerConvert(ParenthesisExpression exp) { + return new ParenthesisExpression(convert(exp.getInnerExpression())); + } + + public Expression innerConvert(ScriptExpression exp) { + List<StatementExpression> lst = new LinkedList<>(); + for (Expression innerExp : exp) { + StatementExpression next = (StatementExpression)branch().convert(innerExp); + if (next != null) { + lst.add(next); + } + } + return new ScriptExpression(lst); + } + + public Expression innerConvert(SelectInputExpression exp) { + List<Pair<String, Expression>> cases = new LinkedList<>(); + for (Pair<String, Expression> pair : exp.getCases()) { + cases.add(new Pair<>(pair.getFirst(), branch().convert(pair.getSecond()))); + } + return new SelectInputExpression(cases); + } + + public Expression innerConvert(StatementExpression exp) { + List<Expression> lst = new LinkedList<>(); + for (Expression innerExp : exp) { + Expression next = convert(innerExp); + if (next != null) { + lst.add(next); + } + } + return new StatementExpression(lst); + } + + public Expression innerConvert(SwitchExpression exp) { + Map<String, Expression> cases = new HashMap<>(); + for (Map.Entry<String, Expression> entry : exp.getCases().entrySet()) { + Expression next = branch().convert(entry.getValue()); + if (next != null) { + cases.put(entry.getKey(), next); + } + } + return new SwitchExpression(cases, branch().convert(exp.getDefaultExpression())); + } + + protected ExpressionConverter branch() { + return this; + } + + @Override + @SuppressWarnings("CloneDoesntDeclareCloneNotSupportedException") + public ExpressionConverter clone() { + try { + return (ExpressionConverter)super.clone(); + } catch (CloneNotSupportedException e) { + throw new UnsupportedOperationException(e); + } + } + + protected abstract boolean shouldConvert(Expression exp); + + protected abstract Expression doConvert(Expression exp); +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizer.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizer.java new file mode 100644 index 00000000000..1f38b402e62 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizer.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.vespa.indexinglanguage.expressions.*; + +import java.util.ArrayList; +import java.util.List; + +/** + * Optimizes expressions by removing expressions that have no effect. + * Typical examples are statements that come before a statement that + * generates a new execution value without regard for the existing one. + */ +public class ExpressionOptimizer extends ExpressionConverter { + @Override + protected boolean shouldConvert(Expression exp) { + return exp instanceof StatementExpression; + } + + @Override + protected Expression doConvert(Expression exp) { + return optimizeStatement((StatementExpression) exp); + } + + private Expression optimizeStatement(StatementExpression statement) { + List<Expression> expressionList = new ArrayList<>(); + List<Expression> candidateList = new ArrayList<>(); + for (Expression exp : statement) { + if (ignoresInput(exp)) { + candidateList.clear(); + } + candidateList.add(convert(exp)); + if (hasSideEffect(exp)) { + expressionList.addAll(candidateList); + candidateList.clear(); + } + } + expressionList.addAll(candidateList); + return new StatementExpression(expressionList); + } + + static boolean hasSideEffect(Expression exp) { + HasSideEffectVisitor visitor = new HasSideEffectVisitor(); + visitor.visit(exp); + return visitor.hasSideEffect; + } + + private static class HasSideEffectVisitor extends ExpressionVisitor { + + boolean hasSideEffect = false; + + @Override + protected void doVisit(Expression exp) { + hasSideEffect |= exp instanceof OutputExpression || + exp instanceof SetVarExpression || + exp instanceof EchoExpression; + } + } + + static boolean ignoresInput(Expression exp) { + if (exp instanceof SwitchExpression || exp instanceof ScriptExpression) { + return false; // Switch and script never ignores input. + } + if (exp instanceof CompositeExpression) { + return new IgnoresInputVisitor().ignoresInput(exp); + } + if (exp instanceof RandomExpression) { + return ((RandomExpression)exp).getMaxValue() != null; + } + return exp instanceof InputExpression || + exp instanceof NowExpression || + exp instanceof SetValueExpression || + exp instanceof HostNameExpression || + exp instanceof GetVarExpression; + } + + private static class IgnoresInputVisitor extends ExpressionConverter { + private boolean ignoresInput = true; + private Expression root = null; + + public boolean ignoresInput(Expression root) { + this.root = root; + convert(root); + return ignoresInput; + } + + @Override + protected boolean shouldConvert(Expression exp) { + if (!ignoresInput) { + return true; // Answer found, skip ahead + } + if (exp == root) { + return false; // Skip root, check children + } + if (exp instanceof StatementExpression) { + for (Expression expression : (StatementExpression) exp) { + if (ExpressionOptimizer.ignoresInput(expression)) { + return true; // Skip children + } + } + ignoresInput = false; + return true; // Answer found, skip children + } + ignoresInput &= ExpressionOptimizer.ignoresInput(exp); + return true; // Children already checked. Skip them. + } + + @Override + protected Expression doConvert(Expression exp) { + return exp; + } + + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcher.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcher.java new file mode 100644 index 00000000000..c24cfcf4f09 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcher.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.vespa.indexinglanguage.expressions.Expression; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ExpressionSearcher<T extends Expression> { + + private final Class<T> searchFor; + + public ExpressionSearcher(Class<T> searchFor) { + this.searchFor = searchFor; + } + + public boolean containedIn(Expression searchIn) { + return searchIn(searchIn) != null; + } + + public T searchIn(Expression searchIn) { + MyConverter searcher = new MyConverter(); + searcher.convert(searchIn); + return searcher.found; + } + + private class MyConverter extends ExpressionConverter { + + T found = null; + + @Override + protected boolean shouldConvert(Expression exp) { + if (searchFor.isInstance(exp)) { + found = searchFor.cast(exp); + return true; // terminate search + } + return false; + } + + @Override + protected Expression doConvert(Expression exp) { + return exp; + } + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitor.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitor.java new file mode 100644 index 00000000000..5cb73b7fba0 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitor.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.vespa.indexinglanguage.expressions.Expression; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class ExpressionVisitor { + + private final MyConverter converter = new MyConverter(); + + public void visit(Expression exp) { + converter.convert(exp); + } + + protected abstract void doVisit(Expression exp); + + private class MyConverter extends ExpressionConverter { + + @Override + protected boolean shouldConvert(Expression exp) { + doVisit(exp); + return false; + } + + @Override + protected Expression doConvert(Expression exp) { + throw new AssertionError(); + } + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateAdapter.java new file mode 100644 index 00000000000..5d3fcd24727 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateAdapter.java @@ -0,0 +1,126 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.*; +import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.FieldValueAdapter; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class FieldPathUpdateAdapter implements UpdateAdapter { + + private final DocumentAdapter adapter; + private final FieldPathUpdate update; + + public FieldPathUpdateAdapter(DocumentAdapter documentAdapter, FieldPathUpdate fieldUpdate) { + adapter = documentAdapter; + update = fieldUpdate; + } + + @Override + public Expression getExpression(Expression expression) { + return expression; + } + + @Override + public DocumentUpdate getOutput() { + Document doc = adapter.getFullOutput(); + DocumentUpdate upd = new DocumentUpdate(doc.getDataType(), doc.getId()); + createUpdatesAt(new ArrayList<>(), adapter.getUpdatableOutput(), 0, upd); + return upd; + } + + @Override + public DataType getInputType(Expression exp, String fieldName) { + return adapter.getInputType(exp, fieldName); + } + + @Override + public FieldValue getInputValue(String fieldName) { + return adapter.getInputValue(fieldName); + } + + @Override + public FieldValue getInputValue(FieldPath fieldPath) { + return adapter.getInputValue(fieldPath); + } + + @Override + public void tryOutputType(Expression exp, String fieldName, DataType valueType) { + adapter.tryOutputType(exp, fieldName, valueType); + } + + @Override + public FieldValueAdapter setOutputValue(Expression exp, String fieldName, FieldValue fieldValue) { + return adapter.setOutputValue(exp, fieldName, fieldValue); + } + + @SuppressWarnings({ "unchecked" }) + private void createUpdatesAt(List<FieldPathEntry> path, FieldValue value, int idx, DocumentUpdate out) { + FieldPath updatePath = update.getFieldPath(); + if (idx < updatePath.size()) { + FieldPathEntry pathEntry = updatePath.get(idx); + FieldPathEntry.Type type = pathEntry.getType(); + if (type == FieldPathEntry.Type.STRUCT_FIELD) { + if (!(value instanceof StructuredFieldValue)) { + throw new IllegalArgumentException("Expected structured field value, got " + + value.getClass().getName() + "."); + } + for (Iterator<Map.Entry<Field, FieldValue>> it = ((StructuredFieldValue)value).iterator(); it.hasNext();) { + Map.Entry<Field, FieldValue> structEntry = it.next(); + List<FieldPathEntry> nextPath = new ArrayList<>(path); + nextPath.add(FieldPathEntry.newStructFieldEntry(structEntry.getKey())); + createUpdatesAt(nextPath, structEntry.getValue(), idx + 1, out); + } + } else if (type == FieldPathEntry.Type.MAP_KEY) { + if (value instanceof WeightedSet) { + WeightedSet wset = (WeightedSet)value; + for (Iterator<FieldValue> it = wset.fieldValueIterator(); it.hasNext();) { + FieldValue wsetEntry = it.next(); + List<FieldPathEntry> nextPath = new ArrayList<>(path); + nextPath.add(FieldPathEntry.newMapLookupEntry(wsetEntry, DataType.INT)); + createUpdatesAt(nextPath, new IntegerFieldValue(wset.get(wsetEntry)), idx + 1, out); + } + } else if (value instanceof MapFieldValue) { + MapFieldValue<FieldValue, FieldValue> map = (MapFieldValue)value; + for (Map.Entry<FieldValue, FieldValue> entry : map.entrySet()) { + List<FieldPathEntry> nextPath = new ArrayList<>(path); + FieldValue nextVal = entry.getValue(); + nextPath.add(FieldPathEntry.newMapLookupEntry(entry.getKey(), nextVal.getDataType())); + createUpdatesAt(nextPath, nextVal, idx + 1, out); + } + } else { + throw new IllegalArgumentException("Expected map or weighted set, got " + + value.getClass().getName() + "."); + } + } else { + path.add(pathEntry); + createUpdatesAt(new ArrayList<>(path), value, idx + 1, out); + } + } else if (update instanceof AddFieldPathUpdate) { + if (!(value instanceof Array)) { + throw new IllegalStateException("Expected array, got " + + value.getClass().getName() + "."); + } + out.addFieldPathUpdate(new AddFieldPathUpdate(update.getDocumentType(), new FieldPath(path).toString(), + update.getOriginalWhereClause(), (Array)value)); + } else if (update instanceof AssignFieldPathUpdate) { + out.addFieldPathUpdate(new AssignFieldPathUpdate(update.getDocumentType(), new FieldPath(path).toString(), + update.getOriginalWhereClause(), value)); + } else if (update instanceof RemoveFieldPathUpdate) { + out.addFieldPathUpdate(new RemoveFieldPathUpdate(update.getDocumentType(), new FieldPath(path).toString(), + update.getOriginalWhereClause())); + } + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateHelper.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateHelper.java new file mode 100644 index 00000000000..361b54e3015 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldPathUpdateHelper.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.FieldPathEntry; +import com.yahoo.document.datatypes.FieldPathIteratorHandler; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class FieldPathUpdateHelper { + + public static boolean isComplete(FieldPathUpdate update) { + if (!(update instanceof AssignFieldPathUpdate)) { + return false; + } + for (FieldPathEntry entry : update.getFieldPath()) { + switch (entry.getType()) { + case STRUCT_FIELD: + case MAP_ALL_KEYS: + case MAP_ALL_VALUES: + continue; + case ARRAY_INDEX: + case MAP_KEY: + case VARIABLE: + return false; + } + } + return true; + } + + public static void applyUpdate(FieldPathUpdate update, Document doc) { + if (update instanceof AddFieldPathUpdate) { + update.applyTo(doc); + } else if (update instanceof AssignFieldPathUpdate) { + AssignFieldPathUpdate assign = (AssignFieldPathUpdate)update; + boolean createMissingPath = assign.getCreateMissingPath(); + boolean removeIfZero = assign.getRemoveIfZero(); + assign.setCreateMissingPath(true); + assign.setRemoveIfZero(false); + + assign.applyTo(doc); + + assign.setCreateMissingPath(createMissingPath); + assign.setRemoveIfZero(removeIfZero); + } else if (update instanceof RemoveFieldPathUpdate) { + doc.iterateNested(update.getFieldPath(), 0, new MyHandler()); + } + } + + public static Document newPartialDocument(DocumentId docId, FieldPathUpdate update) { + Document doc = new Document(update.getDocumentType(), docId); + applyUpdate(update, doc); + return doc; + } + + private static class MyHandler extends FieldPathIteratorHandler { + + @Override + public ModificationStatus doModify(FieldValue fv) { + return ModificationStatus.MODIFIED; + } + + @Override + public boolean createMissingPath() { + return true; + } + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateAdapter.java new file mode 100644 index 00000000000..cf88dc74ed0 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateAdapter.java @@ -0,0 +1,231 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.*; +import com.yahoo.document.update.*; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.FieldValueAdapter; + +import java.util.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class FieldUpdateAdapter implements UpdateAdapter { + + private final DocumentAdapter adapter; + private final Builder builder; + private final Expression optimizedExpression; + + private FieldUpdateAdapter(Expression optimizedExpression, DocumentAdapter adapter, Builder builder) { + this.adapter = adapter; + this.builder = builder; + this.optimizedExpression = optimizedExpression; + } + + @Override + public DocumentUpdate getOutput() { + Document doc = adapter.getUpdatableOutput(); + DocumentUpdate upd = new DocumentUpdate(doc.getDataType(), doc.getId()); + for (Iterator<Map.Entry<Field, FieldValue>> it = doc.iterator(); it.hasNext();) { + Map.Entry<Field, FieldValue> entry = it.next(); + Field field = entry.getKey(); + if (field.getName().equals("sddocname")) { + continue; + } + FieldUpdate fieldUpd = FieldUpdate.create(field); + fieldUpd.addValueUpdates(builder.build(entry.getValue())); + if (!fieldUpd.isEmpty()) { + upd.addFieldUpdate(fieldUpd); + } + } + return upd.isEmpty() ? null : upd; + } + + @Override + public Expression getExpression(Expression expression) { + return optimizedExpression != null ? optimizedExpression : expression; + } + + @Override + public DataType getInputType(Expression exp, String fieldName) { + return adapter.getInputType(exp, fieldName); + } + + @Override + public FieldValue getInputValue(String fieldName) { + return adapter.getInputValue(fieldName); + } + @Override + public FieldValue getInputValue(FieldPath fieldPath) { return adapter.getInputValue(fieldPath); } + + @Override + public void tryOutputType(Expression exp, String fieldName, DataType valueType) { + adapter.tryOutputType(exp, fieldName, valueType); + } + + @Override + public FieldValueAdapter setOutputValue(Expression exp, String fieldName, FieldValue fieldValue) { + return adapter.setOutputValue(exp, fieldName, fieldValue); + } + + public static FieldUpdateAdapter fromPartialUpdate(DocumentAdapter documentAdapter, ValueUpdate valueUpdate) { + return new FieldUpdateAdapter(null, documentAdapter, new PartialBuilder(valueUpdate)); + } + public static FieldUpdateAdapter fromPartialUpdate(Expression expression, DocumentAdapter documentAdapter, ValueUpdate valueUpdate) { + return new FieldUpdateAdapter(expression, documentAdapter, new PartialBuilder(valueUpdate)); + } + + public static FieldUpdateAdapter fromCompleteUpdate(DocumentAdapter documentAdapter) { + return new FieldUpdateAdapter(null, documentAdapter, new CompleteBuilder()); + } + + private interface Builder { + + List<ValueUpdate> build(FieldValue val); + } + + private static class PartialBuilder implements Builder { + + final ValueUpdate update; + + PartialBuilder(ValueUpdate update) { + this.update = update; + } + + @Override + public List<ValueUpdate> build(FieldValue val) { + return createValueUpdates(val, update); + } + + @SuppressWarnings({ "unchecked" }) + List<ValueUpdate> createValueUpdates(FieldValue val, ValueUpdate upd) { + List<ValueUpdate> lst = new ArrayList<>(); + if (upd instanceof ClearValueUpdate) { + lst.add(new ClearValueUpdate()); + } else if (upd instanceof AssignValueUpdate) { + lst.add(new AssignValueUpdate(val)); + } else if (upd instanceof AddValueUpdate) { + if (val instanceof Array) { + lst.addAll(createAddValueUpdateForArray((Array)val, ((AddValueUpdate)upd).getWeight())); + } else if (val instanceof WeightedSet) { + lst.addAll(createAddValueUpdateForWset((WeightedSet)val)); + } else { + // do nothing + } + } else if (upd instanceof ArithmeticValueUpdate) { + lst.add(upd); // leave arithmetics alone + } else if (upd instanceof RemoveValueUpdate) { + if (val instanceof Array) { + lst.addAll(createRemoveValueUpdateForEachElement(((Array)val).fieldValueIterator())); + } else if (val instanceof WeightedSet) { + lst.addAll(createRemoveValueUpdateForEachElement(((WeightedSet)val).fieldValueIterator())); + } else { + // do nothing + } + } else if (upd instanceof MapValueUpdate) { + if (val instanceof Array) { + lst.addAll(createMapValueUpdatesForArray((Array)val, (MapValueUpdate)upd)); + } else if (val instanceof MapFieldValue) { + throw new UnsupportedOperationException("Can not map into a " + val.getClass().getName() + "."); + } else if (val instanceof StructuredFieldValue) { + lst.addAll(createMapValueUpdatesForStruct((StructuredFieldValue)val, (MapValueUpdate)upd)); + } else if (val instanceof WeightedSet) { + lst.addAll(createMapValueUpdatesForWset((WeightedSet)val, (MapValueUpdate)upd)); + } else { + // do nothing + } + } else { + throw new UnsupportedOperationException( + "Value update type " + upd.getClass().getName() + " not supported."); + } + return lst; + } + + @SuppressWarnings({ "unchecked" }) + private List<ValueUpdate> createAddValueUpdateForArray(Array arr, int weight) { + List<ValueUpdate> ret = new ArrayList<>(arr.size()); + for (Iterator<FieldValue> it = arr.fieldValueIterator(); it.hasNext(); ) { + ret.add(new AddValueUpdate(it.next(), weight)); + } + return ret; + } + + @SuppressWarnings({ "unchecked" }) + private List<ValueUpdate> createAddValueUpdateForWset(WeightedSet wset) { + List<ValueUpdate> ret = new ArrayList<>(wset.size()); + for (Iterator<FieldValue> it = wset.fieldValueIterator(); it.hasNext(); ) { + FieldValue key = it.next(); + ret.add(new AddValueUpdate(key, wset.get(key))); + } + return ret; + } + + @SuppressWarnings({ "unchecked" }) + private List<ValueUpdate> createRemoveValueUpdateForEachElement(Iterator<FieldValue> it) { + List<ValueUpdate> ret = new ArrayList<>(); + while (it.hasNext()) { + ret.add(new RemoveValueUpdate(it.next())); + } + return ret; + } + + @SuppressWarnings({ "unchecked" }) + private List<ValueUpdate> createMapValueUpdatesForArray(Array arr, MapValueUpdate upd) { + List<ValueUpdate> ret = new ArrayList<>(); + for (Iterator<FieldValue> it = arr.fieldValueIterator(); it.hasNext();) { + FieldValue childVal = it.next(); + for (ValueUpdate childUpd : createValueUpdates(childVal, upd.getUpdate())) { + ret.add(new MapValueUpdate(childVal, childUpd)); + } + } + return ret; + } + + @SuppressWarnings({ "unchecked" }) + private List<ValueUpdate> createMapValueUpdatesForStruct(StructuredFieldValue struct, MapValueUpdate upd) { + List<ValueUpdate> ret = new ArrayList<>(); + for (Iterator<Map.Entry<Field, FieldValue>> it = struct.iterator(); it.hasNext();) { + Map.Entry<Field, FieldValue> entry = it.next(); + for (ValueUpdate childUpd : createValueUpdates(entry.getValue(), upd.getUpdate())) { + ret.add(new MapValueUpdate(new StringFieldValue(entry.getKey().getName()), childUpd)); + } + } + return ret; + } + + @SuppressWarnings({ "unchecked" }) + private List<ValueUpdate> createMapValueUpdatesForWset(WeightedSet wset, MapValueUpdate upd) { + List<ValueUpdate> ret = new ArrayList<>(); + for (Iterator<FieldValue> it = wset.fieldValueIterator(); it.hasNext();) { + FieldValue childVal = it.next(); + for (ValueUpdate childUpd : createValueUpdates(new IntegerFieldValue(wset.get(childVal)), + upd.getUpdate())) + { + ret.add(new MapValueUpdate(childVal, childUpd)); + } + } + return ret; + } + } + + private static class CompleteBuilder extends PartialBuilder { + + static final ValueUpdate nullMap = new MapValueUpdate(null, null); + static final ValueUpdate nullAssign = new AssignValueUpdate(null); + + CompleteBuilder() { + super(null); + } + + @Override + List<ValueUpdate> createValueUpdates(FieldValue val, ValueUpdate upd) { + if (val instanceof StructuredFieldValue) { + return super.createValueUpdates(val, nullMap); + } else { + return super.createValueUpdates(val, nullAssign); + } + } + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateHelper.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateHelper.java new file mode 100644 index 00000000000..7412b7171d1 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldUpdateHelper.java @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.*; +import com.yahoo.document.update.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class FieldUpdateHelper { + + public static boolean isComplete(Field field, ValueUpdate update) { + if (update instanceof AssignValueUpdate) { + return true; + } + if (!(update instanceof MapValueUpdate)) { + return false; + } + DataType fieldType = field.getDataType(); + if (!(fieldType instanceof StructuredDataType)) { + return false; + } + field = ((StructuredDataType)fieldType).getField(String.valueOf(update.getValue())); + if (field == null) { + return false; + } + return isComplete(field, ((MapValueUpdate)update).getUpdate()); + } + + public static void applyUpdate(Field field, ValueUpdate update, Document doc) { + doc.setFieldValue(field, createFieldValue(field.getDataType().createFieldValue(), update)); + } + + public static Document newPartialDocument(DocumentType docType, DocumentId docId, Field field, ValueUpdate update) { + Document doc = new Document(docType, docId); + applyUpdate(field, update, doc); + return doc; + } + + @SuppressWarnings({ "unchecked" }) + private static FieldValue createFieldValue(FieldValue val, ValueUpdate upd) { + if (upd instanceof ClearValueUpdate) { + return val; + } else if (upd instanceof AssignValueUpdate) { + val.assign(upd.getValue()); + return val; + } else if (upd instanceof AddValueUpdate) { + if (val instanceof Array) { + ((Array)val).add(upd.getValue()); + } else if (val instanceof WeightedSet) { + ((WeightedSet)val).put(upd.getValue(), ((AddValueUpdate)upd).getWeight()); + } + return val; + } else if (upd instanceof ArithmeticValueUpdate) { + if (((ArithmeticValueUpdate)upd).getOperator() == ArithmeticValueUpdate.Operator.DIV && + ((ArithmeticValueUpdate)upd).getOperand().doubleValue() == 0) { + throw new IllegalArgumentException("Division by zero."); + } + val.assign(upd.getValue()); + return val; + } else if (upd instanceof RemoveValueUpdate) { + if (val instanceof Array) { + ((Array)val).add(upd.getValue()); + } else if (val instanceof WeightedSet) { + ((WeightedSet)val).put(upd.getValue(), 1); + } + return val; + } else if (upd instanceof MapValueUpdate) { + if (val instanceof Array) { + return createFieldValue(val, ((MapValueUpdate)upd).getUpdate()); + } else if (val instanceof MapFieldValue) { + throw new UnsupportedOperationException("Can not map into a " + val.getClass().getName() + "."); + } else if (val instanceof StructuredFieldValue) { + Field field = ((StructuredFieldValue)val).getField(String.valueOf(upd.getValue())); + if (field == null) { + throw new IllegalArgumentException("Field '" + upd.getValue() + "' not found."); + } + ((StructuredFieldValue)val).setFieldValue(field, createFieldValue(field.getDataType().createFieldValue(), + ((MapValueUpdate)upd).getUpdate())); + return val; + } else if (val instanceof WeightedSet) { + FieldValue weight = createFieldValue(new IntegerFieldValue(), ((MapValueUpdate)upd).getUpdate()); + if (!(weight instanceof IntegerFieldValue)) { + throw new IllegalArgumentException("Expected integer, got " + weight.getClass().getName() + "."); + } + ((WeightedSet)val).put(upd.getValue(), ((IntegerFieldValue)weight).getInteger()); + return val; + } else { + throw new IllegalArgumentException("Expected multi-value data type, got " + val.getDataType().getName() + "."); + } + } + throw new UnsupportedOperationException("Value update type " + upd.getClass().getName() + " not supported."); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldValueConverter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldValueConverter.java new file mode 100644 index 00000000000..59d5de4a965 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/FieldValueConverter.java @@ -0,0 +1,167 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.*; + +import java.util.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class FieldValueConverter { + + @SuppressWarnings({ "unchecked" }) + public final FieldValue convert(FieldValue value) { + if (value == null) { + return null; + } + if (shouldConvert(value)) { + return doConvert(value); + } + if (value instanceof Array) { + return convertArray((Array)value); + } + if (value instanceof MapFieldValue) { + return convertMap((MapFieldValue)value); + } + if (value instanceof WeightedSet) { + return convertWset((WeightedSet)value); + } + if (value instanceof StructuredFieldValue) { + return convertStructured((StructuredFieldValue)value); + } + return value; + } + + @SuppressWarnings({ "unchecked" }) + private FieldValue convertArray(Array val) { + List<FieldValue> next = new LinkedList<FieldValue>(); + DataType nextType = null; + for (Iterator<FieldValue> it = val.fieldValueIterator(); it.hasNext();) { + FieldValue prevVal = it.next(); + FieldValue nextVal = convert(prevVal); + if (nextVal == null) { + continue; + } + if (nextType == null) { + nextType = nextVal.getDataType(); + } else if (!nextType.isValueCompatible(nextVal)) { + throw new IllegalArgumentException("Expected " + nextType.getName() + ", got " + + nextVal.getDataType().getName() + "."); + } + next.add(nextVal); + } + if (nextType == null) { + return null; + } + Array ret = DataType.getArray(nextType).createFieldValue(); + for (FieldValue nextVal : next) { + ret.add(nextVal); + } + return ret; + } + + @SuppressWarnings({ "unchecked" }) + private FieldValue convertMap(MapFieldValue<FieldValue, FieldValue> val) { + Map<FieldValue, FieldValue> next = new LinkedHashMap<FieldValue, FieldValue>(); + DataType nextKeyType = null, nextValType = null; + for (Map.Entry<FieldValue, FieldValue> entry : val.entrySet()) { + FieldValue prevKey = entry.getKey(); + FieldValue nextKey = convert(prevKey); + if (nextKey == null) { + continue; + } + if (nextKeyType == null) { + nextKeyType = nextKey.getDataType(); + } else if (!nextKeyType.isValueCompatible(nextKey)) { + throw new IllegalArgumentException("Expected " + nextKeyType.getName() + ", got " + + nextKey.getDataType().getName() + "."); + } + FieldValue prevVal = entry.getValue(); + FieldValue nextVal = convert(prevVal); + if (nextVal == null) { + continue; + } + if (nextValType == null) { + nextValType = nextVal.getDataType(); + } else if (!nextValType.isValueCompatible(nextVal)) { + throw new IllegalArgumentException("Expected " + nextValType.getName() + ", got " + + nextVal.getDataType().getName() + "."); + } + next.put(nextKey, nextVal); + } + if (nextKeyType == null || nextValType == null) { + return null; + } + MapFieldValue ret = DataType.getMap(nextKeyType, nextValType).createFieldValue(); + for (Map.Entry<FieldValue, FieldValue> entry : next.entrySet()) { + ret.put(entry.getKey(), entry.getValue()); + } + return ret; + } + + @SuppressWarnings({ "unchecked" }) + private FieldValue convertWset(WeightedSet val) { + Map<FieldValue, Integer> next = new LinkedHashMap<FieldValue, Integer>(); + DataType nextType = null; + for (Iterator<FieldValue> it = val.fieldValueIterator(); it.hasNext();) { + FieldValue prevKey = it.next(); + Integer prevVal = val.get(prevKey); + + FieldValue nextKey = convert(prevKey); + if (nextKey == null) { + continue; + } + if (nextType == null) { + nextType = nextKey.getDataType(); + } else if (!nextType.isValueCompatible(nextKey)) { + throw new IllegalArgumentException("Expected " + nextType.getName() + ", got " + + nextKey.getDataType().getName() + "."); + } + next.put(nextKey, prevVal); + } + if (nextType == null) { + return null; + } + WeightedSet ret = DataType.getWeightedSet(nextType, val.getDataType().createIfNonExistent(), + val.getDataType().removeIfZero()).createFieldValue(); + for (Map.Entry<FieldValue, Integer> entry : next.entrySet()) { + ret.put(entry.getKey(), entry.getValue()); + } + return ret; + } + + private FieldValue convertStructured(StructuredFieldValue val) { + StructuredFieldValue ret = val.getDataType().createFieldValue(); + for (Iterator<Map.Entry<Field, FieldValue>> it = val.iterator(); it.hasNext();) { + Map.Entry<Field, FieldValue> entry = it.next(); + FieldValue prev = entry.getValue(); + FieldValue next = convert(prev); + if (next == null) { + continue; + } + ret.setFieldValue(entry.getKey(), next); + } + return ret; + } + + /** + * Returns whether or not the given {@link FieldValue} should be converted. If this method returns <em>false</em>, + * the converter will proceed to traverse the value itself to see if its internal can be converted. + * + * @param value The value to check. + * @return True to convert, false to traverse. + */ + protected abstract boolean shouldConvert(FieldValue value); + + /** + * Converts the given value. It is IMPERATIVE that the implementation of this method DOES NOT mutate the given + * {@link FieldValue} in place, as that can cause SERIOUS inconsistencies in the parent structures. + * + * @param value The value to convert. + * @return The value to replace the old. + */ + protected abstract FieldValue doConvert(FieldValue value); +}
\ No newline at end of file diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ScriptParser.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ScriptParser.java new file mode 100644 index 00000000000..4643bb05b08 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ScriptParser.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.javacc.FastCharStream; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.indexinglanguage.parser.CharStream; +import com.yahoo.vespa.indexinglanguage.parser.IndexingParser; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import com.yahoo.vespa.indexinglanguage.parser.TokenMgrError; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public final class ScriptParser { + + public static Expression parseExpression(ScriptParserContext config) throws ParseException { + return parse(config, new ParserMethod<Expression>() { + + @Override + public Expression call(IndexingParser parser) throws ParseException { + return parser.root(); + } + }); + } + + public static ScriptExpression parseScript(ScriptParserContext config) throws ParseException { + return parse(config, new ParserMethod<ScriptExpression>() { + + @Override + public ScriptExpression call(IndexingParser parser) throws ParseException { + return parser.script(); + } + }); + } + + public static StatementExpression parseStatement(ScriptParserContext config) throws ParseException { + return parse(config, new ParserMethod<StatementExpression>() { + + @Override + public StatementExpression call(IndexingParser parser) throws ParseException { + try { + return parser.statement(); + } + catch (TokenMgrError e) { + throw new ParseException(e.getMessage()); + } + } + }); + } + + private static interface ParserMethod<T extends Expression> { + + T call(IndexingParser parser) throws ParseException; + } + + private static <T extends Expression> T parse(ScriptParserContext config, ParserMethod<T> method) + throws ParseException { + CharStream input = config.getInputStream(); + IndexingParser parser = new IndexingParser(input); + parser.setAnnotatorConfig(config.getAnnotatorConfig()); + parser.setDefaultFieldName(config.getDefaultFieldName()); + parser.setLinguistics(config.getLinguistcs()); + try { + return method.call(parser); + } catch (ParseException e) { + if (!(input instanceof FastCharStream)) { + throw e; + } + throw new ParseException(((FastCharStream)input).formatException(e.getMessage())); + } finally { + if (parser.token != null && parser.token.next != null) { + input.backup(parser.token.next.image.length()); + } + } + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ScriptParserContext.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ScriptParserContext.java new file mode 100644 index 00000000000..a79ac557754 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ScriptParserContext.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; +import com.yahoo.vespa.indexinglanguage.parser.CharStream; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ScriptParserContext { + + private AnnotatorConfig annotatorConfig = new AnnotatorConfig(); + private Linguistics linguistics; + private String defaultFieldName = null; + private CharStream inputStream = null; + + public ScriptParserContext(Linguistics linguistics) { + this.linguistics = linguistics; + } + + public AnnotatorConfig getAnnotatorConfig() { + return annotatorConfig; + } + + public ScriptParserContext setAnnotatorConfig(AnnotatorConfig config) { + annotatorConfig = new AnnotatorConfig(config); + return this; + } + + public Linguistics getLinguistcs() { + return linguistics; + } + + public ScriptParserContext setLinguistics(Linguistics linguistics) { + this.linguistics = linguistics; + return this; + } + + public String getDefaultFieldName() { + return defaultFieldName; + } + + public ScriptParserContext setDefaultFieldName(String name) { + defaultFieldName = name; + return this; + } + + public CharStream getInputStream() { + return inputStream; + } + + public ScriptParserContext setInputStream(CharStream stream) { + inputStream = stream; + return this; + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java new file mode 100644 index 00000000000..2a00161080e --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.*; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.ValueUpdate; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SimpleAdapterFactory implements AdapterFactory { + public static class SelectExpression { + public Expression selectExpression(DocumentType documentType, String fieldName) { + return null; + } + } + private final SelectExpression expressionSelector; + + public SimpleAdapterFactory() { + this(new SelectExpression()); + } + public SimpleAdapterFactory(SelectExpression expressionSelector) { + this.expressionSelector = expressionSelector; + } + + @Override + public DocumentAdapter newDocumentAdapter(Document doc) { + return newDocumentAdapter(doc, false); + } + + public DocumentAdapter newDocumentAdapter(Document doc, boolean isUpdate) { + if (isUpdate) { + return new SimpleDocumentAdapter(doc); + } + return new SimpleDocumentAdapter(doc, doc); + } + + @Override + public List<UpdateAdapter> newUpdateAdapterList(DocumentUpdate upd) { + List<UpdateAdapter> ret = new ArrayList<>(); + DocumentType docType = upd.getDocumentType(); + DocumentId docId = upd.getId(); + Document complete = new Document(docType, upd.getId()); + for (FieldPathUpdate fieldUpd : upd) { + if (FieldPathUpdateHelper.isComplete(fieldUpd)) { + FieldPathUpdateHelper.applyUpdate(fieldUpd, complete); + } else { + Document partial = FieldPathUpdateHelper.newPartialDocument(docId, fieldUpd); + ret.add(new FieldPathUpdateAdapter(newDocumentAdapter(partial, true), fieldUpd)); + } + } + for (FieldUpdate fieldUpd : upd.getFieldUpdates()) { + Field field = fieldUpd.getField(); + for (ValueUpdate valueUpd : fieldUpd.getValueUpdates()) { + if (FieldUpdateHelper.isComplete(field, valueUpd)) { + FieldUpdateHelper.applyUpdate(field, valueUpd, complete); + } else { + Document partial = FieldUpdateHelper.newPartialDocument(docType, docId, field, valueUpd); + ret.add(FieldUpdateAdapter.fromPartialUpdate(expressionSelector.selectExpression(docType, field.getName()),newDocumentAdapter(partial, true), valueUpd)); + } + } + } + ret.add(FieldUpdateAdapter.fromCompleteUpdate(newDocumentAdapter(complete, true))); + return ret; + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapter.java new file mode 100644 index 00000000000..77914ec9035 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapter.java @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.Field; +import com.yahoo.document.FieldPath; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.VerificationException; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SimpleDocumentAdapter implements DocumentAdapter { + + private final Document input; + private final Document output; + + public SimpleDocumentAdapter(Document input) { + this(input, new Document(input.getDataType(), input.getId())); + } + + public SimpleDocumentAdapter(Document input, Document output) { + this.input = input; + this.output = output; + } + + @Override + public Document getFullOutput() { + return output; + } + + @Override + public Document getUpdatableOutput() { + return output; + } + + @Override + public DataType getInputType(Expression exp, String fieldName) { + try { + return input.getDataType().buildFieldPath(fieldName).getResultingDataType(); + } catch (IllegalArgumentException e) { + throw new VerificationException(exp, "Input field '" + fieldName + "' not found."); + } + } + + @Override + public FieldValue getInputValue(String fieldName) { + try { + return input.getRecursiveValue(fieldName); + } catch (IllegalArgumentException e) { + return null; + } + } + + @Override + public FieldValue getInputValue(FieldPath fieldPath) { + try { + return input.getRecursiveValue(fieldPath); + } catch (IllegalArgumentException e) { + return null; + } + } + + @Override + public void tryOutputType(Expression exp, String fieldName, DataType valueType) { + Field field = output.getDataType().getField(fieldName); + if (field == null) { + throw new VerificationException(exp, "Field '" + fieldName + "' not found."); + } + DataType fieldType = field.getDataType(); + if (!fieldType.isAssignableFrom(valueType)) { + throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to field '" + + fieldName + "' which is " + fieldType.getName() + "."); + } + } + + @SuppressWarnings({ "unchecked" }) + @Override + public SimpleDocumentAdapter setOutputValue(Expression exp, String fieldName, FieldValue fieldValue) { + Field field = output.getField(fieldName); + if (field == null) { + throw new IllegalArgumentException("Field '" + fieldName + "' not found in document type '" + + output.getDataType().getName() + "'."); + } + output.setFieldValue(field, fieldValue); + return this; + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/StringFieldConverter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/StringFieldConverter.java new file mode 100644 index 00000000000..69631c20f95 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/StringFieldConverter.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class StringFieldConverter extends FieldValueConverter { + + @Override + protected final boolean shouldConvert(FieldValue value) { + return value.getDataType().equals(DataType.STRING); + } + + @Override + protected final FieldValue doConvert(FieldValue value) { + return doConvert((StringFieldValue)value); + } + + protected abstract FieldValue doConvert(StringFieldValue value); +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/TypedExpressionConverter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/TypedExpressionConverter.java new file mode 100644 index 00000000000..9e174001aef --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/TypedExpressionConverter.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.vespa.indexinglanguage.expressions.Expression; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class TypedExpressionConverter<T extends Expression> extends ExpressionConverter { + + private final Class<T> expClass; + + public TypedExpressionConverter(Class<T> expClass) { + this.expClass = expClass; + } + + @Override + protected final boolean shouldConvert(Expression exp) { + return expClass.isInstance(exp); + } + + @Override + protected final Expression doConvert(Expression exp) { + return typedConvert(expClass.cast(exp)); + } + + protected abstract Expression typedConvert(T exp); +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/UpdateAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/UpdateAdapter.java new file mode 100644 index 00000000000..f8d08efce24 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/UpdateAdapter.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.DocumentUpdate; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.FieldValueAdapter; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface UpdateAdapter extends FieldValueAdapter { + + public DocumentUpdate getOutput(); + public Expression getExpression(Expression expression); +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java new file mode 100644 index 00000000000..05fbf860295 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; + +/** + * Inserts a "newTransform()" before expressions that "requiresTransform()" + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class ValueTransformProvider extends ExpressionConverter { + + private final Class<? extends Expression> transformClass; + private boolean transformed = false; + private boolean duplicate = false; + + public ValueTransformProvider(Class<? extends Expression> transformClass) { + this.transformClass = transformClass; + } + + @Override + protected final ExpressionConverter branch() { + return clone(); + } + + @Override + protected final boolean shouldConvert(Expression exp) { + if (transformClass.isInstance(exp)) { + if (transformed) { + duplicate = true; + return true; + } + transformed = true; + return false; + } + if (exp.createdOutputType() != null) { + transformed = false; + return false; + } + if (!requiresTransform(exp)) { + return false; + } + if (transformed) { + return false; + } + return true; + } + + @Override + protected final Expression doConvert(Expression exp) { + if (duplicate) { + duplicate = false; + return null; + } + transformed = true; + return new StatementExpression(newTransform(), exp); + } + + protected abstract boolean requiresTransform(Expression exp); + + protected abstract Expression newTransform(); +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java new file mode 100644 index 00000000000..ce56597cc86 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java @@ -0,0 +1,220 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.NumericDataType; +import com.yahoo.document.datatypes.*; +import com.yahoo.vespa.objects.ObjectOperation; +import com.yahoo.vespa.objects.ObjectPredicate; + +import java.math.BigDecimal; +import java.math.MathContext; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ArithmeticExpression extends CompositeExpression { + + public enum Operator { + + ADD(1, "+"), + SUB(1, "-"), + MUL(0, "*"), + DIV(0, "/"), + MOD(0, "%"); + + private final int precedence; + private final String img; + + Operator(int precedence, String img) { + this.precedence = precedence; + this.img = img; + } + + public boolean precedes(Operator op) { + return precedence <= op.precedence; + } + + @Override + public String toString() { + return img; + } + } + + private final Expression lhs; + private final Operator op; + private final Expression rhs; + + public ArithmeticExpression(Expression lhs, Operator op, Expression rhs) { + lhs.getClass(); // throws NullPointerException + op.getClass(); + rhs.getClass(); + this.lhs = lhs; + this.op = op; + this.rhs = rhs; + } + + public Expression getLeftHandSide() { + return lhs; + } + + public Operator getOperator() { + return op; + } + + public Expression getRightHandSide() { + return rhs; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + FieldValue input = ctx.getValue(); + ctx.setValue(evaluate(ctx.setValue(input).execute(lhs).getValue(), + ctx.setValue(input).execute(rhs).getValue())); + } + + @Override + protected void doVerify(VerificationContext ctx) { + DataType input = ctx.getValue(); + ctx.setValue(evaluate(ctx.setValue(input).execute(lhs).getValue(), + ctx.setValue(input).execute(rhs).getValue())); + } + + @Override + public DataType requiredInputType() { + DataType lhsType = lhs.requiredInputType(); + DataType rhsType = rhs.requiredInputType(); + if (lhsType == null) { + return rhsType; + } + if (rhsType == null) { + return lhsType; + } + if (!lhsType.equals(rhsType)) { + throw new VerificationException(this, "Operands require conflicting input types, " + + lhsType.getName() + " vs " + rhsType.getName() + "."); + } + return lhsType; + } + + @Override + public DataType createdOutputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public String toString() { + return lhs + " " + op + " " + rhs; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ArithmeticExpression)) { + return false; + } + ArithmeticExpression exp = (ArithmeticExpression)obj; + if (!lhs.equals(exp.lhs)) { + return false; + } + if (!op.equals(exp.op)) { + return false; + } + if (!rhs.equals(exp.rhs)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + lhs.hashCode() + op.hashCode() + rhs.hashCode(); + } + + private DataType evaluate(DataType lhs, DataType rhs) { + if (lhs == null || rhs == null) { + throw new VerificationException(this, "Attempting to perform arithmetic on a null value."); + } + if (!(lhs instanceof NumericDataType) || + !(rhs instanceof NumericDataType)) + { + throw new VerificationException(this, "Attempting to perform unsupported arithmetic: [" + + lhs.getName() + "] " + op + " [" + rhs.getName() + "]"); + } + if (lhs == DataType.FLOAT || lhs == DataType.DOUBLE || + rhs == DataType.FLOAT || rhs == DataType.DOUBLE) + { + if (lhs == DataType.DOUBLE || rhs == DataType.DOUBLE) { + return DataType.DOUBLE; + } + return DataType.FLOAT; + } + if (lhs == DataType.LONG || rhs == DataType.LONG) { + return DataType.LONG; + } + return DataType.INT; + } + + private FieldValue evaluate(FieldValue lhs, FieldValue rhs) { + if (lhs == null || rhs == null) { + return null; + } + if (!(lhs instanceof NumericFieldValue) || + !(rhs instanceof NumericFieldValue)) + { + throw new IllegalArgumentException("Unsupported operation: [" + lhs.getDataType().getName() + "] " + + op + " [" + rhs.getDataType().getName() + "]"); + } + BigDecimal lhsVal = asBigDecimal((NumericFieldValue)lhs); + BigDecimal rhsVal = asBigDecimal((NumericFieldValue)rhs); + switch (op) { + case ADD: + return createFieldValue(lhs, rhs, lhsVal.add(rhsVal)); + case SUB: + return createFieldValue(lhs, rhs, lhsVal.subtract(rhsVal)); + case MUL: + return createFieldValue(lhs, rhs, lhsVal.multiply(rhsVal)); + case DIV: + return createFieldValue(lhs, rhs, lhsVal.divide(rhsVal, MathContext.DECIMAL64)); + case MOD: + return createFieldValue(lhs, rhs, lhsVal.remainder(rhsVal)); + } + throw new IllegalStateException("Unsupported operation: " + op); + } + + private FieldValue createFieldValue(FieldValue lhs, FieldValue rhs, BigDecimal val) { + if (lhs instanceof FloatFieldValue || lhs instanceof DoubleFieldValue || + rhs instanceof FloatFieldValue || rhs instanceof DoubleFieldValue) + { + if (lhs instanceof DoubleFieldValue || rhs instanceof DoubleFieldValue) { + return new DoubleFieldValue(val.doubleValue()); + } + return new FloatFieldValue(val.floatValue()); + } + if (lhs instanceof LongFieldValue || rhs instanceof LongFieldValue) { + return new LongFieldValue(val.longValue()); + } + return new IntegerFieldValue(val.intValue()); + } + + public static BigDecimal asBigDecimal(NumericFieldValue value) { + if (value instanceof ByteFieldValue) { + return BigDecimal.valueOf(((ByteFieldValue)value).getByte()); + } else if (value instanceof DoubleFieldValue) { + return BigDecimal.valueOf(((DoubleFieldValue)value).getDouble()); + } else if (value instanceof FloatFieldValue) { + return BigDecimal.valueOf(((FloatFieldValue)value).getFloat()); + } else if (value instanceof IntegerFieldValue) { + return BigDecimal.valueOf(((IntegerFieldValue)value).getInteger()); + } else if (value instanceof LongFieldValue) { + return BigDecimal.valueOf(((LongFieldValue)value).getLong()); + } + throw new IllegalArgumentException("Unsupported numeric field value type '" + + value.getClass().getName() + "'."); + } + + @Override + public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { + lhs.select(predicate, operation); + rhs.select(predicate, operation); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpression.java new file mode 100644 index 00000000000..fe3708a7b31 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpression.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class AttributeExpression extends OutputExpression { + + public AttributeExpression(String fieldName) { + super("attribute", fieldName); + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj) && obj instanceof AttributeExpression; + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java new file mode 100644 index 00000000000..d40005bcba1 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.LongFieldValue; +import org.apache.commons.codec.binary.Base64; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Base64DecodeExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + String input = String.valueOf(ctx.getValue()); + if (input.isEmpty()) { + ctx.setValue(new LongFieldValue(Long.MIN_VALUE)); + return; + } + if (input.length() > 12) { + throw new NumberFormatException("Base64 value '" + input + "' is out of range."); + } + byte[] decoded = Base64.decodeBase64(input); + if (decoded == null || decoded.length == 0) { + throw new NumberFormatException("Illegal base64 value '" + input + "'."); + } + long output = 0; + for (int i = decoded.length; --i >= 0;) { + output = (output << 8) + (((int)decoded[i]) & 0xff); + } + ctx.setValue(new LongFieldValue(output)); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return DataType.STRING; + } + + @Override + public DataType createdOutputType() { + return DataType.LONG; + } + + @Override + public String toString() { + return "base64decode"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof Base64DecodeExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java new file mode 100644 index 00000000000..b29971449e4 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import org.apache.commons.codec.binary.Base64; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Base64EncodeExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + long input = ((LongFieldValue)ctx.getValue()).getLong(); + byte[] output = new byte[8]; + for (int i = 0; i < output.length; ++i) { + output[i] = (byte)(input & 0xffL); + input >>>= 8; + } + String encoded = new Base64(0).encodeToString(output); + ctx.setValue(new StringFieldValue(encoded)); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return DataType.LONG; + } + + @Override + public DataType createdOutputType() { + return DataType.STRING; + } + + @Override + public String toString() { + return "base64encode"; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Base64EncodeExpression)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java new file mode 100644 index 00000000000..fe7771f83ec --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java @@ -0,0 +1,163 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.WeightedSet; + +import java.util.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class CatExpression extends ExpressionList<Expression> { + + public CatExpression(Expression... lst) { + super(Arrays.asList(lst)); + } + + public CatExpression(Collection<? extends Expression> lst) { + super(lst); + } + + @Override + protected void doExecute(ExecutionContext ctx) { + FieldValue input = ctx.getValue(); + DataType inputType = input != null ? input.getDataType() : null; + VerificationContext ver = new VerificationContext(ctx); + List<FieldValue> values = new LinkedList<>(); + List<DataType> types = new LinkedList<>(); + for (Expression exp : this) { + FieldValue val = ctx.setValue(input).execute(exp).getValue(); + values.add(val); + + DataType type; + if (val != null) { + type = val.getDataType(); + } else { + type = ver.setValue(inputType).execute(this).getValue(); + } + types.add(type); + } + DataType type = resolveOutputType(types); + ctx.setValue(type == DataType.STRING ? asString(values) : asCollection(type, values)); + } + + @Override + protected void doVerify(VerificationContext ctx) { + DataType input = ctx.getValue(); + List<DataType> types = new LinkedList<>(); + for (Expression exp : this) { + DataType val = ctx.setValue(input).execute(exp).getValue(); + types.add(val); + if (val == null) { + throw new VerificationException(this, "Attempting to concatenate a null value (" + exp + ")."); + } + } + ctx.setValue(resolveOutputType(types)); + } + + @Override + public DataType requiredInputType() { + DataType prev = null; + for (Expression exp : this) { + DataType next = exp.requiredInputType(); + if (next == null) { + // ignore + } else if (prev == null) { + prev = next; + } else if (!prev.isAssignableFrom(next)) { + throw new VerificationException(this, "Operands require conflicting input types, " + + prev.getName() + " vs " + next.getName() + "."); + } + } + return prev; + } + + @Override + public DataType createdOutputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public String toString() { + StringBuilder ret = new StringBuilder(); + for (Iterator<Expression> it = iterator(); it.hasNext();) { + ret.append(it.next()); + if (it.hasNext()) { + ret.append(" . "); + } + } + return ret.toString(); + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj) && obj instanceof CatExpression; + } + + private static DataType resolveOutputType(List<DataType> types) { + DataType ret = null; + for (DataType type : types) { + if (!(type instanceof CollectionDataType)) { + return DataType.STRING; + } + if (ret == null) { + ret = type; + } else if (!ret.isAssignableFrom(type)) { + return DataType.STRING; + } + } + return ret; + } + + private static FieldValue asString(List<FieldValue> outputs) { + StringBuilder ret = new StringBuilder(); + for (FieldValue val : outputs) { + if (val == null) { + return null; + } + ret.append(val.toString()); + } + return new StringFieldValue(ret.toString()); + } + + private static FieldValue asCollection(DataType type, List<FieldValue> values) { + if (type instanceof ArrayDataType) { + return asArray((ArrayDataType)type, values); + } else if (type instanceof WeightedSetDataType) { + return asWset((WeightedSetDataType)type, values); + } else { + throw new UnsupportedOperationException(type.getName()); + } + } + + @SuppressWarnings({ "unchecked" }) + private static FieldValue asArray(ArrayDataType arrType, List<FieldValue> values) { + Array out = arrType.createFieldValue(); + for (FieldValue val : values) { + if (val == null) { + continue; + } + out.addAll((Array)val); + } + return out; + } + + @SuppressWarnings({ "unchecked" }) + private static FieldValue asWset(WeightedSetDataType wsetType, List<FieldValue> values) { + WeightedSet out = wsetType.createFieldValue(); + for (FieldValue val : values) { + if (val == null) { + continue; + } + out.putAll((WeightedSet)val); + } + return out; + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateExpression.java new file mode 100644 index 00000000000..7fb4951bfd9 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateExpression.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ClearStateExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.clear(); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.clear(); + } + + @Override + public DataType requiredInputType() { + return null; + } + + @Override + public DataType createdOutputType() { + return null; + } + + @Override + public String toString() { + return "clear_state"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ClearStateExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java new file mode 100644 index 00000000000..a25a3b1533b --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class CompositeExpression extends Expression { + + protected static String toScriptBlock(Expression exp) { + if (exp instanceof ScriptExpression) { + return exp.toString(); + } + if (exp instanceof StatementExpression) { + return new ScriptExpression((StatementExpression)exp).toString(); + } + return new ScriptExpression(new StatementExpression(exp)).toString(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EchoExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EchoExpression.java new file mode 100644 index 00000000000..67f0c2faef4 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EchoExpression.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; + +import java.io.PrintStream; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class EchoExpression extends Expression { + + private final PrintStream out; + + public EchoExpression() { + this(System.out); + } + + public EchoExpression(PrintStream out) { + this.out = out; + } + + public PrintStream getOutputStream() { + return out; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + out.println(String.valueOf(ctx.getValue())); + } + + @Override + protected void doVerify(VerificationContext ctx) { + // empty + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return null; + } + + @Override + public String toString() { + return "echo"; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof EchoExpression)) { + return false; + } + EchoExpression rhs = (EchoExpression)obj; + if (out != rhs.out) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + out.hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java new file mode 100644 index 00000000000..be70291cb70 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.annotation.*; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.language.process.TokenType; + +import static com.yahoo.language.LinguisticsCase.toLowerCase; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ExactExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + StringFieldValue input = (StringFieldValue)ctx.getValue(); + if (input.getString().isEmpty()) { + return; + } + StringFieldValue output = input.clone(); + ctx.setValue(output); + + String prev = output.getString(); + String next = toLowerCase(prev); + + SpanList root = new SpanList(); + SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS, root); + SpanNode node = new Span(0, prev.length()); + tree.annotate(node, new Annotation(AnnotationTypes.TERM, + next.equals(prev) ? null : new StringFieldValue(next))); + tree.annotate(node, new Annotation(AnnotationTypes.TOKEN_TYPE, + new IntegerFieldValue(TokenType.ALPHABETIC.getValue()))); + root.add(node); + output.setSpanTree(tree); + } + + @Override + protected void doVerify(VerificationContext ctx) { + // empty + } + + @Override + public DataType requiredInputType() { + return DataType.STRING; + } + + @Override + public DataType createdOutputType() { + return null; + } + + @Override + public String toString() { + return "exact"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ExactExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java new file mode 100644 index 00000000000..24527a61901 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java @@ -0,0 +1,130 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.FieldPath; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.detect.Detection; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ExecutionContext implements FieldTypeAdapter, FieldValueAdapter, Cloneable { + + private final Map<String, FieldValue> variables = new HashMap<>(); + private final FieldValueAdapter adapter; + private FieldValue value; + private Language language; + + public ExecutionContext() { + this(null); + } + + public ExecutionContext(FieldValueAdapter adapter) { + this.adapter = adapter; + this.language = Language.UNKNOWN; + } + + public ExecutionContext execute(Expression exp) { + if (exp != null) { + exp.execute(this); + } + return this; + } + + @Override + public DataType getInputType(Expression exp, String fieldName) { + return adapter.getInputType(exp, fieldName); + } + + @Override + public FieldValue getInputValue(String fieldName) { + if (adapter == null) { + throw new IllegalStateException("Can not get field '" + fieldName + "' because adapter is null."); + } + return adapter.getInputValue(fieldName); + } + + @Override + public FieldValue getInputValue(FieldPath fieldPath) { + if (adapter == null) { + throw new IllegalStateException("Can not get field '" + fieldPath + "' because adapter is null."); + } + return adapter.getInputValue(fieldPath); + } + + @Override + public void tryOutputType(Expression exp, String fieldName, DataType valueType) { + adapter.tryOutputType(exp, fieldName, valueType); + } + + @Override + public ExecutionContext setOutputValue(Expression exp, String fieldName, FieldValue fieldValue) { + if (adapter == null) { + throw new IllegalStateException("Can not set field '" + fieldName + "' because adapter is null."); + } + adapter.setOutputValue(exp, fieldName, fieldValue); + return this; + } + + public FieldValueAdapter getAdapter() { + return adapter; + } + + public FieldValue getVariable(String name) { + return variables.get(name); + } + + public ExecutionContext setVariable(String name, FieldValue value) { + variables.put(name, value); + return this; + } + + public Language getLanguage() { + return language; + } + + public ExecutionContext setLanguage(Language language) { + language.getClass(); + this.language = language; + return this; + } + + public Language resolveLanguage(Linguistics linguistics) { + if (language != null && language != Language.UNKNOWN) { + return language; + } + if (linguistics == null) { + return Language.ENGLISH; + } + Detection detection = linguistics.getDetector().detect(String.valueOf(value), null); + if (detection == null) { + return Language.ENGLISH; + } + Language detected = detection.getLanguage(); + if (detected == Language.UNKNOWN) { + return Language.ENGLISH; + } + return detected; + } + + public FieldValue getValue() { + return value; + } + + public ExecutionContext setValue(FieldValue value) { + this.value = value; + return this; + } + + public ExecutionContext clear() { + variables.clear(); + value = null; + return this; + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java new file mode 100644 index 00000000000..f94a100b31d --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java @@ -0,0 +1,217 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.*; +import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import com.yahoo.vespa.objects.Selectable; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class Expression extends Selectable { + + public final FieldValue execute(FieldValue val) { + return execute(new ExecutionContext().setValue(val)); + } + + public final Document execute(AdapterFactory factory, Document doc) { + return execute(factory.newDocumentAdapter(doc)); + } + + public final Document execute(DocumentAdapter adapter) { + execute((FieldValueAdapter)adapter); + return adapter.getFullOutput(); + } + + public static DocumentUpdate execute(Expression expression, AdapterFactory factory, DocumentUpdate upd) { + DocumentUpdate ret = null; + for (UpdateAdapter adapter : factory.newUpdateAdapterList(upd)) { + DocumentUpdate output = adapter.getExpression(expression).execute(adapter); + if (output == null) { + // ignore + } else if (ret != null) { + ret.addAll(output); + } else { + ret = output; + } + } + if (ret != null) { + ret.setCreateIfNonExistent(upd.getCreateIfNonExistent()); + } + return ret; + } + + public final DocumentUpdate execute(UpdateAdapter adapter) { + execute((FieldValueAdapter)adapter); + return adapter.getOutput(); + } + + public final FieldValue execute(FieldValueAdapter adapter) { + return execute(new ExecutionContext(adapter)); + } + + public final FieldValue execute(ExecutionContext ctx) { + DataType inputType = requiredInputType(); + if (inputType != null) { + FieldValue input = ctx.getValue(); + if (input == null) { + return null; + } + if (!inputType.isValueCompatible(input)) { + throw new IllegalArgumentException("Expression '" + this + "' expected " + inputType.getName() + + " input, got " + input.getDataType().getName() + "."); + } + } + doExecute(ctx); + DataType outputType = createdOutputType(); + if (outputType != null) { + FieldValue output = ctx.getValue(); + if (output != null && !outputType.isValueCompatible(output)) { + throw new IllegalStateException("Expression '" + this + "' expected " + outputType.getName() + + " output, got " + output.getDataType().getName() + "."); + } + } + return ctx.getValue(); + } + + protected abstract void doExecute(ExecutionContext ctx); + + public final DataType verify() { + return verify(new VerificationContext()); + } + + public final DataType verify(DataType val) { + return verify(new VerificationContext().setValue(val)); + } + + public final Document verify(Document doc) { + return verify(new SimpleAdapterFactory(), doc); + } + + public final Document verify(AdapterFactory factory, Document doc) { + return verify(factory.newDocumentAdapter(doc)); + } + + public final Document verify(DocumentAdapter adapter) { + verify((FieldTypeAdapter)adapter); + return adapter.getFullOutput(); + } + + public final DocumentUpdate verify(DocumentUpdate upd) { + return verify(new SimpleAdapterFactory(), upd); + } + + public final DocumentUpdate verify(AdapterFactory factory, DocumentUpdate upd) { + DocumentUpdate ret = null; + for (UpdateAdapter adapter : factory.newUpdateAdapterList(upd)) { + DocumentUpdate output = verify(adapter); + if (output == null) { + // ignore + } else if (ret != null) { + ret.addAll(output); + } else { + ret = output; + } + } + return ret; + } + + public final DocumentUpdate verify(UpdateAdapter adapter) { + verify((FieldTypeAdapter)adapter); + return adapter.getOutput(); + } + + public final DataType verify(FieldTypeAdapter adapter) { + return verify(new VerificationContext(adapter)); + } + + public final DataType verify(VerificationContext ctx) { +// System.err.println("enter_verify(exp = '" + this + "', req = " + requiredInputType(ctx) + +// ", in = " + ctx.getValue() + ")"); + DataType inputType = requiredInputType(); + if (inputType != null) { + DataType input = ctx.getValue(); + if (input == null) { + throw new VerificationException(this, "Expected " + inputType.getName() + " input, got null."); + } + if (input.getPrimitiveType() == UnresolvedDataType.INSTANCE) { + throw new VerificationException(this, "Failed to resolve input type."); + } + if (!inputType.isAssignableFrom(input)) { + throw new VerificationException(this, "Expected " + inputType.getName() + " input, got " + + input.getName() + "."); + } + } + doVerify(ctx); + DataType outputType = createdOutputType(); +// System.err.println("exit_verify(exp = '" + this + "', req = " + createdOutputType(ctx) + +// ", out = " + ctx.getValue() + ")"); + if (outputType != null) { + DataType output = ctx.getValue(); + if (output == null) { + throw new VerificationException(this, "Expected " + outputType.getName() + " output, got null."); + } + if (output.getPrimitiveType() == UnresolvedDataType.INSTANCE) { + throw new VerificationException(this, "Failed to resolve output type."); + } + if (!outputType.isAssignableFrom(output)) { + throw new VerificationException(this, "Expected " + outputType.getName() + " output, got " + + output.getName() + "."); + } + } + return ctx.getValue(); + } + + protected abstract void doVerify(VerificationContext ctx); + + public abstract DataType requiredInputType(); + + public abstract DataType createdOutputType(); + + /** Creates an expression with simple lingustics for testing */ + public static Expression fromString(String expression) throws ParseException { + return fromString(expression, new SimpleLinguistics()); + } + + public static Expression fromString(String expression, Linguistics linguistics) throws ParseException { + return newInstance(new ScriptParserContext(linguistics).setInputStream(new IndexingInput(expression))); + } + + public static Expression newInstance(ScriptParserContext context) throws ParseException { + return ScriptParser.parseExpression(context); + } + + protected static boolean equals(Object lhs, Object rhs) { + if (lhs == null) { + if (rhs != null) { + return false; + } + } else { + if (rhs == null) { + return false; + } + if (!lhs.equals(rhs)) { + return false; + } + } + return true; + } + // Convenience For testing + public static Document execute(Expression expression, Document doc) { + return expression.execute(new SimpleAdapterFactory(), doc); + } + public static final DocumentUpdate execute(Expression expression, DocumentUpdate upd) { + return expression.execute(expression, new SimpleAdapterFactory(), upd); + } + public final FieldValue execute() { + return execute(new ExecutionContext()); + } + +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java new file mode 100644 index 00000000000..aec0b2ada28 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DocumentType; +import com.yahoo.vespa.objects.ObjectOperation; +import com.yahoo.vespa.objects.ObjectPredicate; + +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class ExpressionList<T extends Expression> extends CompositeExpression implements Iterable<T> { + + private final List<T> expressions = new LinkedList<T>(); + + protected ExpressionList() { + // empty + } + + protected ExpressionList(Iterable<? extends T> lst) { + for (T exp : lst) { + this.expressions.add(exp); + } + } + + public int size() { + return expressions.size(); + } + + public T get(int idx) { + return expressions.get(idx); + } + + public boolean isEmpty() { + return expressions.isEmpty(); + } + + public List<T> asList() { + return Collections.unmodifiableList(expressions); + } + + @Override + public Iterator<T> iterator() { + return expressions.iterator(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ExpressionList)) { + return false; + } + ExpressionList rhs = (ExpressionList)obj; + if (!expressions.equals(rhs.expressions)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + expressions.hashCode(); + } + + @Override + public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { + for (T exp : expressions) { + exp.select(predicate, operation); + } + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldTypeAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldTypeAdapter.java new file mode 100644 index 00000000000..a7f42178bcf --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldTypeAdapter.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface FieldTypeAdapter { + + public DataType getInputType(Expression exp, String fieldName); + + public void tryOutputType(Expression exp, String fieldName, DataType valueType); +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldValueAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldValueAdapter.java new file mode 100644 index 00000000000..89ed4e005f1 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldValueAdapter.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.FieldPath; +import com.yahoo.document.datatypes.FieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface FieldValueAdapter extends FieldTypeAdapter { + + public FieldValue getInputValue(String fieldName); + public FieldValue getInputValue(FieldPath fieldPath); + + public FieldValueAdapter setOutputValue(Expression exp, String fieldName, FieldValue fieldValue); +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java new file mode 100644 index 00000000000..915092f5e02 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.annotation.*; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; + +import java.util.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class FlattenExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + StringFieldValue input = (StringFieldValue)ctx.getValue(); + SpanTree tree = input.getSpanTree(SpanTrees.LINGUISTICS); + Map<Integer, List<String>> map = new HashMap<>(); + for (Annotation anno : tree) { + SpanNode span = anno.getSpanNode(); + if (span == null) { + continue; + } + if (anno.getType() != AnnotationTypes.TERM) { + continue; + } + FieldValue val = anno.getFieldValue(); + String str; + if (val instanceof StringFieldValue) { + str = ((StringFieldValue)val).getString(); + } else { + str = input.getString().substring(span.getFrom(), span.getTo()); + } + Integer pos = span.getTo(); + List<String> entry = map.get(pos); + if (entry == null) { + entry = new LinkedList<>(); + map.put(pos, entry); + } + entry.add(str); + } + String inputVal = String.valueOf(input); + StringBuilder output = new StringBuilder(); + for (int i = 0, len = inputVal.length(); i <= len; ++i) { + List<String> entry = map.get(i); + if (entry != null) { + Collections.sort(entry); + output.append(entry); + } + if (i < len) { + output.append(inputVal.charAt(i)); + } + } + ctx.setValue(new StringFieldValue(output.toString())); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return DataType.STRING; + } + + @Override + public DataType createdOutputType() { + return DataType.STRING; + } + + @Override + public String toString() { + return "flatten"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof FlattenExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java new file mode 100644 index 00000000000..a74f02e9a4a --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java @@ -0,0 +1,136 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.vespa.indexinglanguage.FieldValueConverter; +import com.yahoo.vespa.objects.ObjectOperation; +import com.yahoo.vespa.objects.ObjectPredicate; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ForEachExpression extends CompositeExpression { + + private final Expression exp; + + public ForEachExpression(Expression exp) { + this.exp = exp; + } + + public Expression getInnerExpression() { + return exp; + } + + @Override + protected void doExecute(final ExecutionContext ctx) { + FieldValue input = ctx.getValue(); + if (input instanceof Array || input instanceof WeightedSet) { + FieldValue next = new MyConverter(ctx, exp).convert(input); + if (next == null) { + VerificationContext vctx = new VerificationContext(ctx); + vctx.setValue(input.getDataType()).execute(this); + next = vctx.getValue().createFieldValue(); + } + ctx.setValue(next); + } else if (input instanceof Struct) { + ctx.setValue(new MyConverter(ctx, exp).convert(input)); + } else { + throw new IllegalArgumentException("Expected Array, Struct or WeightedSet input, got " + + input.getDataType().getName() + "."); + } + } + + @Override + protected void doVerify(VerificationContext ctx) { + DataType input = ctx.getValue(); + if (input instanceof ArrayDataType || input instanceof WeightedSetDataType) { + ctx.setValue(((CollectionDataType)input).getNestedType()).execute(exp); + if (input instanceof ArrayDataType) { + ctx.setValue(DataType.getArray(ctx.getValue())); + } else { + WeightedSetDataType wset = (WeightedSetDataType)input; + ctx.setValue(DataType.getWeightedSet(ctx.getValue(), wset.createIfNonExistent(), wset.removeIfZero())); + } + } else if (input instanceof StructDataType) { + for (Field field : ((StructDataType)input).getFields()) { + DataType fieldType = field.getDataType(); + DataType valueType = ctx.setValue(fieldType).execute(exp).getValue(); + if (!fieldType.isAssignableFrom(valueType)) { + throw new VerificationException(this, "Expected " + fieldType.getName() + " output, got " + + valueType.getName() + "."); + } + } + ctx.setValue(input); + } else { + throw new VerificationException(this, "Expected Array, Struct or WeightedSet input, got " + + input.getName() + "."); + } + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + if (exp.createdOutputType() == null) { + return null; + } + return UnresolvedDataType.INSTANCE; + } + + @Override + public String toString() { + return "for_each { " + exp + " }"; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ForEachExpression)) { + return false; + } + ForEachExpression rhs = (ForEachExpression)obj; + if (!exp.equals(rhs.exp)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + exp.hashCode(); + } + + private static final class MyConverter extends FieldValueConverter { + + final ExecutionContext context; + final Expression expression; + int depth = 0; + + MyConverter(ExecutionContext context, Expression expression) { + this.context = context; + this.expression = expression; + } + + @Override + protected boolean shouldConvert(FieldValue value) { + return ++depth > 1; + } + + @Override + protected FieldValue doConvert(FieldValue value) { + context.setValue(value).execute(expression); + return context.getValue(); + } + } + + @Override + public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { + select(exp, predicate, operation); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java new file mode 100644 index 00000000000..6cc06953437 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; +import com.yahoo.document.StructuredDataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StructuredFieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GetFieldExpression extends Expression { + + private final String fieldName; + + public GetFieldExpression(String fieldName) { + this.fieldName = fieldName; + } + + public String getFieldName() { + return fieldName; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + FieldValue input = ctx.getValue(); + if (!(input instanceof StructuredFieldValue)) { + throw new IllegalArgumentException("Expected structured input, got " + input.getDataType().getName() + "."); + } + StructuredFieldValue struct = (StructuredFieldValue)input; + Field field = struct.getField(fieldName); + if (field == null) { + throw new IllegalArgumentException("Field '" + fieldName + "' not found."); + } + ctx.setValue(struct.getFieldValue(field)); + } + + @Override + protected void doVerify(VerificationContext ctx) { + DataType input = ctx.getValue(); + if (!(input instanceof StructuredDataType)) { + throw new VerificationException(this, "Expected structured input, got " + input.getName() + "."); + } + Field field = ((StructuredDataType)input).getField(fieldName); + if (field == null) { + throw new VerificationException(this, "Field '" + fieldName + "' not found."); + } + ctx.setValue(field.getDataType()); + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public String toString() { + return "get_field " + fieldName; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof GetFieldExpression)) { + return false; + } + GetFieldExpression rhs = (GetFieldExpression)obj; + if (!fieldName.equals(rhs.fieldName)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + fieldName.hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java new file mode 100644 index 00000000000..34f0139037b --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GetVarExpression extends Expression { + + private final String varName; + + public GetVarExpression(String varName) { + this.varName = varName; + } + + public String getVariableName() { + return varName; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setValue(ctx.getVariable(varName)); + } + + @Override + protected void doVerify(VerificationContext ctx) { + DataType input = ctx.getVariable(varName); + if (input == null) { + throw new VerificationException(this, "Variable '" + varName + "' not found."); + } + ctx.setValue(input); + } + + @Override + public DataType requiredInputType() { + return null; + } + + @Override + public DataType createdOutputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public String toString() { + return "get_var " + varName; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof GetVarExpression)) { + return false; + } + GetVarExpression rhs = (GetVarExpression)obj; + if (!varName.equals(rhs.varName)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + varName.hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java new file mode 100644 index 00000000000..b7a49e938b5 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java @@ -0,0 +1,96 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.UpdateAdapter; +import com.yahoo.vespa.objects.ObjectOperation; +import com.yahoo.vespa.objects.ObjectPredicate; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GuardExpression extends CompositeExpression { + + private final Expression exp; + private final boolean shouldExecute; + + public GuardExpression(Expression exp) { + this.exp = exp; + shouldExecute = shouldExecute(exp); + } + + public Expression getInnerExpression() { + return exp; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + if (!shouldExecute && ctx.getAdapter() instanceof UpdateAdapter) { + ctx.setValue(null); + } else { + exp.execute(ctx); + } + } + + @Override + protected void doVerify(VerificationContext ctx) { + exp.verify(ctx); + } + + @Override + public DataType requiredInputType() { + return exp.requiredInputType(); + } + + @Override + public DataType createdOutputType() { + return exp.createdOutputType(); + } + + @Override + public String toString() { + return "guard " + toScriptBlock(exp); + } + + @Override + public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { + select(exp, predicate, operation); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof GuardExpression)) { + return false; + } + GuardExpression rhs = (GuardExpression)obj; + if (!exp.equals(rhs.exp)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + exp.hashCode(); + } + + private static boolean shouldExecute(Expression exp) { + ExecutionGuard guard = new ExecutionGuard(); + guard.visit(exp); + return guard.shouldExecute; + } + + private static class ExecutionGuard extends ExpressionVisitor { + + boolean shouldExecute = false; + + @Override + protected void doVisit(Expression exp) { + if (exp instanceof InputExpression || exp instanceof SetLanguageExpression) { + shouldExecute = true; + } + } + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java new file mode 100644 index 00000000000..11e58883b27 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.LongFieldValue; + +import java.math.BigInteger; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class HexDecodeExpression extends Expression { + + private static final BigInteger ULONG_MAX = new BigInteger("18446744073709551616"); + + @Override + protected void doExecute(ExecutionContext ctx) { + String input = String.valueOf(ctx.getValue()); + if (input.isEmpty()) { + ctx.setValue(new LongFieldValue(Long.MIN_VALUE)); + return; + } + BigInteger output; + try { + output = new BigInteger(input, 16); + } catch (NumberFormatException e) { + throw new NumberFormatException("Illegal hex value '" + input + "'."); + } + if (output.bitLength() > 64) { + throw new NumberFormatException("Hex value '" + input + "' is out of range."); + } + if (output.compareTo(BigInteger.ZERO) == 1 && output.bitLength() == 64) { + output = output.subtract(ULONG_MAX); // flip to negative + } + ctx.setValue(new LongFieldValue(output.longValue())); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return DataType.STRING; + } + + @Override + public DataType createdOutputType() { + return DataType.LONG; + } + + @Override + public String toString() { + return "hexdecode"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof HexDecodeExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java new file mode 100644 index 00000000000..80c7b95308f --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class HexEncodeExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + long input = ((LongFieldValue)ctx.getValue()).getLong(); + ctx.setValue(new StringFieldValue(Long.toHexString(input))); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return DataType.LONG; + } + + @Override + public DataType createdOutputType() { + return DataType.STRING; + } + + @Override + public String toString() { + return "hexencode"; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof HexEncodeExpression)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java new file mode 100644 index 00000000000..30007090b78 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.StringFieldValue; + +import java.net.InetAddress; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class HostNameExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + try { + ctx.setValue(new StringFieldValue(normalizeHostName(InetAddress.getLocalHost().getHostName()))); + } catch (java.net.UnknownHostException e) { + throw new RuntimeException(e); + } + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return null; + } + + @Override + public DataType createdOutputType() { + return DataType.STRING; + } + + @Override + public String toString() { + return "hostname"; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof HostNameExpression)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } + + public static String normalizeHostName(String hostName) { + int pos = hostName.indexOf('.'); + return pos < 0 ? hostName : hostName.substring(0, pos); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java new file mode 100644 index 00000000000..eb7e1c9a005 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java @@ -0,0 +1,216 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.NumericFieldValue; +import com.yahoo.vespa.objects.ObjectOperation; +import com.yahoo.vespa.objects.ObjectPredicate; + +import java.math.BigDecimal; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class IfThenExpression extends CompositeExpression { + + public static enum Comparator { + EQ("=="), + NE("!="), + LT("<"), + LE("<="), + GT(">"), + GE(">="); + + private final String img; + + private Comparator(String img) { + this.img = img; + } + + @Override + public String toString() { + return img; + } + } + + private final Expression lhs; + private final Comparator cmp; + private final Expression rhs; + private final Expression ifTrue; + private final Expression ifFalse; + + public IfThenExpression(Expression lhs, Comparator cmp, Expression rhs, Expression ifTrue) { + this(lhs, cmp, rhs, ifTrue, null); + } + + public IfThenExpression(Expression lhs, Comparator cmp, Expression rhs, Expression ifTrue, Expression ifFalse) { + this.lhs = lhs; + this.cmp = cmp; + this.rhs = rhs; + this.ifTrue = ifTrue; + this.ifFalse = ifFalse; + } + + public Expression getLeftHandSide() { + return lhs; + } + + public Comparator getComparator() { + return cmp; + } + + public Expression getRightHandSide() { + return rhs; + } + + public Expression getIfTrueExpression() { + return ifTrue; + } + + public Expression getIfFalseExpression() { + return ifFalse; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + FieldValue input = ctx.getValue(); + FieldValue lhsVal = ctx.setValue(input).execute(lhs).getValue(); + if (lhsVal == null) { + ctx.setValue(null); + return; + } + FieldValue rhsVal = ctx.setValue(input).execute(rhs).getValue(); + if (rhsVal == null) { + ctx.setValue(null); + return; + } + ctx.setValue(input); + if (isTrue(lhsVal, cmp, rhsVal)) { + ifTrue.execute(ctx); + } else if (ifFalse != null) { + ifFalse.execute(ctx); + } + } + + @Override + protected void doVerify(VerificationContext ctx) { + DataType input = ctx.getValue(); + ctx.setValue(input).execute(lhs); + ctx.setValue(input).execute(rhs); + ctx.setValue(input).execute(ifTrue); + ctx.setValue(input).execute(ifFalse); + ctx.setValue(input); + } + + @Override + public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { + select(lhs, predicate, operation); + select(rhs, predicate, operation); + select(ifTrue, predicate, operation); + select(ifFalse, predicate, operation); + } + + @Override + public DataType requiredInputType() { + DataType input = null; + input = resolveRequiredInputType(input, lhs.requiredInputType()); + input = resolveRequiredInputType(input, rhs.requiredInputType()); + input = resolveRequiredInputType(input, ifTrue.requiredInputType()); + if (ifFalse != null) { + input = resolveRequiredInputType(input, ifFalse.requiredInputType()); + } + return input; + } + + @Override + public DataType createdOutputType() { + return null; + } + + @Override + public String toString() { + StringBuilder ret = new StringBuilder(); + ret.append("if (").append(lhs).append(" ").append(cmp).append(" ").append(rhs).append(") "); + ret.append(toScriptBlock(ifTrue)); + if (ifFalse != null) { + ret.append(" else ").append(toScriptBlock(ifFalse)); + } + return ret.toString(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof IfThenExpression)) { + return false; + } + IfThenExpression exp = (IfThenExpression)obj; + if (!lhs.equals(exp.lhs)) { + return false; + } + if (!cmp.equals(exp.cmp)) { + return false; + } + if (!rhs.equals(exp.rhs)) { + return false; + } + if (!ifTrue.equals(exp.ifTrue)) { + return false; + } + if (!equals(ifFalse, exp.ifFalse)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + int ret = getClass().hashCode() + lhs.hashCode() + cmp.hashCode() + rhs.hashCode() + ifTrue.hashCode(); + if (ifFalse != null) { + ret += ifFalse.hashCode(); + } + return ret; + } + + private DataType resolveRequiredInputType(DataType prev, DataType next) { + if (next == null) { + return prev; + } + if (prev == null) { + return next; + } + if (!prev.equals(next)) { + throw new VerificationException(this, "Operands require conflicting input types, " + + prev.getName() + " vs " + next.getName() + "."); + } + return prev; + } + + private static boolean isTrue(FieldValue lhs, Comparator cmp, FieldValue rhs) { + int res; + if (lhs instanceof NumericFieldValue && rhs instanceof NumericFieldValue) { + BigDecimal lhsVal = ArithmeticExpression.asBigDecimal((NumericFieldValue)lhs); + BigDecimal rhsVal = ArithmeticExpression.asBigDecimal((NumericFieldValue)rhs); + res = lhsVal.compareTo(rhsVal); + } else { + res = lhs.compareTo(rhs); + } + switch (cmp) { + case EQ: + return res == 0; + case NE: + return res != 0; + case GT: + return res > 0; + case GE: + return res >= 0; + case LT: + return res < 0; + case LE: + return res <= 0; + default: + throw new UnsupportedOperationException(cmp.toString()); + } + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IndexExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IndexExpression.java new file mode 100644 index 00000000000..eb7d79abe10 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IndexExpression.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class IndexExpression extends OutputExpression { + + public IndexExpression(String fieldName) { + super("index", fieldName); + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj) && obj instanceof IndexExpression; + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java new file mode 100644 index 00000000000..29957e440a4 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.FieldPath; +import com.yahoo.vespa.objects.ObjectOperation; +import com.yahoo.vespa.objects.ObjectPredicate; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class InputExpression extends Expression { + + private final String fieldName; + private FieldPath fieldPath; + + public InputExpression(String fieldName) { + this.fieldName = fieldName; + } + + public String getFieldName() { + return fieldName; + } + + @Override + protected void doExecute(ExecutionContext ctx) + { + if (fieldPath != null) { + ctx.setValue(ctx.getInputValue(fieldPath)); + } else { + ctx.setValue(ctx.getInputValue(fieldName)); + } + } + + @Override + protected void doVerify(VerificationContext ctx) { + DataType val = ctx.getInputType(this, fieldName); + if (val == null) { + throw new VerificationException(this, "Field '" + fieldName + "' not found."); + } + ctx.setValue(val); + } + + @Override + public DataType requiredInputType() { + return null; + } + + @Override + public DataType createdOutputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public String toString() { + return "input" + (fieldName != null ? " " + fieldName : ""); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof InputExpression)) { + return false; + } + InputExpression rhs = (InputExpression)obj; + if (!equals(fieldName, rhs.fieldName)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + (fieldName != null ? fieldName.hashCode() : 0); + } + + public static class FieldPathOptimizer implements ObjectOperation, ObjectPredicate { + private final DocumentType documentType; + + public FieldPathOptimizer(DocumentType documentType) { + this.documentType = documentType; + } + + @Override + public void execute(Object obj) { + InputExpression exp = (InputExpression) obj; + exp.fieldPath = documentType.buildFieldPath(exp.getFieldName()); + } + + @Override + public boolean check(Object obj) { + return obj instanceof InputExpression; + } + } + + public static class InputFieldNameExtractor implements ObjectOperation, ObjectPredicate { + private List<String> inputFieldNames = new ArrayList<>(1); + + public List<String> getInputFieldNames() { return inputFieldNames; } + + @Override + public void execute(Object obj) { + inputFieldNames.add(((InputExpression) obj).getFieldName()); + } + + @Override + public boolean check(Object obj) { + return obj instanceof InputExpression; + } + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java new file mode 100644 index 00000000000..b440c2e68c8 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.text.StringUtilities; + +import java.util.Iterator; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class JoinExpression extends Expression { + + private final String delimiter; + + public JoinExpression(String delimiter) { + this.delimiter = delimiter; + } + + public String getDelimiter() { + return delimiter; + } + + @SuppressWarnings({ "unchecked" }) + @Override + protected void doExecute(ExecutionContext ctx) { + FieldValue input = ctx.getValue(); + if (!(input instanceof Array)) { + throw new IllegalArgumentException("Expected Array input, got " + input.getDataType().getName() + "."); + } + StringBuilder output = new StringBuilder(); + for (Iterator<FieldValue> it = ((Array)input).fieldValueIterator(); it.hasNext(); ) { + output.append(String.valueOf(it.next())); + if (it.hasNext()) { + output.append(delimiter); + } + } + ctx.setValue(new StringFieldValue(output.toString())); + } + + @Override + protected void doVerify(VerificationContext ctx) { + DataType input = ctx.getValue(); + if (!(input instanceof ArrayDataType)) { + throw new VerificationException(this, "Expected Array input, got " + input.getName() + "."); + } + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return DataType.STRING; + } + + @Override + public String toString() { + return "join \"" + StringUtilities.escape(delimiter, '"') + "\""; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof JoinExpression)) { + return false; + } + JoinExpression rhs = (JoinExpression)obj; + if (!delimiter.equals(rhs.delimiter)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + delimiter.hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseExpression.java new file mode 100644 index 00000000000..272e2c3a3e5 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseExpression.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.StringFieldValue; + +import static com.yahoo.language.LinguisticsCase.toLowerCase; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class LowerCaseExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setValue(new StringFieldValue(toLowerCase(String.valueOf(ctx.getValue())))); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return DataType.STRING; + } + + @Override + public DataType createdOutputType() { + return DataType.STRING; + } + + @Override + public String toString() { + return "lowercase"; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof LowerCaseExpression)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java new file mode 100644 index 00000000000..f648a0e38e4 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Stack;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class MathResolver {
+
+ private final List<Item> items = new LinkedList<>();
+
+ public void push(ArithmeticExpression.Operator op, Expression exp) {
+ op.getClass(); // throws NullPointerException
+ if (items.isEmpty() && op != ArithmeticExpression.Operator.ADD) {
+ throw new IllegalArgumentException("First item in an arithmetic operation must be an addition.");
+ }
+ items.add(new Item(op, exp));
+ }
+
+ public Expression resolve() {
+ Stack<Item> stack = new Stack<>();
+ stack.push(items.remove(0));
+ while (!items.isEmpty()) {
+ Item item = items.remove(0);
+ while (stack.size() > 1 && stack.peek().op.precedes(item.op)) {
+ pop(stack);
+ }
+ stack.push(item);
+ }
+ while (stack.size() > 1) {
+ pop(stack);
+ }
+ return stack.remove(0).exp;
+ }
+
+ private void pop(Stack<Item> stack) {
+ Item rhs = stack.pop();
+ Item lhs = stack.peek();
+ lhs.exp = new ArithmeticExpression(lhs.exp, rhs.op, rhs.exp);
+ }
+
+ private static class Item {
+
+ final ArithmeticExpression.Operator op;
+ Expression exp;
+
+ Item(ArithmeticExpression.Operator op, Expression exp) {
+ this.op = op;
+ this.exp = exp;
+ }
+ }
+}
\ No newline at end of file diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java new file mode 100644 index 00000000000..c5b68e6c978 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.annotation.*; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.GramSplitter; +import com.yahoo.language.process.TokenType; +import com.yahoo.vespa.indexinglanguage.linguistics.LinguisticsAnnotator; + +import java.util.Iterator; + +/** + * A filter which splits incoming text into n-grams + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class NGramExpression extends Expression { + + private final Linguistics linguistics; + private final int gramSize; + + /** + * Creates an executable ngram expression + * + * @param linguistics the gram splitter to use, or null if this is used for representation and will not be executed + * @param gramSize the gram size + */ + public NGramExpression(Linguistics linguistics, int gramSize) { + this.linguistics = linguistics; + this.gramSize = gramSize; + } + + public Linguistics getLinguistics() { + return linguistics; + } + + public int getGramSize() { + return gramSize; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + StringFieldValue input = (StringFieldValue)ctx.getValue(); + SpanList spanList = input.setSpanTree(new SpanTree(SpanTrees.LINGUISTICS)).spanList(); + int lastPosition = 0; + for (Iterator<GramSplitter.Gram> it = linguistics.getGramSplitter().split(input.getString(), gramSize); it.hasNext();) { + GramSplitter.Gram gram = it.next(); + // if there is a gap before this gram, then annotate the gram as punctuation + // (technically it may be of various types, but it does not matter - we just + // need to annotate it somehow (as a non-term) to make sure it is added to the summary) + if (lastPosition < gram.getStart()) { + typedSpan(lastPosition, gram.getStart() - lastPosition, TokenType.PUNCTUATION, spanList); + } + + // annotate gram as a word term + String gramString = gram.extractFrom(input.getString()); + typedSpan(gram.getStart(), gram.getLength(), TokenType.ALPHABETIC, spanList). + annotate(LinguisticsAnnotator.lowerCaseTermAnnotation(gramString, gramString)); + + lastPosition = gram.getStart() + gram.getLength(); + } + // handle punctuation at the end + if (lastPosition < input.toString().length()) { + typedSpan(lastPosition, input.toString().length() - lastPosition, TokenType.PUNCTUATION, spanList); + } + } + + private Span typedSpan(int from, int length, TokenType tokenType, SpanList spanList) { + return (Span)spanList.span(from, length).annotate(AnnotationTypes.TOKEN_TYPE, tokenType.getValue()); + } + + @Override + protected void doVerify(VerificationContext ctx) { + // empty + } + + @Override + public DataType requiredInputType() { + return DataType.STRING; + } + + @Override + public DataType createdOutputType() { + return null; + } + + @Override + public String toString() { + return "ngram " + gramSize; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof NGramExpression)) return false; + + NGramExpression rhs = (NGramExpression)obj; + if (linguistics != rhs.linguistics) return false; + if (gramSize != rhs.gramSize) return false; + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + gramSize; + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java new file mode 100644 index 00000000000..ebdd99b2cfa --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.Transformer; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class NormalizeExpression extends Expression { + + private final Linguistics linguistics; + + public NormalizeExpression(Linguistics linguistics) { + this.linguistics = linguistics; + } + + public Linguistics getLinguistics() { + return linguistics; + } + + @Override + protected void doExecute(ExecutionContext context) { + Transformer transformer = linguistics.getTransformer(); + context.setValue(new StringFieldValue(transformer.accentDrop(String.valueOf(context.getValue()), + context.resolveLanguage(linguistics)))); + } + + @Override + protected void doVerify(VerificationContext context) { + context.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return DataType.STRING; + } + + @Override + public DataType createdOutputType() { + return DataType.STRING; + } + + @Override + public String toString() { + return "normalize"; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof NormalizeExpression)) { + return false; + } + NormalizeExpression rhs = (NormalizeExpression)obj; + if (linguistics != rhs.linguistics) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NowExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NowExpression.java new file mode 100644 index 00000000000..75fb355580c --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NowExpression.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.LongFieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class NowExpression extends Expression { + + private final Timer timer; + + public NowExpression() { + this(SystemTimer.INSTANCE); + } + + public NowExpression(Timer timer) { + this.timer = timer; + } + + public Timer getTimer() { + return timer; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setValue(new LongFieldValue(timer.currentTimeSeconds())); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return null; + } + + @Override + public DataType createdOutputType() { + return DataType.LONG; + } + + @Override + public String toString() { + return "now"; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof NowExpression)) { + return false; + } + NowExpression rhs = (NowExpression)obj; + if (timer != rhs.timer) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + timer.hashCode(); + } + + public static interface Timer { + + public long currentTimeSeconds(); + } + + private static class SystemTimer implements Timer { + + static final Timer INSTANCE = new SystemTimer(); + + @Override + public long currentTimeSeconds() { + return System.currentTimeMillis() / 1000; + } + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java new file mode 100644 index 00000000000..838fb0b3c3b --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.predicate.Predicate; +import com.yahoo.search.predicate.optimization.AndOrSimplifier; +import com.yahoo.search.predicate.optimization.BooleanSimplifier; +import com.yahoo.search.predicate.optimization.ComplexNodeTransformer; +import com.yahoo.search.predicate.optimization.NotNodeReorderer; +import com.yahoo.search.predicate.optimization.PredicateOptions; +import com.yahoo.search.predicate.optimization.PredicateProcessor; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class OptimizePredicateExpression extends Expression { + + private final PredicateProcessor optimizer; + + public OptimizePredicateExpression() { + this(new PredicateOptimizer()); + } + + OptimizePredicateExpression(PredicateProcessor optimizer) { + this.optimizer = optimizer; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + PredicateFieldValue predicate = ((PredicateFieldValue)ctx.getValue()).clone(); + IntegerFieldValue arity = (IntegerFieldValue)ctx.getVariable("arity"); + LongFieldValue lower_bound = (LongFieldValue)ctx.getVariable("lower_bound"); + LongFieldValue upper_bound = (LongFieldValue)ctx.getVariable("upper_bound"); + Long lower = lower_bound != null? lower_bound.getLong() : null; + Long upper = upper_bound != null? upper_bound.getLong() : null; + PredicateOptions options = new PredicateOptions(arity.getInteger(), lower, upper); + predicate.setPredicate(optimizer.process(predicate.getPredicate(), options)); + ctx.setValue(predicate); + } + + @Override + protected void doVerify(VerificationContext ctx) { + checkVariable(ctx, "arity", DataType.INT, true); + checkVariable(ctx, "lower_bound", DataType.LONG, false); + checkVariable(ctx, "upper_bound", DataType.LONG, false); + ctx.setValue(DataType.PREDICATE); + } + + private void checkVariable(VerificationContext ctx, String var, DataType type, boolean required) { + DataType input = ctx.getVariable(var); + if (input == null) { + if (required) { + throw new VerificationException(this, "Variable '" + var + "' must be set."); + } + } else if (input != type) { + throw new VerificationException(this, "Variable '" + var + "' must have type " + type.getName() + "."); + } + } + + @Override + public DataType requiredInputType() { + return DataType.PREDICATE; + } + + @Override + public DataType createdOutputType() { + return null; + } + + @Override + public String toString() { + return "optimize_predicate"; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof OptimizePredicateExpression; + } + + private static class PredicateOptimizer implements PredicateProcessor { + + private final ComplexNodeTransformer complexNodeTransformer = new ComplexNodeTransformer(); + private final BooleanSimplifier booleanSimplifier = new BooleanSimplifier(); + private final AndOrSimplifier andOrSimplifier = new AndOrSimplifier(); + private final NotNodeReorderer notNodeReorderer = new NotNodeReorderer(); + + @Override + public Predicate process(Predicate predicate, PredicateOptions options) { + Predicate processedPredicate = complexNodeTransformer.process(predicate, options); + processedPredicate = booleanSimplifier.process(processedPredicate, options); + processedPredicate = andOrSimplifier.process(processedPredicate, options); + return notNodeReorderer.process(processedPredicate, options); + } + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java new file mode 100644 index 00000000000..063ec3a8955 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class OutputExpression extends Expression { + + private final String image; + private final String fieldName; + + public OutputExpression(String image, String fieldName) { + this.image = image; + this.fieldName = fieldName; + } + + public String getFieldName() { + return fieldName; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setOutputValue(this, fieldName, ctx.getValue()); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.tryOutputType(this, fieldName, ctx.getValue()); + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return null; + } + + @Override + public String toString() { + return image + (fieldName != null ? " " + fieldName : ""); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof OutputExpression)) { + return false; + } + OutputExpression rhs = (OutputExpression)obj; + if (!equals(fieldName, rhs.fieldName)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + (fieldName != null ? fieldName.hashCode() : 0); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java new file mode 100644 index 00000000000..57d833a0748 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.vespa.objects.ObjectOperation; +import com.yahoo.vespa.objects.ObjectPredicate; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ParenthesisExpression extends CompositeExpression { + + private final Expression innerExp; + + public ParenthesisExpression(Expression innerExp) { + this.innerExp = innerExp; + } + + public Expression getInnerExpression() { + return innerExp; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + innerExp.execute(ctx); + } + + @Override + protected void doVerify(VerificationContext ctx) { + innerExp.verify(ctx); + } + + @Override + public DataType requiredInputType() { + return innerExp.requiredInputType(); + } + + @Override + public DataType createdOutputType() { + return innerExp.createdOutputType(); + } + + @Override + public String toString() { + return "(" + innerExp + ")"; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ParenthesisExpression)) { + return false; + } + ParenthesisExpression rhs = (ParenthesisExpression)obj; + if (!innerExp.equals(rhs.innerExp)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + innerExp.hashCode(); + } + + @Override + public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { + select(innerExp, predicate, operation); + } + +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/PassthroughExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/PassthroughExpression.java new file mode 100644 index 00000000000..427c777db5a --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/PassthroughExpression.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +/** + * An output expression which has no search side effects. + * + * @author steinar + */ +public class PassthroughExpression extends OutputExpression { + + private static final String PASSTHROUGH = "passthrough"; + + public PassthroughExpression(String fieldName) { + super(PASSTHROUGH, fieldName); + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj) && obj instanceof PassthroughExpression; + } +}
\ No newline at end of file diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/RandomExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/RandomExpression.java new file mode 100644 index 00000000000..8a5cfc1bb88 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/RandomExpression.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.IntegerFieldValue; + +import java.util.concurrent.ThreadLocalRandom; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class RandomExpression extends Expression { + + private final Integer max; + + public RandomExpression() { + this(null); + } + + public RandomExpression(Integer max) { + this.max = max; + } + + public Integer getMaxValue() { + return max; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + int max; + if (this.max != null) { + max = this.max; + } else { + max = Integer.parseInt(String.valueOf(ctx.getValue())); + } + ctx.setValue(new IntegerFieldValue(ThreadLocalRandom.current().nextInt(max))); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return null; + } + + @Override + public DataType createdOutputType() { + return DataType.INT; + } + + @Override + public String toString() { + return "random" + (max != null ? " " + max : ""); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof RandomExpression)) { + return false; + } + RandomExpression rhs = (RandomExpression)obj; + if (!equals(max, rhs.max)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java new file mode 100644 index 00000000000..79ac7590cac --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.ScriptParser; +import com.yahoo.vespa.indexinglanguage.ScriptParserContext; +import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ScriptExpression extends ExpressionList<StatementExpression> { + + public ScriptExpression() { + super(); + } + + public ScriptExpression(StatementExpression... lst) { + super(Arrays.asList(lst)); + } + + public ScriptExpression(Collection<? extends StatementExpression> lst) { + super(lst); + } + + @Override + protected void doExecute(ExecutionContext ctx) { + FieldValue input = ctx.getValue(); + for (Expression exp : this) { + ctx.setValue(input).execute(exp); + } + ctx.setValue(input); + } + + @Override + protected void doVerify(VerificationContext ctx) { + DataType input = ctx.getValue(); + for (Expression exp : this) { + ctx.setValue(input).execute(exp); + } + ctx.setValue(input); + } + + @Override + public DataType requiredInputType() { + DataType prev = null; + for (Expression exp : this) { + DataType next = exp.requiredInputType(); + if (prev == null) { + prev = next; + } else if (next != null && !prev.isAssignableFrom(next)) { + throw new VerificationException(this, "Statements require conflicting input types, " + + prev.getName() + " vs " + next.getName() + "."); + } + } + return prev; + } + + @Override + public DataType createdOutputType() { + return null; + } + + @Override + public String toString() { + StringBuilder ret = new StringBuilder(); + ret.append("{ "); + for (Iterator<StatementExpression> it = iterator(); it.hasNext();) { + ret.append(it.next()).append(";"); + if (it.hasNext()) { + ret.append(" "); + } + } + ret.append(" }"); + return ret.toString(); + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj) && obj instanceof ScriptExpression; + } + + /** Creates an expression with simple lingustics for testing */ + public static ScriptExpression fromString(String expression) throws ParseException { + return fromString(expression, new SimpleLinguistics()); + } + + public static ScriptExpression fromString(String expression, Linguistics linguistics) throws ParseException { + return newInstance(new ScriptParserContext(linguistics).setInputStream(new IndexingInput(expression))); + } + + public static ScriptExpression newInstance(ScriptParserContext config) throws ParseException { + return ScriptParser.parseScript(config); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java new file mode 100644 index 00000000000..8663dcf7a9d --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java @@ -0,0 +1,107 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.collections.Pair; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.vespa.objects.ObjectOperation; +import com.yahoo.vespa.objects.ObjectPredicate; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SelectInputExpression extends CompositeExpression { + + private final List<Pair<String, Expression>> cases; + + @SafeVarargs + public SelectInputExpression(Pair<String, Expression>... cases) { + this(Arrays.asList(cases)); + } + + public SelectInputExpression(List<Pair<String, Expression>> cases) { + this.cases = cases; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + FieldValue input = ctx.getValue(); + for (Pair<String, Expression> entry : cases) { + FieldValue val = ctx.getInputValue(entry.getFirst()); + if (val != null) { + ctx.setValue(val).execute(entry.getSecond()); + break; + } + } + ctx.setValue(input); + } + + @Override + protected void doVerify(VerificationContext ctx) { + DataType input = ctx.getValue(); + for (Pair<String, Expression> entry : cases) { + DataType val = ctx.getInputType(this, entry.getFirst()); + if (val == null) { + throw new VerificationException(this, "Field '" + entry.getFirst() + "' not found."); + } + ctx.setValue(val).execute(entry.getSecond()); + } + ctx.setValue(input); + } + + @Override + public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { + for (Pair<String, Expression> entry : cases) { + select(entry.getSecond(), predicate, operation); + } + } + + @Override + public DataType requiredInputType() { + return null; + } + + @Override + public DataType createdOutputType() { + return null; + } + + public List<Pair<String, Expression>> getCases() { + return Collections.unmodifiableList(cases); + } + + @Override + public String toString() { + StringBuilder ret = new StringBuilder(); + ret.append("select_input { "); + for (Pair<String, Expression> entry : cases) { + ret.append(entry.getFirst()).append(": "); + Expression exp = entry.getSecond(); + ret.append(exp).append("; "); + } + ret.append("}"); + return ret.toString(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof SelectInputExpression)) { + return false; + } + SelectInputExpression rhs = (SelectInputExpression)obj; + if (!cases.equals(rhs.cases)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + cases.hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageExpression.java new file mode 100644 index 00000000000..35f02ccb05b --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageExpression.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.language.Language; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SetLanguageExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setLanguage(Language.fromLanguageTag(String.valueOf(ctx.getValue()))); + } + + @Override + protected void doVerify(VerificationContext ctx) { + // empty + } + + @Override + public DataType requiredInputType() { + return DataType.STRING; + } + + @Override + public DataType createdOutputType() { + return null; + } + + @Override + public String toString() { + return "set_language"; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof SetLanguageExpression)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueExpression.java new file mode 100644 index 00000000000..edc1fcfa9ee --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueExpression.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.text.StringUtilities; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SetValueExpression extends Expression { + + private final FieldValue value; + + public SetValueExpression(FieldValue value) { + value.getClass(); // throws NullPointerException + this.value = value; + } + + public FieldValue getValue() { + return value; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setValue(value); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(value.getDataType()); + } + + @Override + public DataType requiredInputType() { + return null; + } + + @Override + public DataType createdOutputType() { + return value.getDataType(); + } + + @Override + public String toString() { + if (value instanceof StringFieldValue) { + return "\"" + StringUtilities.escape(value.toString(), '"') + "\""; + } + if (value instanceof LongFieldValue) { + return value.toString() + "L"; + } + return value.toString(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof SetValueExpression)) { + return false; + } + SetValueExpression rhs = (SetValueExpression)obj; + if (!value.equals(rhs.value)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + value.hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java new file mode 100644 index 00000000000..d92831fd6b3 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SetVarExpression extends Expression { + + private final String varName; + + public SetVarExpression(String varName) { + this.varName = varName; + } + + public String getVariableName() { + return varName; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setVariable(varName, ctx.getValue()); + } + + @Override + protected void doVerify(VerificationContext ctx) { + DataType next = ctx.getValue(); + DataType prev = ctx.getVariable(varName); + if (prev != null && !prev.equals(next)) { + throw new VerificationException(this, "Attempting to assign conflicting types to variable '" + varName + + "', " + prev.getName() + " vs " + next.getName() + "."); + } + ctx.setVariable(varName, next); + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return null; + } + + @Override + public String toString() { + return "set_var " + varName; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof SetVarExpression)) { + return false; + } + SetVarExpression rhs = (SetVarExpression)obj; + if (!varName.equals(rhs.varName)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + varName.hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SplitExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SplitExpression.java new file mode 100644 index 00000000000..c2cfcd0cd0e --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SplitExpression.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.text.StringUtilities; + +import java.util.regex.Pattern; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SplitExpression extends Expression { + + private final Pattern splitPattern; + + public SplitExpression(String splitString) { + this.splitPattern = Pattern.compile(splitString); + } + + public Pattern getSplitPattern() { + return splitPattern; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + String input = String.valueOf(ctx.getValue()); + Array<StringFieldValue> output = new Array<>(DataType.getArray(DataType.STRING)); + if (!input.isEmpty()) { + String[] splits = splitPattern.split(input); + for (String split : splits) { + output.add(new StringFieldValue(split)); + } + } + ctx.setValue(output); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return DataType.STRING; + } + + @Override + public DataType createdOutputType() { + return DataType.getArray(DataType.STRING); + } + + @Override + public String toString() { + return "split \"" + StringUtilities.escape(splitPattern.toString(), '"') + "\""; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof SplitExpression)) { + return false; + } + SplitExpression rhs = (SplitExpression)obj; + if (!splitPattern.toString().equals(rhs.splitPattern.toString())) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + splitPattern.toString().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java new file mode 100644 index 00000000000..012ca87ed37 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.ScriptParser; +import com.yahoo.vespa.indexinglanguage.ScriptParserContext; +import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class StatementExpression extends ExpressionList<Expression> { + + public StatementExpression(Expression... lst) { + this(Arrays.asList(lst)); + } + + public StatementExpression(Iterable<Expression> lst) { + super(filterList(lst)); + } + + @Override + protected void doExecute(ExecutionContext ctx) { + for (Expression exp : this) { + ctx.execute(exp); + } + } + + @Override + protected void doVerify(VerificationContext ctx) { + for (Expression exp : this) { + ctx.execute(exp); + } + } + + @Override + public DataType requiredInputType() { + for (Expression exp : this) { + DataType type = exp.requiredInputType(); + if (type != null) { + return type; + } + type = exp.createdOutputType(); + if (type != null) { + return null; + } + } + return null; + } + + @Override + public DataType createdOutputType() { + for (int i = size(); --i >= 0; ) { + DataType type = get(i).createdOutputType(); + if (type != null) { + return type; + } + } + return null; + } + + @Override + public String toString() { + StringBuilder ret = new StringBuilder(); + for (Iterator<Expression> it = iterator(); it.hasNext();) { + ret.append(it.next()); + if (it.hasNext()) { + ret.append(" | "); + } + } + return ret.toString(); + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj) && obj instanceof StatementExpression; + } + + /** Creates an expression with simple lingustics for testing */ + public static StatementExpression fromString(String expression) throws ParseException { + return fromString(expression, new SimpleLinguistics()); + } + + public static StatementExpression fromString(String expression, Linguistics linguistics) throws ParseException { + return newInstance(new ScriptParserContext(linguistics).setInputStream(new IndexingInput(expression))); + } + + public static StatementExpression newInstance(ScriptParserContext config) throws ParseException { + return ScriptParser.parseStatement(config); + } + + private static List<Expression> filterList(Iterable<Expression> lst) { + List<Expression> ret = new LinkedList<>(); + for (Expression exp : lst) { + if (exp instanceof StatementExpression) { + ret.addAll(filterList((StatementExpression)exp)); + } else if (exp != null) { + ret.add(exp); + } + } + return ret; + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringExpression.java new file mode 100644 index 00000000000..bb4351bee77 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringExpression.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.StringFieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SubstringExpression extends Expression { + + private final int from; + private final int to; + + public SubstringExpression(int from, int to) { + if (from < 0 || to < 0 || to < from) { + throw new IndexOutOfBoundsException(); + } + this.from = from; + this.to = to; + } + + public int getFrom() { + return from; + } + + public int getTo() { + return to; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + String input = String.valueOf(ctx.getValue()); + int len = input.length(); + if (from >= len) { + input = ""; + } else if (to >= len) { + input = input.substring(from); + } else { + input = input.substring(from, to); + } + ctx.setValue(new StringFieldValue(input)); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return DataType.STRING; + } + + @Override + public DataType createdOutputType() { + return DataType.STRING; + } + + @Override + public String toString() { + return "substring " + from + " " + to; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof SubstringExpression)) { + return false; + } + SubstringExpression rhs = (SubstringExpression)obj; + if (from != rhs.from) { + return false; + } + if (to != rhs.to) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + + Integer.valueOf(from).hashCode() + + Integer.valueOf(to).hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SummaryExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SummaryExpression.java new file mode 100644 index 00000000000..20c8173ffe2 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SummaryExpression.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SummaryExpression extends OutputExpression { + + public SummaryExpression(String fieldName) { + super("summary", fieldName); + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj) && obj instanceof SummaryExpression; + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java new file mode 100644 index 00000000000..21f633c3aba --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java @@ -0,0 +1,137 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.text.StringUtilities; +import com.yahoo.vespa.objects.ObjectOperation; +import com.yahoo.vespa.objects.ObjectPredicate; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SwitchExpression extends CompositeExpression { + + private final Map<String, Expression> cases = new LinkedHashMap<>(); + private final Expression defaultExp; + + public <T extends Expression> SwitchExpression(Map<String, T> cases) { + this(cases, null); + } + + public <T extends Expression> SwitchExpression(Map<String, T> cases, Expression defaultExp) { + this.defaultExp = defaultExp; + for (Map.Entry<String, T> entry : cases.entrySet()) { + this.cases.put(entry.getKey(), entry.getValue()); + } + } + + public boolean isEmpty() { + return defaultExp == null && cases.isEmpty(); + } + + public Map<String, Expression> getCases() { + return Collections.unmodifiableMap(cases); + } + + public Expression getDefaultExpression() { + return defaultExp; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + FieldValue input = ctx.getValue(); + Expression exp = null; + if (input != null) { + if (!(input instanceof StringFieldValue)) { + throw new IllegalArgumentException("Expected " + DataType.STRING.getName() + " input, got " + + input.getDataType().getName() + "."); + } + exp = cases.get(String.valueOf(input)); + } + if (exp == null) { + exp = defaultExp; + } + if (exp != null) { + exp.execute(ctx); + } + ctx.setValue(input); + } + + @Override + public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { + select(defaultExp, predicate, operation); + for (Expression exp : cases.values()) { + select(exp, predicate, operation); + } + } + + @Override + protected void doVerify(VerificationContext ctx) { + DataType input = ctx.getValue(); + if (input == null) { + throw new VerificationException(this, "Expected " + DataType.STRING.getName() + " input, got null."); + } + if (input != DataType.STRING) { + throw new VerificationException(this, "Expected " + DataType.STRING.getName() + " input, got " + + input.getName() + "."); + } + for (Expression exp : cases.values()) { + ctx.setValue(input).execute(exp); + } + ctx.setValue(input).execute(defaultExp); + ctx.setValue(input); + } + + @Override + public DataType requiredInputType() { + return null; + } + + @Override + public DataType createdOutputType() { + return null; + } + + @Override + public String toString() { + StringBuilder ret = new StringBuilder(); + ret.append("switch { "); + for (Map.Entry<String, Expression> entry : cases.entrySet()) { + ret.append("case \"").append(StringUtilities.escape(entry.getKey(), '"')).append("\": "); + Expression exp = entry.getValue(); + ret.append(exp).append("; "); + } + if (defaultExp != null) { + ret.append("default: ").append(defaultExp).append("; "); + } + ret.append("}"); + return ret.toString(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof SwitchExpression)) { + return false; + } + SwitchExpression rhs = (SwitchExpression)obj; + if (!cases.equals(rhs.cases)) { + return false; + } + if (!equals(defaultExp, rhs.defaultExp)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + (defaultExp != null ? defaultExp.hashCode() : 0); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ThisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ThisExpression.java new file mode 100644 index 00000000000..8ad37b98f4c --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ThisExpression.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ThisExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + // empty + } + + @Override + protected void doVerify(VerificationContext ctx) { + // empty + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public String toString() { + return "this"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ThisExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayExpression.java new file mode 100644 index 00000000000..197ed431955 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayExpression.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.FieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToArrayExpression extends Expression { + + @SuppressWarnings({ "unchecked" }) + @Override + protected void doExecute(ExecutionContext ctx) { + FieldValue input = ctx.getValue(); + DataType inputType = input.getDataType(); + + ArrayDataType outputType = DataType.getArray(inputType); + Array output = outputType.createFieldValue(); + output.add(input); + + ctx.setValue(output); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(DataType.getArray(ctx.getValue())); + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public String toString() { + return "to_array"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ToArrayExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteExpression.java new file mode 100644 index 00000000000..cd01f8251b0 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteExpression.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.ByteFieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToByteExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setValue(new ByteFieldValue(Byte.valueOf(String.valueOf(ctx.getValue())))); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return DataType.BYTE; + } + + @Override + public String toString() { + return "to_byte"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ToByteExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleExpression.java new file mode 100644 index 00000000000..fb54aefe696 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleExpression.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.DoubleFieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToDoubleExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setValue(new DoubleFieldValue(Double.valueOf(String.valueOf(ctx.getValue())))); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return DataType.DOUBLE; + } + + @Override + public String toString() { + return "to_double"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ToDoubleExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatExpression.java new file mode 100644 index 00000000000..ebd866abfa9 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatExpression.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.FloatFieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToFloatExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setValue(new FloatFieldValue(Float.valueOf(String.valueOf(ctx.getValue())))); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return DataType.FLOAT; + } + + @Override + public String toString() { + return "to_float"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ToFloatExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerExpression.java new file mode 100644 index 00000000000..0c900c7756a --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerExpression.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.IntegerFieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToIntegerExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setValue(new IntegerFieldValue(Integer.valueOf(String.valueOf(ctx.getValue())))); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return DataType.INT; + } + + @Override + public String toString() { + return "to_int"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ToIntegerExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongExpression.java new file mode 100644 index 00000000000..63b8b446437 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongExpression.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.LongFieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToLongExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setValue(new LongFieldValue(Long.valueOf(String.valueOf(ctx.getValue())))); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return DataType.LONG; + } + + @Override + public String toString() { + return "to_long"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ToLongExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionExpression.java new file mode 100644 index 00000000000..2b89e05b7b2 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionExpression.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.PositionDataType; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToPositionExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setValue(PositionDataType.fromString(String.valueOf(ctx.getValue()))); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return DataType.STRING; + } + + @Override + public DataType createdOutputType() { + return PositionDataType.INSTANCE; + } + + @Override + public String toString() { + return "to_pos"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ToPositionExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringExpression.java new file mode 100644 index 00000000000..0db289a6b6b --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringExpression.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.StringFieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToStringExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setValue(new StringFieldValue(String.valueOf(ctx.getValue()))); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return DataType.STRING; + } + + @Override + public String toString() { + return "to_string"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ToStringExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetExpression.java new file mode 100644 index 00000000000..f15bac00031 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetExpression.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.WeightedSet; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToWsetExpression extends Expression { + + private final Boolean createIfNonExistent; + private final Boolean removeIfZero; + + public ToWsetExpression(boolean createIfNonExistent, boolean removeIfZero) { + this.createIfNonExistent = createIfNonExistent; + this.removeIfZero = removeIfZero; + } + + public boolean getCreateIfNonExistent() { + return createIfNonExistent; + } + + public boolean getRemoveIfZero() { + return removeIfZero; + } + + @SuppressWarnings({ "unchecked" }) + @Override + protected void doExecute(ExecutionContext ctx) { + FieldValue input = ctx.getValue(); + DataType inputType = input.getDataType(); + + WeightedSetDataType outputType = DataType.getWeightedSet(inputType, createIfNonExistent, removeIfZero); + WeightedSet output = outputType.createFieldValue(); + output.add(input); + + ctx.setValue(output); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(DataType.getWeightedSet(ctx.getValue(), createIfNonExistent, removeIfZero)); + } + + @Override + public DataType requiredInputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public String toString() { + return "to_wset" + + (createIfNonExistent ? " create_if_non_existent" : "") + + (removeIfZero ? " remove_if_zero" : ""); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ToWsetExpression)) { + return false; + } + ToWsetExpression rhs = (ToWsetExpression)obj; + if (createIfNonExistent != rhs.createIfNonExistent) { + return false; + } + if (removeIfZero != rhs.removeIfZero) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + + Boolean.valueOf(createIfNonExistent).hashCode() + + Boolean.valueOf(removeIfZero).hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java new file mode 100644 index 00000000000..272fbde342c --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java @@ -0,0 +1,92 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.StemMode; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; +import com.yahoo.vespa.indexinglanguage.linguistics.LinguisticsAnnotator; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class TokenizeExpression extends Expression { + + private final Linguistics linguistics; + private final AnnotatorConfig config; + + public TokenizeExpression(Linguistics linguistics, AnnotatorConfig config) { + this.linguistics = linguistics; + this.config = config; + } + + public Linguistics getLinguistics() { + return linguistics; + } + + public AnnotatorConfig getConfig() { + return config; + } + + @Override + protected void doExecute(ExecutionContext context) { + StringFieldValue output = ((StringFieldValue)context.getValue()).clone(); + context.setValue(output); + + AnnotatorConfig cfg = new AnnotatorConfig(config); + Language lang = context.resolveLanguage(linguistics); + if (lang != null) { + cfg.setLanguage(lang); + } + LinguisticsAnnotator annotator = new LinguisticsAnnotator(linguistics, cfg); + annotator.annotate(output); + } + + @Override + protected void doVerify(VerificationContext ctx) { + // empty + } + + @Override + public DataType requiredInputType() { + return DataType.STRING; + } + + @Override + public DataType createdOutputType() { + return null; + } + + @Override + public String toString() { + StringBuilder ret = new StringBuilder(); + ret.append("tokenize"); + if (config.getRemoveAccents()) { + ret.append(" normalize"); + } + if (config.getStemMode() != StemMode.NONE) { + ret.append(" stem:\""+config.getStemMode()+"\""); + } + return ret.toString(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof TokenizeExpression)) { + return false; + } + TokenizeExpression rhs = (TokenizeExpression)obj; + if (!config.equals(rhs.config)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode() + config.hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java new file mode 100644 index 00000000000..d544b9218e4 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.StringFieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class TrimExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + ctx.setValue(new StringFieldValue(String.valueOf(ctx.getValue()).trim())); + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return DataType.STRING; + } + + @Override + public DataType createdOutputType() { + return DataType.STRING; + } + + @Override + public String toString() { + return "trim"; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof TrimExpression)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedDataType.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedDataType.java new file mode 100644 index 00000000000..ae0649e5f87 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedDataType.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.datatypes.FieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +final class UnresolvedDataType extends PrimitiveDataType { + + public static final UnresolvedDataType INSTANCE = new UnresolvedDataType(); + + private UnresolvedDataType() { + super("any", -69, UnresolvedFieldValue.class, UnresolvedFieldValue.getFactory()); + } + + @Override + public boolean isValueCompatible(FieldValue value) { + return value != null; + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValue.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValue.java new file mode 100644 index 00000000000..e4216b2b5c7 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValue.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlStream; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class UnresolvedFieldValue extends FieldValue { + private static class Factory extends PrimitiveDataType.Factory { + public FieldValue create() { + return new UnresolvedFieldValue(); + } + } + public static PrimitiveDataType.Factory getFactory() { return new Factory(); } + @Override + public DataType getDataType() { + return UnresolvedDataType.INSTANCE; + } + + @Override + public void printXml(XmlStream xml) { + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + @Override + public void assign(Object o) { + throw new UnsupportedOperationException(); + } + + @Override + public void serialize(Field field, FieldWriter writer) { + throw new UnsupportedOperationException(); + } + + @Override + public void deserialize(Field field, FieldReader reader) { + throw new UnsupportedOperationException(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContext.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContext.java new file mode 100644 index 00000000000..fc6415419a6 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContext.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class VerificationContext implements FieldTypeAdapter, Cloneable { + + private final Map<String, DataType> variables = new HashMap<String, DataType>(); + private final FieldTypeAdapter adapter; + private DataType value; + + public VerificationContext() { + this.adapter = null; + } + + public VerificationContext(FieldTypeAdapter adapter) { + this.adapter = adapter; + } + + public VerificationContext execute(Expression exp) { + if (exp != null) { + exp.verify(this); + } + return this; + } + + @Override + public DataType getInputType(Expression exp, String fieldName) { + return adapter.getInputType(exp, fieldName); + } + + @Override + public void tryOutputType(Expression exp, String fieldName, DataType valueType) { + adapter.tryOutputType(exp, fieldName, valueType); + } + + public DataType getVariable(String name) { + return variables.get(name); + } + + public VerificationContext setVariable(String name, DataType value) { + variables.put(name, value); + return this; + } + + public DataType getValue() { + return value; + } + + public VerificationContext setValue(DataType value) { + this.value = value; + return this; + } + + public VerificationContext clear() { + variables.clear(); + value = null; + return this; + } +} + diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationException.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationException.java new file mode 100644 index 00000000000..c46fe38520c --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationException.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class VerificationException extends RuntimeException { + + private final Expression exp; + + public VerificationException(Expression exp, String msg) { + super(msg); + this.exp = exp; + } + + public Expression getExpression() { + return exp; + } + + @Override + public String toString() { + return getClass().getName() + ": For expression '" + exp + "': " + getMessage(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveExpression.java new file mode 100644 index 00000000000..cdf795ea7d5 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveExpression.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.geo.ZCurve; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ZCurveExpression extends Expression { + + @Override + protected void doExecute(ExecutionContext ctx) { + Struct input = ((Struct)ctx.getValue()); + Integer x = getFieldValue(input, PositionDataType.FIELD_X); + Integer y = getFieldValue(input, PositionDataType.FIELD_Y); + if (x != null && y != null) { + ctx.setValue(new LongFieldValue(ZCurve.encode(x, y))); + } else { + ctx.setValue(null); + } + } + + private static Integer getFieldValue(Struct struct, String fieldName) { + IntegerFieldValue val = (IntegerFieldValue)struct.getFieldValue(fieldName); + return val != null ? val.getInteger() : null; + } + + @Override + protected void doVerify(VerificationContext ctx) { + ctx.setValue(createdOutputType()); + } + + @Override + public DataType requiredInputType() { + return PositionDataType.INSTANCE; + } + + @Override + public DataType createdOutputType() { + return DataType.LONG; + } + + @Override + public String toString() { + return "zcurve"; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ZCurveExpression)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/package-info.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/package-info.java new file mode 100644 index 00000000000..790a58ce651 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java new file mode 100644 index 00000000000..547ba687490 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.linguistics;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.process.StemMode;
+import com.yahoo.vespa.configdefinition.IlscriptsConfig;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class AnnotatorConfig implements Cloneable {
+
+ private Language language;
+ private StemMode stemMode;
+ private boolean removeAccents;
+ private int maxTermOccurences;
+
+ public static final int DEFAULT_MAX_TERM_OCCURRENCES;
+
+ static {
+ IlscriptsConfig defaults = new IlscriptsConfig(new IlscriptsConfig.Builder());
+ DEFAULT_MAX_TERM_OCCURRENCES = defaults.maxtermoccurrences();
+ }
+
+ public AnnotatorConfig() {
+ language = Language.ENGLISH;
+ stemMode = StemMode.NONE;
+ removeAccents = false;
+ maxTermOccurences = DEFAULT_MAX_TERM_OCCURRENCES;
+ }
+
+ public AnnotatorConfig(AnnotatorConfig rhs) {
+ language = rhs.language;
+ stemMode = rhs.stemMode;
+ removeAccents = rhs.removeAccents;
+ maxTermOccurences = rhs.maxTermOccurences;
+ }
+
+ public Language getLanguage() {
+ return language;
+ }
+
+ public AnnotatorConfig setLanguage(Language language) {
+ this.language = language;
+ return this;
+ }
+
+ public StemMode getStemMode() {
+ return stemMode;
+ }
+
+ public AnnotatorConfig setStemMode(StemMode stemMode) {
+ this.stemMode = stemMode;
+ return this;
+ }
+
+ public AnnotatorConfig setStemMode(String name) {
+ this.stemMode = StemMode.valueOf(name);
+ return this;
+ }
+
+ public boolean getRemoveAccents() {
+ return removeAccents;
+ }
+
+ public AnnotatorConfig setRemoveAccents(boolean removeAccents) {
+ this.removeAccents = removeAccents;
+ return this;
+ }
+
+ public int getMaxTermOccurrences() {
+ return maxTermOccurences;
+ }
+
+ public AnnotatorConfig setMaxTermOccurrences(int maxTermCount) {
+ this.maxTermOccurences = maxTermCount;
+ return this;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof AnnotatorConfig)) {
+ return false;
+ }
+ AnnotatorConfig rhs = (AnnotatorConfig)obj;
+ if (!language.equals(rhs.language)) {
+ return false;
+ }
+ if (!stemMode.equals(rhs.stemMode)) {
+ return false;
+ }
+ if (removeAccents != rhs.removeAccents) {
+ return false;
+ }
+ if (maxTermOccurences != rhs.maxTermOccurences) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return getClass().hashCode() + language.hashCode() + stemMode.hashCode() +
+ Boolean.valueOf(removeAccents).hashCode() + maxTermOccurences;
+ }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java new file mode 100644 index 00000000000..70d6168ec22 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java @@ -0,0 +1,164 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.linguistics; + +import java.util.HashMap; +import java.util.Map; + +import com.yahoo.document.annotation.*; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.StemMode; +import com.yahoo.language.process.Token; +import com.yahoo.language.process.Tokenizer; + +import static com.yahoo.language.LinguisticsCase.toLowerCase; + +/** + * This is a tool for adding {@link AnnotationTypes} type annotations to {@link StringFieldValue} objects. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class LinguisticsAnnotator { + + private final Linguistics factory; + private final AnnotatorConfig config; + + private static class TermOccurrences { + final Map<String, Integer> termOccurrences = new HashMap<>(); + final int maxOccurrences; + + public TermOccurrences(int maxOccurences) { + this.maxOccurrences = maxOccurences; + } + + boolean termCountBelowLimit(String term) { + String lowerCasedTerm = toLowerCase(term); + int occurences = termOccurrences.getOrDefault(lowerCasedTerm, 0); + if (occurences >= maxOccurrences) { + return false; + } + + termOccurrences.put(lowerCasedTerm, occurences + 1); + return true; + } + } + + /** + * Constructs a new instance of this annotator. + * + * @param factory the linguistics factory to use when annotating + * @param config the linguistics config to use + */ + public LinguisticsAnnotator(Linguistics factory, AnnotatorConfig config) { + this.factory = factory; + this.config = config; + } + + /** + * Annotates the given string with the appropriate linguistics annotations. + * + * @param text the text to annotate + * @return whether or not anything was annotated + */ + public boolean annotate(StringFieldValue text) { + if (text.getSpanTree(SpanTrees.LINGUISTICS) != null) return true; // Already annotated with LINGUISTICS. + + Tokenizer tokenizer = factory.getTokenizer(); + Iterable<Token> tokens = tokenizer.tokenize(text.getString(), config.getLanguage(), config.getStemMode(), + config.getRemoveAccents()); + TermOccurrences termOccurrences = new TermOccurrences(config.getMaxTermOccurrences()); + SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS); + for (Token token : tokens) { + addAnnotationSpan(text.getString(), tree.spanList(), tokenizer, token, config.getStemMode(), termOccurrences); + } + + if (tree.numAnnotations() == 0) return false; + text.setSpanTree(tree); + return true; + } + + /** + * Creates a TERM annotation which has the lowercase value as annotation (only) if it is different from the + * original. + * + * @param termToLowerCase The term to lower case. + * @param origTerm The original term. + * @return the created TERM annotation. + */ + public static Annotation lowerCaseTermAnnotation(String termToLowerCase, String origTerm) { + String annotationValue = toLowerCase(termToLowerCase); + if (annotationValue.equals(origTerm)) { + return new Annotation(AnnotationTypes.TERM); + } + return new Annotation(AnnotationTypes.TERM, new StringFieldValue(annotationValue)); + } + + private static void addAnnotation(Span here, String term, String orig, TermOccurrences termOccurrences) { + if (termOccurrences.termCountBelowLimit(term)) { + here.annotate(lowerCaseTermAnnotation(term, orig)); + } + } + + private static void addAnnotationSpan(String input, SpanList parent, Tokenizer tokenizer, Token token, StemMode mode, TermOccurrences termOccurrences) { + if (!token.isSpecialToken()) { + if (token.getNumComponents() > 0) { + for (int i = 0; i < token.getNumComponents(); ++i) { + addAnnotationSpan(input, parent, tokenizer, token.getComponent(i), mode, termOccurrences); + } + return; + } + if (!token.isIndexable()) { + return; + } + } + String orig = token.getOrig(); + int pos = (int)token.getOffset(); + if (pos >= input.length()) { + throw new IllegalArgumentException("Token '" + orig + "' has offset " + pos + ", which is outside the " + + "bounds of the input string; " + input); + } + int len = orig.length(); + if (pos + len > input.length()) { + throw new IllegalArgumentException("Token '" + orig + "' has offset " + pos + ", which makes it overflow " + + "the bounds of the input string; " + input); + } + if (mode == StemMode.ALL) { + Span where = parent.span(pos, len); + String lowercasedOrig = toLowerCase(orig); + addAnnotation(where, orig, orig, termOccurrences); + + String lowercasedTerm = lowercasedOrig; + String term = token.getTokenString(); + if (term != null) { + term = tokenizer.getReplacementTerm(term); + } + if (term != null) { + lowercasedTerm = toLowerCase(term); + } + if (! lowercasedOrig.equals(lowercasedTerm)) { + addAnnotation(where, term, orig, termOccurrences); + } + for (int i = 0; i < token.getNumStems(); i++) { + String stem = token.getStem(i); + String lowercasedStem = toLowerCase(stem); + if (! (lowercasedOrig.equals(lowercasedStem) + || lowercasedTerm.equals(lowercasedStem))) + { + addAnnotation(where, stem, orig, termOccurrences); + } + } + } else { + String term = token.getTokenString(); + if (term != null) { + term = tokenizer.getReplacementTerm(term); + } + if (term == null || term.trim().isEmpty()) { + return; + } + if (termOccurrences.termCountBelowLimit(term)) { + parent.span(pos, len).annotate(lowerCaseTermAnnotation(term, token.getOrig())); + } + } + } + +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/package-info.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/package-info.java new file mode 100644 index 00000000000..7d8c8103852 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.indexinglanguage.linguistics; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/package-info.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/package-info.java new file mode 100644 index 00000000000..24261328410 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/IndexingInput.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/IndexingInput.java new file mode 100644 index 00000000000..de9d26547da --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/IndexingInput.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.parser;
+
+import com.yahoo.javacc.FastCharStream;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public final class IndexingInput extends FastCharStream implements CharStream {
+
+ public IndexingInput(String input) {
+ super(input);
+ }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/package-info.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/package-info.java new file mode 100644 index 00000000000..2c222f66069 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/parser/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.indexinglanguage.parser; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/predicate/package-info.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/predicate/package-info.java new file mode 100644 index 00000000000..77050a64119 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/predicate/package-info.java @@ -0,0 +1,3 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@com.yahoo.osgi.annotation.ExportPackage +package com.yahoo.vespa.indexinglanguage.predicate; diff --git a/indexinglanguage/src/main/javacc/IndexingParser.jj b/indexinglanguage/src/main/javacc/IndexingParser.jj new file mode 100644 index 00000000000..a7b17d81c83 --- /dev/null +++ b/indexinglanguage/src/main/javacc/IndexingParser.jj @@ -0,0 +1,822 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// -------------------------------------------------------------------------------- +// +// JavaCC options. +// +// -------------------------------------------------------------------------------- +options { + CACHE_TOKENS = false; + DEBUG_PARSER = false; + ERROR_REPORTING = true; + STATIC = false; + USER_CHAR_STREAM = true; +} + +// -------------------------------------------------------------------------------- +// +// Parser body. +// +// -------------------------------------------------------------------------------- +PARSER_BEGIN(IndexingParser) + +package com.yahoo.vespa.indexinglanguage.parser; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.List; +import java.util.LinkedList; +import java.util.Map; +import java.util.LinkedHashMap; + +import com.yahoo.collections.Pair; +import com.yahoo.document.datatypes.*; +import com.yahoo.text.StringUtilities; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; +import com.yahoo.language.process.StemMode; +import com.yahoo.language.Linguistics; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @version $Id$ + */ +public class IndexingParser { + + private String defaultFieldName; + private Linguistics linguistics; + private AnnotatorConfig annotatorCfg; + + public IndexingParser(String str) { + this(new IndexingInput(str)); + } + + public IndexingParser setDefaultFieldName(String fieldName) { + defaultFieldName = fieldName; + return this; + } + + public IndexingParser setLinguistics(Linguistics linguistics) { + this.linguistics = linguistics; + return this; + } + + public IndexingParser setAnnotatorConfig(AnnotatorConfig cfg) { + annotatorCfg = cfg; + return this; + } + + private static FieldValue parseDouble(String str) { + return new DoubleFieldValue(new BigDecimal(str).doubleValue()); + } + + private static FieldValue parseFloat(String str) { + if (str.endsWith("f") || str.endsWith("F")) { + str = str.substring(0, str.length() - 1); + } + return new FloatFieldValue(new BigDecimal(str).floatValue()); + } + + private static FieldValue parseInteger(String str) { + if (str.startsWith("0x")) { + return new IntegerFieldValue(new BigInteger(str.substring(2), 16).intValue()); + } else { + return new IntegerFieldValue(new BigInteger(str).intValue()); + } + } + + private static FieldValue parseLong(String str) { + if (str.endsWith("l") || str.endsWith("L")) { + str = str.substring(0, str.length() - 1); + } + if (str.startsWith("0x")) { + return new LongFieldValue(new BigInteger(str.substring(2), 16).longValue()); + } else { + return new LongFieldValue(new BigInteger(str).longValue()); + } + } +} + +PARSER_END(IndexingParser) + +SKIP : +{ + " " | "\t" | "\r" | "\f" +} + +SPECIAL_TOKEN : +{ + <COMMENT: "#" (~["\n","\r"])* > +} + +TOKEN : +{ + <INTEGER: (["0"-"9"])+ | ("0" ["x","X"] (["0"-"9","a"-"f","A"-"F"])+)> | + <LONG: <INTEGER> ["l","L"]> | + <DOUBLE: (["0"-"9"])+ ("." (["0"-"9"])*)? (["e","E"] (["+","-"])? (["0"-"9"])+)?> | + <FLOAT: <DOUBLE> ["f", "F"]> +} + +TOKEN : +{ + <NL: "\n"> | + <ADD: "+"> | + <SUB: "-"> | + <MUL: "*"> | + <DIV: "/"> | + <MOD: "%"> | + <EQ: "=="> | + <NE: "!="> | + <LT: "<"> | + <LE: "<="> | + <GT: ">"> | + <GE: ">="> | + <PIPE: "|"> | + <LCURLY: "{"> | + <RCURLY: "}"> | + <LPAREN: "("> | + <RPAREN: ")"> | + <DOT: "."> | + <COMMA: ","> | + <COLON: ":"> | + <SCOLON: ";"> | + <STRING: ("\"" (~["\""] | "\\\"")* "\"") | + ("'" (~["'"] | "\\'")* "'")> | + <ATTRIBUTE: "attribute"> | + <BASE64_DECODE: "base64decode"> | + <BASE64_ENCODE: "base64encode"> | + <CASE: "case"> | + <CASE_DEFAULT: "default"> | + <CLEAR_STATE: "clear_state"> | + <CREATE_IF_NON_EXISTENT: "create_if_non_existent"> | + <ECHO: "echo"> | + <ELSE: "else"> | + <EXACT: "exact"> | + <FLATTEN: "flatten"> | + <FOR_EACH: "for_each"> | + <GET_FIELD: "get_field"> | + <GET_VAR: "get_var"> | + <GUARD: "guard"> | + <HEX_DECODE: "hexdecode"> | + <HEX_ENCODE: "hexencode"> | + <HOST_NAME: "hostname"> | + <IF: "if"> | + <INDEX: "index"> | + <INPUT: "input"> | + <JOIN: "join"> | + <LOWER_CASE: "lowercase"> | + <NGRAM: "ngram"> | + <NORMALIZE: "normalize"> | + <NOW: "now"> | + <OPTIMIZE_PREDICATE: "optimize_predicate"> | + <PASSTHROUGH: "passthrough"> | + <RANDOM: "random"> | + <REMOVE_IF_ZERO: "remove_if_zero"> | + <SELECT_INPUT: "select_input"> | + <SET_LANGUAGE: "set_language"> | + <SET_VAR: "set_var"> | + <SPLIT: "split"> | + <STEM: "stem"> | + <SUBSTRING: "substring"> | + <SUMMARY: "summary"> | + <SWITCH: "switch"> | + <THIS: "this"> | + <TOKENIZE: "tokenize"> | + <TO_ARRAY: "to_array"> | + <TO_BYTE: "to_byte"> | + <TO_DOUBLE: "to_double"> | + <TO_FLOAT: "to_float"> | + <TO_INT: "to_int"> | + <TO_LONG: "to_long"> | + <TO_POS: "to_pos"> | + <TO_STRING: "to_string"> | + <TO_WSET: "to_wset"> | + <TRIM: "trim"> | + <ZCURVE: "zcurve"> | + <IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-"])*> +} + +// -------------------------------------------------------------------------------- +// +// Production rules. +// +// -------------------------------------------------------------------------------- + +Expression root() : +{ + Expression exp; +} +{ + ( exp = statement() [ <SCOLON> ] ) + { + while (exp instanceof ExpressionList && ((ExpressionList)exp).size() == 1) exp = ((ExpressionList)exp).get(0); + return exp; + } +} + +ScriptExpression script() : +{ + StatementExpression exp; + List<StatementExpression> lst = new LinkedList<StatementExpression>(); +} +{ + ( <LCURLY> nl() exp = statement() { lst.add(exp); } nl() + ( <SCOLON> nl() [ exp = statement() { lst.add(exp); } nl() ] )* <RCURLY> ) + { return new ScriptExpression(lst); } +} + +StatementExpression statement() : +{ + Expression exp; + List<Expression> lst = new LinkedList<Expression>(); +} +{ + ( exp = expression() { lst.add(exp); } ( <PIPE> nl() exp = expression() { lst.add(exp); } )* ) + { return new StatementExpression(lst); } +} + +Expression expression() : +{ + Expression exp; + List<Expression> lst = new LinkedList<Expression>(); +} +{ + ( exp = math() { lst.add(exp); } ( <DOT> exp = math() { lst.add(exp); } )* ) + { return lst.size() == 1 ? exp : new CatExpression(lst); } +} + +Expression math() : +{ + ArithmeticExpression.Operator op = ArithmeticExpression.Operator.ADD; + MathResolver math = new MathResolver(); + Expression exp; +} +{ + ( exp = value() { math.push(op, exp); } + ( ( <ADD> { op = ArithmeticExpression.Operator.ADD; } | + <DIV> { op = ArithmeticExpression.Operator.DIV; } | + <MOD> { op = ArithmeticExpression.Operator.MOD; } | + <MUL> { op = ArithmeticExpression.Operator.MUL; } | + <SUB> { op = ArithmeticExpression.Operator.SUB; } ) + exp = value() { math.push(op, exp); } )* ) + { return math.resolve(); } +} + +Expression value() : +{ + Expression val; +} +{ + ( val = attributeExp() | + val = base64DecodeExp() | + val = base64EncodeExp() | + val = clearStateExp() | + val = echoExp() | + val = exactExp() | + val = flattenExp() | + val = forEachExp() | + val = getFieldExp() | + val = getVarExp() | + val = guardExp() | + val = hexDecodeExp() | + val = hexEncodeExp() | + val = hostNameExp() | + val = ifThenExp() | + val = indexExp() | + val = inputExp() | + val = joinExp() | + val = lowerCaseExp() | + val = ngramExp() | + val = normalizeExp() | + val = nowExp() | + val = optimizePredicateExp() | + val = passthroughExp() | + val = randomExp() | + val = script() | + val = selectInputExp() | + val = setLanguageExp() | + val = setValueExp() | + val = setVarExp() | + val = splitExp() | + val = substringExp() | + val = summaryExp() | + val = switchExp() | + val = thisExp() | + val = tokenizeExp() | + val = toArrayExp() | + val = toByteExp() | + val = toDoubleExp() | + val = toFloatExp() | + val = toIntExp() | + val = toLongExp() | + val = toPosExp() | + val = toStringExp() | + val = toWsetExp() | + val = trimExp() | + val = zcurveExp() | + ( <LPAREN> val = statement() <RPAREN> { val = new ParenthesisExpression(val); } ) ) + { return val; } +} + +Expression attributeExp() : +{ + String val = defaultFieldName; +} +{ + ( <ATTRIBUTE> [ val = fieldName() ] ) + { return new AttributeExpression(val); } +} + +Expression base64DecodeExp() : { } +{ + ( <BASE64_DECODE> ) + { return new Base64DecodeExpression(); } +} + +Expression base64EncodeExp() : { } +{ + ( <BASE64_ENCODE> ) + { return new Base64EncodeExpression(); } +} + +Expression clearStateExp() : { } +{ + ( <CLEAR_STATE> ) + { return new ClearStateExpression(); } +} + +Expression echoExp() : { } +{ + ( <ECHO> ) + { return new EchoExpression(); } +} + +Expression exactExp() : { } +{ + ( <EXACT> ) + { return new ExactExpression(); } +} + +Expression flattenExp() : { } +{ + ( <FLATTEN> ) + { return new FlattenExpression(); } +} + +Expression forEachExp() : +{ + Expression val; +} +{ + ( <FOR_EACH> <LCURLY> nl() val = statement() nl() <RCURLY> ) + { return new ForEachExpression(val); } +} + +Expression getFieldExp() : +{ + String val; +} +{ + ( <GET_FIELD> val = identifier() ) + { return new GetFieldExpression(val); } +} + +Expression getVarExp() : +{ + String val; +} +{ + ( <GET_VAR> val = identifier() ) + { return new GetVarExpression(val); } +} + +Expression guardExp() : +{ + Expression val; +} +{ + ( <GUARD> val = script() ) + { return new GuardExpression(val); } +} + +Expression hexDecodeExp() : { } +{ + ( <HEX_DECODE> ) + { return new HexDecodeExpression(); } +} + +Expression hexEncodeExp() : { } +{ + ( <HEX_ENCODE> ) + { return new HexEncodeExpression(); } +} + +Expression hostNameExp() : { } +{ + ( <HOST_NAME> ) + { return new HostNameExpression(); } +} + +Expression ifThenExp() : +{ + Expression lhs, rhs, ifTrue, ifFalse = null; + IfThenExpression.Comparator cmp; +} +{ + ( <IF> <LPAREN> lhs = expression() cmp = ifThenCmp() rhs = expression() <RPAREN> + ifTrue = script() [ <ELSE> ifFalse = script() ] ) + { return new IfThenExpression(lhs, cmp, rhs, ifTrue, ifFalse); } +} + +IfThenExpression.Comparator ifThenCmp() : +{ + IfThenExpression.Comparator val = null; +} +{ + ( <EQ> { val = IfThenExpression.Comparator.EQ; } | + <NE> { val = IfThenExpression.Comparator.NE; } | + <LE> { val = IfThenExpression.Comparator.LE; } | + <LT> { val = IfThenExpression.Comparator.LT; } | + <GE> { val = IfThenExpression.Comparator.GE; } | + <GT> { val = IfThenExpression.Comparator.GT; } ) + { return val; } +} + +Expression indexExp() : +{ + String val = defaultFieldName; +} +{ + ( <INDEX> [ val = fieldName() ] ) + { return new IndexExpression(val); } +} + +Expression inputExp() : +{ + String val = defaultFieldName; +} +{ + ( <INPUT> [ val = identifier() ] ) + { return new InputExpression(val); } +} + +Expression joinExp() : +{ + String val; +} +{ + ( <JOIN> val = string() ) + { return new JoinExpression(val); } +} + +Expression lowerCaseExp() : { } +{ + ( <LOWER_CASE> ) + { return new LowerCaseExpression(); } +} + +Expression ngramExp() : +{ + int gramSize; +} +{ + ( <NGRAM> gramSize = integer() ) + { return new NGramExpression(linguistics, gramSize); } +} + +Expression normalizeExp() : { } +{ + ( <NORMALIZE> ) + { return new NormalizeExpression(linguistics); } +} + +Expression nowExp() : { } +{ + ( <NOW> ) + { return new NowExpression(); } +} + +Expression optimizePredicateExp() : { } +{ + ( <OPTIMIZE_PREDICATE> ) + { return new OptimizePredicateExpression(); } +} + +Expression passthroughExp() : +{ + String val = defaultFieldName; +} +{ + ( <PASSTHROUGH> [ val = fieldName() ] ) + { return new PassthroughExpression(val); } +} + +Expression randomExp() : +{ + Integer val = null; +} +{ + ( <RANDOM> [ LOOKAHEAD(2) val = integer() ] ) + { return new RandomExpression(val); } +} + +Expression selectInputExp() : +{ + List<Pair<String, Expression>> cases = new LinkedList<Pair<String, Expression>>(); + Expression exp; + String str; +} +{ + ( <SELECT_INPUT> <LCURLY> nl() ( str = identifier() <COLON> exp = statement() <SCOLON> nl() + { cases.add(new Pair<String, Expression>(str, exp)); } )+ <RCURLY> ) + { return new SelectInputExpression(cases); } +} + +Expression setLanguageExp() : { } +{ + ( <SET_LANGUAGE> ) + { return new SetLanguageExpression(); } +} + +Expression setValueExp() : +{ + FieldValue val; +} +{ + ( val = fieldValue() ) + { return new SetValueExpression(val); } +} + +Expression setVarExp() : +{ + String val; +} +{ + ( <SET_VAR> val = identifier() ) + { return new SetVarExpression(val); } +} + +Expression splitExp() : +{ + String val; +} +{ + ( <SPLIT> val = string() ) + { return new SplitExpression(val); } +} + +Expression substringExp() : +{ + long from, to; +} +{ + ( <SUBSTRING> from = integer() to = integer() ) + { return new SubstringExpression((int)from, (int)to); } +} + +Expression summaryExp() : +{ + String val = defaultFieldName; +} +{ + ( <SUMMARY> [ val = fieldName() ] ) + { return new SummaryExpression(val); } +} + +Expression switchExp() : +{ + Map<String, Expression> cases = new LinkedHashMap<String, Expression>(); + Expression exp, defaultExp = null; + String str; +} +{ + ( <SWITCH> <LCURLY> nl() + ( <CASE> str = string() <COLON> exp = statement() { cases.put(str, exp); } <SCOLON> nl() )+ + [ <CASE_DEFAULT> <COLON> defaultExp = statement() <SCOLON> nl() ] + <RCURLY> ) + { return new SwitchExpression(cases, defaultExp); } +} + +Expression thisExp() : { } +{ + ( <THIS> ) + { return new ThisExpression(); } +} + +Expression tokenizeExp() : +{ + AnnotatorConfig cfg = annotatorCfg; +} +{ + ( <TOKENIZE> [ cfg = tokenizeCfg() ] ) + { return new TokenizeExpression(linguistics, cfg); } +} + +AnnotatorConfig tokenizeCfg() : +{ + AnnotatorConfig val = new AnnotatorConfig(annotatorCfg); + String str = "SHORTEST"; +} +{ + ( <STEM> ( <COLON> str = string() ) ? { val.setStemMode(str); } | + <NORMALIZE> { val.setRemoveAccents(true); } )+ + { return val; } +} + +Expression toArrayExp() : { } +{ + ( <TO_ARRAY> ) + { return new ToArrayExpression(); } +} + +Expression toByteExp() : { } +{ + ( <TO_BYTE> ) + { return new ToByteExpression(); } +} + +Expression toDoubleExp() : { } +{ + ( <TO_DOUBLE> ) + { return new ToDoubleExpression(); } +} + +Expression toFloatExp() : { } +{ + ( <TO_FLOAT> ) + { return new ToFloatExpression(); } +} + +Expression toIntExp() : { } +{ + ( <TO_INT> ) + { return new ToIntegerExpression(); } +} + +Expression toLongExp() : { } +{ + ( <TO_LONG> ) + { return new ToLongExpression(); } +} + +Expression toPosExp() : { } +{ + ( <TO_POS> ) + { return new ToPositionExpression(); } +} + +Expression toStringExp() : { } +{ + ( <TO_STRING> ) + { return new ToStringExpression(); } +} + +Expression toWsetExp() : +{ + boolean createIfNonExistent = false; + boolean removeIfZero = false; +} +{ + ( <TO_WSET> ( <CREATE_IF_NON_EXISTENT> { createIfNonExistent = true; } | + <REMOVE_IF_ZERO> { removeIfZero = true; } )* ) + { return new ToWsetExpression(createIfNonExistent, removeIfZero); } +} + +Expression trimExp() : { } +{ + ( <TRIM> ) + { return new TrimExpression(); } +} + +Expression zcurveExp() : { } +{ + ( <ZCURVE> ) + { return new ZCurveExpression(); } +} + +String identifier() : +{ + String val; +} +{ + ( val = string() | + ( <ATTRIBUTE> | + <BASE64_DECODE> | + <BASE64_ENCODE> | + <CASE> | + <CASE_DEFAULT> | + <CLEAR_STATE> | + <CREATE_IF_NON_EXISTENT> | + <ECHO> | + <EXACT> | + <ELSE> | + <FLATTEN> | + <FOR_EACH> | + <GET_FIELD> | + <GET_VAR> | + <GUARD> | + <HEX_DECODE> | + <HEX_ENCODE> | + <HOST_NAME> | + <IDENTIFIER> | + <IF> | + <INDEX> | + <INPUT> | + <JOIN> | + <LOWER_CASE> | + <NGRAM> | + <NORMALIZE> | + <NOW> | + <OPTIMIZE_PREDICATE> | + <PASSTHROUGH> | + <RANDOM> | + <REMOVE_IF_ZERO> | + <SELECT_INPUT> | + <SET_LANGUAGE> | + <SET_VAR> | + <SPLIT> | + <STEM> | + <SUBSTRING> | + <SUMMARY> | + <SWITCH> | + <THIS> | + <TO_ARRAY> | + <TO_DOUBLE> | + <TO_FLOAT> | + <TO_INT> | + <TO_LONG> | + <TO_POS> | + <TO_STRING> | + <TO_WSET> | + <TOKENIZE> | + <TRIM> | + <ZCURVE> ) { val = token.image; } ) + { return val; } +} + + +String fieldName() : +{ + StringBuilder builder = new StringBuilder(); + String str; +} +{ + ( str = identifier() { builder.append(str); } ( + LOOKAHEAD(2) <DOT> { builder.append(token.image); } + str = identifier() { builder.append(str); } )* ) + { return builder.toString(); } +} + +FieldValue fieldValue() : +{ + FieldValue val; +} +{ + ( val = numericValue() | val = stringValue() ) + { return val; } +} + +FieldValue numericValue() : +{ + FieldValue val; + String pre = ""; +} +{ + ( [ <ADD> | <SUB> { pre = "-"; } ] + ( <DOUBLE> { val = parseDouble(pre + token.image); } | + <FLOAT> { val = parseFloat(pre + token.image); } | + <INTEGER> { val = parseInteger(pre + token.image); } | + <LONG> { val = parseLong(pre + token.image); } ) ) + { return val; } +} + +FieldValue stringValue() : +{ + String val; +} +{ + ( val = string() ) + { return new StringFieldValue(val); } +} + +String string() : { } +{ + ( <STRING> ) + { return StringUtilities.unescape(token.image.substring(1, token.image.length() - 1)); } +} + +int integer() : +{ + String pre = ""; + int val; +} +{ + ( [ <ADD> | <SUB> { pre = "-"; } ] + <INTEGER> { val = Integer.parseInt(pre + token.image); } ) + { return val; } +} + +void nl() : { } +{ + ( <NL> )* +} + diff --git a/indexinglanguage/src/test/cfg/attributes.cfg b/indexinglanguage/src/test/cfg/attributes.cfg new file mode 100644 index 00000000000..bf9ccd94453 --- /dev/null +++ b/indexinglanguage/src/test/cfg/attributes.cfg @@ -0,0 +1,69 @@ +attribute[22] +attribute[0].name sales +attribute[0].datatype INT32 +attribute[0].collectiontype SINGLE +attribute[1].name pto +attribute[1].datatype INT32 +attribute[1].collectiontype SINGLE +attribute[2].name mid +attribute[2].datatype INT32 +attribute[2].collectiontype ARRAY +attribute[3].name ew +attribute[3].datatype STRING +attribute[3].collectiontype SINGLE +attribute[4].name weight +attribute[4].datatype FLOAT +attribute[4].collectiontype SINGLE +attribute[5].name bgnpfrom +attribute[5].datatype FLOAT +attribute[5].collectiontype SINGLE +attribute[6].name artist +attribute[6].datatype STRING +attribute[6].collectiontype SINGLE +attribute[7].name artistspid +attribute[7].datatype STRING +attribute[7].collectiontype WEIGHTEDSET +attribute[8].name artistspid2 +attribute[8].datatype FLOAT +attribute[8].collectiontype WEIGHTEDSET +attribute[9].name title +attribute[9].datatype STRING +attribute[9].collectiontype SINGLE +attribute[10].name newestedition +attribute[10].datatype UINT32 +attribute[10].collectiontype SINGLE +attribute[11].name year +attribute[11].datatype INT32 +attribute[11].collectiontype ARRAY +attribute[12].name endyear +attribute[12].datatype INT32 +attribute[12].collectiontype ARRAY +attribute[13].name did +attribute[13].datatype INT32 +attribute[13].collectiontype SINGLE +attribute[14].name cbid +attribute[14].datatype INT32 +attribute[14].collectiontype SINGLE +attribute[15].name noupdate +attribute[15].datatype STRING +attribute[15].collectiontype SINGLE +attribute[15].noupdate false +attribute[16].name noupdate2 +attribute[16].datatype STRING +attribute[16].collectiontype SINGLE +attribute[16].noupdate false +attribute[17].name multiposition2d_position +attribute[17].datatype INT64 +attribute[17].collectiontype ARRAY +attribute[18].name extracategories +attribute[18].datatype STRING +attribute[18].collectiontype ARRAY +attribute[19].name default_fieldlength +attribute[19].datatype UINT32 +attribute[19].collectiontype ARRAY +attribute[20].name fmt_fieldlength +attribute[20].datatype UINT32 +attribute[20].collectiontype SINGLE +attribute[21].name categories_fieldlength +attribute[21].datatype UINT32 +attribute[21].collectiontype SINGLE diff --git a/indexinglanguage/src/test/cfg/documentmanager.cfg b/indexinglanguage/src/test/cfg/documentmanager.cfg new file mode 100644 index 00000000000..f93fbf75dbf --- /dev/null +++ b/indexinglanguage/src/test/cfg/documentmanager.cfg @@ -0,0 +1,534 @@ +datatype[20] +datatype[0].id -1245117006 +datatype[0].arraytype[1] +datatype[0].arraytype[0].datatype 0 +datatype[0].weightedsettype[0] +datatype[0].structtype[0] +datatype[0].documenttype[0] +datatype[1].id 1328286588 +datatype[1].arraytype[0] +datatype[1].weightedsettype[1] +datatype[1].weightedsettype[0].datatype 2 +datatype[1].weightedsettype[0].createifnonexistant false +datatype[1].weightedsettype[0].removeifzero false +datatype[1].structtype[0] +datatype[1].documenttype[0] +datatype[2].id 1325751891 +datatype[2].arraytype[0] +datatype[2].weightedsettype[1] +datatype[2].weightedsettype[0].datatype 1 +datatype[2].weightedsettype[0].createifnonexistant false +datatype[2].weightedsettype[0].removeifzero false +datatype[2].structtype[0] +datatype[2].documenttype[0] +datatype[3].id -1486737430 +datatype[3].arraytype[1] +datatype[3].arraytype[0].datatype 2 +datatype[3].weightedsettype[0] +datatype[3].structtype[0] +datatype[3].documenttype[0] +datatype[4].id -1910204744 +datatype[4].arraytype[0] +datatype[4].weightedsettype[0] +datatype[4].structtype[1] +datatype[4].structtype[0].name music.header +datatype[4].structtype[0].version 0 +datatype[4].structtype[0].field[39] +datatype[4].structtype[0].field[0].name bgndata +datatype[4].structtype[0].field[0].id[0] +datatype[4].structtype[0].field[0].datatype 2 +datatype[4].structtype[0].field[1].name sales +datatype[4].structtype[0].field[1].id[0] +datatype[4].structtype[0].field[1].datatype 0 +datatype[4].structtype[0].field[2].name pto +datatype[4].structtype[0].field[2].id[0] +datatype[4].structtype[0].field[2].datatype 0 +datatype[4].structtype[0].field[3].name keys +datatype[4].structtype[0].field[3].id[0] +datatype[4].structtype[0].field[3].datatype 2 +datatype[4].structtype[0].field[4].name mid +datatype[4].structtype[0].field[4].id[0] +datatype[4].structtype[0].field[4].datatype -1245117006 +datatype[4].structtype[0].field[5].name ew +datatype[4].structtype[0].field[5].id[0] +datatype[4].structtype[0].field[5].datatype 2 +datatype[4].structtype[0].field[6].name surl +datatype[4].structtype[0].field[6].id[0] +datatype[4].structtype[0].field[6].datatype 2 +datatype[4].structtype[0].field[7].name userrate +datatype[4].structtype[0].field[7].id[0] +datatype[4].structtype[0].field[7].datatype 0 +datatype[4].structtype[0].field[8].name pid +datatype[4].structtype[0].field[8].id[0] +datatype[4].structtype[0].field[8].datatype 2 +datatype[4].structtype[0].field[9].name weight +datatype[4].structtype[0].field[9].id[0] +datatype[4].structtype[0].field[9].datatype 1 +datatype[4].structtype[0].field[10].name url +datatype[4].structtype[0].field[10].id[0] +datatype[4].structtype[0].field[10].datatype 2 +datatype[4].structtype[0].field[11].name isbn +datatype[4].structtype[0].field[11].id[0] +datatype[4].structtype[0].field[11].datatype 2 +datatype[4].structtype[0].field[12].name fmt +datatype[4].structtype[0].field[12].id[0] +datatype[4].structtype[0].field[12].datatype 2 +datatype[4].structtype[0].field[13].name albumid +datatype[4].structtype[0].field[13].id[0] +datatype[4].structtype[0].field[13].datatype 2 +datatype[4].structtype[0].field[14].name disp_song +datatype[4].structtype[0].field[14].id[0] +datatype[4].structtype[0].field[14].datatype 2 +datatype[4].structtype[0].field[15].name song +datatype[4].structtype[0].field[15].id[0] +datatype[4].structtype[0].field[15].datatype 2 +datatype[4].structtype[0].field[16].name pfrom +datatype[4].structtype[0].field[16].id[0] +datatype[4].structtype[0].field[16].datatype 0 +datatype[4].structtype[0].field[17].name bgnpfrom +datatype[4].structtype[0].field[17].id[0] +datatype[4].structtype[0].field[17].datatype 1 +datatype[4].structtype[0].field[18].name categories +datatype[4].structtype[0].field[18].id[0] +datatype[4].structtype[0].field[18].datatype 2 +datatype[4].structtype[0].field[19].name data +datatype[4].structtype[0].field[19].id[0] +datatype[4].structtype[0].field[19].datatype 2 +datatype[4].structtype[0].field[20].name numreview +datatype[4].structtype[0].field[20].id[0] +datatype[4].structtype[0].field[20].datatype 0 +datatype[4].structtype[0].field[21].name bgnsellers +datatype[4].structtype[0].field[21].id[0] +datatype[4].structtype[0].field[21].datatype 0 +datatype[4].structtype[0].field[22].name image +datatype[4].structtype[0].field[22].id[0] +datatype[4].structtype[0].field[22].datatype 2 +datatype[4].structtype[0].field[23].name artist +datatype[4].structtype[0].field[23].id[0] +datatype[4].structtype[0].field[23].datatype 2 +datatype[4].structtype[0].field[24].name artistspid +datatype[4].structtype[0].field[24].id[0] +datatype[4].structtype[0].field[24].datatype 1328286588 +datatype[4].structtype[0].field[25].name artistspid2 +datatype[4].structtype[0].field[25].id[0] +datatype[4].structtype[0].field[25].datatype 1325751891 +datatype[4].structtype[0].field[26].name artistspid3 +datatype[4].structtype[0].field[26].id[0] +datatype[4].structtype[0].field[26].datatype 1328286588 +datatype[4].structtype[0].field[27].name title +datatype[4].structtype[0].field[27].id[0] +datatype[4].structtype[0].field[27].datatype 2 +datatype[4].structtype[0].field[28].name newestedition +datatype[4].structtype[0].field[28].id[0] +datatype[4].structtype[0].field[28].datatype 0 +datatype[4].structtype[0].field[29].name bgnpto +datatype[4].structtype[0].field[29].id[0] +datatype[4].structtype[0].field[29].datatype 2 +datatype[4].structtype[0].field[30].name year +datatype[4].structtype[0].field[30].id[0] +datatype[4].structtype[0].field[30].datatype -1245117006 +datatype[4].structtype[0].field[31].name endyear +datatype[4].structtype[0].field[31].id[0] +datatype[4].structtype[0].field[31].datatype -1245117006 +datatype[4].structtype[0].field[32].name did +datatype[4].structtype[0].field[32].id[0] +datatype[4].structtype[0].field[32].datatype 0 +datatype[4].structtype[0].field[33].name scorekey +datatype[4].structtype[0].field[33].id[0] +datatype[4].structtype[0].field[33].datatype 0 +datatype[4].structtype[0].field[34].name cbid +datatype[4].structtype[0].field[34].id[0] +datatype[4].structtype[0].field[34].datatype 0 +datatype[4].structtype[0].field[35].name titles +datatype[4].structtype[0].field[35].id[0] +datatype[4].structtype[0].field[35].datatype -1486737430 +datatype[4].structtype[0].field[36].name noupdate +datatype[4].structtype[0].field[36].id[0] +datatype[4].structtype[0].field[36].datatype 2 +datatype[4].structtype[0].field[37].name noupdate2 +datatype[4].structtype[0].field[37].id[0] +datatype[4].structtype[0].field[37].datatype 2 +datatype[4].structtype[0].field[38].name multiposition2d +datatype[4].structtype[0].field[38].id[0] +datatype[4].structtype[0].field[38].datatype -1486737430 +datatype[4].documenttype[0] +datatype[5].id 993120973 +datatype[5].arraytype[0] +datatype[5].weightedsettype[0] +datatype[5].structtype[1] +datatype[5].structtype[0].name music.body +datatype[5].structtype[0].version 0 +datatype[5].structtype[0].field[0] +datatype[5].documenttype[0] +datatype[6].id 1412693671 +datatype[6].arraytype[0] +datatype[6].weightedsettype[0] +datatype[6].structtype[0] +datatype[6].documenttype[1] +datatype[6].documenttype[0].name music +datatype[6].documenttype[0].version 0 +datatype[6].documenttype[0].inherits[0] +datatype[6].documenttype[0].headerstruct -1910204744 +datatype[6].documenttype[0].bodystruct 993120973 +datatype[7].id -1801920207 +datatype[7].arraytype[0] +datatype[7].weightedsettype[0] +datatype[7].structtype[1] +datatype[7].structtype[0].name music_summary.header +datatype[7].structtype[0].version 0 +datatype[7].structtype[0].field[40] +datatype[7].structtype[0].field[0].name distance +datatype[7].structtype[0].field[0].id[0] +datatype[7].structtype[0].field[0].datatype 0 +datatype[7].structtype[0].field[1].name sddocname +datatype[7].structtype[0].field[1].id[0] +datatype[7].structtype[0].field[1].datatype 2 +datatype[7].structtype[0].field[2].name bgndata +datatype[7].structtype[0].field[2].id[0] +datatype[7].structtype[0].field[2].datatype 2 +datatype[7].structtype[0].field[3].name sales +datatype[7].structtype[0].field[3].id[0] +datatype[7].structtype[0].field[3].datatype 0 +datatype[7].structtype[0].field[4].name pto +datatype[7].structtype[0].field[4].id[0] +datatype[7].structtype[0].field[4].datatype 0 +datatype[7].structtype[0].field[5].name mid +datatype[7].structtype[0].field[5].id[0] +datatype[7].structtype[0].field[5].datatype 2 +datatype[7].structtype[0].field[6].name ew +datatype[7].structtype[0].field[6].id[0] +datatype[7].structtype[0].field[6].datatype 2 +datatype[7].structtype[0].field[7].name surl +datatype[7].structtype[0].field[7].id[0] +datatype[7].structtype[0].field[7].datatype 2 +datatype[7].structtype[0].field[8].name userrate +datatype[7].structtype[0].field[8].id[0] +datatype[7].structtype[0].field[8].datatype 0 +datatype[7].structtype[0].field[9].name pid +datatype[7].structtype[0].field[9].id[0] +datatype[7].structtype[0].field[9].datatype 2 +datatype[7].structtype[0].field[10].name weight +datatype[7].structtype[0].field[10].id[0] +datatype[7].structtype[0].field[10].datatype 1 +datatype[7].structtype[0].field[11].name url +datatype[7].structtype[0].field[11].id[0] +datatype[7].structtype[0].field[11].datatype 2 +datatype[7].structtype[0].field[12].name isbn +datatype[7].structtype[0].field[12].id[0] +datatype[7].structtype[0].field[12].datatype 2 +datatype[7].structtype[0].field[13].name fmt +datatype[7].structtype[0].field[13].id[0] +datatype[7].structtype[0].field[13].datatype 2 +datatype[7].structtype[0].field[14].name albumid +datatype[7].structtype[0].field[14].id[0] +datatype[7].structtype[0].field[14].datatype 2 +datatype[7].structtype[0].field[15].name disp_song +datatype[7].structtype[0].field[15].id[0] +datatype[7].structtype[0].field[15].datatype 2 +datatype[7].structtype[0].field[16].name song +datatype[7].structtype[0].field[16].id[0] +datatype[7].structtype[0].field[16].datatype 2 +datatype[7].structtype[0].field[17].name pfrom +datatype[7].structtype[0].field[17].id[0] +datatype[7].structtype[0].field[17].datatype 0 +datatype[7].structtype[0].field[18].name bgnpfrom +datatype[7].structtype[0].field[18].id[0] +datatype[7].structtype[0].field[18].datatype 1 +datatype[7].structtype[0].field[19].name categories +datatype[7].structtype[0].field[19].id[0] +datatype[7].structtype[0].field[19].datatype 2 +datatype[7].structtype[0].field[20].name data +datatype[7].structtype[0].field[20].id[0] +datatype[7].structtype[0].field[20].datatype 2 +datatype[7].structtype[0].field[21].name numreview +datatype[7].structtype[0].field[21].id[0] +datatype[7].structtype[0].field[21].datatype 0 +datatype[7].structtype[0].field[22].name bgnsellers +datatype[7].structtype[0].field[22].id[0] +datatype[7].structtype[0].field[22].datatype 0 +datatype[7].structtype[0].field[23].name image +datatype[7].structtype[0].field[23].id[0] +datatype[7].structtype[0].field[23].datatype 2 +datatype[7].structtype[0].field[24].name artist +datatype[7].structtype[0].field[24].id[0] +datatype[7].structtype[0].field[24].datatype 2 +datatype[7].structtype[0].field[25].name artistspid +datatype[7].structtype[0].field[25].id[0] +datatype[7].structtype[0].field[25].datatype 2 +datatype[7].structtype[0].field[26].name artistspid3 +datatype[7].structtype[0].field[26].id[0] +datatype[7].structtype[0].field[26].datatype 2 +datatype[7].structtype[0].field[27].name title +datatype[7].structtype[0].field[27].id[0] +datatype[7].structtype[0].field[27].datatype 2 +datatype[7].structtype[0].field[28].name newestedition +datatype[7].structtype[0].field[28].id[0] +datatype[7].structtype[0].field[28].datatype 0 +datatype[7].structtype[0].field[29].name bgnpto +datatype[7].structtype[0].field[29].id[0] +datatype[7].structtype[0].field[29].datatype 2 +datatype[7].structtype[0].field[30].name year +datatype[7].structtype[0].field[30].id[0] +datatype[7].structtype[0].field[30].datatype 2 +datatype[7].structtype[0].field[31].name endyear +datatype[7].structtype[0].field[31].id[0] +datatype[7].structtype[0].field[31].datatype 2 +datatype[7].structtype[0].field[32].name did +datatype[7].structtype[0].field[32].id[0] +datatype[7].structtype[0].field[32].datatype 0 +datatype[7].structtype[0].field[33].name scorekey +datatype[7].structtype[0].field[33].id[0] +datatype[7].structtype[0].field[33].datatype 0 +datatype[7].structtype[0].field[34].name cbid +datatype[7].structtype[0].field[34].id[0] +datatype[7].structtype[0].field[34].datatype 0 +datatype[7].structtype[0].field[35].name titles +datatype[7].structtype[0].field[35].id[0] +datatype[7].structtype[0].field[35].datatype 2 +datatype[7].structtype[0].field[36].name ranklog +datatype[7].structtype[0].field[36].id[0] +datatype[7].structtype[0].field[36].datatype 2 +datatype[7].structtype[0].field[37].name rankfeatures +datatype[7].structtype[0].field[37].id[0] +datatype[7].structtype[0].field[37].datatype 2 +datatype[7].structtype[0].field[38].name summaryfeatures +datatype[7].structtype[0].field[38].id[0] +datatype[7].structtype[0].field[38].datatype 2 +datatype[7].structtype[0].field[39].name documentid +datatype[7].structtype[0].field[39].id[0] +datatype[7].structtype[0].field[39].datatype 2 +datatype[7].documenttype[0] +datatype[8].id -1728551034 +datatype[8].arraytype[0] +datatype[8].weightedsettype[0] +datatype[8].structtype[1] +datatype[8].structtype[0].name music_summary.body +datatype[8].structtype[0].version 0 +datatype[8].structtype[0].field[0] +datatype[8].documenttype[0] +datatype[9].id 1601149518 +datatype[9].arraytype[0] +datatype[9].weightedsettype[0] +datatype[9].structtype[0] +datatype[9].documenttype[1] +datatype[9].documenttype[0].name music_summary +datatype[9].documenttype[0].version 0 +datatype[9].documenttype[0].inherits[0] +datatype[9].documenttype[0].headerstruct -1801920207 +datatype[9].documenttype[0].bodystruct -1728551034 +datatype[10].id 1509154821 +datatype[10].arraytype[0] +datatype[10].weightedsettype[0] +datatype[10].structtype[1] +datatype[10].structtype[0].name music_index.header +datatype[10].structtype[0].version 0 +datatype[10].structtype[0].field[19] +datatype[10].structtype[0].field[0].name sddocname +datatype[10].structtype[0].field[0].id[0] +datatype[10].structtype[0].field[0].datatype -1486737430 +datatype[10].structtype[0].field[1].name sales +datatype[10].structtype[0].field[1].id[0] +datatype[10].structtype[0].field[1].datatype -1245117006 +datatype[10].structtype[0].field[2].name pto +datatype[10].structtype[0].field[2].id[0] +datatype[10].structtype[0].field[2].datatype -1245117006 +datatype[10].structtype[0].field[3].name keys +datatype[10].structtype[0].field[3].id[0] +datatype[10].structtype[0].field[3].datatype -1486737430 +datatype[10].structtype[0].field[4].name mid +datatype[10].structtype[0].field[4].id[0] +datatype[10].structtype[0].field[4].datatype -1245117006 +datatype[10].structtype[0].field[5].name ew +datatype[10].structtype[0].field[5].id[0] +datatype[10].structtype[0].field[5].datatype -1486737430 +datatype[10].structtype[0].field[6].name fmt +datatype[10].structtype[0].field[6].id[0] +datatype[10].structtype[0].field[6].datatype -1486737430 +datatype[10].structtype[0].field[7].name song +datatype[10].structtype[0].field[7].id[0] +datatype[10].structtype[0].field[7].datatype -1486737430 +datatype[10].structtype[0].field[8].name categories +datatype[10].structtype[0].field[8].id[0] +datatype[10].structtype[0].field[8].datatype -1486737430 +datatype[10].structtype[0].field[9].name artist +datatype[10].structtype[0].field[9].id[0] +datatype[10].structtype[0].field[9].datatype -1486737430 +datatype[10].structtype[0].field[10].name artistspid3 +datatype[10].structtype[0].field[10].id[0] +datatype[10].structtype[0].field[10].datatype -1486737430 +datatype[10].structtype[0].field[11].name title +datatype[10].structtype[0].field[11].id[0] +datatype[10].structtype[0].field[11].datatype -1486737430 +datatype[10].structtype[0].field[12].name newestedition +datatype[10].structtype[0].field[12].id[0] +datatype[10].structtype[0].field[12].datatype -1245117006 +datatype[10].structtype[0].field[13].name year +datatype[10].structtype[0].field[13].id[0] +datatype[10].structtype[0].field[13].datatype -1245117006 +datatype[10].structtype[0].field[14].name endyear +datatype[10].structtype[0].field[14].id[0] +datatype[10].structtype[0].field[14].datatype -1245117006 +datatype[10].structtype[0].field[15].name did +datatype[10].structtype[0].field[15].id[0] +datatype[10].structtype[0].field[15].datatype -1245117006 +datatype[10].structtype[0].field[16].name scorekey +datatype[10].structtype[0].field[16].id[0] +datatype[10].structtype[0].field[16].datatype -1245117006 +datatype[10].structtype[0].field[17].name cbid +datatype[10].structtype[0].field[17].id[0] +datatype[10].structtype[0].field[17].datatype -1245117006 +datatype[10].structtype[0].field[18].name titles +datatype[10].structtype[0].field[18].id[0] +datatype[10].structtype[0].field[18].datatype -1486737430 +datatype[10].documenttype[0] +datatype[11].id -1997730982 +datatype[11].arraytype[0] +datatype[11].weightedsettype[0] +datatype[11].structtype[1] +datatype[11].structtype[0].name music_index.body +datatype[11].structtype[0].version 0 +datatype[11].structtype[0].field[0] +datatype[11].documenttype[0] +datatype[12].id 2108744186 +datatype[12].arraytype[0] +datatype[12].weightedsettype[0] +datatype[12].structtype[0] +datatype[12].documenttype[1] +datatype[12].documenttype[0].name music_index +datatype[12].documenttype[0].version 0 +datatype[12].documenttype[0].inherits[0] +datatype[12].documenttype[0].headerstruct 1509154821 +datatype[12].documenttype[0].bodystruct -1997730982 +datatype[13].id 58874399 +datatype[13].arraytype[1] +datatype[13].arraytype[0].datatype 4 +datatype[13].weightedsettype[0] +datatype[13].structtype[0] +datatype[13].documenttype[0] +datatype[14].id -1497398149 +datatype[14].arraytype[0] +datatype[14].weightedsettype[0] +datatype[14].structtype[1] +datatype[14].structtype[0].name music_attribute.header +datatype[14].structtype[0].version 0 +datatype[14].structtype[0].field[22] +datatype[14].structtype[0].field[0].name sales +datatype[14].structtype[0].field[0].id[0] +datatype[14].structtype[0].field[0].datatype 0 +datatype[14].structtype[0].field[1].name pto +datatype[14].structtype[0].field[1].id[0] +datatype[14].structtype[0].field[1].datatype 0 +datatype[14].structtype[0].field[2].name mid +datatype[14].structtype[0].field[2].id[0] +datatype[14].structtype[0].field[2].datatype -1245117006 +datatype[14].structtype[0].field[3].name ew +datatype[14].structtype[0].field[3].id[0] +datatype[14].structtype[0].field[3].datatype 2 +datatype[14].structtype[0].field[4].name weight +datatype[14].structtype[0].field[4].id[0] +datatype[14].structtype[0].field[4].datatype 1 +datatype[14].structtype[0].field[5].name bgnpfrom +datatype[14].structtype[0].field[5].id[0] +datatype[14].structtype[0].field[5].datatype 1 +datatype[14].structtype[0].field[6].name artist +datatype[14].structtype[0].field[6].id[0] +datatype[14].structtype[0].field[6].datatype 2 +datatype[14].structtype[0].field[7].name artistspid +datatype[14].structtype[0].field[7].id[0] +datatype[14].structtype[0].field[7].datatype 1328286588 +datatype[14].structtype[0].field[8].name artistspid2 +datatype[14].structtype[0].field[8].id[0] +datatype[14].structtype[0].field[8].datatype 1325751891 +datatype[14].structtype[0].field[9].name title +datatype[14].structtype[0].field[9].id[0] +datatype[14].structtype[0].field[9].datatype 2 +datatype[14].structtype[0].field[10].name newestedition +datatype[14].structtype[0].field[10].id[0] +datatype[14].structtype[0].field[10].datatype 0 +datatype[14].structtype[0].field[11].name year +datatype[14].structtype[0].field[11].id[0] +datatype[14].structtype[0].field[11].datatype -1245117006 +datatype[14].structtype[0].field[12].name endyear +datatype[14].structtype[0].field[12].id[0] +datatype[14].structtype[0].field[12].datatype -1245117006 +datatype[14].structtype[0].field[13].name did +datatype[14].structtype[0].field[13].id[0] +datatype[14].structtype[0].field[13].datatype 0 +datatype[14].structtype[0].field[14].name cbid +datatype[14].structtype[0].field[14].id[0] +datatype[14].structtype[0].field[14].datatype 0 +datatype[14].structtype[0].field[15].name noupdate +datatype[14].structtype[0].field[15].id[0] +datatype[14].structtype[0].field[15].datatype 2 +datatype[14].structtype[0].field[16].name noupdate2 +datatype[14].structtype[0].field[16].id[0] +datatype[14].structtype[0].field[16].datatype 2 +datatype[14].structtype[0].field[17].name multiposition2d_position +datatype[14].structtype[0].field[17].id[0] +datatype[14].structtype[0].field[17].datatype 58874399 +datatype[14].structtype[0].field[18].name extracategories +datatype[14].structtype[0].field[18].id[0] +datatype[14].structtype[0].field[18].datatype -1486737430 +datatype[14].structtype[0].field[19].name default_fieldlength +datatype[14].structtype[0].field[19].id[0] +datatype[14].structtype[0].field[19].datatype -1245117006 +datatype[14].structtype[0].field[20].name fmt_fieldlength +datatype[14].structtype[0].field[20].id[0] +datatype[14].structtype[0].field[20].datatype 0 +datatype[14].structtype[0].field[21].name categories_fieldlength +datatype[14].structtype[0].field[21].id[0] +datatype[14].structtype[0].field[21].datatype 0 +datatype[14].documenttype[0] +datatype[15].id 1243829584 +datatype[15].arraytype[0] +datatype[15].weightedsettype[0] +datatype[15].structtype[1] +datatype[15].structtype[0].name music_attribute.body +datatype[15].structtype[0].version 0 +datatype[15].structtype[0].field[0] +datatype[15].documenttype[0] +datatype[16].id 1990571588 +datatype[16].arraytype[0] +datatype[16].weightedsettype[0] +datatype[16].structtype[0] +datatype[16].documenttype[1] +datatype[16].documenttype[0].name music_attribute +datatype[16].documenttype[0].version 0 +datatype[16].documenttype[0].inherits[0] +datatype[16].documenttype[0].headerstruct -1497398149 +datatype[16].documenttype[0].bodystruct 1243829584 +datatype[17].id -592896846 +datatype[17].arraytype[0] +datatype[17].weightedsettype[0] +datatype[17].structtype[1] +datatype[17].structtype[0].name indexingdocument.header +datatype[17].structtype[0].version 0 +datatype[17].structtype[0].field[3] +datatype[17].structtype[0].field[0].name index +datatype[17].structtype[0].field[0].id[0] +datatype[17].structtype[0].field[0].datatype 8 +datatype[17].structtype[0].field[1].name summary +datatype[17].structtype[0].field[1].id[0] +datatype[17].structtype[0].field[1].datatype 8 +datatype[17].structtype[0].field[2].name attribute +datatype[17].structtype[0].field[2].id[0] +datatype[17].structtype[0].field[2].datatype 8 +datatype[17].documenttype[0] +datatype[18].id -2093772985 +datatype[18].arraytype[0] +datatype[18].weightedsettype[0] +datatype[18].structtype[1] +datatype[18].structtype[0].name indexingdocument.body +datatype[18].structtype[0].version 0 +datatype[18].structtype[0].field[0] +datatype[18].documenttype[0] +datatype[19].id -1831281171 +datatype[19].arraytype[0] +datatype[19].weightedsettype[0] +datatype[19].structtype[0] +datatype[19].documenttype[1] +datatype[19].documenttype[0].name indexingdocument +datatype[19].documenttype[0].version 0 +datatype[19].documenttype[0].inherits[0] +datatype[19].documenttype[0].headerstruct -592896846 +datatype[19].documenttype[0].bodystruct -2093772985 diff --git a/indexinglanguage/src/test/cfg/documentmanager_inherit.cfg b/indexinglanguage/src/test/cfg/documentmanager_inherit.cfg new file mode 100644 index 00000000000..428c9049212 --- /dev/null +++ b/indexinglanguage/src/test/cfg/documentmanager_inherit.cfg @@ -0,0 +1,216 @@ +datatype[19] +datatype[0].id 2006483754 +datatype[0].arraytype[0] +datatype[0].weightedsettype[0] +datatype[0].structtype[1] +datatype[0].structtype[0].name newssummary.header +datatype[0].structtype[0].version 0 +datatype[0].structtype[0].field[4] +datatype[0].structtype[0].field[0].name uri +datatype[0].structtype[0].field[0].id[0] +datatype[0].structtype[0].field[0].datatype 2 +datatype[0].structtype[0].field[1].name where +datatype[0].structtype[0].field[1].id[0] +datatype[0].structtype[0].field[1].datatype 2 +datatype[0].structtype[0].field[2].name title +datatype[0].structtype[0].field[2].id[0] +datatype[0].structtype[0].field[2].datatype 2 +datatype[0].structtype[0].field[3].name weight +datatype[0].structtype[0].field[3].id[0] +datatype[0].structtype[0].field[3].datatype 1 +datatype[0].documenttype[0] +datatype[1].id -2059783233 +datatype[1].arraytype[0] +datatype[1].weightedsettype[0] +datatype[1].structtype[1] +datatype[1].structtype[0].name newssummary.body +datatype[1].structtype[0].version 0 +datatype[1].structtype[0].field[0] +datatype[1].documenttype[0] +datatype[2].id -756330891 +datatype[2].arraytype[0] +datatype[2].weightedsettype[0] +datatype[2].structtype[0] +datatype[2].documenttype[1] +datatype[2].documenttype[0].name newssummary +datatype[2].documenttype[0].version 0 +datatype[2].documenttype[0].inherits[0] +datatype[2].documenttype[0].headerstruct 2006483754 +datatype[2].documenttype[0].bodystruct -2059783233 +datatype[3].id 2010790819 +datatype[3].arraytype[0] +datatype[3].weightedsettype[0] +datatype[3].structtype[1] +datatype[3].structtype[0].name newssummary_summary.header +datatype[3].structtype[0].version 0 +datatype[3].structtype[0].field[6] +datatype[3].structtype[0].field[0].name sddocname +datatype[3].structtype[0].field[0].id[0] +datatype[3].structtype[0].field[0].datatype 2 +datatype[3].structtype[0].field[1].name uri +datatype[3].structtype[0].field[1].id[0] +datatype[3].structtype[0].field[1].datatype 2 +datatype[3].structtype[0].field[2].name title +datatype[3].structtype[0].field[2].id[0] +datatype[3].structtype[0].field[2].datatype 2 +datatype[3].structtype[0].field[3].name weight +datatype[3].structtype[0].field[3].id[0] +datatype[3].structtype[0].field[3].datatype 1 +datatype[3].structtype[0].field[4].name ranklog +datatype[3].structtype[0].field[4].id[0] +datatype[3].structtype[0].field[4].datatype 2 +datatype[3].structtype[0].field[5].name documentid +datatype[3].structtype[0].field[5].id[0] +datatype[3].structtype[0].field[5].datatype 2 +datatype[3].documenttype[0] +datatype[4].id 760329848 +datatype[4].arraytype[0] +datatype[4].weightedsettype[0] +datatype[4].structtype[1] +datatype[4].structtype[0].name newssummary_summary.body +datatype[4].structtype[0].version 0 +datatype[4].structtype[0].field[0] +datatype[4].documenttype[0] +datatype[5].id -1535558628 +datatype[5].arraytype[0] +datatype[5].weightedsettype[0] +datatype[5].structtype[0] +datatype[5].documenttype[1] +datatype[5].documenttype[0].name newssummary_summary +datatype[5].documenttype[0].version 0 +datatype[5].documenttype[0].inherits[0] +datatype[5].documenttype[0].headerstruct 2010790819 +datatype[5].documenttype[0].bodystruct 760329848 +datatype[6].id -1486737430 +datatype[6].arraytype[1] +datatype[6].arraytype[0].datatype 2 +datatype[6].weightedsettype[0] +datatype[6].structtype[0] +datatype[6].documenttype[0] +datatype[7].id -296931593 +datatype[7].arraytype[0] +datatype[7].weightedsettype[0] +datatype[7].structtype[1] +datatype[7].structtype[0].name newssummary_index.header +datatype[7].structtype[0].version 0 +datatype[7].structtype[0].field[2] +datatype[7].structtype[0].field[0].name sddocname +datatype[7].structtype[0].field[0].id[0] +datatype[7].structtype[0].field[0].datatype -1486737430 +datatype[7].structtype[0].field[1].name title +datatype[7].structtype[0].field[1].id[0] +datatype[7].structtype[0].field[1].datatype -1486737430 +datatype[7].documenttype[0] +datatype[8].id -2066649396 +datatype[8].arraytype[0] +datatype[8].weightedsettype[0] +datatype[8].structtype[1] +datatype[8].structtype[0].name newssummary_index.body +datatype[8].structtype[0].version 0 +datatype[8].structtype[0].field[0] +datatype[8].documenttype[0] +datatype[9].id 1957994312 +datatype[9].arraytype[0] +datatype[9].weightedsettype[0] +datatype[9].structtype[0] +datatype[9].documenttype[1] +datatype[9].documenttype[0].name newssummary_index +datatype[9].documenttype[0].version 0 +datatype[9].documenttype[0].inherits[0] +datatype[9].documenttype[0].headerstruct -296931593 +datatype[9].documenttype[0].bodystruct -2066649396 +datatype[10].id -1089205651 +datatype[10].arraytype[0] +datatype[10].weightedsettype[0] +datatype[10].structtype[1] +datatype[10].structtype[0].name newssummary_attribute.header +datatype[10].structtype[0].version 0 +datatype[10].structtype[0].field[1] +datatype[10].structtype[0].field[0].name weight +datatype[10].structtype[0].field[0].id[0] +datatype[10].structtype[0].field[0].datatype 1 +datatype[10].documenttype[0] +datatype[11].id 761573314 +datatype[11].arraytype[0] +datatype[11].weightedsettype[0] +datatype[11].structtype[1] +datatype[11].structtype[0].name newssummary_attribute.body +datatype[11].structtype[0].version 0 +datatype[11].structtype[0].field[0] +datatype[11].documenttype[0] +datatype[12].id -1613882222 +datatype[12].arraytype[0] +datatype[12].weightedsettype[0] +datatype[12].structtype[0] +datatype[12].documenttype[1] +datatype[12].documenttype[0].name newssummary_attribute +datatype[12].documenttype[0].version 0 +datatype[12].documenttype[0].inherits[0] +datatype[12].documenttype[0].headerstruct -1089205651 +datatype[12].documenttype[0].bodystruct 761573314 +datatype[13].id 2098419674 +datatype[13].arraytype[0] +datatype[13].weightedsettype[0] +datatype[13].structtype[1] +datatype[13].structtype[0].name newsarticle.header +datatype[13].structtype[0].version 0 +datatype[13].structtype[0].field[1] +datatype[13].structtype[0].field[0].name city +datatype[13].structtype[0].field[0].id[0] +datatype[13].structtype[0].field[0].datatype 2 +datatype[13].documenttype[0] +datatype[14].id 197293167 +datatype[14].arraytype[0] +datatype[14].weightedsettype[0] +datatype[14].structtype[1] +datatype[14].structtype[0].name newsarticle.body +datatype[14].structtype[0].version 0 +datatype[14].structtype[0].field[0] +datatype[14].documenttype[0] +datatype[15].id -1710661691 +datatype[15].arraytype[0] +datatype[15].weightedsettype[0] +datatype[15].structtype[0] +datatype[15].documenttype[1] +datatype[15].documenttype[0].name newsarticle +datatype[15].documenttype[0].version 0 +datatype[15].documenttype[0].inherits[1] +datatype[15].documenttype[0].inherits[0].name newssummary +datatype[15].documenttype[0].inherits[0].version 0 +datatype[15].documenttype[0].headerstruct 2098419674 +datatype[15].documenttype[0].bodystruct 197293167 +datatype[16].id -592896846 +datatype[16].arraytype[0] +datatype[16].weightedsettype[0] +datatype[16].structtype[1] +datatype[16].structtype[0].name indexingdocument.header +datatype[16].structtype[0].version 0 +datatype[16].structtype[0].field[3] +datatype[16].structtype[0].field[0].name index +datatype[16].structtype[0].field[0].id[0] +datatype[16].structtype[0].field[0].datatype 8 +datatype[16].structtype[0].field[1].name summary +datatype[16].structtype[0].field[1].id[0] +datatype[16].structtype[0].field[1].datatype 8 +datatype[16].structtype[0].field[2].name attribute +datatype[16].structtype[0].field[2].id[0] +datatype[16].structtype[0].field[2].datatype 8 +datatype[16].documenttype[0] +datatype[17].id -2093772985 +datatype[17].arraytype[0] +datatype[17].weightedsettype[0] +datatype[17].structtype[1] +datatype[17].structtype[0].name indexingdocument.body +datatype[17].structtype[0].version 0 +datatype[17].structtype[0].field[0] +datatype[17].documenttype[0] +datatype[18].id -1831281171 +datatype[18].arraytype[0] +datatype[18].weightedsettype[0] +datatype[18].structtype[0] +datatype[18].documenttype[1] +datatype[18].documenttype[0].name indexingdocument +datatype[18].documenttype[0].version 0 +datatype[18].documenttype[0].inherits[0] +datatype[18].documenttype[0].headerstruct -592896846 +datatype[18].documenttype[0].bodystruct -2093772985 diff --git a/indexinglanguage/src/test/cfg/exactmatch/documentmanager.cfg b/indexinglanguage/src/test/cfg/exactmatch/documentmanager.cfg new file mode 100644 index 00000000000..6a4cc7e7d37 --- /dev/null +++ b/indexinglanguage/src/test/cfg/exactmatch/documentmanager.cfg @@ -0,0 +1,167 @@ +datatype[16] +datatype[0].id 125394903 +datatype[0].arraytype[0] +datatype[0].weightedsettype[0] +datatype[0].structtype[1] +datatype[0].structtype[0].name exactmatch.header +datatype[0].structtype[0].version 0 +datatype[0].structtype[0].field[1] +datatype[0].structtype[0].field[0].name field3 +datatype[0].structtype[0].field[0].id[0] +datatype[0].structtype[0].field[0].datatype 2 +datatype[0].documenttype[0] +datatype[1].id 588535724 +datatype[1].arraytype[0] +datatype[1].weightedsettype[0] +datatype[1].structtype[1] +datatype[1].structtype[0].name exactmatch.body +datatype[1].structtype[0].version 0 +datatype[1].structtype[0].field[0] +datatype[1].documenttype[0] +datatype[2].id -21255576 +datatype[2].arraytype[0] +datatype[2].weightedsettype[0] +datatype[2].structtype[0] +datatype[2].documenttype[1] +datatype[2].documenttype[0].name exactmatch +datatype[2].documenttype[0].version 0 +datatype[2].documenttype[0].inherits[0] +datatype[2].documenttype[0].headerstruct 125394903 +datatype[2].documenttype[0].bodystruct 588535724 +datatype[3].id 694033232 +datatype[3].arraytype[0] +datatype[3].weightedsettype[0] +datatype[3].structtype[1] +datatype[3].structtype[0].name exactmatch_summary.header +datatype[3].structtype[0].version 0 +datatype[3].structtype[0].field[4] +datatype[3].structtype[0].field[0].name sddocname +datatype[3].structtype[0].field[0].id[0] +datatype[3].structtype[0].field[0].datatype 2 +datatype[3].structtype[0].field[1].name field3 +datatype[3].structtype[0].field[1].id[0] +datatype[3].structtype[0].field[1].datatype 2 +datatype[3].structtype[0].field[2].name ranklog +datatype[3].structtype[0].field[2].id[0] +datatype[3].structtype[0].field[2].datatype 2 +datatype[3].structtype[0].field[3].name documentid +datatype[3].structtype[0].field[3].id[0] +datatype[3].structtype[0].field[3].datatype 2 +datatype[3].documenttype[0] +datatype[4].id -273441435 +datatype[4].arraytype[0] +datatype[4].weightedsettype[0] +datatype[4].structtype[1] +datatype[4].structtype[0].name exactmatch_summary.body +datatype[4].structtype[0].version 0 +datatype[4].structtype[0].field[0] +datatype[4].documenttype[0] +datatype[5].id 280229135 +datatype[5].arraytype[0] +datatype[5].weightedsettype[0] +datatype[5].structtype[0] +datatype[5].documenttype[1] +datatype[5].documenttype[0].name exactmatch_summary +datatype[5].documenttype[0].version 0 +datatype[5].documenttype[0].inherits[0] +datatype[5].documenttype[0].headerstruct 694033232 +datatype[5].documenttype[0].bodystruct -273441435 +datatype[6].id -1486737430 +datatype[6].arraytype[1] +datatype[6].arraytype[0].datatype 2 +datatype[6].weightedsettype[0] +datatype[6].structtype[0] +datatype[6].documenttype[0] +datatype[7].id -1330702876 +datatype[7].arraytype[0] +datatype[7].weightedsettype[0] +datatype[7].structtype[1] +datatype[7].structtype[0].name exactmatch_index.header +datatype[7].structtype[0].version 0 +datatype[7].structtype[0].field[2] +datatype[7].structtype[0].field[0].name sddocname +datatype[7].structtype[0].field[0].id[0] +datatype[7].structtype[0].field[0].datatype -1486737430 +datatype[7].structtype[0].field[1].name field3 +datatype[7].structtype[0].field[1].id[0] +datatype[7].structtype[0].field[1].datatype -1486737430 +datatype[7].documenttype[0] +datatype[8].id 1248472313 +datatype[8].arraytype[0] +datatype[8].weightedsettype[0] +datatype[8].structtype[1] +datatype[8].structtype[0].name exactmatch_index.body +datatype[8].structtype[0].version 0 +datatype[8].structtype[0].field[0] +datatype[8].documenttype[0] +datatype[9].id -1843463941 +datatype[9].arraytype[0] +datatype[9].weightedsettype[0] +datatype[9].structtype[0] +datatype[9].documenttype[1] +datatype[9].documenttype[0].name exactmatch_index +datatype[9].documenttype[0].version 0 +datatype[9].documenttype[0].inherits[0] +datatype[9].documenttype[0].headerstruct -1330702876 +datatype[9].documenttype[0].bodystruct 1248472313 +datatype[10].id 522105562 +datatype[10].arraytype[0] +datatype[10].weightedsettype[0] +datatype[10].structtype[1] +datatype[10].structtype[0].name exactmatch_attribute.header +datatype[10].structtype[0].version 0 +datatype[10].structtype[0].field[0] +datatype[10].documenttype[0] +datatype[11].id -555184273 +datatype[11].arraytype[0] +datatype[11].weightedsettype[0] +datatype[11].structtype[1] +datatype[11].structtype[0].name exactmatch_attribute.body +datatype[11].structtype[0].version 0 +datatype[11].structtype[0].field[0] +datatype[11].documenttype[0] +datatype[12].id -398564155 +datatype[12].arraytype[0] +datatype[12].weightedsettype[0] +datatype[12].structtype[0] +datatype[12].documenttype[1] +datatype[12].documenttype[0].name exactmatch_attribute +datatype[12].documenttype[0].version 0 +datatype[12].documenttype[0].inherits[0] +datatype[12].documenttype[0].headerstruct 522105562 +datatype[12].documenttype[0].bodystruct -555184273 +datatype[13].id -592896846 +datatype[13].arraytype[0] +datatype[13].weightedsettype[0] +datatype[13].structtype[1] +datatype[13].structtype[0].name indexingdocument.header +datatype[13].structtype[0].version 0 +datatype[13].structtype[0].field[3] +datatype[13].structtype[0].field[0].name index +datatype[13].structtype[0].field[0].id[0] +datatype[13].structtype[0].field[0].datatype -1843463941 +datatype[13].structtype[0].field[1].name summary +datatype[13].structtype[0].field[1].id[0] +datatype[13].structtype[0].field[1].datatype 280229135 +datatype[13].structtype[0].field[2].name attribute +datatype[13].structtype[0].field[2].id[0] +datatype[13].structtype[0].field[2].datatype -398564155 +datatype[13].documenttype[0] +datatype[14].id -2093772985 +datatype[14].arraytype[0] +datatype[14].weightedsettype[0] +datatype[14].structtype[1] +datatype[14].structtype[0].name indexingdocument.body +datatype[14].structtype[0].version 0 +datatype[14].structtype[0].field[0] +datatype[14].documenttype[0] +datatype[15].id -1831281171 +datatype[15].arraytype[0] +datatype[15].weightedsettype[0] +datatype[15].structtype[0] +datatype[15].documenttype[1] +datatype[15].documenttype[0].name indexingdocument +datatype[15].documenttype[0].version 0 +datatype[15].documenttype[0].inherits[0] +datatype[15].documenttype[0].headerstruct -592896846 +datatype[15].documenttype[0].bodystruct -2093772985 diff --git a/indexinglanguage/src/test/cfg/exactmatch/indexingdocument.cfg b/indexinglanguage/src/test/cfg/exactmatch/indexingdocument.cfg new file mode 100644 index 00000000000..761bd4ff532 --- /dev/null +++ b/indexinglanguage/src/test/cfg/exactmatch/indexingdocument.cfg @@ -0,0 +1,20 @@ +indexingdoc[1] +indexingdoc[0].name exactmatch +indexingdoc[0].source[1] +indexingdoc[0].source[0] exactmatch +indexingdoc[0].context[2] +indexingdoc[0].context[0].fullname sddocname.sddocname +indexingdoc[0].context[0].shortname sddocname +indexingdoc[0].context[0].datatype 2 +indexingdoc[0].context[1].fullname field3.field3 +indexingdoc[0].context[1].shortname field3 +indexingdoc[0].context[1].datatype 2 +indexingdoc[0].summary[4] +indexingdoc[0].summary[0].name sddocname +indexingdoc[0].summary[0].datatype 2 +indexingdoc[0].summary[1].name field3 +indexingdoc[0].summary[1].datatype 2 +indexingdoc[0].summary[2].name ranklog +indexingdoc[0].summary[2].datatype 2 +indexingdoc[0].summary[3].name documentid +indexingdoc[0].summary[3].datatype 2 diff --git a/indexinglanguage/src/test/cfg/fileio/documentmanager.cfg b/indexinglanguage/src/test/cfg/fileio/documentmanager.cfg new file mode 100644 index 00000000000..bd7a5645dd8 --- /dev/null +++ b/indexinglanguage/src/test/cfg/fileio/documentmanager.cfg @@ -0,0 +1,425 @@ +datatype[17] +datatype[0].id -1216487824 +datatype[0].arraytype[1] +datatype[0].arraytype[0].datatype 0 +datatype[0].weightedsettype[0] +datatype[0].structtype[0] +datatype[0].documenttype[0] +datatype[1].id -1910204744 +datatype[1].arraytype[0] +datatype[1].weightedsettype[0] +datatype[1].structtype[1] +datatype[1].structtype[0].name music.header +datatype[1].structtype[0].version 0 +datatype[1].structtype[0].field[33] +datatype[1].structtype[0].field[0].name bgndata +datatype[1].structtype[0].field[0].id[0] +datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[1].name sales +datatype[1].structtype[0].field[1].id[0] +datatype[1].structtype[0].field[1].datatype 0 +datatype[1].structtype[0].field[2].name pto +datatype[1].structtype[0].field[2].id[0] +datatype[1].structtype[0].field[2].datatype 0 +datatype[1].structtype[0].field[3].name keys +datatype[1].structtype[0].field[3].id[0] +datatype[1].structtype[0].field[3].datatype 2 +datatype[1].structtype[0].field[4].name mid +datatype[1].structtype[0].field[4].id[0] +datatype[1].structtype[0].field[4].datatype -1216487824 +datatype[1].structtype[0].field[5].name ew +datatype[1].structtype[0].field[5].id[0] +datatype[1].structtype[0].field[5].datatype 2 +datatype[1].structtype[0].field[6].name surl +datatype[1].structtype[0].field[6].id[0] +datatype[1].structtype[0].field[6].datatype 2 +datatype[1].structtype[0].field[7].name userrate +datatype[1].structtype[0].field[7].id[0] +datatype[1].structtype[0].field[7].datatype 0 +datatype[1].structtype[0].field[8].name pid +datatype[1].structtype[0].field[8].id[0] +datatype[1].structtype[0].field[8].datatype 2 +datatype[1].structtype[0].field[9].name weight +datatype[1].structtype[0].field[9].id[0] +datatype[1].structtype[0].field[9].datatype 1 +datatype[1].structtype[0].field[10].name url +datatype[1].structtype[0].field[10].id[0] +datatype[1].structtype[0].field[10].datatype 2 +datatype[1].structtype[0].field[11].name isbn +datatype[1].structtype[0].field[11].id[0] +datatype[1].structtype[0].field[11].datatype 2 +datatype[1].structtype[0].field[12].name fmt +datatype[1].structtype[0].field[12].id[0] +datatype[1].structtype[0].field[12].datatype 2 +datatype[1].structtype[0].field[13].name albumid +datatype[1].structtype[0].field[13].id[0] +datatype[1].structtype[0].field[13].datatype 2 +datatype[1].structtype[0].field[14].name disp_song +datatype[1].structtype[0].field[14].id[0] +datatype[1].structtype[0].field[14].datatype 2 +datatype[1].structtype[0].field[15].name song +datatype[1].structtype[0].field[15].id[0] +datatype[1].structtype[0].field[15].datatype 2 +datatype[1].structtype[0].field[16].name pfrom +datatype[1].structtype[0].field[16].id[0] +datatype[1].structtype[0].field[16].datatype 0 +datatype[1].structtype[0].field[17].name bgnpfrom +datatype[1].structtype[0].field[17].id[0] +datatype[1].structtype[0].field[17].datatype 1 +datatype[1].structtype[0].field[18].name categories +datatype[1].structtype[0].field[18].id[0] +datatype[1].structtype[0].field[18].datatype 2 +datatype[1].structtype[0].field[19].name data +datatype[1].structtype[0].field[19].id[0] +datatype[1].structtype[0].field[19].datatype 2 +datatype[1].structtype[0].field[20].name numreview +datatype[1].structtype[0].field[20].id[0] +datatype[1].structtype[0].field[20].datatype 0 +datatype[1].structtype[0].field[21].name bgnsellers +datatype[1].structtype[0].field[21].id[0] +datatype[1].structtype[0].field[21].datatype 0 +datatype[1].structtype[0].field[22].name image +datatype[1].structtype[0].field[22].id[0] +datatype[1].structtype[0].field[22].datatype 2 +datatype[1].structtype[0].field[23].name artist +datatype[1].structtype[0].field[23].id[0] +datatype[1].structtype[0].field[23].datatype 2 +datatype[1].structtype[0].field[24].name artistspid +datatype[1].structtype[0].field[24].id[0] +datatype[1].structtype[0].field[24].datatype 18 +datatype[1].structtype[0].field[25].name title +datatype[1].structtype[0].field[25].id[0] +datatype[1].structtype[0].field[25].datatype 2 +datatype[1].structtype[0].field[26].name newestedition +datatype[1].structtype[0].field[26].id[0] +datatype[1].structtype[0].field[26].datatype 0 +datatype[1].structtype[0].field[27].name bgnpto +datatype[1].structtype[0].field[27].id[0] +datatype[1].structtype[0].field[27].datatype 2 +datatype[1].structtype[0].field[28].name year +datatype[1].structtype[0].field[28].id[0] +datatype[1].structtype[0].field[28].datatype -1216487824 +datatype[1].structtype[0].field[29].name did +datatype[1].structtype[0].field[29].id[0] +datatype[1].structtype[0].field[29].datatype 0 +datatype[1].structtype[0].field[30].name scorekey +datatype[1].structtype[0].field[30].id[0] +datatype[1].structtype[0].field[30].datatype 0 +datatype[1].structtype[0].field[31].name cbid +datatype[1].structtype[0].field[31].id[0] +datatype[1].structtype[0].field[31].datatype 0 +datatype[1].structtype[0].field[32].name indexingdocument +datatype[1].structtype[0].field[32].id[0] +datatype[1].structtype[0].field[32].datatype 8 +datatype[1].documenttype[0] +datatype[2].id 993120973 +datatype[2].arraytype[0] +datatype[2].weightedsettype[0] +datatype[2].structtype[1] +datatype[2].structtype[0].name music.body +datatype[2].structtype[0].version 0 +datatype[2].structtype[0].field[0] +datatype[2].documenttype[0] +datatype[3].id 1412693671 +datatype[3].arraytype[0] +datatype[3].weightedsettype[0] +datatype[3].structtype[0] +datatype[3].documenttype[1] +datatype[3].documenttype[0].name music +datatype[3].documenttype[0].version 0 +datatype[3].documenttype[0].inherits[0] +datatype[3].documenttype[0].headerstruct -1910204744 +datatype[3].documenttype[0].bodystruct 993120973 +datatype[4].id -1801920207 +datatype[4].arraytype[0] +datatype[4].weightedsettype[0] +datatype[4].structtype[1] +datatype[4].structtype[0].name music_summary.header +datatype[4].structtype[0].version 0 +datatype[4].structtype[0].field[32] +datatype[4].structtype[0].field[0].name sddocname +datatype[4].structtype[0].field[0].id[0] +datatype[4].structtype[0].field[0].datatype 2 +datatype[4].structtype[0].field[1].name bgndata +datatype[4].structtype[0].field[1].id[0] +datatype[4].structtype[0].field[1].datatype 2 +datatype[4].structtype[0].field[2].name sales +datatype[4].structtype[0].field[2].id[0] +datatype[4].structtype[0].field[2].datatype 0 +datatype[4].structtype[0].field[3].name pto +datatype[4].structtype[0].field[3].id[0] +datatype[4].structtype[0].field[3].datatype 0 +datatype[4].structtype[0].field[4].name mid +datatype[4].structtype[0].field[4].id[0] +datatype[4].structtype[0].field[4].datatype 0 +datatype[4].structtype[0].field[5].name ew +datatype[4].structtype[0].field[5].id[0] +datatype[4].structtype[0].field[5].datatype 2 +datatype[4].structtype[0].field[6].name surl +datatype[4].structtype[0].field[6].id[0] +datatype[4].structtype[0].field[6].datatype 2 +datatype[4].structtype[0].field[7].name userrate +datatype[4].structtype[0].field[7].id[0] +datatype[4].structtype[0].field[7].datatype 0 +datatype[4].structtype[0].field[8].name pid +datatype[4].structtype[0].field[8].id[0] +datatype[4].structtype[0].field[8].datatype 2 +datatype[4].structtype[0].field[9].name weight +datatype[4].structtype[0].field[9].id[0] +datatype[4].structtype[0].field[9].datatype 1 +datatype[4].structtype[0].field[10].name url +datatype[4].structtype[0].field[10].id[0] +datatype[4].structtype[0].field[10].datatype 2 +datatype[4].structtype[0].field[11].name isbn +datatype[4].structtype[0].field[11].id[0] +datatype[4].structtype[0].field[11].datatype 2 +datatype[4].structtype[0].field[12].name fmt +datatype[4].structtype[0].field[12].id[0] +datatype[4].structtype[0].field[12].datatype 2 +datatype[4].structtype[0].field[13].name albumid +datatype[4].structtype[0].field[13].id[0] +datatype[4].structtype[0].field[13].datatype 2 +datatype[4].structtype[0].field[14].name disp_song +datatype[4].structtype[0].field[14].id[0] +datatype[4].structtype[0].field[14].datatype 2 +datatype[4].structtype[0].field[15].name song +datatype[4].structtype[0].field[15].id[0] +datatype[4].structtype[0].field[15].datatype 2 +datatype[4].structtype[0].field[16].name pfrom +datatype[4].structtype[0].field[16].id[0] +datatype[4].structtype[0].field[16].datatype 0 +datatype[4].structtype[0].field[17].name bgnpfrom +datatype[4].structtype[0].field[17].id[0] +datatype[4].structtype[0].field[17].datatype 1 +datatype[4].structtype[0].field[18].name categories +datatype[4].structtype[0].field[18].id[0] +datatype[4].structtype[0].field[18].datatype 2 +datatype[4].structtype[0].field[19].name data +datatype[4].structtype[0].field[19].id[0] +datatype[4].structtype[0].field[19].datatype 2 +datatype[4].structtype[0].field[20].name numreview +datatype[4].structtype[0].field[20].id[0] +datatype[4].structtype[0].field[20].datatype 0 +datatype[4].structtype[0].field[21].name bgnsellers +datatype[4].structtype[0].field[21].id[0] +datatype[4].structtype[0].field[21].datatype 0 +datatype[4].structtype[0].field[22].name image +datatype[4].structtype[0].field[22].id[0] +datatype[4].structtype[0].field[22].datatype 2 +datatype[4].structtype[0].field[23].name artist +datatype[4].structtype[0].field[23].id[0] +datatype[4].structtype[0].field[23].datatype 2 +datatype[4].structtype[0].field[24].name title +datatype[4].structtype[0].field[24].id[0] +datatype[4].structtype[0].field[24].datatype 2 +datatype[4].structtype[0].field[25].name newestedition +datatype[4].structtype[0].field[25].id[0] +datatype[4].structtype[0].field[25].datatype 0 +datatype[4].structtype[0].field[26].name bgnpto +datatype[4].structtype[0].field[26].id[0] +datatype[4].structtype[0].field[26].datatype 2 +datatype[4].structtype[0].field[27].name year +datatype[4].structtype[0].field[27].id[0] +datatype[4].structtype[0].field[27].datatype 0 +datatype[4].structtype[0].field[28].name did +datatype[4].structtype[0].field[28].id[0] +datatype[4].structtype[0].field[28].datatype 0 +datatype[4].structtype[0].field[29].name scorekey +datatype[4].structtype[0].field[29].id[0] +datatype[4].structtype[0].field[29].datatype 0 +datatype[4].structtype[0].field[30].name cbid +datatype[4].structtype[0].field[30].id[0] +datatype[4].structtype[0].field[30].datatype 0 +datatype[4].structtype[0].field[31].name ranklog +datatype[4].structtype[0].field[31].id[0] +datatype[4].structtype[0].field[31].datatype 2 +datatype[4].documenttype[0] +datatype[5].id -1728551034 +datatype[5].arraytype[0] +datatype[5].weightedsettype[0] +datatype[5].structtype[1] +datatype[5].structtype[0].name music_summary.body +datatype[5].structtype[0].version 0 +datatype[5].structtype[0].field[0] +datatype[5].documenttype[0] +datatype[6].id 1601149518 +datatype[6].arraytype[0] +datatype[6].weightedsettype[0] +datatype[6].structtype[0] +datatype[6].documenttype[1] +datatype[6].documenttype[0].name music_summary +datatype[6].documenttype[0].version 0 +datatype[6].documenttype[0].inherits[0] +datatype[6].documenttype[0].headerstruct -1801920207 +datatype[6].documenttype[0].bodystruct -1728551034 +datatype[7].id 1000775434 +datatype[7].arraytype[1] +datatype[7].arraytype[0].datatype 2 +datatype[7].weightedsettype[0] +datatype[7].structtype[0] +datatype[7].documenttype[0] +datatype[8].id 1509154821 +datatype[8].arraytype[0] +datatype[8].weightedsettype[0] +datatype[8].structtype[1] +datatype[8].structtype[0].name music_index.header +datatype[8].structtype[0].version 0 +datatype[8].structtype[0].field[16] +datatype[8].structtype[0].field[0].name sddocname +datatype[8].structtype[0].field[0].id[0] +datatype[8].structtype[0].field[0].datatype 1000775434 +datatype[8].structtype[0].field[1].name sales +datatype[8].structtype[0].field[1].id[0] +datatype[8].structtype[0].field[1].datatype -1216487824 +datatype[8].structtype[0].field[2].name pto +datatype[8].structtype[0].field[2].id[0] +datatype[8].structtype[0].field[2].datatype -1216487824 +datatype[8].structtype[0].field[3].name keys +datatype[8].structtype[0].field[3].id[0] +datatype[8].structtype[0].field[3].datatype 1000775434 +datatype[8].structtype[0].field[4].name mid +datatype[8].structtype[0].field[4].id[0] +datatype[8].structtype[0].field[4].datatype -1216487824 +datatype[8].structtype[0].field[5].name ew +datatype[8].structtype[0].field[5].id[0] +datatype[8].structtype[0].field[5].datatype 1000775434 +datatype[8].structtype[0].field[6].name fmt +datatype[8].structtype[0].field[6].id[0] +datatype[8].structtype[0].field[6].datatype 1000775434 +datatype[8].structtype[0].field[7].name song +datatype[8].structtype[0].field[7].id[0] +datatype[8].structtype[0].field[7].datatype 1000775434 +datatype[8].structtype[0].field[8].name categories +datatype[8].structtype[0].field[8].id[0] +datatype[8].structtype[0].field[8].datatype 1000775434 +datatype[8].structtype[0].field[9].name artist +datatype[8].structtype[0].field[9].id[0] +datatype[8].structtype[0].field[9].datatype 1000775434 +datatype[8].structtype[0].field[10].name title +datatype[8].structtype[0].field[10].id[0] +datatype[8].structtype[0].field[10].datatype 1000775434 +datatype[8].structtype[0].field[11].name newestedition +datatype[8].structtype[0].field[11].id[0] +datatype[8].structtype[0].field[11].datatype -1216487824 +datatype[8].structtype[0].field[12].name year +datatype[8].structtype[0].field[12].id[0] +datatype[8].structtype[0].field[12].datatype -1216487824 +datatype[8].structtype[0].field[13].name did +datatype[8].structtype[0].field[13].id[0] +datatype[8].structtype[0].field[13].datatype -1216487824 +datatype[8].structtype[0].field[14].name scorekey +datatype[8].structtype[0].field[14].id[0] +datatype[8].structtype[0].field[14].datatype -1216487824 +datatype[8].structtype[0].field[15].name cbid +datatype[8].structtype[0].field[15].id[0] +datatype[8].structtype[0].field[15].datatype -1216487824 +datatype[8].documenttype[0] +datatype[9].id -1997730982 +datatype[9].arraytype[0] +datatype[9].weightedsettype[0] +datatype[9].structtype[1] +datatype[9].structtype[0].name music_index.body +datatype[9].structtype[0].version 0 +datatype[9].structtype[0].field[0] +datatype[9].documenttype[0] +datatype[10].id 2108744186 +datatype[10].arraytype[0] +datatype[10].weightedsettype[0] +datatype[10].structtype[0] +datatype[10].documenttype[1] +datatype[10].documenttype[0].name music_index +datatype[10].documenttype[0].version 0 +datatype[10].documenttype[0].inherits[0] +datatype[10].documenttype[0].headerstruct 1509154821 +datatype[10].documenttype[0].bodystruct -1997730982 +datatype[11].id -1497398149 +datatype[11].arraytype[0] +datatype[11].weightedsettype[0] +datatype[11].structtype[1] +datatype[11].structtype[0].name music_attribute.header +datatype[11].structtype[0].version 0 +datatype[11].structtype[0].field[10] +datatype[11].structtype[0].field[0].name sales +datatype[11].structtype[0].field[0].id[0] +datatype[11].structtype[0].field[0].datatype 0 +datatype[11].structtype[0].field[1].name pto +datatype[11].structtype[0].field[1].id[0] +datatype[11].structtype[0].field[1].datatype 0 +datatype[11].structtype[0].field[2].name mid +datatype[11].structtype[0].field[2].id[0] +datatype[11].structtype[0].field[2].datatype -1216487824 +datatype[11].structtype[0].field[3].name weight +datatype[11].structtype[0].field[3].id[0] +datatype[11].structtype[0].field[3].datatype 1 +datatype[11].structtype[0].field[4].name bgnpfrom +datatype[11].structtype[0].field[4].id[0] +datatype[11].structtype[0].field[4].datatype 1 +datatype[11].structtype[0].field[5].name artistspid +datatype[11].structtype[0].field[5].id[0] +datatype[11].structtype[0].field[5].datatype 18 +datatype[11].structtype[0].field[6].name newestedition +datatype[11].structtype[0].field[6].id[0] +datatype[11].structtype[0].field[6].datatype 0 +datatype[11].structtype[0].field[7].name year +datatype[11].structtype[0].field[7].id[0] +datatype[11].structtype[0].field[7].datatype -1216487824 +datatype[11].structtype[0].field[8].name did +datatype[11].structtype[0].field[8].id[0] +datatype[11].structtype[0].field[8].datatype 0 +datatype[11].structtype[0].field[9].name cbid +datatype[11].structtype[0].field[9].id[0] +datatype[11].structtype[0].field[9].datatype 0 +datatype[11].documenttype[0] +datatype[12].id 1243829584 +datatype[12].arraytype[0] +datatype[12].weightedsettype[0] +datatype[12].structtype[1] +datatype[12].structtype[0].name music_attribute.body +datatype[12].structtype[0].version 0 +datatype[12].structtype[0].field[0] +datatype[12].documenttype[0] +datatype[13].id 1990571588 +datatype[13].arraytype[0] +datatype[13].weightedsettype[0] +datatype[13].structtype[0] +datatype[13].documenttype[1] +datatype[13].documenttype[0].name music_attribute +datatype[13].documenttype[0].version 0 +datatype[13].documenttype[0].inherits[0] +datatype[13].documenttype[0].headerstruct -1497398149 +datatype[13].documenttype[0].bodystruct 1243829584 +datatype[14].id -592896846 +datatype[14].arraytype[0] +datatype[14].weightedsettype[0] +datatype[14].structtype[1] +datatype[14].structtype[0].name indexingdocument.header +datatype[14].structtype[0].version 0 +datatype[14].structtype[0].field[3] +datatype[14].structtype[0].field[0].name index +datatype[14].structtype[0].field[0].id[0] +datatype[14].structtype[0].field[0].datatype 8 +datatype[14].structtype[0].field[1].name summary +datatype[14].structtype[0].field[1].id[0] +datatype[14].structtype[0].field[1].datatype 8 +datatype[14].structtype[0].field[2].name attribute +datatype[14].structtype[0].field[2].id[0] +datatype[14].structtype[0].field[2].datatype 8 +datatype[14].documenttype[0] +datatype[15].id -2093772985 +datatype[15].arraytype[0] +datatype[15].weightedsettype[0] +datatype[15].structtype[1] +datatype[15].structtype[0].name indexingdocument.body +datatype[15].structtype[0].version 0 +datatype[15].structtype[0].field[0] +datatype[15].documenttype[0] +datatype[16].id -1831281171 +datatype[16].arraytype[0] +datatype[16].weightedsettype[0] +datatype[16].structtype[0] +datatype[16].documenttype[1] +datatype[16].documenttype[0].name indexingdocument +datatype[16].documenttype[0].version 0 +datatype[16].documenttype[0].inherits[0] +datatype[16].documenttype[0].headerstruct -592896846 +datatype[16].documenttype[0].bodystruct -2093772985 diff --git a/indexinglanguage/src/test/cfg/fileio/indexingdocument.cfg b/indexinglanguage/src/test/cfg/fileio/indexingdocument.cfg new file mode 100644 index 00000000000..d41a56f65fb --- /dev/null +++ b/indexinglanguage/src/test/cfg/fileio/indexingdocument.cfg @@ -0,0 +1,166 @@ +indexingdoc[1] +indexingdoc[0].name music +indexingdoc[0].source[1] +indexingdoc[0].source[0] music +indexingdoc[0].context[16] +indexingdoc[0].context[0].fullname sddocname.sddocname +indexingdoc[0].context[0].shortname sddocname +indexingdoc[0].context[0].datatype 2 +indexingdoc[0].context[1].fullname sales.sales +indexingdoc[0].context[1].shortname sales +indexingdoc[0].context[1].datatype 0 +indexingdoc[0].context[2].fullname pto.pto +indexingdoc[0].context[2].shortname pto +indexingdoc[0].context[2].datatype 0 +indexingdoc[0].context[3].fullname default.keys +indexingdoc[0].context[3].shortname keys +indexingdoc[0].context[3].datatype 2 +indexingdoc[0].context[4].fullname mid.mid +indexingdoc[0].context[4].shortname mid +indexingdoc[0].context[4].datatype 0 +indexingdoc[0].context[5].fullname default.ew +indexingdoc[0].context[5].shortname ew +indexingdoc[0].context[5].datatype 2 +indexingdoc[0].context[6].fullname fmt.fmt +indexingdoc[0].context[6].shortname fmt +indexingdoc[0].context[6].datatype 2 +indexingdoc[0].context[7].fullname default.song +indexingdoc[0].context[7].shortname song +indexingdoc[0].context[7].datatype 2 +indexingdoc[0].context[8].fullname categories.categories +indexingdoc[0].context[8].shortname categories +indexingdoc[0].context[8].datatype 2 +indexingdoc[0].context[9].fullname default.artist +indexingdoc[0].context[9].shortname artist +indexingdoc[0].context[9].datatype 2 +indexingdoc[0].context[10].fullname default.title +indexingdoc[0].context[10].shortname title +indexingdoc[0].context[10].datatype 2 +indexingdoc[0].context[11].fullname newestedition.newestedition +indexingdoc[0].context[11].shortname newestedition +indexingdoc[0].context[11].datatype 0 +indexingdoc[0].context[12].fullname year.year +indexingdoc[0].context[12].shortname year +indexingdoc[0].context[12].datatype 0 +indexingdoc[0].context[13].fullname did.did +indexingdoc[0].context[13].shortname did +indexingdoc[0].context[13].datatype 0 +indexingdoc[0].context[14].fullname scorekey.scorekey +indexingdoc[0].context[14].shortname scorekey +indexingdoc[0].context[14].datatype 0 +indexingdoc[0].context[15].fullname cbid.cbid +indexingdoc[0].context[15].shortname cbid +indexingdoc[0].context[15].datatype 0 +indexingdoc[0].summary[32] +indexingdoc[0].summary[0].name sddocname +indexingdoc[0].summary[0].datatype 2 +indexingdoc[0].summary[1].name bgndata +indexingdoc[0].summary[1].datatype 2 +indexingdoc[0].summary[2].name sales +indexingdoc[0].summary[2].datatype 0 +indexingdoc[0].summary[3].name pto +indexingdoc[0].summary[3].datatype 0 +indexingdoc[0].summary[4].name mid +indexingdoc[0].summary[4].datatype 0 +indexingdoc[0].summary[5].name ew +indexingdoc[0].summary[5].datatype 2 +indexingdoc[0].summary[6].name surl +indexingdoc[0].summary[6].datatype 2 +indexingdoc[0].summary[7].name userrate +indexingdoc[0].summary[7].datatype 0 +indexingdoc[0].summary[8].name pid +indexingdoc[0].summary[8].datatype 2 +indexingdoc[0].summary[9].name weight +indexingdoc[0].summary[9].datatype 1 +indexingdoc[0].summary[10].name url +indexingdoc[0].summary[10].datatype 2 +indexingdoc[0].summary[11].name isbn +indexingdoc[0].summary[11].datatype 2 +indexingdoc[0].summary[12].name fmt +indexingdoc[0].summary[12].datatype 2 +indexingdoc[0].summary[13].name albumid +indexingdoc[0].summary[13].datatype 2 +indexingdoc[0].summary[14].name disp_song +indexingdoc[0].summary[14].datatype 2 +indexingdoc[0].summary[15].name song +indexingdoc[0].summary[15].datatype 2 +indexingdoc[0].summary[16].name pfrom +indexingdoc[0].summary[16].datatype 0 +indexingdoc[0].summary[17].name bgnpfrom +indexingdoc[0].summary[17].datatype 1 +indexingdoc[0].summary[18].name categories +indexingdoc[0].summary[18].datatype 2 +indexingdoc[0].summary[19].name data +indexingdoc[0].summary[19].datatype 2 +indexingdoc[0].summary[20].name numreview +indexingdoc[0].summary[20].datatype 0 +indexingdoc[0].summary[21].name bgnsellers +indexingdoc[0].summary[21].datatype 0 +indexingdoc[0].summary[22].name image +indexingdoc[0].summary[22].datatype 2 +indexingdoc[0].summary[23].name artist +indexingdoc[0].summary[23].datatype 2 +indexingdoc[0].summary[24].name title +indexingdoc[0].summary[24].datatype 2 +indexingdoc[0].summary[25].name newestedition +indexingdoc[0].summary[25].datatype 0 +indexingdoc[0].summary[26].name bgnpto +indexingdoc[0].summary[26].datatype 2 +indexingdoc[0].summary[27].name year +indexingdoc[0].summary[27].datatype 0 +indexingdoc[0].summary[28].name did +indexingdoc[0].summary[28].datatype 0 +indexingdoc[0].summary[29].name scorekey +indexingdoc[0].summary[29].datatype 0 +indexingdoc[0].summary[30].name cbid +indexingdoc[0].summary[30].datatype 0 +indexingdoc[0].summary[31].name ranklog +indexingdoc[0].summary[31].datatype 2 +indexingdoc[0].attribute[13] +indexingdoc[0].attribute[0].name sales +indexingdoc[0].attribute[0].datatype 0 +indexingdoc[0].attribute[0].containertype SINGLE +indexingdoc[0].attribute[1].name pto +indexingdoc[0].attribute[1].datatype 0 +indexingdoc[0].attribute[1].containertype SINGLE +indexingdoc[0].attribute[2].name mid +indexingdoc[0].attribute[2].datatype 0 +indexingdoc[0].attribute[2].containertype ARRAY +indexingdoc[0].attribute[3].name weight +indexingdoc[0].attribute[3].datatype 1 +indexingdoc[0].attribute[3].containertype SINGLE +indexingdoc[0].attribute[4].name bgnpfrom +indexingdoc[0].attribute[4].datatype 1 +indexingdoc[0].attribute[4].containertype SINGLE +indexingdoc[0].attribute[5].name artistspid +indexingdoc[0].attribute[5].datatype 2 +indexingdoc[0].attribute[5].containertype WEIGHTEDSET +indexingdoc[0].attribute[5].containerproperty[0] +indexingdoc[0].attribute[6].name newestedition +indexingdoc[0].attribute[6].datatype 0 +indexingdoc[0].attribute[6].containertype SINGLE +indexingdoc[0].attribute[7].name year +indexingdoc[0].attribute[7].datatype 0 +indexingdoc[0].attribute[7].containertype ARRAY +indexingdoc[0].attribute[8].name did +indexingdoc[0].attribute[8].datatype 0 +indexingdoc[0].attribute[8].containertype SINGLE +indexingdoc[0].attribute[9].name cbid +indexingdoc[0].attribute[9].datatype 0 +indexingdoc[0].attribute[9].containertype SINGLE +indexingdoc[0].attribute[10].name artistspid_c +indexingdoc[0].attribute[10].datatype 2 +indexingdoc[0].attribute[10].containertype WEIGHTEDSET +indexingdoc[0].attribute[10].containerproperty[1] +indexingdoc[0].attribute[10].containerproperty[0].propertyname "createifnonexistant" +indexingdoc[0].attribute[11].name artistspid_r +indexingdoc[0].attribute[11].datatype 2 +indexingdoc[0].attribute[11].containertype WEIGHTEDSET +indexingdoc[0].attribute[11].containerproperty[1] +indexingdoc[0].attribute[11].containerproperty[0].propertyname "removeifzero" +indexingdoc[0].attribute[12].name artistspid_c_r +indexingdoc[0].attribute[12].datatype 2 +indexingdoc[0].attribute[12].containertype WEIGHTEDSET +indexingdoc[0].attribute[12].containerproperty[2] +indexingdoc[0].attribute[12].containerproperty[0].propertyname "createifnonexistant" +indexingdoc[0].attribute[12].containerproperty[1].propertyname "removeifzero" diff --git a/indexinglanguage/src/test/cfg/indexingdocument.cfg b/indexinglanguage/src/test/cfg/indexingdocument.cfg new file mode 100644 index 00000000000..5bebe6e9d22 --- /dev/null +++ b/indexinglanguage/src/test/cfg/indexingdocument.cfg @@ -0,0 +1,210 @@ +indexingdoc[1] +indexingdoc[0].name music +indexingdoc[0].source[1] +indexingdoc[0].source[0] music +indexingdoc[0].context[19] +indexingdoc[0].context[0].fullname sddocname.sddocname +indexingdoc[0].context[0].shortname sddocname +indexingdoc[0].context[0].datatype 2 +indexingdoc[0].context[1].fullname sales.sales +indexingdoc[0].context[1].shortname sales +indexingdoc[0].context[1].datatype 0 +indexingdoc[0].context[2].fullname pto.pto +indexingdoc[0].context[2].shortname pto +indexingdoc[0].context[2].datatype 0 +indexingdoc[0].context[3].fullname default.keys +indexingdoc[0].context[3].shortname keys +indexingdoc[0].context[3].datatype 2 +indexingdoc[0].context[4].fullname mid.mid +indexingdoc[0].context[4].shortname mid +indexingdoc[0].context[4].datatype 0 +indexingdoc[0].context[5].fullname default.ew +indexingdoc[0].context[5].shortname ew +indexingdoc[0].context[5].datatype 2 +indexingdoc[0].context[6].fullname fmt.fmt +indexingdoc[0].context[6].shortname fmt +indexingdoc[0].context[6].datatype 2 +indexingdoc[0].context[7].fullname default.song +indexingdoc[0].context[7].shortname song +indexingdoc[0].context[7].datatype 2 +indexingdoc[0].context[8].fullname categories.categories +indexingdoc[0].context[8].shortname categories +indexingdoc[0].context[8].datatype 2 +indexingdoc[0].context[9].fullname default.artist +indexingdoc[0].context[9].shortname artist +indexingdoc[0].context[9].datatype 2 +indexingdoc[0].context[10].fullname artistspid3.artistspid3 +indexingdoc[0].context[10].shortname artistspid3 +indexingdoc[0].context[10].datatype 2 +indexingdoc[0].context[11].fullname default.title +indexingdoc[0].context[11].shortname title +indexingdoc[0].context[11].datatype 2 +indexingdoc[0].context[12].fullname newestedition.newestedition +indexingdoc[0].context[12].shortname newestedition +indexingdoc[0].context[12].datatype 0 +indexingdoc[0].context[13].fullname year.year +indexingdoc[0].context[13].shortname year +indexingdoc[0].context[13].datatype 0 +indexingdoc[0].context[14].fullname endyear.endyear +indexingdoc[0].context[14].shortname endyear +indexingdoc[0].context[14].datatype 0 +indexingdoc[0].context[15].fullname did.did +indexingdoc[0].context[15].shortname did +indexingdoc[0].context[15].datatype 0 +indexingdoc[0].context[16].fullname scorekey.scorekey +indexingdoc[0].context[16].shortname scorekey +indexingdoc[0].context[16].datatype 0 +indexingdoc[0].context[17].fullname cbid.cbid +indexingdoc[0].context[17].shortname cbid +indexingdoc[0].context[17].datatype 0 +indexingdoc[0].context[18].fullname titles.titles +indexingdoc[0].context[18].shortname titles +indexingdoc[0].context[18].datatype 2 +indexingdoc[0].summary[40] +indexingdoc[0].summary[0].name distance +indexingdoc[0].summary[0].datatype 0 +indexingdoc[0].summary[1].name sddocname +indexingdoc[0].summary[1].datatype 2 +indexingdoc[0].summary[2].name bgndata +indexingdoc[0].summary[2].datatype 2 +indexingdoc[0].summary[3].name sales +indexingdoc[0].summary[3].datatype 0 +indexingdoc[0].summary[4].name pto +indexingdoc[0].summary[4].datatype 0 +indexingdoc[0].summary[5].name mid +indexingdoc[0].summary[5].datatype 2 +indexingdoc[0].summary[6].name ew +indexingdoc[0].summary[6].datatype 2 +indexingdoc[0].summary[7].name surl +indexingdoc[0].summary[7].datatype 2 +indexingdoc[0].summary[8].name userrate +indexingdoc[0].summary[8].datatype 0 +indexingdoc[0].summary[9].name pid +indexingdoc[0].summary[9].datatype 2 +indexingdoc[0].summary[10].name weight +indexingdoc[0].summary[10].datatype 1 +indexingdoc[0].summary[11].name url +indexingdoc[0].summary[11].datatype 2 +indexingdoc[0].summary[12].name isbn +indexingdoc[0].summary[12].datatype 2 +indexingdoc[0].summary[13].name fmt +indexingdoc[0].summary[13].datatype 2 +indexingdoc[0].summary[14].name albumid +indexingdoc[0].summary[14].datatype 2 +indexingdoc[0].summary[15].name disp_song +indexingdoc[0].summary[15].datatype 2 +indexingdoc[0].summary[16].name song +indexingdoc[0].summary[16].datatype 2 +indexingdoc[0].summary[17].name pfrom +indexingdoc[0].summary[17].datatype 0 +indexingdoc[0].summary[18].name bgnpfrom +indexingdoc[0].summary[18].datatype 1 +indexingdoc[0].summary[19].name categories +indexingdoc[0].summary[19].datatype 2 +indexingdoc[0].summary[20].name data +indexingdoc[0].summary[20].datatype 2 +indexingdoc[0].summary[21].name numreview +indexingdoc[0].summary[21].datatype 0 +indexingdoc[0].summary[22].name bgnsellers +indexingdoc[0].summary[22].datatype 0 +indexingdoc[0].summary[23].name image +indexingdoc[0].summary[23].datatype 2 +indexingdoc[0].summary[24].name artist +indexingdoc[0].summary[24].datatype 2 +indexingdoc[0].summary[25].name artistspid +indexingdoc[0].summary[25].datatype 2 +indexingdoc[0].summary[26].name artistspid3 +indexingdoc[0].summary[26].datatype 2 +indexingdoc[0].summary[27].name title +indexingdoc[0].summary[27].datatype 2 +indexingdoc[0].summary[28].name newestedition +indexingdoc[0].summary[28].datatype 0 +indexingdoc[0].summary[29].name bgnpto +indexingdoc[0].summary[29].datatype 2 +indexingdoc[0].summary[30].name year +indexingdoc[0].summary[30].datatype 2 +indexingdoc[0].summary[31].name endyear +indexingdoc[0].summary[31].datatype 2 +indexingdoc[0].summary[32].name did +indexingdoc[0].summary[32].datatype 0 +indexingdoc[0].summary[33].name scorekey +indexingdoc[0].summary[33].datatype 0 +indexingdoc[0].summary[34].name cbid +indexingdoc[0].summary[34].datatype 0 +indexingdoc[0].summary[35].name titles +indexingdoc[0].summary[35].datatype 2 +indexingdoc[0].summary[36].name ranklog +indexingdoc[0].summary[36].datatype 2 +indexingdoc[0].summary[37].name rankfeatures +indexingdoc[0].summary[37].datatype 2 +indexingdoc[0].summary[38].name summaryfeatures +indexingdoc[0].summary[38].datatype 2 +indexingdoc[0].summary[39].name documentid +indexingdoc[0].summary[39].datatype 2 +indexingdoc[0].attribute[22] +indexingdoc[0].attribute[0].name sales +indexingdoc[0].attribute[0].datatype 0 +indexingdoc[0].attribute[0].containertype SINGLE +indexingdoc[0].attribute[1].name pto +indexingdoc[0].attribute[1].datatype 0 +indexingdoc[0].attribute[1].containertype SINGLE +indexingdoc[0].attribute[2].name mid +indexingdoc[0].attribute[2].datatype 0 +indexingdoc[0].attribute[2].containertype ARRAY +indexingdoc[0].attribute[3].name ew +indexingdoc[0].attribute[3].datatype 2 +indexingdoc[0].attribute[3].containertype SINGLE +indexingdoc[0].attribute[4].name weight +indexingdoc[0].attribute[4].datatype 1 +indexingdoc[0].attribute[4].containertype SINGLE +indexingdoc[0].attribute[5].name bgnpfrom +indexingdoc[0].attribute[5].datatype 1 +indexingdoc[0].attribute[5].containertype SINGLE +indexingdoc[0].attribute[6].name artist +indexingdoc[0].attribute[6].datatype 2 +indexingdoc[0].attribute[6].containertype SINGLE +indexingdoc[0].attribute[7].name artistspid +indexingdoc[0].attribute[7].datatype 2 +indexingdoc[0].attribute[7].containertype WEIGHTEDSET +indexingdoc[0].attribute[8].name artistspid2 +indexingdoc[0].attribute[8].datatype 1 +indexingdoc[0].attribute[8].containertype WEIGHTEDSET +indexingdoc[0].attribute[9].name title +indexingdoc[0].attribute[9].datatype 2 +indexingdoc[0].attribute[9].containertype SINGLE +indexingdoc[0].attribute[10].name newestedition +indexingdoc[0].attribute[10].datatype 0 +indexingdoc[0].attribute[10].containertype SINGLE +indexingdoc[0].attribute[11].name year +indexingdoc[0].attribute[11].datatype 0 +indexingdoc[0].attribute[11].containertype ARRAY +indexingdoc[0].attribute[12].name endyear +indexingdoc[0].attribute[12].datatype 0 +indexingdoc[0].attribute[12].containertype ARRAY +indexingdoc[0].attribute[13].name did +indexingdoc[0].attribute[13].datatype 0 +indexingdoc[0].attribute[13].containertype SINGLE +indexingdoc[0].attribute[14].name cbid +indexingdoc[0].attribute[14].datatype 0 +indexingdoc[0].attribute[14].containertype SINGLE +indexingdoc[0].attribute[15].name noupdate +indexingdoc[0].attribute[15].datatype 2 +indexingdoc[0].attribute[15].containertype SINGLE +indexingdoc[0].attribute[16].name noupdate2 +indexingdoc[0].attribute[16].datatype 2 +indexingdoc[0].attribute[16].containertype SINGLE +indexingdoc[0].attribute[17].name multiposition2d_position +indexingdoc[0].attribute[17].datatype 4 +indexingdoc[0].attribute[17].containertype ARRAY +indexingdoc[0].attribute[18].name extracategories +indexingdoc[0].attribute[18].datatype 2 +indexingdoc[0].attribute[18].containertype ARRAY +indexingdoc[0].attribute[19].name default_fieldlength +indexingdoc[0].attribute[19].datatype 0 +indexingdoc[0].attribute[19].containertype ARRAY +indexingdoc[0].attribute[20].name fmt_fieldlength +indexingdoc[0].attribute[20].datatype 0 +indexingdoc[0].attribute[20].containertype SINGLE +indexingdoc[0].attribute[21].name categories_fieldlength +indexingdoc[0].attribute[21].datatype 0 +indexingdoc[0].attribute[21].containertype SINGLE diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentTestCase.java new file mode 100644 index 00000000000..188dfb3f8f5 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentTestCase.java @@ -0,0 +1,141 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.*; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +@SuppressWarnings({ "unchecked" }) +public class DocumentTestCase { + + @Test + public void requireThatArrayOfStructIsProcessedCorrectly() throws ParseException { + DocumentType docType = new DocumentType("my_input"); + docType.addField(new Field("my_str", DataType.getArray(DataType.STRING))); + docType.addField(new Field("my_pos", DataType.getArray(PositionDataType.INSTANCE))); + + Document doc = new Document(docType, "doc:scheme:"); + Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING)); + arr.add(new StringFieldValue("6;9")); + doc.setFieldValue("my_str", arr); + + assertNotNull(doc = Expression.execute(Expression.fromString("input my_str | for_each { to_pos } | index my_pos"), doc)); + assertNotNull(doc.getFieldValue("my_str")); + FieldValue val = doc.getFieldValue("my_pos"); + assertNotNull(val); + assertEquals(DataType.getArray(PositionDataType.INSTANCE), val.getDataType()); + assertTrue(val instanceof Array); + arr = (Array)val; + assertEquals(1, arr.size()); + assertNotNull(val = arr.getFieldValue(0)); + assertEquals(PositionDataType.INSTANCE, val.getDataType()); + assertTrue(val instanceof Struct); + Struct pos = (Struct)val; + assertEquals(new IntegerFieldValue(6), PositionDataType.getXValue(pos)); + assertEquals(new IntegerFieldValue(9), PositionDataType.getYValue(pos)); + } + + @Test + public void requireThatConcatenationWorks() throws ParseException { + DocumentType docType = new DocumentType("my_input"); + docType.addField(new Field("arr_a", DataType.getArray(DataType.STRING))); + docType.addField(new Field("arr_b", DataType.getArray(DataType.STRING))); + docType.addField(new Field("out", DataType.getArray(DataType.STRING))); + + Expression exp = Expression.fromString("input arr_a . input arr_b | index out"); + { + Document doc = new Document(docType, "doc:scheme:"); + assertNotNull(doc = Expression.execute(exp, doc)); + FieldValue val = doc.getFieldValue("out"); + assertNotNull(val); + assertEquals(DataType.getArray(DataType.STRING), val.getDataType()); + assertTrue(val instanceof Array); + Array arr = (Array)val; + assertEquals(0, arr.size()); + } + { + Document doc = new Document(docType, "doc:scheme:"); + Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING)); + arr.add(new StringFieldValue("a1")); + doc.setFieldValue("arr_a", arr); + + assertNotNull(doc = Expression.execute(exp, doc)); + FieldValue val = doc.getFieldValue("out"); + assertNotNull(val); + assertEquals(DataType.getArray(DataType.STRING), val.getDataType()); + assertTrue(val instanceof Array); + arr = (Array)val; + assertEquals(1, arr.size()); + } + { + Document doc = new Document(docType, "doc:scheme:"); + Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING)); + arr.add(new StringFieldValue("a1")); + arr.add(new StringFieldValue("a2")); + doc.setFieldValue("arr_a", arr); + arr = new Array<StringFieldValue>(DataType.getArray(DataType.STRING)); + arr.add(new StringFieldValue("b1")); + doc.setFieldValue("arr_b", arr); + + assertNotNull(doc = Expression.execute(exp, doc)); + FieldValue val = doc.getFieldValue("out"); + assertNotNull(val); + assertEquals(DataType.getArray(DataType.STRING), val.getDataType()); + assertTrue(val instanceof Array); + arr = (Array)val; + assertEquals(3, arr.size()); + } + } + + @Test + public void requireThatConcatenationOfEmbracedStatementsWorks() throws ParseException { + DocumentType docType = new DocumentType("my_input"); + docType.addField(new Field("str_a", DataType.STRING)); + docType.addField(new Field("str_b", DataType.STRING)); + docType.addField(new Field("out", DataType.getArray(DataType.STRING))); + + Expression exp = Expression.fromString("(input str_a | split ',') . (input str_b | split ',') | index out"); + { + Document doc = new Document(docType, "doc:scheme:"); + assertNotNull(doc = Expression.execute(exp, doc)); + FieldValue val = doc.getFieldValue("out"); + assertNotNull(val); + assertEquals(DataType.getArray(DataType.STRING), val.getDataType()); + assertTrue(val instanceof Array); + Array arr = (Array)val; + assertEquals(0, arr.size()); + } + { + Document doc = new Document(docType, "doc:scheme:"); + doc.setFieldValue("str_a", new StringFieldValue("a1")); + + assertNotNull(doc = Expression.execute(exp, doc)); + FieldValue val = doc.getFieldValue("out"); + assertNotNull(val); + assertEquals(DataType.getArray(DataType.STRING), val.getDataType()); + assertTrue(val instanceof Array); + Array arr = (Array)val; + assertEquals(1, arr.size()); + } + { + Document doc = new Document(docType, "doc:scheme:"); + doc.setFieldValue("str_a", new StringFieldValue("a1,a2")); + doc.setFieldValue("str_b", new StringFieldValue("b1")); + + assertNotNull(doc = Expression.execute(exp, doc)); + FieldValue val = doc.getFieldValue("out"); + assertNotNull(val); + assertEquals(DataType.getArray(DataType.STRING), val.getDataType()); + assertTrue(val instanceof Array); + Array arr = (Array)val; + assertEquals(3, arr.size()); + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToPathUpdateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToPathUpdateTestCase.java new file mode 100644 index 00000000000..feef07a46bd --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToPathUpdateTestCase.java @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class DocumentToPathUpdateTestCase { + + @Test + public void requireThatIntegerAssignIsConverted() { + DocumentType docType = new DocumentType("my_type"); + docType.addField(new Field("my_int", DataType.INT)); + + FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "my_int", "", new IntegerFieldValue(69)); + Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd); + assertNotNull(doc); + doc.setFieldValue("my_int", new IntegerFieldValue(96)); + + DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldPathUpdates().size()); + assertNotNull(upd = docUpd.getFieldPathUpdates().get(0)); + + assertTrue(upd instanceof AssignFieldPathUpdate); + assertEquals("my_int", upd.getOriginalFieldPath()); + assertEquals(new IntegerFieldValue(96), ((AssignFieldPathUpdate)upd).getNewValue()); + } + + @Test + public void requireThatStringAssignIsConverted() { + DocumentType docType = new DocumentType("my_type"); + docType.addField(new Field("my_str", DataType.STRING)); + + FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "my_str", "", new StringFieldValue("69")); + Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd); + assertNotNull(doc); + doc.setFieldValue("my_str", new StringFieldValue("96")); + + DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldPathUpdates().size()); + assertNotNull(upd = docUpd.getFieldPathUpdates().get(0)); + + assertTrue(upd instanceof AssignFieldPathUpdate); + assertEquals("my_str", upd.getOriginalFieldPath()); + assertEquals(new StringFieldValue("96"), ((AssignFieldPathUpdate)upd).getNewValue()); + } + + @Test + public void requireThatStructAssignIsConverted() { + DocumentType docType = new DocumentType("my_type"); + StructDataType structType = new StructDataType("my_struct"); + structType.addField(new Field("b", DataType.INT)); + docType.addField(new Field("a", structType)); + + Struct struct = structType.createFieldValue(); + struct.setFieldValue("b", new IntegerFieldValue(69)); + FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "a", "", struct); + Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("a"); + assertTrue(obj instanceof Struct); + struct = (Struct)obj; + struct.setFieldValue("b", new IntegerFieldValue(96)); + + DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldPathUpdates().size()); + assertNotNull(upd = docUpd.getFieldPathUpdates().get(0)); + + assertTrue(upd instanceof AssignFieldPathUpdate); + assertEquals("a", upd.getOriginalFieldPath()); + assertEquals(struct, ((AssignFieldPathUpdate)upd).getNewValue()); + } + + @Test + public void requireThatStructElementAssignIsConverted() { + DocumentType docType = new DocumentType("my_type"); + StructDataType structType = new StructDataType("my_struct"); + structType.addField(new Field("b", DataType.INT)); + docType.addField(new Field("a", structType)); + + FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "a.b", "", new IntegerFieldValue(69)); + Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("a"); + assertTrue(obj instanceof Struct); + Struct struct = (Struct)obj; + struct.setFieldValue("b", new IntegerFieldValue(96)); + + DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldPathUpdates().size()); + assertNotNull(upd = docUpd.getFieldPathUpdates().get(0)); + + assertTrue(upd instanceof AssignFieldPathUpdate); + assertEquals("a.b", upd.getOriginalFieldPath()); + obj = ((AssignFieldPathUpdate)upd).getNewValue(); + assertTrue(obj instanceof IntegerFieldValue); + assertEquals(96, ((IntegerFieldValue)obj).getInteger()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToValueUpdateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToValueUpdateTestCase.java new file mode 100644 index 00000000000..ecf43080b4b --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToValueUpdateTestCase.java @@ -0,0 +1,340 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.*; +import com.yahoo.document.update.*; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +@SuppressWarnings({ "unchecked" }) +public class DocumentToValueUpdateTestCase { + + @Test + public void requireThatSddocnameFieldIsIgnored() { + DocumentType docType = new DocumentType("my_type"); + docType.addField(new Field("sddocname", DataType.STRING)); + + ValueUpdate valueUpd = ValueUpdate.createAssign(new StringFieldValue("69")); + Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("sddocname"), valueUpd); + doc.setFieldValue("sddocname", new StringFieldValue("96")); + + UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); + assertNull(adapter.getOutput()); + } + + @Test + public void requireThatEmptyFieldUpdatesAreIgnored() { + DocumentType docType = new DocumentType("my_type"); + docType.addField(new Field("my_int", DataType.INT)); + docType.addField(new Field("my_str", DataType.STRING)); + + ValueUpdate valueUpd = ValueUpdate.createAssign(new IntegerFieldValue(69)); + Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_int"), valueUpd); + doc.setFieldValue("my_int", new IntegerFieldValue(96)); + + UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); + DocumentUpdate docUpd = adapter.getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldUpdates().size()); + assertEquals("my_int", docUpd.getFieldUpdate(0).getField().getName()); + } + + @Test + public void requireThatIntegerAssignIsConverted() { + DocumentType docType = new DocumentType("my_type"); + docType.addField(new Field("my_int", DataType.INT)); + + ValueUpdate valueUpd = ValueUpdate.createAssign(new IntegerFieldValue(69)); + Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_int"), valueUpd); + doc.setFieldValue("my_int", new IntegerFieldValue(96)); + + UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); + DocumentUpdate docUpd = adapter.getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldUpdates().size()); + + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + assertNotNull(fieldUpd); + + assertEquals(docType.getField("my_int"), fieldUpd.getField()); + assertEquals(1, fieldUpd.getValueUpdates().size()); + assertNotNull(valueUpd = fieldUpd.getValueUpdate(0)); + assertTrue(valueUpd instanceof AssignValueUpdate); + assertEquals(new IntegerFieldValue(96), valueUpd.getValue()); + } + + @Test + public void requireThatClearFieldIsConverted() { + DocumentType docType = new DocumentType("my_type"); + docType.addField(new Field("my_int", DataType.INT)); + + ValueUpdate valueUpd = ValueUpdate.createClear(); + Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_int"), valueUpd); + assertNotNull(doc.getFieldValue("my_int")); + + UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); + DocumentUpdate docUpd = adapter.getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldUpdates().size()); + + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + assertNotNull(fieldUpd); + + assertEquals(docType.getField("my_int"), fieldUpd.getField()); + assertEquals(1, fieldUpd.getValueUpdates().size()); + assertNotNull(valueUpd = fieldUpd.getValueUpdate(0)); + assertTrue(valueUpd instanceof ClearValueUpdate); + } + + @Test + public void requireThatStringAssignIsConverted() { + DocumentType docType = new DocumentType("my_type"); + docType.addField(new Field("my_str", DataType.STRING)); + + ValueUpdate valueUpd = ValueUpdate.createAssign(new StringFieldValue("69")); + Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_str"), valueUpd); + doc.setFieldValue("my_str", new StringFieldValue("96")); + + UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); + DocumentUpdate docUpd = adapter.getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldUpdates().size()); + + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + assertNotNull(fieldUpd); + + assertEquals(docType.getField("my_str"), fieldUpd.getField()); + assertEquals(1, fieldUpd.getValueUpdates().size()); + assertNotNull(valueUpd = fieldUpd.getValueUpdate(0)); + assertTrue(valueUpd instanceof AssignValueUpdate); + assertEquals(new StringFieldValue("96"), valueUpd.getValue()); + } + + @Test + public void requireThatStructAssignIsConverted() { + DocumentType docType = new DocumentType("my_type"); + StructDataType structType = new StructDataType("my_struct"); + structType.addField(new Field("b", DataType.INT)); + docType.addField(new Field("a", structType)); + + Struct struct = structType.createFieldValue(); + struct.setFieldValue("b", new IntegerFieldValue(69)); + ValueUpdate valueUpd = ValueUpdate.createAssign(struct); + Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("a"), valueUpd); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("a"); + assertTrue(obj instanceof Struct); + struct = (Struct)obj; + struct.setFieldValue("b", new IntegerFieldValue(96)); + + UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); + DocumentUpdate docUpd = adapter.getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldUpdates().size()); + + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + assertNotNull(fieldUpd); + + assertEquals(docType.getField("a"), fieldUpd.getField()); + assertEquals(1, fieldUpd.getValueUpdates().size()); + assertNotNull(valueUpd = fieldUpd.getValueUpdate(0)); + assertTrue(valueUpd instanceof AssignValueUpdate); + assertEquals(struct, valueUpd.getValue()); + } + + @Test + public void requireThatArrayElementAddIsConverted() { + DocumentType docType = new DocumentType("my_type"); + ArrayDataType arrType = DataType.getArray(DataType.STRING); + docType.addField(new Field("my_arr", arrType)); + + ValueUpdate valueUpd = ValueUpdate.createAdd(new StringFieldValue("foo")); + Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_arr"), valueUpd); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("my_arr"); + assertTrue(obj instanceof Array); + Array<StringFieldValue> arr = (Array<StringFieldValue>)obj; + arr.set(0, new StringFieldValue("bar")); + + UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); + DocumentUpdate docUpd = adapter.getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldUpdates().size()); + + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + assertNotNull(fieldUpd); + + assertEquals(docType.getField("my_arr"), fieldUpd.getField()); + assertEquals(1, fieldUpd.getValueUpdates().size()); + assertNotNull(valueUpd = fieldUpd.getValueUpdate(0)); + assertTrue(valueUpd instanceof AddValueUpdate); + assertEquals(new StringFieldValue("bar"), valueUpd.getValue()); + } + + @Test + public void requireThatArrayElementRemoveIsConverted() { + DocumentType docType = new DocumentType("my_type"); + ArrayDataType arrType = DataType.getArray(DataType.STRING); + docType.addField(new Field("my_arr", arrType)); + + ValueUpdate valueUpd = ValueUpdate.createRemove(new StringFieldValue("foo")); + Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_arr"), valueUpd); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("my_arr"); + assertTrue(obj instanceof Array); + Array<StringFieldValue> arr = (Array<StringFieldValue>)obj; + arr.set(0, new StringFieldValue("bar")); + + UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); + DocumentUpdate docUpd = adapter.getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldUpdates().size()); + + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + assertNotNull(fieldUpd); + + assertEquals(docType.getField("my_arr"), fieldUpd.getField()); + assertEquals(1, fieldUpd.getValueUpdates().size()); + assertNotNull(valueUpd = fieldUpd.getValueUpdate(0)); + assertTrue(valueUpd instanceof RemoveValueUpdate); + assertEquals(new StringFieldValue("bar"), valueUpd.getValue()); + } + + @Test + public void requireThatArrayOfStructElementAddIsConverted() { + DocumentType docType = new DocumentType("my_type"); + StructDataType structType = new StructDataType("my_struct"); + structType.addField(new Field("b", DataType.INT)); + docType.addField(new Field("a", DataType.getArray(structType))); + + Struct struct = structType.createFieldValue(); + struct.setFieldValue("b", new IntegerFieldValue(69)); + ValueUpdate valueUpd = ValueUpdate.createAdd(struct); + Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("a"), valueUpd); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("a"); + assertTrue(obj instanceof Array); + Array<Struct> arr = (Array<Struct>)obj; + struct = structType.createFieldValue(); + struct.setFieldValue("b", new IntegerFieldValue(96)); + arr.set(0, struct); + + UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); + DocumentUpdate docUpd = adapter.getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldUpdates().size()); + + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + assertNotNull(fieldUpd); + + assertEquals(docType.getField("a"), fieldUpd.getField()); + assertEquals(1, fieldUpd.getValueUpdates().size()); + assertNotNull(valueUpd = fieldUpd.getValueUpdate(0)); + assertTrue(valueUpd instanceof AddValueUpdate); + assertEquals(struct, valueUpd.getValue()); + } + + @Test + public void requireThatWsetElementAssignIsConverted() { + DocumentType docType = new DocumentType("my_type"); + WeightedSetDataType wsetType = DataType.getWeightedSet(DataType.STRING); + docType.addField(new Field("my_wset", wsetType)); + + ValueUpdate valueUpd = ValueUpdate.createMap(new StringFieldValue("foo"), ValueUpdate.createAssign(new IntegerFieldValue(69))); + Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_wset"), valueUpd); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("my_wset"); + assertTrue(obj instanceof WeightedSet); + WeightedSet<StringFieldValue> wset = (WeightedSet<StringFieldValue>)obj; + wset.put(new StringFieldValue("foo"), 96); + + UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); + DocumentUpdate docUpd = adapter.getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldUpdates().size()); + + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + assertNotNull(fieldUpd); + + assertEquals(docType.getField("my_wset"), fieldUpd.getField()); + assertEquals(1, fieldUpd.getValueUpdates().size()); + assertNotNull(valueUpd = fieldUpd.getValueUpdate(0)); + assertTrue(valueUpd instanceof MapValueUpdate); + assertEquals(new StringFieldValue("foo"), valueUpd.getValue()); + assertNotNull(valueUpd = ((MapValueUpdate)valueUpd).getUpdate()); + assertTrue(valueUpd instanceof AssignValueUpdate); + assertEquals(new IntegerFieldValue(96), valueUpd.getValue()); + } + + @Test + public void requireThatWsetElementAddIsConverted() { + DocumentType docType = new DocumentType("my_type"); + WeightedSetDataType wsetType = DataType.getWeightedSet(DataType.STRING); + docType.addField(new Field("my_wset", wsetType)); + + ValueUpdate valueUpd = ValueUpdate.createAdd(new StringFieldValue("foo"), 69); + Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_wset"), valueUpd); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("my_wset"); + assertTrue(obj instanceof WeightedSet); + WeightedSet<StringFieldValue> wset = (WeightedSet<StringFieldValue>)obj; + wset.put(new StringFieldValue("foo"), 96); + + UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); + DocumentUpdate docUpd = adapter.getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldUpdates().size()); + + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + assertNotNull(fieldUpd); + + assertEquals(docType.getField("my_wset"), fieldUpd.getField()); + assertEquals(1, fieldUpd.getValueUpdates().size()); + assertNotNull(valueUpd = fieldUpd.getValueUpdate(0)); + assertTrue(valueUpd instanceof AddValueUpdate); + assertEquals(new StringFieldValue("foo"), valueUpd.getValue()); + assertEquals(96, ((AddValueUpdate)valueUpd).getWeight()); + } + + @Test + public void requireThatWsetElementRemoveIsConverted() { + DocumentType docType = new DocumentType("my_type"); + WeightedSetDataType wsetType = DataType.getWeightedSet(DataType.STRING); + docType.addField(new Field("my_wset", wsetType)); + + ValueUpdate valueUpd = ValueUpdate.createRemove(new StringFieldValue("foo")); + Document doc = FieldUpdateHelper.newPartialDocument(docType, null, docType.getField("my_wset"), valueUpd); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("my_wset"); + assertTrue(obj instanceof WeightedSet); + WeightedSet<StringFieldValue> wset = (WeightedSet<StringFieldValue>)obj; + wset.remove(new StringFieldValue("foo")); + wset.add(new StringFieldValue("bar")); + + UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); + DocumentUpdate docUpd = adapter.getOutput(); + assertNotNull(docUpd); + assertEquals(1, docUpd.getFieldUpdates().size()); + + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); + assertNotNull(fieldUpd); + + assertEquals(docType.getField("my_wset"), fieldUpd.getField()); + assertEquals(1, fieldUpd.getValueUpdates().size()); + assertNotNull(valueUpd = fieldUpd.getValueUpdate(0)); + assertTrue(valueUpd instanceof RemoveValueUpdate); + assertEquals(new StringFieldValue("bar"), valueUpd.getValue()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentUpdateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentUpdateTestCase.java new file mode 100644 index 00000000000..aec8d0332c2 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentUpdateTestCase.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.update.AddValueUpdate; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.ValueUpdate; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class DocumentUpdateTestCase { + + @Test + public void requireThatArrayOfStructElementAddIsProcessedCorrectly() throws ParseException { + DocumentType docType = new DocumentType("my_input"); + docType.addField(new Field("my_str", DataType.getArray(DataType.STRING))); + docType.addField(new Field("my_pos", DataType.getArray(PositionDataType.INSTANCE))); + + DocumentUpdate docUpdate = new DocumentUpdate(docType, "doc:scheme:"); + docUpdate.addFieldUpdate(FieldUpdate.createAdd(docType.getField("my_str"), new StringFieldValue("6;9"))); + docUpdate = Expression.execute(Expression.fromString("input my_str | for_each { to_pos } | index my_pos"), docUpdate); + + assertNotNull(docUpdate); + assertEquals(0, docUpdate.getFieldPathUpdates().size()); + assertEquals(1, docUpdate.getFieldUpdates().size()); + + FieldUpdate fieldUpd = docUpdate.getFieldUpdate(0); + assertNotNull(fieldUpd); + assertEquals(docType.getField("my_pos"), fieldUpd.getField()); + assertEquals(1, fieldUpd.getValueUpdates().size()); + + ValueUpdate valueUpd = fieldUpd.getValueUpdate(0); + assertNotNull(valueUpd); + assertTrue(valueUpd instanceof AddValueUpdate); + + Object val = valueUpd.getValue(); + assertNotNull(val); + assertTrue(val instanceof Struct); + Struct pos = (Struct)val; + assertEquals(PositionDataType.INSTANCE, pos.getDataType()); + assertEquals(new IntegerFieldValue(6), PositionDataType.getXValue(pos)); + assertEquals(new IntegerFieldValue(9), PositionDataType.getYValue(pos)); + } + + @Test + public void requireThatCreateIfNonExistentFlagIsPropagated() throws ParseException { + DocumentType docType = new DocumentType("my_input"); + docType.addField(new Field("my_str", DataType.getArray(DataType.STRING))); + DocumentUpdate upd = new DocumentUpdate(docType, "doc:scheme:"); + upd.addFieldUpdate(FieldUpdate.createAdd(docType.getField("my_str"), new StringFieldValue("foo"))); + upd.setCreateIfNonExistent(true); + + upd = Expression.execute(Expression.fromString("input my_str | index my_str"), upd); + assertTrue(upd.getCreateIfNonExistent()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java new file mode 100644 index 00000000000..cd5f17c3b98 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java @@ -0,0 +1,281 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.collections.Pair; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression; +import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Base64DecodeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Base64EncodeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.CatExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression; +import com.yahoo.vespa.indexinglanguage.expressions.CompositeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.EchoExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ExecutionContext; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; +import com.yahoo.vespa.indexinglanguage.expressions.GetFieldExpression; +import com.yahoo.vespa.indexinglanguage.expressions.GetVarExpression; +import com.yahoo.vespa.indexinglanguage.expressions.GuardExpression; +import com.yahoo.vespa.indexinglanguage.expressions.HexDecodeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.HexEncodeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.HostNameExpression; +import com.yahoo.vespa.indexinglanguage.expressions.IfThenExpression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.JoinExpression; +import com.yahoo.vespa.indexinglanguage.expressions.LowerCaseExpression; +import com.yahoo.vespa.indexinglanguage.expressions.NormalizeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.NowExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OptimizePredicateExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ParenthesisExpression; +import com.yahoo.vespa.indexinglanguage.expressions.RandomExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SelectInputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SplitExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SubstringExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SwitchExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ThisExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToArrayExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToByteExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToDoubleExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToFloatExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToIntegerExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToLongExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToStringExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToWsetExpression; +import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.TrimExpression; +import com.yahoo.vespa.indexinglanguage.expressions.VerificationContext; +import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ExpressionConverterTestCase { + + @SuppressWarnings("unchecked") + @Test + public void requireThatAllExpressionTypesCanBeTraversed() { + assertConvertable(new ArithmeticExpression(new InputExpression("foo"), ArithmeticExpression.Operator.ADD, + new InputExpression("bar"))); + assertConvertable(new AttributeExpression("foo")); + assertConvertable(new Base64DecodeExpression()); + assertConvertable(new Base64EncodeExpression()); + assertConvertable(new CatExpression(new InputExpression("foo"), new IndexExpression("bar"))); + assertConvertable(new ClearStateExpression()); + assertConvertable(new EchoExpression()); + assertConvertable(new ForEachExpression(new IndexExpression("foo"))); + assertConvertable(new GetFieldExpression("foo")); + assertConvertable(new GetVarExpression("foo")); + assertConvertable(new GuardExpression(new IndexExpression("foo"))); + assertConvertable(new HexDecodeExpression()); + assertConvertable(new HexEncodeExpression()); + assertConvertable(new HostNameExpression()); + assertConvertable(new IfThenExpression(new InputExpression("foo"), IfThenExpression.Comparator.EQ, + new InputExpression("bar"), + new IndexExpression("baz"), + new IndexExpression("cox"))); + assertConvertable(new IndexExpression("foo")); + assertConvertable(new InputExpression("foo")); + assertConvertable(new JoinExpression("foo")); + assertConvertable(new LowerCaseExpression()); + assertConvertable(new NormalizeExpression(new SimpleLinguistics())); + assertConvertable(new NowExpression()); + assertConvertable(new OptimizePredicateExpression()); + assertConvertable(new ParenthesisExpression(new InputExpression("foo"))); + assertConvertable(new RandomExpression(69)); + assertConvertable(new ScriptExpression(new StatementExpression(new InputExpression("foo")))); + assertConvertable(new SelectInputExpression(new Pair<String, Expression>("foo", new IndexExpression("bar")), + new Pair<String, Expression>("bar", new IndexExpression("foo")))); + assertConvertable(new SetLanguageExpression()); + assertConvertable(new SetValueExpression(new IntegerFieldValue(69))); + assertConvertable(new SetVarExpression("foo")); + assertConvertable(new SplitExpression("foo")); + assertConvertable(new StatementExpression(new InputExpression("foo"))); + assertConvertable(new SubstringExpression(6, 9)); + assertConvertable(new SummaryExpression("foo")); + assertConvertable(new SwitchExpression(Collections.singletonMap("foo", (Expression)new IndexExpression("bar")), + new InputExpression("baz"))); + assertConvertable(new ThisExpression()); + assertConvertable(new ToArrayExpression()); + assertConvertable(new ToByteExpression()); + assertConvertable(new ToDoubleExpression()); + assertConvertable(new ToFloatExpression()); + assertConvertable(new ToIntegerExpression()); + assertConvertable(new TokenizeExpression(new SimpleLinguistics(), new AnnotatorConfig())); + assertConvertable(new ToLongExpression()); + assertConvertable(new ToPositionExpression()); + assertConvertable(new ToStringExpression()); + assertConvertable(new ToWsetExpression(false, false)); + assertConvertable(new TrimExpression()); + assertConvertable(new ZCurveExpression()); + } + + @Test + public void requireThatScriptElementsCanBeRemoved() { + StatementExpression foo = new StatementExpression(new AttributeExpression("foo")); + StatementExpression bar = new StatementExpression(new AttributeExpression("bar")); + ScriptExpression before = new ScriptExpression(foo, bar); + + Expression after = new SearchReplace(foo, null).convert(before); + assertTrue(after instanceof ScriptExpression); + assertEquals(1, ((ScriptExpression)after).size()); + assertEquals(bar, ((ScriptExpression)after).get(0)); + + after = new SearchReplace(bar, null).convert(before); + assertTrue(after instanceof ScriptExpression); + assertEquals(1, ((ScriptExpression)after).size()); + assertEquals(foo, ((ScriptExpression)after).get(0)); + } + + @Test + public void requireThatSwitchElementsCanBeRemoved() { + Map<String, Expression> cases = new HashMap<>(); + Expression foo = new AttributeExpression("foo"); + Expression bar = new AttributeExpression("bar"); + cases.put("foo", foo); + cases.put("bar", bar); + SwitchExpression before = new SwitchExpression(cases); + + Expression after = new SearchReplace(foo, null).convert(before); + assertTrue(after instanceof SwitchExpression); + assertEquals(1, ((SwitchExpression)after).getCases().size()); + + after = new SearchReplace(bar, null).convert(before); + assertTrue(after instanceof SwitchExpression); + assertEquals(1, ((SwitchExpression)after).getCases().size()); + } + + @Test + public void requireThatUnknownCompositeThrows() { + try { + new MyTraverser().convert(new MyComposite()); + fail(); + } catch (UnsupportedOperationException e) { + assertEquals(NoSuchMethodException.class, e.getCause().getClass()); + } + } + + @Test + public void requireThatConversionExceptionCanBeThrown() { + final RuntimeException expectedCause = new RuntimeException(); + try { + new ExpressionConverter() { + + @Override + protected boolean shouldConvert(Expression exp) { + return exp instanceof AttributeExpression; + } + + @Override + protected Expression doConvert(Expression exp) { + throw expectedCause; + } + }.convert(new StatementExpression(new AttributeExpression("foo"))); + fail(); + } catch (RuntimeException e) { + assertSame(expectedCause, e); + } + } + + @Test + public void requireThatCatConversionIgnoresNull() { + Expression exp = new ExpressionConverter() { + + @Override + protected boolean shouldConvert(Expression exp) { + return exp instanceof AttributeExpression; + } + + @Override + protected Expression doConvert(Expression exp) { + return null; + } + }.convert(new CatExpression(new AttributeExpression("foo"), new IndexExpression("bar"))); + assertEquals(new CatExpression(new IndexExpression("bar")), exp); + } + + private static void assertConvertable(Expression exp) { + MyTraverser traverser = new MyTraverser(); + assertEquals(traverser.convert(exp), exp); + } + + private static class SearchReplace extends ExpressionConverter { + + final Expression searchFor; + final Expression replaceWith; + + private SearchReplace(Expression searchFor, Expression replaceWith) { + this.searchFor = searchFor; + this.replaceWith = replaceWith; + } + + @Override + protected boolean shouldConvert(Expression exp) { + return exp == searchFor; + } + + @Override + protected Expression doConvert(Expression exp) { + return replaceWith; + } + } + + private static class MyTraverser extends ExpressionConverter { + + @Override + protected boolean shouldConvert(Expression exp) { + return false; + } + + @Override + protected Expression doConvert(Expression exp) { + return exp; + } + } + + private static class MyComposite extends CompositeExpression { + + @Override + protected void doExecute(ExecutionContext ctx) { + + } + + @Override + protected void doVerify(VerificationContext ctx) { + + } + + @Override + public DataType requiredInputType() { + return null; + } + + @Override + public DataType createdOutputType() { + return null; + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizerTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizerTestCase.java new file mode 100644 index 00000000000..2258c7d5a13 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionOptimizerTestCase.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:magnarn@yahoo-inc.com">Magnar Nedland</a> + */ +public class ExpressionOptimizerTestCase { + @Test + public void requireThatSimpleExpressionsAreNotChanged() { + assertNotOptimized("input foo"); + } + + @Test + public void requireThatStatementsBeforeOneThatIgnoresInputAreRemoved() { + checkStatementThatIgnoresInput(new InputExpression("foo")); + checkStatementThatIgnoresInput(new NowExpression()); + checkStatementThatIgnoresInput(new SetValueExpression(new IntegerFieldValue(42))); + checkStatementThatIgnoresInput(new HostNameExpression()); + checkStatementThatIgnoresInput(new RandomExpression(42)); + checkStatementThatIgnoresInput(new ArithmeticExpression( + new NowExpression(), ArithmeticExpression.Operator.ADD, new NowExpression())); + checkStatementThatIgnoresInput(new CatExpression(new NowExpression(), new NowExpression())); + checkStatementThatIgnoresInput(new ParenthesisExpression(new NowExpression())); + assertOptimized("input foo | (trim | now)", "(now)"); + assertOptimized("input foo | get_var bar", "get_var bar"); + assertOptimized("1 | index foo | now | 0", "1 | index foo | 0"); + } + + @Test + public void requireThatStatementsBeforeOneThatConsidersInputAreKept() { + assertNotOptimized("input foo | random"); + assertNotOptimized("input foo | trim + now"); + assertNotOptimized("input foo | (trim + now) - now"); + assertNotOptimized("input foo | (now . trim)"); + assertNotOptimized("input foo | (trim)"); + assertNotOptimized("input foo | (trim | trim)"); + assertNotOptimized("input foo | switch {case 'bar': now; case 'foo': now; }"); + assertNotOptimized("{ input test | { summary test; }; }"); + assertNotOptimized("1 | set_var foo | 0 | set_var bar"); + assertNotOptimized("1 | index foo | 0 | index bar"); + assertNotOptimized("1 | echo | 0 | echo"); + assertNotOptimized("'foo' | if (1 < 2) { now | summary } else { 42 | summary } | summary"); + assertNotOptimized("{ 0 | set_var tmp; " + + " input foo | split ';' | for_each { to_int + get_var tmp | set_var tmp };" + + " get_var tmp | attribute bar; }"); + assertNotOptimized("0 | set_var tmp | " + + "input foo | split ';' | for_each { to_int + get_var tmp | set_var tmp } | " + + "get_var tmp | attribute bar"); + } + + private void checkStatementThatIgnoresInput(Expression exp) { + assertOptimized(new StatementExpression(new InputExpression("xyzzy"), exp), exp); + assertOptimized(new StatementExpression(new InputExpression("xyzzy"), exp, new LowerCaseExpression()), + new StatementExpression(exp, new LowerCaseExpression())); + } + + private void assertOptimized(Expression input, Expression expected) { + assertEquals(expected.toString(), new ExpressionOptimizer().convert(input).toString()); + } + + private void assertOptimized(String input, String expected) { + try { + assertOptimized(Expression.fromString(input), Expression.fromString(expected)); + } catch (ParseException e) { + fail(e.getMessage()); + } + } + + private void assertNotOptimized(String script) { + assertOptimized(script, script); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcherTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcherTestCase.java new file mode 100644 index 00000000000..2e47ccb7ea3 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionSearcherTestCase.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.collections.Pair; +import com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression; +import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.CatExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; +import com.yahoo.vespa.indexinglanguage.expressions.GuardExpression; +import com.yahoo.vespa.indexinglanguage.expressions.IfThenExpression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ParenthesisExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SelectInputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SwitchExpression; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ExpressionSearcherTestCase { + + private static final ExpressionSearcher searcher = new ExpressionSearcher<>(IndexExpression.class); + + @Test + public void requireThatExpressionsCanBeFound() { + IndexExpression exp = new IndexExpression("foo"); + assertFound(exp, new ArithmeticExpression(exp, ArithmeticExpression.Operator.ADD, + new AttributeExpression("foo"))); + assertFound(exp, new ArithmeticExpression(new AttributeExpression("foo"), + ArithmeticExpression.Operator.ADD, exp)); + assertFound(exp, new CatExpression(exp)); + assertFound(exp, new CatExpression(new AttributeExpression("foo"), exp)); + assertFound(exp, new ForEachExpression(exp)); + assertFound(exp, new GuardExpression(exp)); + assertFound(exp, new IfThenExpression(exp, + IfThenExpression.Comparator.EQ, + new AttributeExpression("foo"), + new AttributeExpression("bar"), + new AttributeExpression("baz"))); + assertFound(exp, new IfThenExpression(new AttributeExpression("foo"), + IfThenExpression.Comparator.EQ, + exp, + new AttributeExpression("bar"), + new AttributeExpression("baz"))); + assertFound(exp, new IfThenExpression(new AttributeExpression("foo"), + IfThenExpression.Comparator.EQ, + new AttributeExpression("bar"), + exp, + new AttributeExpression("baz"))); + assertFound(exp, new IfThenExpression(new AttributeExpression("foo"), + IfThenExpression.Comparator.EQ, + new AttributeExpression("bar"), + new AttributeExpression("baz"), + exp)); + assertFound(exp, new ParenthesisExpression(exp)); + assertFound(exp, new ScriptExpression(new StatementExpression(exp))); + assertFound(exp, new ScriptExpression(new StatementExpression(new AttributeExpression("foo")), + new StatementExpression(exp))); + assertFound(exp, new SelectInputExpression( + Arrays.asList(new Pair<String, Expression>("foo", exp), + new Pair<String, Expression>("bar", new AttributeExpression("bar"))))); + assertFound(exp, new SelectInputExpression( + Arrays.asList(new Pair<String, Expression>("foo", new AttributeExpression("bar")), + new Pair<String, Expression>("bar", exp)))); + assertFound(exp, new StatementExpression(exp)); + assertFound(exp, new StatementExpression(new AttributeExpression("foo"), exp)); + assertFound(exp, new SwitchExpression( + Collections.singletonMap("foo", exp), + new AttributeExpression("bar"))); + assertFound(exp, new SwitchExpression( + Collections.singletonMap("foo", new AttributeExpression("bar")), + exp)); + } + + private static void assertFound(IndexExpression searchFor, Expression searchIn) { + assertTrue(searcher.containedIn(searchIn)); + assertSame(searchFor, searcher.searchIn(searchIn)); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitorTestCase.java new file mode 100644 index 00000000000..95a916cd0ad --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionVisitorTestCase.java @@ -0,0 +1,141 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.collections.Pair; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression; +import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Base64DecodeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Base64EncodeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.CatExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression; +import com.yahoo.vespa.indexinglanguage.expressions.EchoExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; +import com.yahoo.vespa.indexinglanguage.expressions.GetFieldExpression; +import com.yahoo.vespa.indexinglanguage.expressions.GetVarExpression; +import com.yahoo.vespa.indexinglanguage.expressions.GuardExpression; +import com.yahoo.vespa.indexinglanguage.expressions.HexDecodeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.HexEncodeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.HostNameExpression; +import com.yahoo.vespa.indexinglanguage.expressions.IfThenExpression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.JoinExpression; +import com.yahoo.vespa.indexinglanguage.expressions.LowerCaseExpression; +import com.yahoo.vespa.indexinglanguage.expressions.NormalizeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.NowExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OptimizePredicateExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ParenthesisExpression; +import com.yahoo.vespa.indexinglanguage.expressions.RandomExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SelectInputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SplitExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SubstringExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SwitchExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ThisExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToArrayExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToByteExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToDoubleExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToFloatExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToIntegerExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToLongExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToStringExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ToWsetExpression; +import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.TrimExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ExpressionVisitorTestCase { + + @SuppressWarnings("unchecked") + @Test + public void requireThatAllExpressionsAreVisited() { + assertCount(3, new ArithmeticExpression(new InputExpression("foo"), ArithmeticExpression.Operator.ADD, + new InputExpression("bar"))); + assertCount(1, new AttributeExpression("foo")); + assertCount(1, new Base64DecodeExpression()); + assertCount(1, new Base64EncodeExpression()); + assertCount(3, new CatExpression(new InputExpression("foo"), new IndexExpression("bar"))); + assertCount(1, new ClearStateExpression()); + assertCount(1, new EchoExpression()); + assertCount(2, new ForEachExpression(new IndexExpression("foo"))); + assertCount(1, new GetFieldExpression("foo")); + assertCount(1, new GetVarExpression("foo")); + assertCount(2, new GuardExpression(new IndexExpression("foo"))); + assertCount(1, new HexDecodeExpression()); + assertCount(1, new HexEncodeExpression()); + assertCount(1, new HostNameExpression()); + assertCount(5, new IfThenExpression(new InputExpression("foo"), IfThenExpression.Comparator.EQ, + new InputExpression("bar"), + new IndexExpression("baz"), + new IndexExpression("cox"))); + assertCount(1, new IndexExpression("foo")); + assertCount(1, new InputExpression("foo")); + assertCount(1, new JoinExpression("foo")); + assertCount(1, new LowerCaseExpression()); + assertCount(1, new NormalizeExpression(new SimpleLinguistics())); + assertCount(1, new NowExpression()); + assertCount(1, new OptimizePredicateExpression()); + assertCount(2, new ParenthesisExpression(new InputExpression("foo"))); + assertCount(1, new RandomExpression(69)); + assertCount(3, new ScriptExpression(new StatementExpression(new InputExpression("foo")))); + assertCount(3, new SelectInputExpression(new Pair<String, Expression>("foo", new IndexExpression("bar")), + new Pair<String, Expression>("bar", new IndexExpression("foo")))); + assertCount(1, new SetLanguageExpression()); + assertCount(1, new SetValueExpression(new IntegerFieldValue(69))); + assertCount(1, new SetVarExpression("foo")); + assertCount(1, new SplitExpression("foo")); + assertCount(2, new StatementExpression(new InputExpression("foo"))); + assertCount(1, new SummaryExpression("foo")); + assertCount(1, new SubstringExpression(6, 9)); + assertCount(3, new SwitchExpression(Collections.singletonMap("foo", (Expression)new IndexExpression("bar")), + new InputExpression("baz"))); + assertCount(1, new ThisExpression()); + assertCount(1, new ToArrayExpression()); + assertCount(1, new ToByteExpression()); + assertCount(1, new ToDoubleExpression()); + assertCount(1, new ToFloatExpression()); + assertCount(1, new ToIntegerExpression()); + assertCount(1, new TokenizeExpression(new SimpleLinguistics(), new AnnotatorConfig())); + assertCount(1, new ToLongExpression()); + assertCount(1, new ToPositionExpression()); + assertCount(1, new ToStringExpression()); + assertCount(1, new ToWsetExpression(false, false)); + assertCount(1, new TrimExpression()); + assertCount(1, new ZCurveExpression()); + } + + private static void assertCount(int expectedCount, Expression exp) { + MyVisitor visitor = new MyVisitor(); + visitor.visit(exp); + assertEquals(expectedCount, visitor.count); + } + + private static class MyVisitor extends ExpressionVisitor { + + int count = 0; + + @Override + protected void doVisit(Expression exp) { + ++count; + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/FieldValueConverterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/FieldValueConverterTestCase.java new file mode 100644 index 00000000000..96140be7500 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/FieldValueConverterTestCase.java @@ -0,0 +1,423 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.StructDataType; +import com.yahoo.document.datatypes.*; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlStream; +import org.junit.Test; + +import java.util.Iterator; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class FieldValueConverterTestCase { + + @Test + public void requireThatNullIsIgnored() { + assertNull(new FieldValueConverter() { + + @Override + protected boolean shouldConvert(FieldValue value) { + throw new AssertionError(); + } + + @Override + protected FieldValue doConvert(FieldValue value) { + throw new AssertionError(); + } + }.convert(null)); + } + + @Test + public void requireThatUnknownTypesFallThrough() { + FieldValue val = new MyFieldValue(); + assertSame(val, new FieldValueConverter() { + + @Override + protected boolean shouldConvert(FieldValue value) { + return false; + } + + @Override + protected FieldValue doConvert(FieldValue value) { + throw new AssertionError(); + } + }.convert(val)); + } + + @Test + public void requireThatSingleValueIsConverted() { + StringFieldValue before = new StringFieldValue("69"); + FieldValue after = new StringMarker().doConvert(before); + + assertTrue(after instanceof StringFieldValue); + assertEquals(new StringFieldValue("69'"), after); + } + + @Test + public void requireThatArrayElementsAreConverted() { + Array<StringFieldValue> before = new Array<>(DataType.getArray(DataType.STRING)); + before.add(new StringFieldValue("6")); + before.add(new StringFieldValue("9")); + FieldValue after = new StringMarker().convert(before); + + assertTrue(after instanceof Array); + assertEquals(new StringFieldValue("6'"), ((Array)after).get(0)); + assertEquals(new StringFieldValue("9'"), ((Array)after).get(1)); + } + + @Test + public void requireThatConvertedArrayElementCompatibilityIsEnforced() { + Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING)); + StringFieldValue foo = new StringFieldValue("foo"); + StringFieldValue bar = new StringFieldValue("bar"); + arr.add(foo); + arr.add(bar); + try { + new SearchReplace(foo, new IntegerFieldValue(69)).convert(arr); + fail(); + } catch (IllegalArgumentException e) { + + } + try { + new SearchReplace(bar, new IntegerFieldValue(69)).convert(arr); + fail(); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void requireThatMapEntriesAreConverted() { + MapFieldValue<StringFieldValue, StringFieldValue> before = + new MapFieldValue<>(DataType.getMap(DataType.STRING, DataType.STRING)); + before.put(new StringFieldValue("6"), new StringFieldValue("9")); + before.put(new StringFieldValue("9"), new StringFieldValue("6")); + FieldValue after = new StringMarker().convert(before); + + assertTrue(after instanceof MapFieldValue); + assertEquals(new StringFieldValue("6'"), ((MapFieldValue)after).get(new StringFieldValue("9'"))); + assertEquals(new StringFieldValue("9'"), ((MapFieldValue)after).get(new StringFieldValue("6'"))); + } + + @Test + public void requireThatMapElementsCanBeRemoved() { + MapFieldValue<StringFieldValue, StringFieldValue> before = + new MapFieldValue<>(DataType.getMap(DataType.STRING, DataType.STRING)); + StringFieldValue foo = new StringFieldValue("foo"); + StringFieldValue bar = new StringFieldValue("bar"); + StringFieldValue baz = new StringFieldValue("baz"); + StringFieldValue cox = new StringFieldValue("cox"); + before.put(foo, bar); + before.put(baz, cox); + + FieldValue after = new SearchReplace(foo, null).convert(before); + assertTrue(after instanceof MapFieldValue); + assertEquals(1, ((MapFieldValue)after).size()); + + after = new SearchReplace(bar, null).convert(before); + assertTrue(after instanceof MapFieldValue); + assertEquals(1, ((MapFieldValue)after).size()); + + after = new SearchReplace(baz, null).convert(before); + assertTrue(after instanceof MapFieldValue); + assertEquals(1, ((MapFieldValue)after).size()); + + after = new SearchReplace(cox, null).convert(before); + assertTrue(after instanceof MapFieldValue); + assertEquals(1, ((MapFieldValue)after).size()); + } + + @Test + public void requireThatConvertedMapElementCompatibilityIsEnforced() { + MapFieldValue<StringFieldValue, StringFieldValue> before = + new MapFieldValue<>(DataType.getMap(DataType.STRING, DataType.STRING)); + StringFieldValue foo = new StringFieldValue("foo"); + StringFieldValue bar = new StringFieldValue("bar"); + StringFieldValue baz = new StringFieldValue("baz"); + StringFieldValue cox = new StringFieldValue("cox"); + before.put(foo, bar); + before.put(baz, cox); + + try { + new SearchReplace(foo, new IntegerFieldValue(69)).convert(before); + fail(); + } catch (IllegalArgumentException e) { + + } + try { + new SearchReplace(bar, new IntegerFieldValue(69)).convert(before); + fail(); + } catch (IllegalArgumentException e) { + + } + try { + new SearchReplace(baz, new IntegerFieldValue(69)).convert(before); + fail(); + } catch (IllegalArgumentException e) { + + } + try { + new SearchReplace(cox, new IntegerFieldValue(69)).convert(before); + fail(); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void requireThatWsetElementsAreConverted() { + WeightedSet<StringFieldValue> before = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING)); + before.put(new StringFieldValue("6"), 69); + before.put(new StringFieldValue("9"), 96); + FieldValue after = new StringMarker().convert(before); + + assertTrue(after instanceof WeightedSet); + @SuppressWarnings("unchecked") + WeightedSet<StringFieldValue> w = (WeightedSet<StringFieldValue>) after; + assertEquals(2, w.size()); + assertTrue(w.contains(new StringFieldValue("9'"))); + assertEquals(96, w.get(new StringFieldValue("9'")).intValue()); + assertTrue(w.contains(new StringFieldValue("6'"))); + assertEquals(69, w.get(new StringFieldValue("6'")).intValue()); + } + + @Test + public void requireThatWsetElementsCanBeRemoved() { + WeightedSet<StringFieldValue> before = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING)); + StringFieldValue foo = new StringFieldValue("foo"); + StringFieldValue bar = new StringFieldValue("bar"); + before.put(foo, 6); + before.put(bar, 9); + + FieldValue after = new SearchReplace(foo, null).convert(before); + assertTrue(after instanceof WeightedSet); + assertEquals(1, ((WeightedSet)after).size()); + + after = new SearchReplace(bar, null).convert(before); + assertTrue(after instanceof WeightedSet); + assertEquals(1, ((WeightedSet)after).size()); + } + + @Test + public void requireThatConvertedWsetElementCompatibilityIsEnforced() { + WeightedSet<StringFieldValue> before = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING)); + StringFieldValue foo = new StringFieldValue("foo"); + StringFieldValue bar = new StringFieldValue("bar"); + before.put(foo, 6); + before.put(bar, 9); + + try { + new SearchReplace(foo, new IntegerFieldValue(69)).convert(before); + fail(); + } catch (IllegalArgumentException e) { + + } + try { + new SearchReplace(bar, new IntegerFieldValue(69)).convert(before); + fail(); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void requireThatEmptyCollectionsAreConvertedToNull() { + FieldValueConverter converter = new FieldValueConverter() { + + @Override + protected boolean shouldConvert(FieldValue value) { + return false; + } + + @Override + protected FieldValue doConvert(FieldValue value) { + throw new AssertionError(); + } + }; + assertNull(converter.convert(new Array(DataType.getArray(DataType.STRING)))); + assertNull(converter.convert(new MapFieldValue(DataType.getMap(DataType.STRING, DataType.STRING)))); + assertNull(converter.convert(new WeightedSet<StringFieldValue>(DataType.getWeightedSet(DataType.STRING)))); + } + + @Test + public void requireThatStructElementsAreConverted() { + StructDataType type = new StructDataType("foo"); + type.addField(new Field("bar", DataType.STRING)); + type.addField(new Field("baz", DataType.STRING)); + Struct before = type.createFieldValue(); + before.setFieldValue("bar", new StringFieldValue("6")); + before.setFieldValue("baz", new StringFieldValue("9")); + FieldValue after = new StringMarker().convert(before); + + assertTrue(after instanceof Struct); + assertEquals(new StringFieldValue("6'"), ((Struct)after).getFieldValue("bar")); + assertEquals(new StringFieldValue("9'"), ((Struct)after).getFieldValue("baz")); + } + + @Test + public void requireThatStructElementsCanBeRemoved() { + StructDataType type = new StructDataType("foo"); + type.addField(new Field("bar", DataType.STRING)); + type.addField(new Field("baz", DataType.STRING)); + Struct before = type.createFieldValue(); + StringFieldValue barVal = new StringFieldValue("6"); + StringFieldValue bazVal = new StringFieldValue("9"); + before.setFieldValue("bar", barVal); + before.setFieldValue("baz", bazVal); + + FieldValue after = new SearchReplace(barVal, null).convert(before); + assertTrue(after instanceof Struct); + assertNull(((Struct)after).getFieldValue("bar")); + assertEquals(bazVal, ((Struct)after).getFieldValue("baz")); + + after = new SearchReplace(bazVal, null).convert(before); + assertTrue(after instanceof Struct); + assertEquals(barVal, ((Struct)after).getFieldValue("bar")); + assertNull(((Struct)after).getFieldValue("baz")); + } + + @Test + public void requireThatStructArrayElementsAreConverted() { + StructDataType type = new StructDataType("foo"); + type.addField(new Field("bar", DataType.STRING)); + type.addField(new Field("baz", DataType.STRING)); + Array<Struct> before = new Array<>(DataType.getArray(type)); + Struct elem = type.createFieldValue(); + elem.setFieldValue("bar", new StringFieldValue("6")); + elem.setFieldValue("baz", new StringFieldValue("9")); + before.add(elem); + elem = type.createFieldValue(); + elem.setFieldValue("bar", new StringFieldValue("9")); + elem.setFieldValue("baz", new StringFieldValue("6")); + before.add(elem); + FieldValue after = new StringMarker().convert(before); + + assertTrue(after instanceof Array); + FieldValue val = ((Array)after).getFieldValue(0); + assertTrue(val instanceof Struct); + assertEquals(new StringFieldValue("6'"), ((Struct)val).getFieldValue("bar")); + assertEquals(new StringFieldValue("9'"), ((Struct)val).getFieldValue("baz")); + val = ((Array)after).getFieldValue(1); + assertTrue(val instanceof Struct); + assertEquals(new StringFieldValue("9'"), ((Struct)val).getFieldValue("bar")); + assertEquals(new StringFieldValue("6'"), ((Struct)val).getFieldValue("baz")); + } + + @Test + public void requireThatArrayChangeNestedType() { + Array<StringFieldValue> before = new Array<>(DataType.getArray(DataType.STRING)); + before.add(new StringFieldValue("69")); + FieldValue after = new IntegerParser().convert(before); + + assertTrue(after instanceof Array); + assertEquals(DataType.getArray(DataType.INT), after.getDataType()); + } + + @Test + public void requireThatMapChangeNestedType() { + MapFieldValue<StringFieldValue, StringFieldValue> before = + new MapFieldValue<>( + DataType.getMap(DataType.STRING, DataType.STRING)); + before.put(new StringFieldValue("6"), new StringFieldValue("9")); + FieldValue after = new IntegerParser().convert(before); + + assertTrue(after instanceof MapFieldValue); + assertEquals(DataType.getMap(DataType.INT, DataType.INT), after.getDataType()); + } + + @Test + public void requireThatWsetChangeNestedType() { + assertWsetChangeNestedType(false, false); + assertWsetChangeNestedType(false, true); + assertWsetChangeNestedType(true, false); + assertWsetChangeNestedType(true, true); + } + + private static void assertWsetChangeNestedType(boolean createIfNonExistent, boolean removeIfZero) { + WeightedSet<StringFieldValue> before = + new WeightedSet<>(DataType.getWeightedSet(DataType.STRING, createIfNonExistent, removeIfZero)); + before.put(new StringFieldValue("6"), 9); + FieldValue after = new IntegerParser().convert(before); + + assertTrue(after instanceof WeightedSet); + assertEquals(DataType.getWeightedSet(DataType.INT, createIfNonExistent, removeIfZero), after.getDataType()); + } + + private static class StringMarker extends StringFieldConverter { + + @Override + protected FieldValue doConvert(StringFieldValue value) { + return new StringFieldValue(value.toString() + "'"); + } + } + + private static class IntegerParser extends StringFieldConverter { + + @Override + protected FieldValue doConvert(StringFieldValue value) { + return new IntegerFieldValue(Integer.valueOf(value.toString())); + } + } + + private static class SearchReplace extends FieldValueConverter { + + final FieldValue searchFor; + final FieldValue replaceWith; + + private SearchReplace(FieldValue searchFor, FieldValue replaceWith) { + this.searchFor = searchFor; + this.replaceWith = replaceWith; + } + + @Override + protected boolean shouldConvert(FieldValue value) { + return value == searchFor; + } + + @Override + protected FieldValue doConvert(FieldValue value) { + return replaceWith; + } + } + + private static class MyFieldValue extends FieldValue { + + @Override + public DataType getDataType() { + return null; + } + + @Override + public void printXml(XmlStream xml) { + + } + + @Override + public void clear() { + + } + + @Override + public void assign(Object obj) { + + } + + @Override + public void serialize(Field field, FieldWriter writer) { + + } + + @Override + public void deserialize(Field field, FieldReader reader) { + + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/PathUpdateToDocumentTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/PathUpdateToDocumentTestCase.java new file mode 100644 index 00000000000..3472920dcdf --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/PathUpdateToDocumentTestCase.java @@ -0,0 +1,141 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.*; +import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class PathUpdateToDocumentTestCase { + + @Test + public void requireThatIntegerFieldsAreConverted() { + DocumentType docType = new DocumentType("my_type"); + docType.addField(new Field("my_int", DataType.INT)); + + FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "my_int", "", new IntegerFieldValue(69)); + Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd); + assertNotNull(doc); + + assertEquals(69, ((IntegerFieldValue)doc.getFieldValue("my_int")).getInteger()); + } + + @Test + public void requireThatStringFieldsAreConverted() { + DocumentType docType = new DocumentType("my_type"); + docType.addField(new Field("my_str", DataType.STRING)); + + FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "my_str", "", new StringFieldValue("69")); + Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd); + assertNotNull(doc); + + assertEquals("69", ((StringFieldValue)doc.getFieldValue("my_str")).getString()); + } + + @SuppressWarnings({ "unchecked" }) + @Test + public void requireThatArrayFieldsAreConverted() { + DocumentType docType = new DocumentType("my_type"); + ArrayDataType arrType = DataType.getArray(DataType.INT); + docType.addField(new Field("my_arr", arrType)); + + Array<IntegerFieldValue> arrVal = arrType.createFieldValue(); + arrVal.add(new IntegerFieldValue(6)); + arrVal.add(new IntegerFieldValue(9)); + FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "my_arr", "", arrVal); + + Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("my_arr"); + assertTrue(obj instanceof Array); + Array arr = (Array)obj; + assertEquals(2, arr.size()); + assertEquals(new IntegerFieldValue(6), arr.get(0)); + assertEquals(new IntegerFieldValue(9), arr.get(1)); + } + + @Test + public void requireThatWsetKeysAreConverted() { + DocumentType docType = new DocumentType("my_type"); + WeightedSetDataType wsetType = DataType.getWeightedSet(DataType.STRING); + docType.addField(new Field("my_wset", wsetType)); + + FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "my_wset{69}", "", new IntegerFieldValue(96)); + + Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("my_wset"); + assertTrue(obj instanceof WeightedSet); + WeightedSet wset = (WeightedSet)obj; + assertEquals(1, wset.size()); + assertEquals(96, wset.get(new StringFieldValue("69")).intValue()); + } + + @Test + public void requireThatNestedStructsAreConverted() { + DocumentType docType = new DocumentType("my_type"); + StructDataType structType = new StructDataType("my_struct"); + structType.addField(new Field("b", DataType.INT)); + docType.addField(new Field("a", structType)); + + FieldPathUpdate upd = new AssignFieldPathUpdate(docType, "a.b", "", new IntegerFieldValue(69)); + + Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("a"); + assertTrue(obj instanceof Struct); + Struct struct = (Struct)obj; + assertEquals(new IntegerFieldValue(69), struct.getFieldValue("b")); + } + + @SuppressWarnings({ "unchecked" }) + @Test + public void requireThatAddIsConverted() { + DocumentType docType = new DocumentType("my_type"); + ArrayDataType arrType = DataType.getArray(DataType.INT); + docType.addField(new Field("my_arr", arrType)); + + Array<IntegerFieldValue> arrVal = arrType.createFieldValue(); + arrVal.add(new IntegerFieldValue(6)); + arrVal.add(new IntegerFieldValue(9)); + FieldPathUpdate upd = new AddFieldPathUpdate(docType, "my_arr", "", arrVal); + + Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("my_arr"); + assertTrue(obj instanceof Array); + Array arr = (Array)obj; + assertEquals(2, arr.size()); + assertEquals(new IntegerFieldValue(6), arr.get(0)); + assertEquals(new IntegerFieldValue(9), arr.get(1)); + } + + @Test + public void requireThatRemoveIsConverted() { + DocumentType docType = new DocumentType("my_type"); + ArrayDataType arrType = DataType.getArray(DataType.INT); + docType.addField(new Field("my_arr", arrType)); + + FieldPathUpdate upd = new RemoveFieldPathUpdate(docType, "my_arr", ""); + + Document doc = FieldPathUpdateHelper.newPartialDocument(null, upd); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("my_arr"); + assertTrue(obj instanceof Array); + Array arr = (Array)obj; + assertEquals(0, arr.size()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptParserTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptParserTestCase.java new file mode 100644 index 00000000000..5b034ec90b6 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptParserTestCase.java @@ -0,0 +1,100 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.expressions.EchoExpression; +import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class ScriptParserTestCase { + + @Test + public void requireThatExpressionParserCanBeInvoked() throws ParseException { + try { + ScriptParser.parseExpression(newContext("foo")); + } catch (ParseException e) { + assertException(e, "Encountered \" <IDENTIFIER> \"foo \"\" at line 1, column 1."); + } + assertEquals(new InputExpression("foo"), + ScriptParser.parseExpression(newContext("input foo"))); + assertEquals(new StatementExpression(new InputExpression("foo"), new EchoExpression()), + ScriptParser.parseExpression(newContext("input foo | echo"))); + assertEquals(new ScriptExpression(new StatementExpression(new InputExpression("foo")), + new StatementExpression(new EchoExpression())), + ScriptParser.parseExpression(newContext("{ input foo; echo }"))); + } + + @Test + public void requireThatStatementParserCanBeInvoked() throws ParseException { + try { + ScriptParser.parseStatement(newContext("foo")); + } catch (ParseException e) { + assertException(e, "Encountered \" <IDENTIFIER> \"foo \"\" at line 1, column 1."); + } + assertEquals(new StatementExpression(new InputExpression("foo")), + ScriptParser.parseStatement(newContext("input foo"))); + assertEquals(new StatementExpression(new InputExpression("foo"), new EchoExpression()), + ScriptParser.parseStatement(newContext("input foo | echo"))); + assertEquals(new StatementExpression(new ScriptExpression(new StatementExpression(new InputExpression("foo")), + new StatementExpression(new EchoExpression()))), + ScriptParser.parseStatement(newContext("{ input foo; echo }"))); + } + + @Test + public void requireThatScriptParserCanBeInvoked() throws ParseException { + try { + ScriptParser.parseScript(newContext("foo")); + } catch (ParseException e) { + assertException(e, "Encountered \" <IDENTIFIER> \"foo \"\" at line 1, column 1."); + } + try { + ScriptParser.parseScript(newContext("input foo")); + } catch (ParseException e) { + assertException(e, "Encountered \" \"input\" \"input \"\" at line 1, column 1."); + } + try { + ScriptParser.parseScript(newContext("input foo | echo")); + } catch (ParseException e) { + assertException(e, "Encountered \" \"input\" \"input \"\" at line 1, column 1."); + } + assertEquals(new ScriptExpression(new StatementExpression(new InputExpression("foo")), + new StatementExpression(new EchoExpression())), + ScriptParser.parseScript(newContext("{ input foo; echo }"))); + } + + @Test + public void requireThatStatementParserBacksUpStream() throws ParseException { + ScriptParserContext config = newContext("input foo input bar"); + assertEquals(new StatementExpression(new InputExpression("foo")), ScriptParser.parseStatement(config)); + assertEquals(new StatementExpression(new InputExpression("bar")), ScriptParser.parseStatement(config)); + } + + @Test + public void requireThatScriptParserBacksUpStream() throws ParseException { + ScriptParserContext config = newContext("{ input foo }{ input bar }"); + assertEquals(new ScriptExpression(new StatementExpression(new InputExpression("foo"))), + ScriptParser.parseScript(config)); + assertEquals(new ScriptExpression(new StatementExpression(new InputExpression("bar"))), + ScriptParser.parseScript(config)); + } + + private static void assertException(ParseException e, String expectedMessage) throws ParseException { + if (!e.getMessage().startsWith(expectedMessage)) { + throw e; + } + } + + private static ScriptParserContext newContext(String input) { + return new ScriptParserContext(new SimpleLinguistics()).setInputStream(new IndexingInput(input)); + } + +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java new file mode 100644 index 00000000000..41a020e1384 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ScriptTestCase { + + private final DocumentType type; + + public ScriptTestCase() { + type = new DocumentType("mytype"); + type.addField("in-1", DataType.STRING); + type.addField("in-2", DataType.STRING); + type.addField("out-1", DataType.STRING); + type.addField("out-2", DataType.STRING); + } + + @Test + public void requireThatScriptExecutesStatements() { + Document input = new Document(type, "doc:scheme:"); + input.setFieldValue("in-1", new StringFieldValue("6")); + input.setFieldValue("in-2", new StringFieldValue("9")); + + Expression exp = new ScriptExpression( + new StatementExpression(new InputExpression("in-1"), new AttributeExpression("out-1")), + new StatementExpression(new InputExpression("in-2"), new AttributeExpression("out-2"))); + Document output = Expression.execute(exp, input); + assertNotNull(output); + assertEquals(new StringFieldValue("6"), output.getFieldValue("out-1")); + assertEquals(new StringFieldValue("9"), output.getFieldValue("out-2")); + } + + @Test + public void requireThatEachStatementHasEmptyInput() { + Document input = new Document(type, "doc:scheme:"); + input.setFieldValue(input.getField("in-1"), new StringFieldValue("69")); + + Expression exp = new ScriptExpression( + new StatementExpression(new InputExpression("in-1"), new AttributeExpression("out-1")), + new StatementExpression(new AttributeExpression("out-2"))); + try { + exp.verify(input); + fail(); + } catch (VerificationException e) { + assertTrue(e.getExpression() instanceof ScriptExpression); + assertEquals("Expected any input, got null.", e.getMessage()); + } + } + + @Test + public void requireThatFactoryMethodWorks() throws ParseException { + Document input = new Document(type, "doc:scheme:"); + input.setFieldValue("in-1", new StringFieldValue("FOO")); + + Document output = Expression.execute(Expression.fromString("input 'in-1' | { index 'out-1'; lowercase | index 'out-2' }"), input); + assertNotNull(output); + assertEquals(new StringFieldValue("FOO"), output.getFieldValue("out-1")); + assertEquals(new StringFieldValue("foo"), output.getFieldValue("out-2")); + } + + @Test + public void requireThatIfExpressionPassesOriginalInputAlong() throws ParseException { + Document input = new Document(type, "doc:scheme:"); + Document output = Expression.execute(Expression.fromString("'foo' | if (1 < 2) { 'bar' | index 'out-1' } else { 'baz' | index 'out-1' } | index 'out-1'"), input); + assertNotNull(output); + assertEquals(new StringFieldValue("foo"), output.getFieldValue("out-1")); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactoryTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactoryTestCase.java new file mode 100644 index 00000000000..13b2d1c3249 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactoryTestCase.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.ValueUpdate; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertNotNull; + +/** + * @author <a href="mailto:magnarn@yahoo-inc.com">Magnar Nedland</a> + */ +public class SimpleAdapterFactoryTestCase { + + @Test + public void requireThatCompleteUpdatesAreCombined() { + DocumentType docType = new DocumentType("my_type"); + DocumentUpdate update = new DocumentUpdate(docType, "doc:foo:1"); + Field field1 = new Field("int1", DataType.INT); + Field field2 = new Field("int2", DataType.INT); + Field field3 = new Field("int3", DataType.INT); + docType.addField(field1); + docType.addField(field2); + docType.addField(field3); + update.addFieldUpdate(FieldUpdate.createAssign(field1, new IntegerFieldValue(10))); + update.addFieldUpdate(FieldUpdate.createAssign(field2, new IntegerFieldValue(20))); + update.addFieldUpdate(FieldUpdate.createIncrement(field3, 30)); + + SimpleAdapterFactory factory = new SimpleAdapterFactory(); + List<UpdateAdapter> adapters = factory.newUpdateAdapterList(update); + assertEquals(2, adapters.size()); + } + + @Test + public void requireThatFieldUpdateCanHaveManyPartialUpdatesForOneField() { + DocumentType docType = new DocumentType("my_type"); + DocumentUpdate docUpdate = new DocumentUpdate(docType, "doc:foo:1"); + Field field = new Field("my_int", DataType.INT); + docType.addField(field); + FieldUpdate fieldUpdate = FieldUpdate.create(field); + fieldUpdate.addValueUpdate(ValueUpdate.createIncrement(1)); + fieldUpdate.addValueUpdate(ValueUpdate.createIncrement(2)); + fieldUpdate.addValueUpdate(ValueUpdate.createIncrement(4)); + docUpdate.addFieldUpdate(fieldUpdate); + + SimpleAdapterFactory factory = new SimpleAdapterFactory(); + List<UpdateAdapter> adapters = factory.newUpdateAdapterList(docUpdate); + assertEquals(4, adapters.size()); + + UpdateAdapter adapter = adapters.get(0); + assertNotNull(adapter); + assertEquals(new IntegerFieldValue(1), adapter.getInputValue("my_int")); + assertNotNull(adapter = adapters.get(1)); + assertEquals(new IntegerFieldValue(2), adapter.getInputValue("my_int")); + assertNotNull(adapter = adapters.get(2)); + assertEquals(new IntegerFieldValue(4), adapter.getInputValue("my_int")); + assertNotNull(adapter = adapters.get(3)); + assertNull(adapter.getInputValue("my_int")); // always add an adapter for complete updates + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapterTestCase.java new file mode 100644 index 00000000000..ec9505e6ca8 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleDocumentAdapterTestCase.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.vespa.indexinglanguage.expressions.VerificationException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SimpleDocumentAdapterTestCase { + + @Test + public void requireThatStructFieldsCanBeAccessed() { + DataType barType = DataType.STRING; + FieldValue bar = barType.createFieldValue("bar"); + + StructDataType fooType = new StructDataType("my_struct"); + fooType.addField(new Field("bar", barType)); + Struct foo = new Struct(fooType); + foo.setFieldValue("bar", bar); + + DocumentType docType = new DocumentType("my_doc"); + docType.addField("foo", fooType); + Document doc = new Document(docType, "doc:scheme:"); + doc.setFieldValue("foo", foo); + + DocumentAdapter adapter = new SimpleDocumentAdapter(doc); + assertEquals(fooType, adapter.getInputType(null, "foo")); + assertEquals(foo, adapter.getInputValue("foo")); + assertEquals(barType, adapter.getInputType(null, "foo.bar")); + assertEquals(bar, adapter.getInputValue("foo.bar")); + } + + @Test + public void requireThatUnknownFieldsReturnNull() { + DocumentType docType = new DocumentType("my_doc"); + Document doc = new Document(docType, "doc:scheme:"); + + DocumentAdapter adapter = new SimpleDocumentAdapter(doc); + try { + adapter.getInputType(null, "foo"); + fail(); + } catch (VerificationException e) { + assertEquals("Input field 'foo' not found.", e.getMessage()); + } + assertNull(adapter.getInputValue("foo")); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleTestAdapter.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleTestAdapter.java new file mode 100644 index 00000000000..9f25f376a2a --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleTestAdapter.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.FieldPath; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.FieldValueAdapter; +import com.yahoo.vespa.indexinglanguage.expressions.VerificationException; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SimpleTestAdapter implements FieldValueAdapter { + + private final Map<String, DataType> types = new HashMap<String, DataType>(); + private final Map<String, FieldValue> values = new HashMap<String, FieldValue>(); + + public SimpleTestAdapter(Field... fields) { + for (Field field : fields) { + types.put(field.getName(), field.getDataType()); + } + } + + public SimpleTestAdapter createField(Field field) { + types.put(field.getName(), field.getDataType()); + return this; + } + + @Override + public DataType getInputType(Expression exp, String fieldName) { + return types.get(fieldName); + } + + @Override + public FieldValue getInputValue(String fieldName) { + return values.get(fieldName); + } + + @Override + public FieldValue getInputValue(FieldPath fieldPath) { + return values.get(fieldPath.toString()); + } + + @Override + public void tryOutputType(Expression exp, String fieldName, DataType valueType) { + DataType fieldType = types.get(fieldName); + if (fieldType == null) { + throw new VerificationException(exp, "Field '" + fieldName + "' not found."); + } + if (!fieldType.isAssignableFrom(valueType)) { + throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to field '" + + fieldName + "' which is " + fieldType.getName() + "."); + } + } + + @Override + public SimpleTestAdapter setOutputValue(Expression exp, String fieldName, FieldValue fieldValue) { + values.put(fieldName, fieldValue); + return this; + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/TypedExpressionConverterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/TypedExpressionConverterTestCase.java new file mode 100644 index 00000000000..0e1e42ff391 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/TypedExpressionConverterTestCase.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.vespa.indexinglanguage.expressions.*; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class TypedExpressionConverterTestCase { + + @Test + public void requireThatOnlyExpressionsOfGivenTypeIsConverted() { + assertConvert(new AttributeExpression("foo"), + new IndexExpression("foo")); + assertConvert(new StatementExpression(new AttributeExpression("foo")), + new StatementExpression(new IndexExpression("foo"))); + assertConvert(new SummaryExpression("foo"), + new SummaryExpression("foo")); + assertConvert(new StatementExpression(new SummaryExpression("foo")), + new StatementExpression(new SummaryExpression("foo"))); + } + + private static void assertConvert(Expression before, Expression expectedAfter) { + assertEquals(expectedAfter, new MyConverter().convert(before)); + } + + private static class MyConverter extends TypedExpressionConverter<AttributeExpression> { + + public MyConverter() { + super(AttributeExpression.class); + } + + @Override + protected Expression typedConvert(AttributeExpression exp) { + return new IndexExpression(exp.getFieldName()); + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueTransformProviderTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueTransformProviderTestCase.java new file mode 100644 index 00000000000..fdcfd05736e --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueTransformProviderTestCase.java @@ -0,0 +1,173 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.collections.Pair; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ValueTransformProviderTestCase { + + @Test + public void requireThatInsertWorks() { + assertProvided(new StatementExpression(new IndexExpression("foo")), + new StatementExpression(new LowerCaseExpression(), + new IndexExpression("foo"))); + } + + @Test + public void requireThatInsertionIsJustInTime() { + assertProvided(new StatementExpression(new TrimExpression(), + new IndexExpression("foo")), + new StatementExpression(new TrimExpression(), + new LowerCaseExpression(), + new IndexExpression("foo"))); + } + + @Test + public void requireThatExistingTransformIsDetected() { + assertProvided(new StatementExpression(new LowerCaseExpression(), + new IndexExpression("foo")), + new StatementExpression(new LowerCaseExpression(), + new IndexExpression("foo"))); + } + + @Test + public void requireThatExistingRedundantTransformIsRemoved() { + assertProvided(new StatementExpression(new IndexExpression("foo"), + new LowerCaseExpression(), + new IndexExpression("bar")), + new StatementExpression(new LowerCaseExpression(), + new IndexExpression("foo"), + new IndexExpression("bar"))); + } + + @Test + public void requireThatNoRedundantTransformIsInserted() { + assertProvided(new StatementExpression(new IndexExpression("foo"), + new IndexExpression("bar")), + new StatementExpression(new LowerCaseExpression(), + new IndexExpression("foo"), + new IndexExpression("bar"))); + } + + @Test + public void requireThatExistingTransformIsPreserved() { + assertProvided(new StatementExpression(new LowerCaseExpression(), + new IndexExpression("foo"), + new IndexExpression("bar")), + new StatementExpression(new LowerCaseExpression(), + new IndexExpression("foo"), + new IndexExpression("bar"))); + } + + @Test + public void requireThatCompositeInsertWorks() { + assertProvided(new StatementExpression(new ForEachExpression(new IndexExpression("foo"))), + new StatementExpression(new ForEachExpression(new StatementExpression( + new LowerCaseExpression(), + new IndexExpression("foo"))))); + } + + @Test + public void requireThatStatementsAreManagedSeparately() { + assertProvided(new ScriptExpression(new StatementExpression(new IndexExpression("foo")), + new StatementExpression(new IndexExpression("bar"))), + new ScriptExpression(new StatementExpression(new LowerCaseExpression(), + new IndexExpression("foo")), + new StatementExpression(new LowerCaseExpression(), + new IndexExpression("bar")))); + } + + @Test + public void requireThatIfThenBranchesAreManagedSeparately() { + assertProvided(new StatementExpression(new IfThenExpression(new IndexExpression("a"), + IfThenExpression.Comparator.EQ, + new IndexExpression("b"), + new IndexExpression("c"), + new IndexExpression("d")), + new IndexExpression("e")), + new StatementExpression(new IfThenExpression(new StatementExpression(new LowerCaseExpression(), + new IndexExpression("a")), + IfThenExpression.Comparator.EQ, + new StatementExpression(new LowerCaseExpression(), + new IndexExpression("b")), + new StatementExpression(new LowerCaseExpression(), + new IndexExpression("c")), + new StatementExpression(new LowerCaseExpression(), + new IndexExpression("d"))), + new LowerCaseExpression(), + new IndexExpression("e"))); + } + + @Test + public void requireThatSelectInputBranchesAreManagedSeparately() { + List<Pair<String, Expression>> before = new LinkedList<Pair<String, Expression>>(); + before.add(new Pair<String, Expression>("a", new IndexExpression("b"))); + before.add(new Pair<String, Expression>("c", new IndexExpression("d"))); + + List<Pair<String, Expression>> after = new LinkedList<Pair<String, Expression>>(); + after.add(new Pair<String, Expression>("a", new StatementExpression(new LowerCaseExpression(), + new IndexExpression("b")))); + after.add(new Pair<String, Expression>("c", new StatementExpression(new LowerCaseExpression(), + new IndexExpression("d")))); + + assertProvided(new StatementExpression(new SelectInputExpression(before), + new IndexExpression("e")), + new StatementExpression(new SelectInputExpression(after), + new LowerCaseExpression(), + new IndexExpression("e"))); + } + + @Test + public void requireThatSwitchBranchesAreManagedSeparately() { + Map<String, Expression> before = new LinkedHashMap<String, Expression>(); + before.put("a", new IndexExpression("b")); + before.put("c", new IndexExpression("d")); + + Map<String, Expression> after = new LinkedHashMap<String, Expression>(); + after.put("a", new StatementExpression(new LowerCaseExpression(), + new IndexExpression("b"))); + after.put("c", new StatementExpression(new LowerCaseExpression(), + new IndexExpression("d"))); + + assertProvided(new StatementExpression(new SwitchExpression(before, + new IndexExpression("e")), + new IndexExpression("f")), + new StatementExpression(new SwitchExpression(after, + new StatementExpression(new LowerCaseExpression(), + new IndexExpression("e"))), + new LowerCaseExpression(), + new IndexExpression("f"))); + } + + private void assertProvided(Expression before, Expression after) { + assertEquals(after, new MyProvider().convert(before)); + } + + private static class MyProvider extends ValueTransformProvider { + + MyProvider() { + super(LowerCaseExpression.class); + } + + @Override + protected boolean requiresTransform(Expression exp) { + return exp instanceof IndexExpression; + } + + @Override + protected Expression newTransform() { + return new LowerCaseExpression(); + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueUpdateToDocumentTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueUpdateToDocumentTestCase.java new file mode 100644 index 00000000000..333c07b9e2d --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ValueUpdateToDocumentTestCase.java @@ -0,0 +1,156 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.*; +import com.yahoo.document.update.ValueUpdate; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ValueUpdateToDocumentTestCase { + + @Test + public void requireThatIntegerFieldsAreConverted() { + DocumentType docType = new DocumentType("my_type"); + Field field = new Field("my_int", DataType.INT); + docType.addField(field); + + ValueUpdate update = ValueUpdate.createAssign(new IntegerFieldValue(42)); + Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update); + assertNotNull(doc); + + assertEquals(42, ((IntegerFieldValue)doc.getFieldValue("my_int")).getInteger()); + } + + + @Test + public void requireThatClearValueUpdatesAreConverted() { + DocumentType docType = new DocumentType("my_type"); + Field field = new Field("my_int", DataType.INT); + docType.addField(field); + + ValueUpdate update = ValueUpdate.createClear(); + Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update); + assertNotNull(doc); + + assertNotNull(doc.getFieldValue("my_int")); + assertEquals(new IntegerFieldValue(), doc.getFieldValue("my_int")); + } + + + @Test + public void requireThatStringFieldsAreConverted() { + DocumentType docType = new DocumentType("my_type"); + Field field = new Field("my_str", DataType.STRING); + docType.addField(field); + + ValueUpdate update = ValueUpdate.createAssign(new StringFieldValue("42")); + Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update); + assertNotNull(doc); + + assertEquals("42", ((StringFieldValue)doc.getFieldValue("my_str")).getString()); + } + + @SuppressWarnings({ "unchecked" }) + @Test + public void requireThatArrayFieldsAreConverted() { + DocumentType docType = new DocumentType("my_type"); + ArrayDataType arrType = DataType.getArray(DataType.INT); + Field field = new Field("my_arr", arrType); + docType.addField(field); + + Array<IntegerFieldValue> arrVal = arrType.createFieldValue(); + arrVal.add(new IntegerFieldValue(6)); + arrVal.add(new IntegerFieldValue(9)); + ValueUpdate update = ValueUpdate.createAssign(arrVal); + + Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("my_arr"); + assertTrue(obj instanceof Array); + Array arr = (Array)obj; + assertEquals(2, arr.size()); + assertEquals(new IntegerFieldValue(6), arr.get(0)); + assertEquals(new IntegerFieldValue(9), arr.get(1)); + } + + @Test + public void requireThatWsetKeysAreConverted() { + DocumentType docType = new DocumentType("my_type"); + WeightedSetDataType wsetType = DataType.getWeightedSet(DataType.STRING); + Field field = new Field("my_wset", wsetType); + docType.addField(field); + + ValueUpdate update = ValueUpdate.createMap(new StringFieldValue("69"), ValueUpdate.createAssign(new IntegerFieldValue(96))); + + Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("my_wset"); + assertTrue(obj instanceof WeightedSet); + WeightedSet wset = (WeightedSet)obj; + assertEquals(1, wset.size()); + assertEquals(96, wset.get(new StringFieldValue("69")).intValue()); + } + + @Test + public void requireThatNestedStructsAreConverted() { + DocumentType docType = new DocumentType("my_type"); + StructDataType structType = new StructDataType("my_struct"); + structType.addField(new Field("b", DataType.INT)); + Field field = new Field("a", structType); + docType.addField(field); + + ValueUpdate update = ValueUpdate.createMap(new StringFieldValue("b"), ValueUpdate.createAssign(new IntegerFieldValue(42))); + Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("a"); + assertTrue(obj instanceof Struct); + Struct struct = (Struct)obj; + assertEquals(new IntegerFieldValue(42), struct.getFieldValue("b")); + } + + @SuppressWarnings({ "unchecked" }) + @Test + public void requireThatAddIsConverted() { + DocumentType docType = new DocumentType("my_type"); + ArrayDataType arrType = DataType.getArray(DataType.INT); + Field field = new Field("my_arr", arrType); + docType.addField(field); + + ValueUpdate update = ValueUpdate.createMap(new IntegerFieldValue(0), ValueUpdate.createAdd(new IntegerFieldValue(6))); + + Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("my_arr"); + assertTrue(obj instanceof Array); + Array arr = (Array)obj; + assertEquals(1, arr.size()); + assertEquals(new IntegerFieldValue(6), arr.get(0)); + } + + @Test + public void requireThatRemoveIsConverted() { + DocumentType docType = new DocumentType("my_type"); + ArrayDataType arrType = DataType.getArray(DataType.INT); + Field field = new Field("my_arr", arrType); + docType.addField(field); + + ValueUpdate update = ValueUpdate.createClear(); + + Document doc = FieldUpdateHelper.newPartialDocument(docType, new DocumentId("doc:foo:1"), field, update); + assertNotNull(doc); + + FieldValue obj = doc.getFieldValue("my_arr"); + assertTrue(obj instanceof Array); + Array arr = (Array)obj; + assertEquals(0, arr.size()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticTestCase.java new file mode 100644 index 00000000000..51d2a3717a9 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticTestCase.java @@ -0,0 +1,205 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression.Operator; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ArithmeticTestCase { + + @Test + public void requireThatAccessorsWork() { + ArithmeticExpression exp = newArithmetic(6, Operator.ADD, 9); + assertEquals(newLong(6), exp.getLeftHandSide()); + assertEquals(Operator.ADD, exp.getOperator()); + assertEquals(newLong(9), exp.getRightHandSide()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = newArithmetic(6, Operator.ADD, 9); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(newArithmetic(1, Operator.DIV, 1))); + assertFalse(exp.equals(newArithmetic(6, Operator.DIV, 1))); + assertFalse(exp.equals(newArithmetic(6, Operator.ADD, 1))); + assertEquals(exp, newArithmetic(6, Operator.ADD, 9)); + assertEquals(exp.hashCode(), newArithmetic(6, Operator.ADD, 9).hashCode()); + } + + @Test + public void requireThatConstructorDoesNotAcceptNull() { + try { + newArithmetic(null, Operator.ADD, new SimpleExpression()); + fail(); + } catch (NullPointerException e) { + + } + try { + newArithmetic(new SimpleExpression(), null, new SimpleExpression()); + fail(); + } catch (NullPointerException e) { + + } + try { + newArithmetic(new SimpleExpression(), Operator.ADD, null); + fail(); + } catch (NullPointerException e) { + + } + } + + @Test + public void requireThatVerifyCallsAreForwarded() { + assertVerify(SimpleExpression.newOutput(DataType.INT), Operator.ADD, + SimpleExpression.newOutput(DataType.INT), null); + assertVerifyThrows(SimpleExpression.newOutput(null), Operator.ADD, + SimpleExpression.newOutput(DataType.INT), null, + "Attempting to perform arithmetic on a null value."); + assertVerifyThrows(SimpleExpression.newOutput(DataType.INT), Operator.ADD, + SimpleExpression.newOutput(null), null, + "Attempting to perform arithmetic on a null value."); + assertVerifyThrows(SimpleExpression.newOutput(null), Operator.ADD, + SimpleExpression.newOutput(null), null, + "Attempting to perform arithmetic on a null value."); + assertVerifyThrows(SimpleExpression.newOutput(DataType.INT), Operator.ADD, + SimpleExpression.newOutput(DataType.STRING), null, + "Attempting to perform unsupported arithmetic: [int] + [string]"); + assertVerifyThrows(SimpleExpression.newOutput(DataType.STRING), Operator.ADD, + SimpleExpression.newOutput(DataType.STRING), null, + "Attempting to perform unsupported arithmetic: [string] + [string]"); + } + + @Test + public void requireThatOperandInputCanBeNull() { + SimpleExpression reqNull = new SimpleExpression(); + SimpleExpression reqInt = new SimpleExpression().setRequiredInput(DataType.INT); + assertNull(newArithmetic(reqNull, Operator.ADD, reqNull).requiredInputType()); + assertEquals(DataType.INT, newArithmetic(reqInt, Operator.ADD, reqNull).requiredInputType()); + assertEquals(DataType.INT, newArithmetic(reqInt, Operator.ADD, reqInt).requiredInputType()); + assertEquals(DataType.INT, newArithmetic(reqNull, Operator.ADD, reqInt).requiredInputType()); + } + + @Test + public void requireThatOperandsAreInputCompatible() { + assertVerify(new SimpleExpression().setRequiredInput(DataType.INT), Operator.ADD, + new SimpleExpression().setRequiredInput(DataType.INT), DataType.INT); + assertVerifyThrows(new SimpleExpression().setRequiredInput(DataType.INT), Operator.ADD, + new SimpleExpression().setRequiredInput(DataType.STRING), null, + "Operands require conflicting input types, int vs string."); + } + + @Test + public void requireThatResultIsCalculated() { + for (int i = 0; i < 50; ++i) { + LongFieldValue lhs = new LongFieldValue(i); + LongFieldValue rhs = new LongFieldValue(100 - i); + assertResult(lhs, Operator.ADD, rhs, new LongFieldValue(lhs.getLong() + rhs.getLong())); + assertResult(lhs, Operator.SUB, rhs, new LongFieldValue(lhs.getLong() - rhs.getLong())); + assertResult(lhs, Operator.DIV, rhs, new LongFieldValue(lhs.getLong() / rhs.getLong())); + assertResult(lhs, Operator.MOD, rhs, new LongFieldValue(lhs.getLong() % rhs.getLong())); + assertResult(lhs, Operator.MUL, rhs, new LongFieldValue(lhs.getLong() * rhs.getLong())); + } + } + + @Test + public void requireThatArithmeticWithNullEvaluatesToNull() { + assertNull(newArithmetic(new SimpleExpression(), Operator.ADD, + new SetValueExpression(new LongFieldValue(69))).execute()); + assertNull(newArithmetic(new SetValueExpression(new LongFieldValue(69)), Operator.ADD, + new SimpleExpression()).execute()); + } + + @Test + public void requireThatNonNumericOperandThrows() { + try { + newArithmetic(new SetValueExpression(new IntegerFieldValue(6)), Operator.ADD, + new SetValueExpression(new StringFieldValue("9"))).execute(); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Unsupported operation: [int] + [string]", e.getMessage()); + } + try { + newArithmetic(new SetValueExpression(new StringFieldValue("6")), Operator.ADD, + new SetValueExpression(new IntegerFieldValue(9))).execute(); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Unsupported operation: [string] + [int]", e.getMessage()); + } + } + + @Test + public void requireThatProperNumericalTypeIsUsed() { + for (Operator op : Operator.values()) { + assertType(DataType.INT, op, DataType.INT, DataType.INT); + assertType(DataType.LONG, op, DataType.INT, DataType.LONG); + assertType(DataType.LONG, op, DataType.LONG, DataType.LONG); + assertType(DataType.INT, op, DataType.LONG, DataType.LONG); + + assertType(DataType.FLOAT, op, DataType.FLOAT, DataType.FLOAT); + assertType(DataType.DOUBLE, op, DataType.FLOAT, DataType.DOUBLE); + assertType(DataType.DOUBLE, op, DataType.DOUBLE, DataType.DOUBLE); + assertType(DataType.FLOAT, op, DataType.DOUBLE, DataType.DOUBLE); + + assertType(DataType.INT, op, DataType.FLOAT, DataType.FLOAT); + assertType(DataType.INT, op, DataType.DOUBLE, DataType.DOUBLE); + } + } + + private void assertResult(FieldValue lhs, Operator op, FieldValue rhs, FieldValue expected) { + assertEquals(expected, evaluate(new SetValueExpression(lhs), op, + new SetValueExpression(rhs))); + } + + private void assertType(DataType lhs, Operator op, DataType rhs, DataType expected) { + assertEquals(expected, newArithmetic(SimpleExpression.newOutput(lhs), op, + SimpleExpression.newOutput(rhs)).verify()); + assertEquals(expected, newArithmetic(lhs.createFieldValue(6), op, + rhs.createFieldValue(9)).execute().getDataType()); + } + + private static FieldValue evaluate(Expression lhs, Operator op, Expression rhs) { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + new ArithmeticExpression(lhs, op, rhs).execute(ctx); + return ctx.getValue(); + } + + private static ArithmeticExpression newArithmetic(long lhs, Operator op, long rhs) { + return newArithmetic(new LongFieldValue(lhs), op, new LongFieldValue(rhs)); + } + + private static ArithmeticExpression newArithmetic(FieldValue lhs, Operator op, FieldValue rhs) { + return newArithmetic(new SetValueExpression(lhs), op, new SetValueExpression(rhs)); + } + + private static ArithmeticExpression newArithmetic(Expression lhs, Operator op, Expression rhs) { + return new ArithmeticExpression(lhs, op, rhs); + } + + private static SetValueExpression newLong(long val) { + return new SetValueExpression(new LongFieldValue(val)); + } + + private static void assertVerify(Expression lhs, Operator op, Expression rhs, DataType val) { + new ArithmeticExpression(lhs, op, rhs).verify(val); + } + + private static void assertVerifyThrows(Expression lhs, Operator op, Expression rhs, DataType val, + String expectedException) { + try { + new ArithmeticExpression(lhs, op, rhs).verify(val); + fail(); + } catch (VerificationException e) { + assertEquals(expectedException, e.getMessage()); + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpressionTestCase.java new file mode 100644 index 00000000000..097e0f21bc1 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpressionTestCase.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertExecute; +import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertVerify; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class AttributeExpressionTestCase { + + @Test + public void requireThatAccessorsWork() { + AttributeExpression exp = new AttributeExpression("foo"); + assertEquals("foo", exp.getFieldName()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new AttributeExpression("foo"); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new AttributeExpression("bar"))); + assertFalse(exp.equals(new IndexExpression("foo"))); + assertEquals(exp, new AttributeExpression("foo")); + assertEquals(exp.hashCode(), new AttributeExpression("foo").hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + assertVerify(new AttributeExpression("foo")); + } + + @Test + public void requireThatExpressionCanBeExecuted() { + assertExecute(new AttributeExpression("foo")); + } +}
\ No newline at end of file diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java new file mode 100644 index 00000000000..3ed01e464be --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Base64DecodeTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new Base64DecodeExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new Base64DecodeExpression()); + assertEquals(exp.hashCode(), new Base64DecodeExpression().hashCode()); + } + + @Test + public void requireThatInputIsDecoded() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("zcIoHQ")); + new Base64DecodeExpression().execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof LongFieldValue); + assertEquals(489210573L, ((LongFieldValue)val).getLong()); + } + + @Test + public void requireThatEmptyStringDecodesToLongMinValue() { + assertEquals(new LongFieldValue(Long.MIN_VALUE), + new Base64DecodeExpression().execute(new StringFieldValue(""))); + } + + @Test + public void requireThatInputDoesNotExceedMaxLength() { + try { + new Base64DecodeExpression().execute(new StringFieldValue("abcdefghijlkm")); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Base64 value 'abcdefghijlkm' is out of range.", e.getMessage()); + } + } + + @Test + public void requireThatIllegalInputThrows() { + try { + new Base64DecodeExpression().execute(new StringFieldValue("???")); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Illegal base64 value '???'.", e.getMessage()); + } + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new Base64DecodeExpression(); + assertVerify(DataType.STRING, exp, DataType.LONG); + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.LONG, exp, "Expected string input, got long."); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeTestCase.java new file mode 100644 index 00000000000..3f7b92cba5c --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeTestCase.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Base64EncodeTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new Base64EncodeExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new Base64EncodeExpression()); + assertEquals(exp.hashCode(), new Base64EncodeExpression().hashCode()); + } + + @Test + public void requireThatInputIsEncoded() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new LongFieldValue(489210573L)); + new Base64EncodeExpression().execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof StringFieldValue); + assertEquals("zcIoHQAAAAA=", ((StringFieldValue)val).getString()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new Base64EncodeExpression(); + assertVerify(DataType.LONG, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected long input, got null."); + assertVerifyThrows(DataType.STRING, exp, "Expected long input, got string."); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java new file mode 100644 index 00000000000..89b8c89a48f --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java @@ -0,0 +1,247 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.*; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class CatTestCase { + + @Test + public void requireThatAccessorsWork() { + Expression foo = new AttributeExpression("foo"); + Expression bar = new AttributeExpression("bar"); + CatExpression exp = new CatExpression(foo, bar); + assertEquals(2, exp.size()); + assertSame(foo, exp.get(0)); + assertSame(bar, exp.get(1)); + assertEquals(Arrays.asList(foo, bar), exp.asList()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression foo = new AttributeExpression("foo"); + Expression bar = new AttributeExpression("bar"); + Expression exp = new CatExpression(foo, bar); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new StatementExpression(foo, bar))); + assertFalse(exp.equals(new CatExpression())); + assertFalse(exp.equals(new CatExpression(foo))); + assertFalse(exp.equals(new CatExpression(bar, foo))); + assertEquals(exp, new CatExpression(foo, bar)); + assertEquals(exp.hashCode(), new CatExpression(foo, bar).hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + assertVerify(new SetValueExpression(new StringFieldValue("foo")), + new SetValueExpression(new StringFieldValue("bar")), null); + assertVerify(new SimpleExpression().setRequiredInput(DataType.STRING), + new SimpleExpression().setRequiredInput(DataType.STRING), DataType.STRING); + assertVerifyThrows(new SimpleExpression().setCreatedOutput(null), + new SimpleExpression().setCreatedOutput(DataType.STRING), null, + "Attempting to concatenate a null value "); + assertVerifyThrows(new SimpleExpression().setRequiredInput(DataType.STRING), + new SimpleExpression().setRequiredInput(DataType.INT), null, + "Operands require conflicting input types, string vs int."); + assertVerifyThrows(new SimpleExpression().setRequiredInput(DataType.STRING), + new SimpleExpression().setRequiredInput(DataType.STRING), null, + "Expected string input, got null."); + assertVerifyThrows(new SimpleExpression().setRequiredInput(DataType.STRING), + new SimpleExpression().setRequiredInput(DataType.STRING), DataType.INT, + "Expected string input, got int."); + } + + @Test + public void requireThatPrimitivesAreConcatenated() { + assertEquals(new StringFieldValue("69"), evaluate(new StringFieldValue("6"), new StringFieldValue("9"))); + assertEquals(new StringFieldValue("69"), evaluate(new StringFieldValue("6"), new IntegerFieldValue(9))); + assertEquals(new StringFieldValue("69"), evaluate(new IntegerFieldValue(6), new IntegerFieldValue(9))); + assertEquals(new StringFieldValue("69"), evaluate(new IntegerFieldValue(6), new StringFieldValue("9"))); + } + + @Test + public void requireThatPrimitivesCanNotBeNull() { + assertNull(evaluate(DataType.STRING, new StringFieldValue("69"), DataType.STRING, null)); + assertNull(evaluate(DataType.STRING, null, DataType.STRING, new StringFieldValue("69"))); + assertNull(evaluate(DataType.INT, new IntegerFieldValue(69), DataType.INT, null)); + assertNull(evaluate(DataType.INT, null, DataType.INT, new IntegerFieldValue(69))); + } + + @Test + public void requireThatPrimitiveVerificationWorks() { + assertEquals(DataType.STRING, evaluate(DataType.getArray(DataType.STRING), DataType.STRING)); + assertEquals(DataType.STRING, evaluate(DataType.STRING, DataType.getArray(DataType.STRING))); + assertEquals(DataType.STRING, evaluate(DataType.getWeightedSet(DataType.STRING), DataType.STRING)); + assertEquals(DataType.STRING, evaluate(DataType.STRING, DataType.getWeightedSet(DataType.STRING))); + } + + @Test + public void requireThatInputValueIsAvailableToAllInnerExpressions() { + assertEquals(new StringFieldValue("foobarfoo"), + new StatementExpression(new SetValueExpression(new StringFieldValue("foo")), + new CatExpression(new ThisExpression(), + new SetValueExpression(new StringFieldValue("bar")), + new ThisExpression())).execute()); + } + + @Test + public void requiredThatRequiredInputTypeAllowsNull() { + assertVerify(new SetValueExpression(new StringFieldValue("foo")), new TrimExpression(), DataType.STRING); + assertVerify(new TrimExpression(), new SetValueExpression(new StringFieldValue("foo")), DataType.STRING); + } + + @Test + public void requireThatArraysAreConcatenated() { + Array<StringFieldValue> lhs = new Array<>(DataType.getArray(DataType.STRING)); + lhs.add(new StringFieldValue("6")); + Array<StringFieldValue> rhs = new Array<>(DataType.getArray(DataType.STRING)); + rhs.add(new StringFieldValue("9")); + + FieldValue val = evaluate(lhs, rhs); + assertTrue(val instanceof Array); + + Array arr = (Array)val; + assertEquals(2, arr.size()); + assertEquals(new StringFieldValue("6"), arr.get(0)); + assertEquals(new StringFieldValue("9"), arr.get(1)); + } + + @Test + public void requireThatArraysCanBeNull() { + DataType type = DataType.getArray(DataType.STRING); + Array<StringFieldValue> arr = new Array<>(type); + arr.add(new StringFieldValue("9")); + + FieldValue val = evaluate(type, null, type, arr); + assertEquals(type, val.getDataType()); + assertEquals(1, ((Array)val).size()); + assertEquals(new StringFieldValue("9"), ((Array)val).get(0)); + + val = evaluate(type, arr, type, null); + assertEquals(type, val.getDataType()); + assertEquals(1, ((Array)val).size()); + assertEquals(new StringFieldValue("9"), ((Array)val).get(0)); + } + + @Test + public void requireThatWsetsAreConcatenated() { + WeightedSet<StringFieldValue> lhs = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING)); + lhs.put(new StringFieldValue("6"), 9); + WeightedSet<StringFieldValue> rhs = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING)); + rhs.put(new StringFieldValue("9"), 6); + + FieldValue val = evaluate(lhs, rhs); + assertTrue(val instanceof WeightedSet); + + WeightedSet wset = (WeightedSet)val; + assertEquals(2, wset.size()); + assertEquals(Integer.valueOf(9), wset.get(new StringFieldValue("6"))); + assertEquals(Integer.valueOf(6), wset.get(new StringFieldValue("9"))); + } + + @Test + public void requireThatWsetsCanBeNull() { + DataType type = DataType.getWeightedSet(DataType.STRING); + WeightedSet<StringFieldValue> wset = new WeightedSet<>(type); + wset.put(new StringFieldValue("6"), 9); + + FieldValue val = evaluate(type, null, type, wset); + assertEquals(type, val.getDataType()); + assertEquals(1, ((WeightedSet)val).size()); + assertEquals(Integer.valueOf(9), ((WeightedSet)val).get(new StringFieldValue("6"))); + + val = evaluate(type, wset, type, null); + assertEquals(type, val.getDataType()); + assertEquals(1, ((WeightedSet)val).size()); + assertEquals(Integer.valueOf(9), ((WeightedSet)val).get(new StringFieldValue("6"))); + } + + @Test + public void requireThatCollectionTypesMustBeCompatible() { + assertEquals(DataType.getArray(DataType.STRING), evaluate(DataType.getArray(DataType.STRING), + DataType.getArray(DataType.STRING))); + assertEquals(DataType.STRING, evaluate(DataType.getArray(DataType.STRING), DataType.getArray(DataType.INT))); + assertEquals(DataType.STRING, + evaluate(DataType.getArray(DataType.STRING), DataType.getWeightedSet(DataType.STRING))); + + assertEquals(DataType.getWeightedSet(DataType.STRING), evaluate(DataType.getWeightedSet(DataType.STRING), + DataType.getWeightedSet(DataType.STRING))); + assertEquals(DataType.STRING, + evaluate(DataType.getWeightedSet(DataType.STRING), DataType.getWeightedSet(DataType.INT))); + assertEquals(DataType.STRING, + evaluate(DataType.getWeightedSet(DataType.STRING), DataType.getArray(DataType.STRING))); + } + + @Test + public void requireThatCollectionValuesMustBeCompatible() { + { + Array<StringFieldValue> arrA = new Array<>(DataType.getArray(DataType.STRING)); + arrA.add(new StringFieldValue("6")); + Array<IntegerFieldValue> arrB = new Array<>(DataType.getArray(DataType.INT)); + arrB.add(new IntegerFieldValue(9)); + assertEquals(new StringFieldValue(arrA.toString() + arrB.toString()), evaluate(arrA, arrB)); + assertEquals(new StringFieldValue(arrB.toString() + arrA.toString()), evaluate(arrB, arrA)); + } + { + Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING)); + arr.add(new StringFieldValue("6")); + WeightedSet<StringFieldValue> wset = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING)); + wset.add(new StringFieldValue("9")); + assertEquals(new StringFieldValue(arr.toString() + wset.toString()), evaluate(arr, wset)); + assertEquals(new StringFieldValue(wset.toString() + arr.toString()), evaluate(wset, arr)); + } + { + WeightedSet<StringFieldValue> wsetA = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING)); + wsetA.add(new StringFieldValue("6")); + WeightedSet<IntegerFieldValue> wsetB = new WeightedSet<>(DataType.getWeightedSet(DataType.INT)); + wsetB.add(new IntegerFieldValue(9)); + assertEquals(new StringFieldValue(wsetA.toString() + wsetB.toString()), evaluate(wsetA, wsetB)); + assertEquals(new StringFieldValue(wsetB.toString() + wsetA.toString()), evaluate(wsetB, wsetA)); + } + } + + private static void assertVerify(Expression expA, Expression expB, DataType val) { + new CatExpression(expA, expB).verify(val); + } + + private static void assertVerifyThrows(Expression expA, Expression expB, DataType val, String expectedException) { + try { + new CatExpression(expA, expB).verify(val); + fail(); + } catch (VerificationException e) { + if (!e.getMessage().startsWith(expectedException)) { + assertEquals(expectedException, e.getMessage()); + } + } + } + + private static FieldValue evaluate(FieldValue valA, FieldValue valB) { + return evaluate(valA.getDataType(), valA, valB.getDataType(), valB); + } + + private static FieldValue evaluate(DataType typeA, FieldValue valA, DataType typeB, FieldValue valB) { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter(new Field("a", typeA), + new Field("b", typeB))); + ctx.setOutputValue(null, "a", valA); + ctx.setOutputValue(null, "b", valB); + new CatExpression(new InputExpression("a"), new InputExpression("b")).execute(ctx); + return ctx.getValue(); + } + + private static DataType evaluate(DataType typeA, DataType typeB) { + SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("a", typeA), new Field("b", typeB)); + VerificationContext ctx = new VerificationContext(adapter); + new CatExpression(new InputExpression("a"), new InputExpression("b")).verify(ctx); + return ctx.getValue(); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateTestCase.java new file mode 100644 index 00000000000..4fdb63a934e --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateTestCase.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ClearStateTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new ClearStateExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new ClearStateExpression()); + assertEquals(exp.hashCode(), new ClearStateExpression().hashCode()); + } + + @Test + public void requireThatExecutionContextIsCleared() { + MyExecution ctx = new MyExecution(); + ctx.execute(new ClearStateExpression()); + assertTrue(ctx.cleared); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ClearStateExpression(); + assertVerify(null, exp, null); + assertVerify(DataType.INT, exp, null); + assertVerify(DataType.STRING, exp, null); + } + + @Test + public void requireThatVerificationContextIsCleared() { + MyVerification ctx = new MyVerification(); + ctx.execute(new ClearStateExpression()); + assertTrue(ctx.cleared); + } + + private static class MyExecution extends ExecutionContext { + + boolean cleared = false; + + @Override + public ExecutionContext clear() { + cleared = true; + return this; + } + } + + private static class MyVerification extends VerificationContext { + + boolean cleared = false; + + @Override + public VerificationContext clear() { + cleared = true; + return this; + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpressionTestCase.java new file mode 100644 index 00000000000..0ca70441b65 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpressionTestCase.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class CompositeExpressionTestCase { + + @Test + public void requireThatToScriptBlockOutputIsParsable() throws ParseException { + Expression exp = new SetValueExpression(new IntegerFieldValue(69)); + assertScript("{ 69; }", exp); + assertScript("{ 69; }", new StatementExpression(exp)); + assertScript("{ 69; }", new ScriptExpression(new StatementExpression(exp))); + } + + private static void assertScript(String expectedScript, Expression exp) throws ParseException { + String str = CompositeExpression.toScriptBlock(exp); + assertEquals(expectedScript, str); + assertNotNull(ScriptExpression.fromString(str)); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/EchoTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/EchoTestCase.java new file mode 100644 index 00000000000..9e1ae2e5350 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/EchoTestCase.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class EchoTestCase { + + @Test + public void requireThatAccessorsWork() { + assertSame(System.out, new EchoExpression().getOutputStream()); + + PrintStream out = new PrintStream(System.out); + assertSame(out, new EchoExpression(out).getOutputStream()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + PrintStream out = new PrintStream(System.out); + Expression exp = new EchoExpression(out); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new EchoExpression())); + assertFalse(exp.equals(new EchoExpression(new PrintStream(System.err)))); + assertEquals(exp, new EchoExpression(out)); + assertEquals(exp.hashCode(), new EchoExpression(out).hashCode()); + } + + @Test + public void requireThatValueIsEchoed() { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("69")); + new EchoExpression(new PrintStream(out)).execute(ctx); + + assertEquals("69" + System.getProperty("line.separator"), out.toString()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new EchoExpression(); + assertVerify(DataType.INT, exp, DataType.INT); + assertVerify(DataType.STRING, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected any input, got null."); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java new file mode 100644 index 00000000000..63447145613 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.annotation.*; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import java.util.Iterator; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ExactTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new ExactExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new ExactExpression()); + assertEquals(exp.hashCode(), new ExactExpression().hashCode()); + } + + @Test + public void requireThatValueIsNotChanged() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("FOO")); + new ExactExpression().execute(ctx); + + assertEquals("FOO", String.valueOf(ctx.getValue())); + } + + @Test + public void requireThatValueIsAnnotated() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("FOO")); + new ExactExpression().execute(ctx); + + assertAnnotation(0, 3, new StringFieldValue("foo"), (StringFieldValue)ctx.getValue()); + } + + @Test + public void requireThatThereIsNoSegmentation() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("FOO BAR")); + new ExactExpression().execute(ctx); + + assertAnnotation(0, 7, new StringFieldValue("foo bar"), (StringFieldValue)ctx.getValue()); + } + + @Test + public void requireThatRedundantAnnotationValueIsIgnored() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("foo")); + new ExactExpression().execute(ctx); + + assertAnnotation(0, 3, null, (StringFieldValue)ctx.getValue()); + } + + @Test + public void requireThatEmptyStringsAreNotAnnotated() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("")); + new ExactExpression().execute(ctx); + + assertNull(((StringFieldValue)ctx.getValue()).getSpanTree(SpanTrees.LINGUISTICS)); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ExactExpression(); + assertVerify(DataType.STRING, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + } + + private static void assertAnnotation(int expectedFrom, int expectedLen, StringFieldValue expectedVal, + StringFieldValue actualVal) + { + SpanTree tree = actualVal.getSpanTree(SpanTrees.LINGUISTICS); + assertNotNull(tree); + SpanList root = (SpanList)tree.getRoot(); + assertNotNull(root); + + Iterator<SpanNode> nodeIt = root.childIterator(); + assertTrue(nodeIt.hasNext()); + SpanNode node = nodeIt.next(); + assertNotNull(node); + assertEquals(expectedFrom, node.getFrom()); + assertEquals(expectedLen, node.getLength()); + + Iterator<Annotation> annoIt = tree.iterator(node); + assertTrue(annoIt.hasNext()); + Annotation anno = annoIt.next(); + assertNotNull(anno); + assertEquals(expectedVal, anno.getFieldValue()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContextTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContextTestCase.java new file mode 100644 index 00000000000..770f2f5662c --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContextTestCase.java @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ExecutionContextTestCase { + + @Test + public void requireThatValueCanBeSet() { + ExecutionContext ctx = new ExecutionContext(); + FieldValue val = new StringFieldValue("foo"); + ctx.setValue(val); + assertSame(val, ctx.getValue()); + } + + @Test + public void requireThatVariablesCanBeSet() { + ExecutionContext ctx = new ExecutionContext(); + FieldValue val = new StringFieldValue("foo"); + ctx.setVariable("foo", val); + assertSame(val, ctx.getVariable("foo")); + } + + @Test + public void requireThatLanguageCanBeSet() { + ExecutionContext ctx = new ExecutionContext(); + ctx.setLanguage(Language.ARABIC); + assertEquals(Language.ARABIC, ctx.getLanguage()); + } + + @Test + public void requireThatNullLanguageThrowsException() { + ExecutionContext ctx = new ExecutionContext(); + try { + ctx.setLanguage(null); + fail(); + } catch (NullPointerException e) { + + } + } + + @Test + public void requireThatClearRemovesValue() { + ExecutionContext ctx = new ExecutionContext(); + ctx.setValue(new StringFieldValue("foo")); + ctx.clear(); + assertNull(ctx.getValue()); + } + + @Test + public void requireThatClearRemovesVariables() { + ExecutionContext ctx = new ExecutionContext(); + ctx.setVariable("foo", new StringFieldValue("foo")); + ctx.clear(); + assertNull(ctx.getVariable("foo")); + } + + @Test + public void requireThatClearDoesNotClearLanguage() { + ExecutionContext ctx = new ExecutionContext(); + ctx.setLanguage(Language.ARABIC); + ctx.clear(); + assertEquals(Language.ARABIC, ctx.getLanguage()); + } + + @Test + public void requireThatResolveLanguageDefaultsToEnglishWithoutLinguistics() { + ExecutionContext ctx = new ExecutionContext(); + assertEquals(Language.ENGLISH, ctx.resolveLanguage(null)); + } + + @Test + public void requireThatResolveLanguageDefaultsToEnglishWithoutValue() { + ExecutionContext ctx = new ExecutionContext(); + assertEquals(Language.ENGLISH, ctx.resolveLanguage(new SimpleLinguistics())); + } + + @Test + public void requireThatLanguageCanBeResolved() { + ExecutionContext ctx = new ExecutionContext(); + ctx.setValue(new StringFieldValue("\u3072\u3089\u304c\u306a")); + assertEquals(Language.JAPANESE, ctx.resolveLanguage(new SimpleLinguistics())); + ctx.setValue(new StringFieldValue("\ud55c\uae00\uacfc")); + assertEquals(Language.KOREAN, ctx.resolveLanguage(new SimpleLinguistics())); + } + + @Test + public void requireThatExplicitLanguagePreventsDetection() { + ExecutionContext ctx = new ExecutionContext(); + ctx.setLanguage(Language.ARABIC); + ctx.setValue(new StringFieldValue("\u3072\u3089\u304c\u306a")); + assertEquals(Language.ARABIC, ctx.resolveLanguage(new SimpleLinguistics())); + ctx.setValue(new StringFieldValue("\ud55c\uae00\uacfc")); + assertEquals(Language.ARABIC, ctx.resolveLanguage(new SimpleLinguistics())); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssert.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssert.java new file mode 100644 index 00000000000..62173461744 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssert.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; + +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +class ExpressionAssert { + + public static void assertVerifyCtx(VerificationContext ctx, Expression exp, DataType expectedValueAfter) { + assertEquals(expectedValueAfter, exp.verify(ctx)); + } + + public static void assertVerify(DataType valueBefore, Expression exp, DataType expectedValueAfter) { + assertVerifyCtx(new VerificationContext().setValue(valueBefore), exp, expectedValueAfter); + } + + public static void assertVerifyThrows(DataType valueBefore, Expression exp, String expectedException) { + assertVerifyCtxThrows(new VerificationContext().setValue(valueBefore), exp, expectedException); + } + + public static void assertVerifyCtxThrows(VerificationContext ctx, Expression exp, String expectedException) { + try { + exp.verify(ctx); + fail(); + } catch (VerificationException e) { + if (!Pattern.matches(expectedException, e.getMessage())) { + assertEquals(expectedException, e.getMessage()); + } + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssertTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssertTestCase.java new file mode 100644 index 00000000000..c4f7872cfcc --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssertTestCase.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.assertNotNull; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class ExpressionAssertTestCase { + + @Test + public void requireThatAssertVerifyMethodThrowsWhenAppropriate() { + Throwable thrown = null; + try { + assertVerify(DataType.INT, new SimpleExpression(), DataType.STRING); + } catch (Throwable t) { + thrown = t; + } + assertNotNull(thrown); + + thrown = null; + try { + assertVerifyThrows(DataType.INT, new SimpleExpression(), + "unchecked expected exception message"); + } catch (Throwable t) { + thrown = t; + } + assertNotNull(thrown); + + thrown = null; + try { + assertVerifyThrows(DataType.INT, SimpleExpression.newRequired(DataType.STRING), + "wrong expected exception message"); + } catch (Throwable t) { + thrown = t; + } + assertNotNull(thrown); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionTestCase.java new file mode 100644 index 00000000000..172bed83862 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionTestCase.java @@ -0,0 +1,102 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class ExpressionTestCase { + + @Test + public void requireThatInputTypeIsCheckedBeforeExecute() { + assertExecute(newRequiredInput(DataType.INT), null); + assertExecute(newRequiredInput(DataType.INT), new IntegerFieldValue(69)); + assertExecuteThrows(newRequiredInput(DataType.INT), new StringFieldValue("foo"), + new IllegalArgumentException("expected int input, got string")); + } + + @Test + public void requireThatOutputTypeIsCheckedAfterExecute() { + assertExecute(newCreatedOutput(DataType.INT, (FieldValue)null), null); + assertExecute(newCreatedOutput(DataType.INT, new IntegerFieldValue(69)), null); + assertExecuteThrows(newCreatedOutput(DataType.INT, new StringFieldValue("foo")), null, + new IllegalStateException("expected int output, got string")); + } + + @Test + public void requireThatInputTypeIsCheckedBeforeVerify() { + assertVerify(newRequiredInput(DataType.INT), DataType.INT); + assertVerifyThrows(newRequiredInput(DataType.INT), null, + "Expected int input, got null."); + assertVerifyThrows(newRequiredInput(DataType.INT), UnresolvedDataType.INSTANCE, + "Failed to resolve input type."); + assertVerifyThrows(newRequiredInput(DataType.INT), DataType.STRING, + "Expected int input, got string."); + } + + @Test + public void requireThatOutputTypeIsCheckedAfterVerify() { + assertVerify(newCreatedOutput(DataType.INT, DataType.INT), null); + assertVerifyThrows(newCreatedOutput(DataType.INT, (DataType)null), null, + "Expected int output, got null."); + assertVerifyThrows(newCreatedOutput(DataType.INT, UnresolvedDataType.INSTANCE), null, + "Failed to resolve output type."); + assertVerifyThrows(newCreatedOutput(DataType.INT, DataType.STRING), null, + "Expected int output, got string."); + } + + @Test + public void requireThatEqualsMethodWorks() { + assertTrue(Expression.equals(null, null)); + assertTrue(Expression.equals(1, 1)); + assertFalse(Expression.equals(1, 2)); + assertFalse(Expression.equals(1, null)); + assertFalse(Expression.equals(null, 2)); + } + + private static Expression newRequiredInput(DataType requiredInput) { + return new SimpleExpression().setRequiredInput(requiredInput); + } + + private static Expression newCreatedOutput(DataType createdOutput, FieldValue actualOutput) { + return new SimpleExpression().setCreatedOutput(createdOutput).setExecuteValue(actualOutput); + } + + private static Expression newCreatedOutput(DataType createdOutput, DataType actualOutput) { + return new SimpleExpression().setCreatedOutput(createdOutput).setVerifyValue(actualOutput); + } + + private static void assertExecute(Expression exp, FieldValue val) { + exp.execute(val); + } + + private static void assertExecuteThrows(Expression exp, FieldValue val, Exception expectedException) { + try { + exp.execute(val); + fail(); + } catch (RuntimeException e) { + assertEquals(expectedException.getClass(), e.getClass()); + assertTrue(e.getMessage().contains(expectedException.getMessage())); + } + } + + private static void assertVerify(Expression exp, DataType val) { + exp.verify(val); + } + + private static void assertVerifyThrows(Expression exp, DataType val, String expectedException) { + try { + exp.verify(val); + fail(); + } catch (VerificationException e) { + assertEquals(expectedException, e.getMessage()); + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenTestCase.java new file mode 100644 index 00000000000..6e6ca016aa4 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenTestCase.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.annotation.*; +import com.yahoo.document.datatypes.StringFieldValue; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class FlattenTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new FlattenExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new FlattenExpression()); + assertEquals(exp.hashCode(), new FlattenExpression().hashCode()); + } + + @Test + public void requireThatAnnotationsAreFlattened() { + SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS); + tree.annotate(new Span(0, 3), new Annotation(AnnotationTypes.TERM, new StringFieldValue("oof"))); + tree.annotate(new Span(4, 3), new Annotation(AnnotationTypes.TERM, new StringFieldValue("rab"))); + tree.annotate(new Span(8, 3), new Annotation(AnnotationTypes.TERM, new StringFieldValue("zab"))); + + StringFieldValue val = new StringFieldValue("foo bar baz"); + val.setSpanTree(tree); + + assertEquals(new StringFieldValue("foo[oof] bar[rab] baz[zab]"), new FlattenExpression().execute(val)); + } + + @Test + public void requireThatNonTermAnnotationsAreIgnored() { + SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS); + tree.annotate(new Span(0, 3), new Annotation(AnnotationTypes.STEM, new StringFieldValue("oof"))); + + StringFieldValue val = new StringFieldValue("foo"); + val.setSpanTree(tree); + + assertEquals(new StringFieldValue("foo"), new FlattenExpression().execute(val)); + } + + @Test + public void requireThatNonSpanAnnotationsAreIgnored() { + SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS); + tree.annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("oof"))); + + StringFieldValue val = new StringFieldValue("foo"); + val.setSpanTree(tree); + + assertEquals(new StringFieldValue("foo"), new FlattenExpression().execute(val)); + } + + @Test + public void requireThatAnnotationsAreSorted() { + SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS); + tree.annotate(new Span(0, 3), new Annotation(AnnotationTypes.TERM, new StringFieldValue("cox"))); + tree.annotate(new Span(0, 3), new Annotation(AnnotationTypes.TERM, new StringFieldValue("baz"))); + tree.annotate(new Span(0, 3), new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar"))); + + StringFieldValue val = new StringFieldValue("foo"); + val.setSpanTree(tree); + + assertEquals(new StringFieldValue("foo[bar, baz, cox]"), new FlattenExpression().execute(val)); + } + + @Test + public void requireThatAnnotationsWithoutFieldValueUseOriginalSpan() { + SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS); + tree.annotate(new Span(0, 3), new Annotation(AnnotationTypes.TERM)); + + StringFieldValue val = new StringFieldValue("foo"); + val.setSpanTree(tree); + + assertEquals(new StringFieldValue("foo[foo]"), new FlattenExpression().execute(val)); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new FlattenExpression(); + assertVerify(DataType.STRING, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java new file mode 100644 index 00000000000..2bf00c98490 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java @@ -0,0 +1,254 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; +import com.yahoo.document.StructDataType; +import com.yahoo.document.datatypes.*; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import java.util.LinkedList; +import java.util.List; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ForEachTestCase { + + @Test + public void requireThatAccessorsWork() { + Expression innerExp = new AttributeExpression("foo"); + ForEachExpression exp = new ForEachExpression(innerExp); + assertSame(innerExp, exp.getInnerExpression()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression innerExp = new AttributeExpression("foo"); + Expression exp = new ForEachExpression(innerExp); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new ForEachExpression(new AttributeExpression("bar")))); + assertEquals(exp, new ForEachExpression(innerExp)); + assertEquals(exp.hashCode(), new ForEachExpression(innerExp).hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ForEachExpression(SimpleExpression.newConversion(DataType.INT, DataType.STRING)); + assertVerify(DataType.getArray(DataType.INT), exp, DataType.getArray(DataType.STRING)); + assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected Array, Struct or WeightedSet input, got int."); + assertVerifyThrows(DataType.getArray(DataType.STRING), exp, "Expected int input, got string."); + } + + @Test + public void requireThatStructFieldCompatibilityIsVerified() { + StructDataType type = new StructDataType("my_struct"); + type.addField(new Field("foo", DataType.INT)); + assertVerify(type, new ForEachExpression(new SimpleExpression()), type); + assertVerifyThrows(type, new ForEachExpression(SimpleExpression.newConversion(DataType.STRING, DataType.INT)), + "Expected string input, got int."); + assertVerifyThrows(type, new ForEachExpression(SimpleExpression.newConversion(DataType.INT, DataType.STRING)), + "Expected int output, got string."); + } + + @Test + public void requireThatEachTokenIsExecutedSeparately() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING)); + arr.add(new StringFieldValue("6")); + arr.add(new StringFieldValue("9")); + ctx.setValue(arr); + + MyCollector exp = new MyCollector(); + new ForEachExpression(exp).execute(ctx); + + assertEquals(2, exp.lst.size()); + + FieldValue val = exp.lst.get(0); + assertTrue(val instanceof StringFieldValue); + assertEquals("6", ((StringFieldValue)val).getString()); + + val = exp.lst.get(1); + assertTrue(val instanceof StringFieldValue); + assertEquals("9", ((StringFieldValue)val).getString()); + } + + @Test + public void requireThatCreatedOutputTypeDependsOnInnerExpression() { + assertNull(new ForEachExpression(new SimpleExpression()).createdOutputType()); + assertNotNull(new ForEachExpression(new SetValueExpression(new IntegerFieldValue(69))).createdOutputType()); + } + + @Test + public void requireThatArrayCanBeConverted() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + Array<StringFieldValue> before = new Array<>(DataType.getArray(DataType.STRING)); + before.add(new StringFieldValue("6")); + before.add(new StringFieldValue("9")); + ctx.setValue(before); + + new ForEachExpression(new ToIntegerExpression()).execute(ctx); + FieldValue val = ctx.getValue(); + assertTrue(val instanceof Array); + + Array after = (Array)val; + assertEquals(2, after.size()); + assertEquals(new IntegerFieldValue(6), after.get(0)); + assertEquals(new IntegerFieldValue(9), after.get(1)); + } + + @Test + public void requireThatEmptyArrayCanBeConverted() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new Array<StringFieldValue>(DataType.getArray(DataType.STRING))); + + new ForEachExpression(new ToIntegerExpression()).execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof Array); + assertEquals(DataType.INT, ((Array)val).getDataType().getNestedType()); + assertTrue(((Array)val).isEmpty()); + } + + @Test + public void requireThatIllegalInputValueThrows() { + try { + new ForEachExpression(new SimpleExpression()).execute(new StringFieldValue("foo")); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Expected Array, Struct or WeightedSet input, got string.", e.getMessage()); + } + } + + @Test + public void requireThatArrayWithNullCanBeConverted() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING)); + arr.add(new StringFieldValue("foo")); + ctx.setValue(arr); + + new ForEachExpression(SimpleExpression.newConversion(DataType.STRING, DataType.INT) + .setExecuteValue(null)).execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof Array); + assertEquals(DataType.INT, ((Array)val).getDataType().getNestedType()); + assertTrue(((Array)val).isEmpty()); + } + + @Test + public void requireThatWsetCanBeConverted() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + WeightedSet<StringFieldValue> before = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING)); + before.put(new StringFieldValue("6"), 9); + before.put(new StringFieldValue("9"), 6); + ctx.setValue(before); + + new ForEachExpression(new ToIntegerExpression()).execute(ctx); + FieldValue val = ctx.getValue(); + assertTrue(val instanceof WeightedSet); + + WeightedSet after = (WeightedSet)val; + assertEquals(2, after.size()); + assertEquals(new Integer(9), after.get(new IntegerFieldValue(6))); + assertEquals(new Integer(6), after.get(new IntegerFieldValue(9))); + } + + @Test + public void requireThatEmptyWsetCanBeConverted() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new WeightedSet<StringFieldValue>(DataType.getWeightedSet(DataType.STRING))); + + new ForEachExpression(new ToIntegerExpression()).execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof WeightedSet); + assertEquals(DataType.INT, ((WeightedSet)val).getDataType().getNestedType()); + assertTrue(((WeightedSet)val).isEmpty()); + } + + @Test + public void requireThatStructContentCanBeConverted() { + StructDataType type = new StructDataType("my_type"); + type.addField(new Field("my_str", DataType.STRING)); + Struct struct = new Struct(type); + struct.setFieldValue("my_str", new StringFieldValue(" foo ")); + + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(struct); + + new ForEachExpression(new TrimExpression()).execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof Struct); + assertEquals(type, val.getDataType()); + assertEquals(new StringFieldValue("foo"), ((Struct)val).getFieldValue("my_str")); + } + + @Test + public void requireThatIncompatibleStructFieldsFailToValidate() { + StructDataType type = new StructDataType("my_type"); + type.addField(new Field("my_int", DataType.INT)); + + VerificationContext ctx = new VerificationContext(new SimpleTestAdapter()); + ctx.setValue(type); + + try { + new ForEachExpression(new ToArrayExpression()).verify(ctx); + fail(); + } catch (VerificationException e) { + assertEquals("Expected int output, got Array<int>.", e.getMessage()); + } + } + + @Test + public void requireThatIncompatibleStructFieldsFailToExecute() { + StructDataType type = new StructDataType("my_type"); + type.addField(new Field("my_int", DataType.INT)); + Struct struct = new Struct(type); + struct.setFieldValue("my_int", new IntegerFieldValue(69)); + + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(struct); + + try { + new ForEachExpression(new ToArrayExpression()).execute(ctx); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Class class com.yahoo.document.datatypes.Array not applicable to an class " + + "com.yahoo.document.datatypes.IntegerFieldValue instance.", e.getMessage()); + } + } + + private static class MyCollector extends Expression { + + List<FieldValue> lst = new LinkedList<>(); + + @Override + protected void doExecute(ExecutionContext ctx) { + lst.add(ctx.getValue()); + } + + @Override + protected void doVerify(VerificationContext ctx) { + + } + + @Override + public DataType requiredInputType() { + return null; + } + + @Override + public DataType createdOutputType() { + return null; + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldTestCase.java new file mode 100644 index 00000000000..1cb55e288da --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldTestCase.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.vespa.indexinglanguage.SimpleDocumentAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GetFieldTestCase { + + @Test + public void requireThatAccessorsWork() { + GetFieldExpression exp = new GetFieldExpression("foo"); + assertEquals("foo", exp.getFieldName()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new GetFieldExpression("foo"); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new GetFieldExpression("bar"))); + assertEquals(exp, new GetFieldExpression("foo")); + assertEquals(exp.hashCode(), new GetFieldExpression("foo").hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + StructDataType type = new StructDataType("my_struct"); + type.addField(new Field("foo", DataType.STRING)); + Expression exp = new GetFieldExpression("foo"); + assertVerify(type, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected structured input, got int."); + assertVerifyThrows(type, new GetFieldExpression("bar"), "Field 'bar' not found."); + } + + @Test + public void requireThatStructFieldsCanBeRead() { + DataType barType = DataType.STRING; + FieldValue bar = barType.createFieldValue("bar"); + + StructDataType fooType = new StructDataType("my_struct"); + fooType.addField(new Field("bar", barType)); + Struct foo = new Struct(fooType); + foo.setFieldValue("bar", bar); + + DocumentType docType = new DocumentType("my_doc"); + docType.addField("foo", fooType); + Document doc = new Document(docType, "doc:scheme:"); + doc.setFieldValue("foo", foo); + + ExecutionContext ctx = new ExecutionContext(new SimpleDocumentAdapter(doc)); + assertEquals(bar, new StatementExpression(new InputExpression("foo"), + new GetFieldExpression("bar")).execute(ctx)); + } + + @Test + public void requireThatIllegalInputThrows() { + try { + new GetFieldExpression("foo").execute(new StringFieldValue("bar")); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Expected structured input, got string.", e.getMessage()); + } + } + + @Test + public void requireThatUnknownFieldThrows() { + try { + new GetFieldExpression("foo").execute(new StructDataType("my_struct").createFieldValue()); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Field 'foo' not found.", e.getMessage()); + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarTestCase.java new file mode 100644 index 00000000000..8455a47d781 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarTestCase.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GetVarTestCase { + + @Test + public void requireThatAccessorsWorks() { + GetVarExpression exp = new GetVarExpression("foo"); + assertEquals("foo", exp.getVariableName()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new GetVarExpression("foo"); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new GetVarExpression("bar"))); + assertEquals(exp, new GetVarExpression("foo")); + assertEquals(exp.hashCode(), new GetVarExpression("foo").hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + VerificationContext ctx = new VerificationContext(); + ctx.setVariable("foo", DataType.STRING); + + assertEquals(DataType.STRING, new GetVarExpression("foo").verify(ctx)); + try { + new GetVarExpression("bar").verify(ctx); + fail(); + } catch (VerificationException e) { + assertEquals("Variable 'bar' not found.", e.getMessage()); + } + } + + @Test + public void requireThatSymbolIsRead() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setVariable("in", new IntegerFieldValue(69)); + new GetVarExpression("in").execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof IntegerFieldValue); + assertEquals(69, ((IntegerFieldValue)val).getInteger()); + } + + @Test + public void requireThatGetVarCanBeUsedToImplementSum() throws ParseException { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setOutputValue(null, "in", new StringFieldValue("0;1;2;3;4;5;6;7;8;9")); + ScriptExpression.fromString("{ 0 | set_var tmp; " + + " input in | split ';' | for_each { to_int + get_var tmp | set_var tmp };" + + " get_var tmp | attribute out; }").execute(ctx); + + FieldValue val = ctx.getInputValue("out"); + assertTrue(val instanceof IntegerFieldValue); + assertEquals(45, ((IntegerFieldValue)val).getInteger()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java new file mode 100644 index 00000000000..b6e7b1c7b41 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java @@ -0,0 +1,126 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.update.AssignValueUpdate; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.ValueUpdate; +import com.yahoo.language.Language; +import com.yahoo.vespa.indexinglanguage.SimpleAdapterFactory; +import com.yahoo.vespa.indexinglanguage.UpdateAdapter; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import org.junit.Test; + +import java.util.List; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GuardTestCase { + + @Test + public void requireThatAccessorsWork() { + Expression innerExp = new AttributeExpression("foo"); + GuardExpression exp = new GuardExpression(innerExp); + assertSame(innerExp, exp.getInnerExpression()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression innerExp = new AttributeExpression("foo"); + Expression exp = new GuardExpression(innerExp); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new GuardExpression(new AttributeExpression("bar")))); + assertEquals(exp, new GuardExpression(innerExp)); + assertEquals(exp.hashCode(), new GuardExpression(innerExp).hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new GuardExpression(SimpleExpression.newConversion(DataType.INT, DataType.STRING)); + assertVerify(DataType.INT, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected int input, got null."); + assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string."); + } + + @Test + public void requireThatInputFieldsAreIncludedByDocument() throws ParseException { + DocumentType docType = new DocumentType("my_input"); + docType.addField(new Field("my_lng", DataType.LONG)); + docType.addField(new Field("my_str", DataType.STRING)); + + Document doc = new Document(docType, "doc:scheme:"); + doc.setFieldValue("my_str", new StringFieldValue("69")); + assertNotNull(doc = Expression.execute(Expression.fromString("guard { input my_str | to_int | attribute my_lng }"), doc)); + assertEquals(new LongFieldValue(69), doc.getFieldValue("my_lng")); + } + + @Test + public void requireThatInputFieldsAreIncludedByUpdate() throws ParseException { + DocumentType docType = new DocumentType("my_input"); + docType.addField(new Field("my_lng", DataType.LONG)); + docType.addField(new Field("my_str", DataType.STRING)); + + DocumentUpdate docUpdate = new DocumentUpdate(docType, "doc:scheme:"); + docUpdate.addFieldUpdate(FieldUpdate.createAssign(docType.getField("my_str"), new StringFieldValue("69"))); + assertNotNull(docUpdate = Expression.execute(Expression.fromString("guard { input my_str | to_int | attribute my_lng }"), docUpdate)); + + assertEquals(0, docUpdate.getFieldPathUpdates().size()); + assertEquals(1, docUpdate.getFieldUpdates().size()); + + FieldUpdate fieldUpd = docUpdate.getFieldUpdate(0); + assertNotNull(fieldUpd); + assertEquals(docType.getField("my_lng"), fieldUpd.getField()); + assertEquals(1, fieldUpd.getValueUpdates().size()); + + ValueUpdate valueUpd = fieldUpd.getValueUpdate(0); + assertNotNull(valueUpd); + assertTrue(valueUpd instanceof AssignValueUpdate); + assertEquals(new LongFieldValue(69), valueUpd.getValue()); + } + + @Test + public void requireThatConstFieldsAreIncludedByDocument() throws ParseException { + DocumentType docType = new DocumentType("my_input"); + docType.addField(new Field("my_lng", DataType.LONG)); + docType.addField(new Field("my_str", DataType.STRING)); + + Document doc = new Document(docType, "doc:scheme:"); + doc.setFieldValue("my_str", new StringFieldValue("foo")); + assertNotNull(doc = Expression.execute(Expression.fromString("guard { now | attribute my_lng }"), doc)); + assertTrue(doc.getFieldValue("my_lng") instanceof LongFieldValue); + } + + @Test + public void requireThatConstFieldsAreSkippedByUpdate() throws ParseException { + DocumentType docType = new DocumentType("my_input"); + docType.addField(new Field("my_int", DataType.INT)); + docType.addField(new Field("my_str", DataType.STRING)); + + DocumentUpdate docUpdate = new DocumentUpdate(docType, "doc:scheme:"); + docUpdate.addFieldUpdate(FieldUpdate.createAssign(docType.getField("my_str"), new StringFieldValue("foo"))); + assertNull(Expression.execute(Expression.fromString("guard { now | attribute my_int }"), docUpdate)); + } + + @Test + public void requireThatLanguageCanBeSetByUpdate() throws ParseException { + DocumentType docType = new DocumentType("my_input"); + docType.addField(new Field("my_str", DataType.STRING)); + DocumentUpdate docUpdate = new DocumentUpdate(docType, "doc:scheme:"); + docUpdate.addFieldUpdate(FieldUpdate.createAssign(docType.getField("my_str"), new StringFieldValue("foo"))); + + SimpleAdapterFactory factory = new SimpleAdapterFactory(); + List<UpdateAdapter> lst = factory.newUpdateAdapterList(docUpdate); + assertEquals(1, lst.size()); + + ExecutionContext ctx = new ExecutionContext(lst.get(0)); + Expression.fromString("guard { 'en' | set_language }").execute(ctx); + assertEquals(Language.ENGLISH, ctx.getLanguage()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeTestCase.java new file mode 100644 index 00000000000..69083d48101 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeTestCase.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class HexDecodeTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new HexDecodeExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new HexDecodeExpression()); + assertEquals(exp.hashCode(), new HexDecodeExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new HexDecodeExpression(); + assertVerify(DataType.STRING, exp, DataType.LONG); + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.LONG, exp, "Expected string input, got long."); + } + + @Test + public void requireInputIsDecoded() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("1d28c2cd")); + new HexDecodeExpression().execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof LongFieldValue); + assertEquals(489210573L, ((LongFieldValue)val).getLong()); + } + + @Test + public void requireThatLargeInputIsDecoded() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("ff7a3c87fd74abff")); + new HexDecodeExpression().execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof LongFieldValue); + assertEquals(-37651092108694529L, ((LongFieldValue)val).getLong()); + } + + @Test + public void requireThatEmptyStringDecodesToLongMinValue() { + assertEquals(new LongFieldValue(Long.MIN_VALUE), + new HexDecodeExpression().execute(new StringFieldValue(""))); + } + + @Test + public void requireThatInputDoesNotExceedMaxLength() { + try { + new HexDecodeExpression().execute(new StringFieldValue("1ffffffffffffffff")); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Hex value '1ffffffffffffffff' is out of range.", e.getMessage()); + } + } + + @Test + public void requireThatIllegalInputThrows() { + try { + new HexDecodeExpression().execute(new StringFieldValue("???")); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Illegal hex value '???'.", e.getMessage()); + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeTestCase.java new file mode 100644 index 00000000000..4873e428de8 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeTestCase.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class HexEncodeTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new HexEncodeExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new HexEncodeExpression()); + assertEquals(exp.hashCode(), new HexEncodeExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new HexEncodeExpression(); + assertVerify(DataType.LONG, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected long input, got null."); + assertVerifyThrows(DataType.STRING, exp, "Expected long input, got string."); + } + + @Test + public void requireThatInputIsEncoded() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new LongFieldValue(489210573L)); + new HexEncodeExpression().execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof StringFieldValue); + assertEquals("1d28c2cd", ((StringFieldValue)val).getString()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameTestCase.java new file mode 100644 index 00000000000..713a8d7bcb0 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameTestCase.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class HostNameTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new HostNameExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new HostNameExpression()); + assertEquals(exp.hashCode(), new HostNameExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new HostNameExpression(); + assertVerify(null, exp, DataType.STRING); + assertVerify(DataType.INT, exp, DataType.STRING); + assertVerify(DataType.STRING, exp, DataType.STRING); + } + + @Test + public void requireThatHostnameIsSet() throws UnknownHostException { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + new HostNameExpression().execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof StringFieldValue); + assertEquals(HostNameExpression.normalizeHostName(InetAddress.getLocalHost().getHostName()), + ((StringFieldValue)val).getString()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java new file mode 100644 index 00000000000..b59564e9600 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java @@ -0,0 +1,319 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.*; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static com.yahoo.vespa.indexinglanguage.expressions.IfThenExpression.Comparator; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class IfThenTestCase { + + @Test + public void requireThatAccessorsWork() { + Expression lhs = new AttributeExpression("lhs"); + Expression rhs = new AttributeExpression("rhs"); + Expression ifTrue = new AttributeExpression("ifTrue"); + Expression ifFalse = new AttributeExpression("ifFalse"); + IfThenExpression exp = new IfThenExpression(lhs, Comparator.EQ, rhs, ifTrue, ifFalse); + assertSame(lhs, exp.getLeftHandSide()); + assertSame(rhs, exp.getRightHandSide()); + assertEquals(Comparator.EQ, exp.getComparator()); + assertSame(ifTrue, exp.getIfTrueExpression()); + assertSame(ifFalse, exp.getIfFalseExpression()); + } + + @Test + public void requireThatRequiredInputTypeCompatibilityIsVerified() { + Expression exp = newRequiredInput(DataType.STRING, Comparator.EQ, DataType.STRING, + DataType.STRING, DataType.STRING); + assertVerify(DataType.STRING, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, newRequiredInput(DataType.INT, Comparator.EQ, DataType.STRING, + DataType.STRING, DataType.STRING), + "Operands require conflicting input types, int vs string."); + assertVerifyThrows(null, newRequiredInput(DataType.STRING, Comparator.EQ, DataType.INT, + DataType.STRING, DataType.STRING), + "Operands require conflicting input types, string vs int."); + assertVerifyThrows(null, newRequiredInput(DataType.STRING, Comparator.EQ, DataType.STRING, + DataType.INT, DataType.STRING), + "Operands require conflicting input types, string vs int."); + assertVerifyThrows(null, newRequiredInput(DataType.STRING, Comparator.EQ, DataType.STRING, + DataType.STRING, DataType.INT), + "Operands require conflicting input types, string vs int."); + } + + @Test + public void requireThatExpressionCanBeVerified() { + assertVerify(DataType.STRING, new FlattenExpression(), DataType.STRING); + assertVerifyThrows(null, new FlattenExpression(), + "Expected string input, got null."); + assertVerifyThrows(DataType.INT, new FlattenExpression(), + "Expected string input, got int."); + } + + @Test + public void requireThatIfFalseDefaultsToNull() { + IfThenExpression exp = new IfThenExpression(new AttributeExpression("lhs"), Comparator.EQ, + new AttributeExpression("rhs"), new AttributeExpression("ifTrue")); + assertNull(exp.getIfFalseExpression()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression lhs = new AttributeExpression("lhs"); + Expression rhs = new AttributeExpression("rhs"); + Expression ifTrue = new AttributeExpression("ifTrue"); + Expression ifFalse = new AttributeExpression("ifFalse"); + Expression exp = new IfThenExpression(lhs, Comparator.EQ, rhs, ifTrue, ifFalse); + + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new IfThenExpression(new IndexExpression("lhs"), Comparator.NE, + new IndexExpression("rhs"), + new IndexExpression("ifTrue"), + new IndexExpression("ifFalse")))); + assertFalse(exp.equals(new IfThenExpression(new AttributeExpression("lhs"), Comparator.NE, + new IndexExpression("rhs"), + new IndexExpression("ifTrue"), + new IndexExpression("ifFalse")))); + assertFalse(exp.equals(new IfThenExpression(new AttributeExpression("lhs"), Comparator.EQ, + new IndexExpression("rhs"), + new IndexExpression("ifTrue"), + new IndexExpression("ifFalse")))); + assertFalse(exp.equals(new IfThenExpression(new AttributeExpression("lhs"), Comparator.EQ, + new AttributeExpression("rhs"), + new IndexExpression("ifTrue"), + new IndexExpression("ifFalse")))); + assertFalse(exp.equals(new IfThenExpression(new AttributeExpression("lhs"), Comparator.EQ, + new AttributeExpression("rhs"), + new AttributeExpression("ifTrue"), + new IndexExpression("ifFalse")))); + assertEquals(exp, new IfThenExpression(new AttributeExpression("lhs"), Comparator.EQ, + new AttributeExpression("rhs"), + new AttributeExpression("ifTrue"), + new AttributeExpression("ifFalse"))); + assertEquals(exp.hashCode(), new IfThenExpression(new AttributeExpression("lhs"), Comparator.EQ, + new AttributeExpression("rhs"), + new AttributeExpression("ifTrue"), + new AttributeExpression("ifFalse")).hashCode()); + } + + @Test + public void requireThatAllChildrenSeeInputValue() { + FieldValueAdapter adapter = createTestAdapter(); + new StatementExpression(new SetValueExpression(new IntegerFieldValue(69)), + new IfThenExpression(new AttributeExpression("lhs"), + Comparator.EQ, + new AttributeExpression("rhs"), + new AttributeExpression("ifTrue"), + new AttributeExpression("ifFalse"))).execute(adapter); + assertEquals(new IntegerFieldValue(69), adapter.getInputValue("lhs")); + assertEquals(new IntegerFieldValue(69), adapter.getInputValue("rhs")); + assertEquals(new IntegerFieldValue(69), adapter.getInputValue("ifTrue")); + assertNull(null, adapter.getInputValue("ifFalse")); + + adapter = createTestAdapter(); + new StatementExpression(new SetValueExpression(new IntegerFieldValue(69)), + new IfThenExpression(new AttributeExpression("lhs"), + Comparator.NE, + new AttributeExpression("rhs"), + new AttributeExpression("ifTrue"), + new AttributeExpression("ifFalse"))).execute(adapter); + assertEquals(new IntegerFieldValue(69), adapter.getInputValue("lhs")); + assertEquals(new IntegerFieldValue(69), adapter.getInputValue("rhs")); + assertNull(null, adapter.getInputValue("ifTrue")); + assertEquals(new IntegerFieldValue(69), adapter.getInputValue("ifFalse")); + } + + @Test + public void requireThatElseExpIsOptional() { + ExecutionContext ctx = new ExecutionContext(); + Expression exp = new IfThenExpression(new SetValueExpression(new IntegerFieldValue(6)), + Comparator.GT, + new SetValueExpression(new IntegerFieldValue(9)), + new SetValueExpression(new StringFieldValue("69"))); + FieldValue val = ctx.setValue(new IntegerFieldValue(96)).execute(exp).getValue(); + assertTrue(val instanceof IntegerFieldValue); + assertEquals(96, ((IntegerFieldValue)val).getInteger()); + } + + @Test + public void requireThatNonNumericValuesUseFieldValueCompareTo() { + FieldValue small = new MyFieldValue(6); + FieldValue large = new MyFieldValue(9); + + assertCmpTrue(small, Comparator.EQ, small); + assertCmpFalse(small, Comparator.NE, small); + assertCmpFalse(small, Comparator.GT, small); + assertCmpTrue(small, Comparator.GE, small); + assertCmpFalse(small, Comparator.LT, small); + assertCmpTrue(small, Comparator.LE, small); + + assertCmpFalse(small, Comparator.EQ, large); + assertCmpTrue(small, Comparator.NE, large); + assertCmpFalse(small, Comparator.GT, large); + assertCmpFalse(small, Comparator.GE, large); + assertCmpTrue(small, Comparator.LT, large); + assertCmpTrue(small, Comparator.LE, large); + } + + @Test + public void requireThatNumericValuesSupportNumericCompareTo() { + List<NumericFieldValue> sixes = Arrays.asList(new ByteFieldValue((byte)6), + new DoubleFieldValue(6.0), + new FloatFieldValue(6.0f), + new IntegerFieldValue(6), + new LongFieldValue(6L)); + List<NumericFieldValue> nines = Arrays.asList(new ByteFieldValue((byte)9), + new DoubleFieldValue(9.0), + new FloatFieldValue(9.0f), + new IntegerFieldValue(9), + new LongFieldValue(9L)); + for (NumericFieldValue lhs : sixes) { + for (NumericFieldValue rhs : sixes) { + assertCmpTrue(lhs, Comparator.EQ, rhs); + assertCmpFalse(lhs, Comparator.NE, rhs); + assertCmpFalse(lhs, Comparator.GT, rhs); + assertCmpTrue(lhs, Comparator.GE, rhs); + assertCmpFalse(lhs, Comparator.LT, rhs); + assertCmpTrue(lhs, Comparator.LE, rhs); + } + for (NumericFieldValue rhs : nines) { + assertCmpFalse(lhs, Comparator.EQ, rhs); + assertCmpTrue(lhs, Comparator.NE, rhs); + assertCmpFalse(lhs, Comparator.GT, rhs); + assertCmpFalse(lhs, Comparator.GE, rhs); + assertCmpTrue(lhs, Comparator.LT, rhs); + assertCmpTrue(lhs, Comparator.LE, rhs); + } + } + for (NumericFieldValue lhs : nines) { + for (NumericFieldValue rhs : nines) { + assertCmpTrue(lhs, Comparator.EQ, rhs); + assertCmpFalse(lhs, Comparator.NE, rhs); + assertCmpFalse(lhs, Comparator.GT, rhs); + assertCmpTrue(lhs, Comparator.GE, rhs); + assertCmpFalse(lhs, Comparator.LT, rhs); + assertCmpTrue(lhs, Comparator.LE, rhs); + } + for (NumericFieldValue rhs : sixes) { + assertCmpFalse(lhs, Comparator.EQ, rhs); + assertCmpTrue(lhs, Comparator.NE, rhs); + assertCmpTrue(lhs, Comparator.GT, rhs); + assertCmpTrue(lhs, Comparator.GE, rhs); + assertCmpFalse(lhs, Comparator.LT, rhs); + assertCmpFalse(lhs, Comparator.LE, rhs); + } + } + } + + @Test + public void requireThatNullLeftOrRightHandSideEvaluatesToNull() { + Expression exp = new IfThenExpression(new GetVarExpression("lhs"), Comparator.EQ, new GetVarExpression("rhs"), + new SetValueExpression(new StringFieldValue("true")), + new SetValueExpression(new StringFieldValue("false"))); + assertEquals(new StringFieldValue("true"), + exp.execute(new ExecutionContext().setVariable("lhs", new IntegerFieldValue(69)) + .setVariable("rhs", new IntegerFieldValue(69)))); + assertEquals(new StringFieldValue("false"), + exp.execute(new ExecutionContext().setVariable("lhs", new IntegerFieldValue(6)) + .setVariable("rhs", new IntegerFieldValue(9)))); + assertNull(exp.execute(new ExecutionContext().setVariable("lhs", new IntegerFieldValue(69)))); + assertNull(exp.execute(new ExecutionContext().setVariable("rhs", new IntegerFieldValue(69)))); + } + + private static void assertCmpTrue(FieldValue lhs, Comparator cmp, FieldValue rhs) { + assertTrue(evaluateIfThen(lhs, cmp, rhs)); + } + + private static void assertCmpFalse(FieldValue lhs, Comparator cmp, FieldValue rhs) { + assertFalse(evaluateIfThen(lhs, cmp, rhs)); + } + + private static boolean evaluateIfThen(FieldValue lhs, Comparator cmp, FieldValue rhs) { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + new StatementExpression( + new SetValueExpression(new IntegerFieldValue(1)), + new IfThenExpression(new SetValueExpression(lhs), cmp, new SetValueExpression(rhs), + new SetVarExpression("true"), + new SetVarExpression("false"))).execute(ctx); + return ctx.getVariable("true") != null; + } + + private static FieldValueAdapter createTestAdapter() { + return new SimpleTestAdapter(new Field("lhs", DataType.INT), + new Field("rhs", DataType.INT), + new Field("ifTrue", DataType.INT), + new Field("ifFalse", DataType.INT)); + } + + private static Expression newRequiredInput(DataType lhs, Comparator cmp, DataType rhs, DataType ifTrue, + DataType ifFalse) { + return new IfThenExpression(new SimpleExpression().setRequiredInput(lhs), cmp, + new SimpleExpression().setRequiredInput(rhs), + new SimpleExpression().setRequiredInput(ifTrue), + ifFalse != null ? new SimpleExpression().setRequiredInput(ifFalse) : null); + } + + private static class MyFieldValue extends FieldValue { + + final Integer val; + + MyFieldValue(int val) { + this.val = val; + } + + @Override + public DataType getDataType() { + return null; + } + + @Override + public void printXml(XmlStream xml) { + + } + + @Override + public void clear() { + + } + + @Override + public void assign(Object o) { + + } + + @Override + public void serialize(Field field, FieldWriter writer) { + + } + + @Override + public void deserialize(Field field, FieldReader reader) { + + } + + @Override + public int compareTo(FieldValue rhs) { + if (!(rhs instanceof MyFieldValue)) { + throw new AssertionError(); + } + return val.compareTo(((MyFieldValue)rhs).val); + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IndexExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IndexExpressionTestCase.java new file mode 100644 index 00000000000..5097a14b870 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IndexExpressionTestCase.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertExecute; +import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertVerify; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class IndexExpressionTestCase { + + @Test + public void requireThatAccessorsWork() { + IndexExpression exp = new IndexExpression("foo"); + assertEquals("foo", exp.getFieldName()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression lhs = new IndexExpression("foo"); + assertFalse(lhs.equals(new Object())); + assertFalse(lhs.equals(new AttributeExpression("foo"))); + assertFalse(lhs.equals(new IndexExpression("bar"))); + assertEquals(lhs, new IndexExpression("foo")); + assertEquals(lhs.hashCode(), new IndexExpression("foo").hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + assertVerify(new IndexExpression("foo")); + } + + @Test + public void requireThatExpressionCanBeExecuted() { + assertExecute(new IndexExpression("foo")); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/InputTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/InputTestCase.java new file mode 100644 index 00000000000..ed53f50444c --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/InputTestCase.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.vespa.indexinglanguage.SimpleDocumentAdapter; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class InputTestCase { + + @Test + public void requireThatAccessorsWork() { + InputExpression exp = new InputExpression("foo"); + assertEquals("foo", exp.getFieldName()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new InputExpression(null); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new InputExpression("foo"))); + assertEquals(exp, new InputExpression(null)); + assertEquals(exp.hashCode(), new InputExpression(null).hashCode()); + + exp = new InputExpression("foo"); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new InputExpression("bar"))); + assertEquals(exp, new InputExpression("foo")); + assertEquals(exp.hashCode(), new InputExpression("foo").hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("foo", DataType.STRING)); + adapter.setOutputValue(null, "foo", new StringFieldValue("69")); + assertEquals(DataType.STRING, new InputExpression("foo").verify(adapter)); + try { + new InputExpression("bar").verify(adapter); + fail(); + } catch (VerificationException e) { + assertEquals("Field 'bar' not found.", e.getMessage()); + } + } + + @Test + public void requireThatFieldIsRead() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter(new Field("in", DataType.STRING))); + ctx.setOutputValue(null, "in", new StringFieldValue("69")); + new InputExpression("in").execute(ctx); + + assertEquals(new StringFieldValue("69"), ctx.getValue()); + } + + @Test + public void requireThatStructFieldsCanBeRead() { + DataType barType = DataType.STRING; + FieldValue bar = barType.createFieldValue("bar"); + + StructDataType fooType = new StructDataType("my_struct"); + fooType.addField(new Field("bar", barType)); + Struct foo = new Struct(fooType); + foo.setFieldValue("bar", bar); + + DocumentType docType = new DocumentType("my_doc"); + docType.addField("foo", fooType); + Document doc = new Document(docType, "doc:scheme:"); + doc.setFieldValue("foo", foo); + + ExecutionContext ctx = new ExecutionContext(new SimpleDocumentAdapter(doc)); + assertEquals(foo, new InputExpression("foo").execute(ctx)); + assertEquals(bar, new InputExpression("foo.bar").execute(ctx)); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/JoinTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/JoinTestCase.java new file mode 100644 index 00000000000..09abbf95688 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/JoinTestCase.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class JoinTestCase { + + @Test + public void requireThatAccessorsWork() { + JoinExpression exp = new JoinExpression("foo"); + assertEquals("foo", exp.getDelimiter()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new JoinExpression("foo"); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new JoinExpression("bar"))); + assertEquals(exp, new JoinExpression("foo")); + assertEquals(exp.hashCode(), new JoinExpression("foo").hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new JoinExpression(";"); + assertVerify(DataType.getArray(DataType.INT), exp, DataType.STRING); + assertVerify(DataType.getArray(DataType.STRING), exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected any input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected Array input, got int."); + } + + @Test + public void requireThatValueIsJoined() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + Array<StringFieldValue> arr = new Array<>(DataType.getArray(DataType.STRING)); + arr.add(new StringFieldValue("6")); + arr.add(new StringFieldValue("9")); + ctx.setValue(arr); + + new JoinExpression(";").execute(ctx); + assertEquals(new StringFieldValue("6;9"), ctx.getValue()); + } + + @Test + public void requireThatNonArrayInputThrows() { + try { + new JoinExpression(";").execute(new StringFieldValue("foo")); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Expected Array input, got string.", e.getMessage()); + } + } + + @Test + public void requireThatAccessorWorks() { + JoinExpression exp = new JoinExpression(";"); + assertEquals(";", exp.getDelimiter()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseTestCase.java new file mode 100644 index 00000000000..96d83847fa8 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseTestCase.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class LowerCaseTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new LowerCaseExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new LowerCaseExpression()); + assertEquals(exp.hashCode(), new LowerCaseExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new LowerCaseExpression(); + assertVerify(DataType.STRING, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + } + + @Test + public void requireThatStringIsLowerCased() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("FOO")); + new LowerCaseExpression().execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof StringFieldValue); + assertEquals("foo", ((StringFieldValue)val).getString()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolverTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolverTestCase.java new file mode 100644 index 00000000000..b4f03dacd64 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolverTestCase.java @@ -0,0 +1,122 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class MathResolverTestCase { + + // -------------------------------------------------------------------------------- + // + // Tests + // + // -------------------------------------------------------------------------------- + + @Test + public void requireThatOperatorsWork() { + assertEquals(3, evaluate(1, ArithmeticExpression.Operator.ADD, 2)); + assertEquals(1, evaluate(3, ArithmeticExpression.Operator.SUB, 2)); + assertEquals(2, evaluate(4, ArithmeticExpression.Operator.DIV, 2)); + assertEquals(1, evaluate(3, ArithmeticExpression.Operator.MOD, 2)); + assertEquals(4, evaluate(2, ArithmeticExpression.Operator.MUL, 2)); + } + + @Test + public void requireThatOperatorPrecedenceIsCorrect() { + assertEquals(3, evaluate(1, ArithmeticExpression.Operator.ADD, 1, ArithmeticExpression.Operator.ADD, 1)); + assertEquals(1, evaluate(1, ArithmeticExpression.Operator.ADD, 1, ArithmeticExpression.Operator.SUB, 1)); + assertEquals(4, evaluate(2, ArithmeticExpression.Operator.ADD, 4, ArithmeticExpression.Operator.DIV, 2)); + assertEquals(2, evaluate(1, ArithmeticExpression.Operator.ADD, 3, ArithmeticExpression.Operator.MOD, 2)); + assertEquals(3, evaluate(1, ArithmeticExpression.Operator.ADD, 1, ArithmeticExpression.Operator.MUL, 2)); + + assertEquals(1, evaluate(1, ArithmeticExpression.Operator.SUB, 1, ArithmeticExpression.Operator.ADD, 1)); + assertEquals(-1, evaluate(1, ArithmeticExpression.Operator.SUB, 1, ArithmeticExpression.Operator.SUB, 1)); + assertEquals(-1, evaluate(1, ArithmeticExpression.Operator.SUB, 4, ArithmeticExpression.Operator.DIV, 2)); + assertEquals(1, evaluate(2, ArithmeticExpression.Operator.SUB, 3, ArithmeticExpression.Operator.MOD, 2)); + assertEquals(-1, evaluate(1, ArithmeticExpression.Operator.SUB, 1, ArithmeticExpression.Operator.MUL, 2)); + + assertEquals(3, evaluate(4, ArithmeticExpression.Operator.DIV, 2, ArithmeticExpression.Operator.ADD, 1)); + assertEquals(1, evaluate(4, ArithmeticExpression.Operator.DIV, 2, ArithmeticExpression.Operator.SUB, 1)); + assertEquals(2, evaluate(4, ArithmeticExpression.Operator.DIV, 2, ArithmeticExpression.Operator.DIV, 1)); + assertEquals(2, evaluate(4, ArithmeticExpression.Operator.DIV, 2, ArithmeticExpression.Operator.MOD, 3)); + assertEquals(2, evaluate(4, ArithmeticExpression.Operator.DIV, 2, ArithmeticExpression.Operator.MUL, 1)); + + assertEquals(2, evaluate(3, ArithmeticExpression.Operator.MOD, 2, ArithmeticExpression.Operator.ADD, 1)); + assertEquals(0, evaluate(3, ArithmeticExpression.Operator.MOD, 2, ArithmeticExpression.Operator.SUB, 1)); + assertEquals(1, evaluate(3, ArithmeticExpression.Operator.MOD, 2, ArithmeticExpression.Operator.DIV, 1)); + assertEquals(1, evaluate(3, ArithmeticExpression.Operator.MOD, 2, ArithmeticExpression.Operator.MOD, 2)); + assertEquals(1, evaluate(3, ArithmeticExpression.Operator.MOD, 2, ArithmeticExpression.Operator.MUL, 1)); + + assertEquals(3, evaluate(1, ArithmeticExpression.Operator.MUL, 2, ArithmeticExpression.Operator.ADD, 1)); + assertEquals(1, evaluate(1, ArithmeticExpression.Operator.MUL, 2, ArithmeticExpression.Operator.SUB, 1)); + assertEquals(2, evaluate(2, ArithmeticExpression.Operator.MUL, 2, ArithmeticExpression.Operator.DIV, 2)); + assertEquals(0, evaluate(1, ArithmeticExpression.Operator.MUL, 2, ArithmeticExpression.Operator.MOD, 2)); + assertEquals(4, evaluate(1, ArithmeticExpression.Operator.MUL, 2, ArithmeticExpression.Operator.MUL, 2)); + } + + @Test + public void requireThatFirstOperatorIsAdd() { + MathResolver resolver = new MathResolver(); + for (ArithmeticExpression.Operator type : ArithmeticExpression.Operator.values()) { + if (type == ArithmeticExpression.Operator.ADD) { + continue; + } + try { + resolver.push(type, newInteger(69)); + } catch (IllegalArgumentException e) { + assertEquals("First item in an arithmetic operation must be an addition.", e.getMessage()); + } + } + } + + @Test + public void requireThatNullOperatorThrowsException() { + try { + new MathResolver().push(null, newInteger(69)); + fail(); + } catch (NullPointerException e) { + + } + } + + // -------------------------------------------------------------------------------- + // + // Utilities + // + // -------------------------------------------------------------------------------- + + private static Expression newInteger(int val) { + return new SetValueExpression(new IntegerFieldValue(val)); + } + + private static int evaluate(Expression exp) { + FieldValue val = new ExecutionContext(new SimpleTestAdapter()).execute(exp).getValue(); + assertTrue(val instanceof IntegerFieldValue); + return ((IntegerFieldValue)val).getInteger(); + } + + private static int evaluate(int lhs, ArithmeticExpression.Operator op, int rhs) { + MathResolver resolver = new MathResolver(); + resolver.push(ArithmeticExpression.Operator.ADD, newInteger(lhs)); + resolver.push(op, newInteger(rhs)); + return evaluate(resolver.resolve()); + } + + private static int evaluate(int valA, + ArithmeticExpression.Operator opB, int valC, + ArithmeticExpression.Operator opD, int valE) + { + MathResolver resolver = new MathResolver(); + resolver.push(ArithmeticExpression.Operator.ADD, newInteger(valA)); + resolver.push(opB, newInteger(valC)); + resolver.push(opD, newInteger(valE)); + return evaluate(resolver.resolve()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java new file mode 100644 index 00000000000..c4cd1716940 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.annotation.*; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.TokenType; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Iterator; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author bratseth + */ +public class NGramTestCase { + + @Test + public void requireThatAccessorsWork() { + Linguistics linguistics = new SimpleLinguistics(); + NGramExpression exp = new NGramExpression(linguistics, 69); + assertSame(linguistics, exp.getLinguistics()); + assertEquals(69, exp.getGramSize()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Linguistics linguistics = new SimpleLinguistics(); + NGramExpression exp = new NGramExpression(linguistics, 69); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new NGramExpression(Mockito.mock(Linguistics.class), 96))); + assertFalse(exp.equals(new NGramExpression(linguistics, 96))); + assertEquals(exp, new NGramExpression(linguistics, 69)); + assertEquals(exp.hashCode(), new NGramExpression(new SimpleLinguistics(), 69).hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new NGramExpression(new SimpleLinguistics(), 69); + assertVerify(DataType.STRING, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + } + + @Test + public void testNGrams() { + ExecutionContext context = new ExecutionContext(new SimpleTestAdapter()); + context.setValue(new StringFieldValue("en gul Bille sang... ")); + new NGramExpression(new SimpleLinguistics(), 3).execute(context); + + StringFieldValue value = (StringFieldValue)context.getValue(); + assertEquals("Grams are pure annotations - field value is unchanged", "en gul Bille sang... ", + value.getString()); + SpanTree gramTree = value.getSpanTree(SpanTrees.LINGUISTICS); + assertNotNull(gramTree); + SpanList grams = (SpanList)gramTree.getRoot(); + Iterator<SpanNode> i = grams.childIterator(); + assertSpan(0, 2, true, i, gramTree); // en + assertSpan(2, 1, false, i, gramTree); // <space> + assertSpan(3, 3, true, i, gramTree); // gul + assertSpan(6, 1, false, i, gramTree); // <space> + assertSpan(7, 3, true, i, gramTree, "bil"); // Bil + assertSpan(8, 3, true, i, gramTree); + assertSpan(9, 3, true, i, gramTree); + assertSpan(12, 1, false, i, gramTree); // <space> + assertSpan(13, 3, true, i, gramTree); + assertSpan(14, 3, true, i, gramTree); + assertSpan(17, 4, false, i, gramTree); // <...space> + assertFalse(i.hasNext()); + } + + private void assertSpan(int from, int length, boolean gram, Iterator<SpanNode> i, SpanTree tree) { + assertSpan(from, length, gram, i, tree, null); + } + + private void assertSpan(int from, int length, boolean gram, Iterator<SpanNode> i, SpanTree tree, String termValue) { + if (!i.hasNext()) { + fail("No more spans"); + } + SpanNode gramSpan = i.next(); + assertEquals("gram start", from, gramSpan.getFrom()); + assertEquals("gram length", length, gramSpan.getLength()); + assertTrue(gramSpan.isLeafNode()); + Iterator<Annotation> annotations = tree.iterator(gramSpan); + Annotation typeAnnotation = annotations.next(); + assertEquals(AnnotationTypes.TOKEN_TYPE, typeAnnotation.getType()); + int typeInt = ((IntegerFieldValue)typeAnnotation.getFieldValue()).getInteger(); + if (gram) { + assertEquals(TokenType.ALPHABETIC.getValue(), typeInt); + Annotation termAnnotation = annotations.next(); + assertEquals(AnnotationTypes.TERM, termAnnotation.getType()); + if (termValue == null) { + assertNull(termAnnotation.getFieldValue()); + } else { + assertEquals(termValue, ((StringFieldValue)termAnnotation.getFieldValue()).getString()); + } + } else { // gap between grams + assertEquals(TokenType.PUNCTUATION.getValue(), typeInt); + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java new file mode 100644 index 00000000000..65f966f9691 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; +import org.mockito.Mockito; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class NormalizeTestCase { + + @Test + public void requireThatAccessorsWork() { + Linguistics linguistics = new SimpleLinguistics(); + NormalizeExpression exp = new NormalizeExpression(linguistics); + assertSame(linguistics, exp.getLinguistics()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Linguistics linguistics = new SimpleLinguistics(); + NormalizeExpression exp = new NormalizeExpression(linguistics); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new NormalizeExpression(Mockito.mock(Linguistics.class)))); + assertEquals(exp, new NormalizeExpression(linguistics)); + assertEquals(exp.hashCode(), new NormalizeExpression(linguistics).hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new NormalizeExpression(new SimpleLinguistics()); + assertVerify(DataType.STRING, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + } + + @Test + public void requireThatInputIsNormalized() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setLanguage(Language.ENGLISH); + ctx.setValue(new StringFieldValue("b\u00e9yonc\u00e8")); + new NormalizeExpression(new SimpleLinguistics()).execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof StringFieldValue); + assertEquals("beyonce", ((StringFieldValue)val).getString()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NowTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NowTestCase.java new file mode 100644 index 00000000000..dce37a1ef4a --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NowTestCase.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class NowTestCase { + + @Test + public void requireThatAccessorsWork() { + MyTimer timer = new MyTimer(); + NowExpression exp = new NowExpression(timer); + assertSame(timer, exp.getTimer()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + MyTimer timer = new MyTimer(); + NowExpression exp = new NowExpression(timer); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new NowExpression(new MyTimer()))); + assertEquals(exp, new NowExpression(timer)); + assertEquals(exp.hashCode(), new NowExpression(timer).hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new NowExpression(); + assertVerify(null, exp, DataType.LONG); + assertVerify(DataType.INT, exp, DataType.LONG); + assertVerify(DataType.STRING, exp, DataType.LONG); + } + + @Test + public void requireThatCurrentTimeIsSet() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + new NowExpression(new MyTimer()).execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof LongFieldValue); + assertEquals(69L, ((LongFieldValue)val).getLong()); + } + + private class MyTimer implements NowExpression.Timer { + + @Override + public long currentTimeSeconds() { + return 69L; + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateTestCase.java new file mode 100644 index 00000000000..c835eca4057 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateTestCase.java @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.predicate.Predicate; +import org.junit.Test; +import org.mockito.Mockito; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyCtx; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyCtxThrows; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class OptimizePredicateTestCase { + + @Test + public void requireThatOptimizerIsCalledWithCloneOfInput() { + final Predicate predicateA = Mockito.mock(Predicate.class); + final Predicate predicateB = Mockito.mock(Predicate.class); + final PredicateFieldValue input = new PredicateFieldValue(predicateA); + ExecutionContext ctx = new ExecutionContext() + .setValue(input) + .setVariable("arity", new IntegerFieldValue(10)); + FieldValue output = new OptimizePredicateExpression( + (predicate, options) -> { + assertNotSame(predicateA, predicate); + return predicateB; + } + ).execute(ctx); + assertNotSame(output, input); + assertTrue(output instanceof PredicateFieldValue); + assertSame(predicateB, ((PredicateFieldValue)output).getPredicate()); + } + + @Test + public void requireThatPredicateOptionsAreSet() { + final Predicate predicate = Mockito.mock(Predicate.class); + final PredicateFieldValue input = new PredicateFieldValue(predicate); + ExecutionContext ctx = new ExecutionContext() + .setValue(input) + .setVariable("arity", new IntegerFieldValue(10)); + new OptimizePredicateExpression((predicate1, options) -> { + assertEquals(10, options.getArity()); + assertEquals(0x8000000000000000L, options.getLowerBound()); + assertEquals(0x7fffffffffffffffL, options.getUpperBound()); + return predicate1; + }).execute(ctx); + ctx.setVariable("upper_bound", new LongFieldValue(1000)); + ctx.setVariable("lower_bound", new LongFieldValue(0)); + new OptimizePredicateExpression( + (value, options) -> { + assertEquals(10, options.getArity()); + assertEquals(0, options.getLowerBound()); + assertEquals(1000, options.getUpperBound()); + return value; + } + ).execute(ctx); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new OptimizePredicateExpression(); + assertTrue(exp.equals(exp)); + assertFalse(exp.equals(new Object())); + assertTrue(exp.equals(new OptimizePredicateExpression())); + assertEquals(exp.hashCode(), new OptimizePredicateExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new OptimizePredicateExpression(); + assertVerifyThrows(null, exp, "Expected predicate input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected predicate input, got int."); + assertVerifyThrows(DataType.PREDICATE, exp, "Variable 'arity' must be set."); + + VerificationContext context = new VerificationContext().setValue(DataType.PREDICATE); + context.setVariable("arity", DataType.STRING); + assertVerifyCtxThrows(context, exp, "Variable 'arity' must have type int."); + context.setVariable("arity", DataType.INT); + assertVerifyCtx(context, exp, DataType.PREDICATE); + context.setVariable("lower_bound", DataType.INT); + assertVerifyCtxThrows(context, exp, "Variable 'lower_bound' must have type long."); + context.setVariable("lower_bound", DataType.LONG); + assertVerifyCtx(context, exp, DataType.PREDICATE); + context.setVariable("upper_bound", DataType.INT); + assertVerifyCtxThrows(context, exp, "Variable 'upper_bound' must have type long."); + context.setVariable("upper_bound", DataType.LONG); + assertVerifyCtx(context, exp, DataType.PREDICATE); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssert.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssert.java new file mode 100644 index 00000000000..2e6befea363 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssert.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +class OutputAssert { + + public static void assertExecute(OutputExpression exp) { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter(new Field(exp.getFieldName(), DataType.STRING))); + ctx.setValue(new StringFieldValue("69")); + ctx.execute(exp); + + FieldValue out = ctx.getInputValue(exp.getFieldName()); + assertTrue(out instanceof StringFieldValue); + assertEquals("69", ((StringFieldValue)out).getString()); + } + + public static void assertVerify(OutputExpression exp) { + assertVerify(new MyAdapter(null), DataType.INT, exp); + assertVerify(new MyAdapter(null), DataType.STRING, exp); + assertVerifyThrows(new MyAdapter(null), null, exp, "Expected any input, got null."); + assertVerifyThrows(new MyAdapter(new VerificationException(null, "foo")), DataType.INT, exp, "foo"); + } + + public static void assertVerify(FieldTypeAdapter adapter, DataType value, Expression exp) { + assertEquals(value, new VerificationContext(adapter).setValue(value).execute(exp).getValue()); + } + + public static void assertVerifyThrows(FieldTypeAdapter adapter, DataType value, Expression exp, + String expectedException) + { + try { + new VerificationContext(adapter).setValue(value).execute(exp); + fail(); + } catch (VerificationException e) { + assertEquals(expectedException, e.getMessage()); + } + } + + private static class MyAdapter implements FieldTypeAdapter { + + final RuntimeException e; + + MyAdapter(RuntimeException e) { + this.e = e; + } + + @Override + public DataType getInputType(Expression exp, String fieldName) { + throw new AssertionError(); + } + + @Override + public void tryOutputType(Expression exp, String fieldName, DataType valueType) { + if (e != null) { + throw e; + } + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssertTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssertTestCase.java new file mode 100644 index 00000000000..dd1a0bdaa5d --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssertTestCase.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertVerifyThrows; +import static org.junit.Assert.assertNotNull; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class OutputAssertTestCase { + + @Test + public void requireThatAssertVerifyMethodThrowsWhenAppropriate() { + Throwable thrown = null; + try { + assertVerify(null, DataType.INT, SimpleExpression.newRequired(DataType.STRING)); + } catch (Throwable t) { + thrown = t; + } + assertNotNull(thrown); + + thrown = null; + try { + assertVerifyThrows(null, DataType.INT, SimpleExpression.newRequired(DataType.INT), + "unchecked expected exception message"); + } catch (Throwable t) { + thrown = t; + } + assertNotNull(thrown); + + thrown = null; + try { + assertVerifyThrows(null, DataType.INT, SimpleExpression.newRequired(DataType.STRING), + "wrong expected exception message"); + } catch (Throwable t) { + thrown = t; + } + assertNotNull(thrown); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisTestCase.java new file mode 100644 index 00000000000..2e7bf148d60 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisTestCase.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ParenthesisTestCase { + + @Test + public void requireThatAccessorsWork() { + Expression innerExp = new AttributeExpression("foo"); + ParenthesisExpression exp = new ParenthesisExpression(innerExp); + assertSame(innerExp, exp.getInnerExpression()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression innerExp = new AttributeExpression("foo"); + Expression exp = new ParenthesisExpression(innerExp); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new ParenthesisExpression(new AttributeExpression("bar")))); + assertEquals(exp, new ParenthesisExpression(innerExp)); + assertEquals(exp.hashCode(), new ParenthesisExpression(innerExp).hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ParenthesisExpression(SimpleExpression.newConversion(DataType.INT, DataType.STRING)); + assertVerify(DataType.INT, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected int input, got null."); + assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string."); + } + + @Test + public void requireThatNestedExpressionIsRun() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter(new Field("in", DataType.STRING))); + ctx.setOutputValue(null, "in", new StringFieldValue("69")); + new ParenthesisExpression(new InputExpression("in")).execute(ctx); + + assertTrue(ctx.getValue() instanceof StringFieldValue); + assertEquals("69", ((StringFieldValue)ctx.getValue()).getString()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/RandomTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/RandomTestCase.java new file mode 100644 index 00000000000..7ccda30d957 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/RandomTestCase.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class RandomTestCase { + + @Test + public void requireThatAccessorsWork() { + RandomExpression exp = new RandomExpression(69); + assertEquals(Integer.valueOf(69), exp.getMaxValue()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new RandomExpression(69); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new RandomExpression(96))); + assertEquals(exp, new RandomExpression(69)); + assertEquals(exp.hashCode(), new RandomExpression(69).hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new RandomExpression(); + assertVerify(null, exp, DataType.INT); + assertVerify(DataType.INT, exp, DataType.INT); + assertVerify(DataType.STRING, exp, DataType.INT); + } + + @Test + public void requireThatRandomValueIsSet() { + for (int i = 0; i < 666; ++i) { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + new RandomExpression(69).execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof IntegerFieldValue); + assertTrue(((IntegerFieldValue)val).getInteger() < 69); + } + } + + @Test + public void requireThatInputValueIsParsedAsMaxIfNoneIsConfigured() { + for (int i = 0; i < 666; ++i) { + ExecutionContext ctx = new ExecutionContext().setValue(new IntegerFieldValue(69)); + new RandomExpression().execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof IntegerFieldValue); + assertTrue(((IntegerFieldValue)val).getInteger() < 69); + } + } + + @Test + public void requireThatIllegalInputThrowsNumberFormatException() { + try { + new RandomExpression().execute(new ExecutionContext()); + fail(); + } catch (NumberFormatException e) { + + } + try { + new RandomExpression().execute(new ExecutionContext().setValue(new StringFieldValue("foo"))); + fail(); + } catch (NumberFormatException e) { + + } + } + + @Test + public void requireThatDefaultMaxIsNull() { + assertNull(new RandomExpression().getMaxValue()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java new file mode 100644 index 00000000000..e204596bbcf --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import java.util.Arrays; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ScriptTestCase { + + @Test + public void requireThatAccessorsWork() { + ScriptExpression exp = newScript(); + assertTrue(exp.isEmpty()); + assertEquals(0, exp.size()); + + StatementExpression foo = newStatement(new AttributeExpression("foo")); + StatementExpression bar = newStatement(new AttributeExpression("bar")); + exp = newScript(foo, bar); + assertFalse(exp.isEmpty()); + assertEquals(2, exp.size()); + assertSame(foo, exp.get(0)); + assertSame(bar, exp.get(1)); + assertEquals(Arrays.asList(foo, bar), exp.asList()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + StatementExpression foo = newStatement(new AttributeExpression("foo")); + StatementExpression bar = newStatement(new AttributeExpression("bar")); + Expression exp = newScript(foo, bar); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new CatExpression(foo, bar))); + assertFalse(exp.equals(newScript(newStatement(new IndexExpression("foo"))))); + assertFalse(exp.equals(newScript(newStatement(new IndexExpression("foo")), + newStatement(new IndexExpression("bar"))))); + assertFalse(exp.equals(newScript(newStatement(foo), + newStatement(new IndexExpression("bar"))))); + assertEquals(exp, newScript(foo, bar)); + assertEquals(exp.hashCode(), newScript(foo, bar).hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = newScript(newStatement(SimpleExpression.newConversion(DataType.INT, DataType.STRING))); + assertVerify(DataType.INT, exp, DataType.INT); // does not touch output + assertVerifyThrows(null, exp, "Expected int input, got null."); + assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string."); + + assertVerifyThrows(null, newScript(newStatement(SimpleExpression.newConversion(DataType.INT, DataType.STRING)), + newStatement(SimpleExpression.newConversion(DataType.STRING, DataType.INT))), + "Statements require conflicting input types, int vs string."); + } + + @Test + public void requireThatInputValueIsAvailableToAllStatements() { + SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out-1", DataType.INT), + new Field("out-2", DataType.INT)); + newStatement(new SetValueExpression(new IntegerFieldValue(69)), + newScript(newStatement(new AttributeExpression("out-1"), + new AttributeExpression("out-2")))).execute(adapter); + assertEquals(new IntegerFieldValue(69), adapter.getInputValue("out-1")); + assertEquals(new IntegerFieldValue(69), adapter.getInputValue("out-2")); + } + + @Test + public void requireThatScriptEvaluatesToInputValue() { + SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out", DataType.INT)); + newStatement(new SetValueExpression(new IntegerFieldValue(6)), + newScript(newStatement(new SetValueExpression(new IntegerFieldValue(9)))), + new AttributeExpression("out")).execute(adapter); + assertEquals(new IntegerFieldValue(6), adapter.getInputValue("out")); + } + + @Test + public void requireThatVariablesAreAvailableInScript() { + SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out", DataType.INT)); + newScript(newStatement(new SetValueExpression(new IntegerFieldValue(69)), + new SetVarExpression("tmp")), + newStatement(new GetVarExpression("tmp"), + new AttributeExpression("out"))).execute(adapter); + assertEquals(new IntegerFieldValue(69), adapter.getInputValue("out")); + } + + @Test + public void requireThatVariablesAreAvailableOutsideScript() { + SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out", DataType.INT)); + newStatement(newScript(newStatement(new SetValueExpression(new IntegerFieldValue(69)), + new SetVarExpression("tmp"))), + new GetVarExpression("tmp"), + new AttributeExpression("out")).execute(adapter); + assertEquals(new IntegerFieldValue(69), adapter.getInputValue("out")); + } + + @Test + public void requireThatVariablesReplaceOthersOutsideScript() { + SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("out", DataType.INT)); + newStatement(new SetValueExpression(new IntegerFieldValue(6)), + new SetVarExpression("tmp"), + newScript(newStatement(new SetValueExpression(new IntegerFieldValue(9)), + new SetVarExpression("tmp"))), + new GetVarExpression("tmp"), + new AttributeExpression("out")).execute(adapter); + assertEquals(new IntegerFieldValue(9), adapter.getInputValue("out")); + } + + private static ScriptExpression newScript(StatementExpression... args) { + return new ScriptExpression(args); + } + + private static StatementExpression newStatement(Expression... args) { + return new StatementExpression(args); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java new file mode 100644 index 00000000000..a19b1a8f349 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.collections.Pair; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SelectInputTestCase { + + @Test + public void requireThatAccessorsWork() { + List<Pair<String, Expression>> cases = new LinkedList<>(); + Pair<String, Expression> foo = new Pair<String, Expression>("foo", new AttributeExpression("foo")); + Pair<String, Expression> bar = new Pair<String, Expression>("bar", new AttributeExpression("bar")); + cases.add(foo); + cases.add(bar); + SelectInputExpression exp = new SelectInputExpression(cases); + + assertEquals(2, exp.getCases().size()); + assertSame(foo, exp.getCases().get(0)); + assertSame(bar, exp.getCases().get(1)); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression foo = new AttributeExpression("foo"); + Expression exp = newSelectInput(foo, "bar", "baz"); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(newSelectInput(new IndexExpression("foo")))); + assertFalse(exp.equals(newSelectInput(new IndexExpression("foo"), "bar"))); + assertFalse(exp.equals(newSelectInput(new IndexExpression("foo"), "bar", "baz"))); + assertFalse(exp.equals(newSelectInput(foo))); + assertFalse(exp.equals(newSelectInput(foo, "bar"))); + assertEquals(exp, newSelectInput(foo, "bar", "baz")); + assertEquals(exp.hashCode(), newSelectInput(foo, "bar", "baz").hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + SimpleTestAdapter adapter = new SimpleTestAdapter(); + adapter.createField(new Field("my_int", DataType.INT)); + adapter.createField(new Field("my_str", DataType.STRING)); + + Expression exp = newSelectInput(new AttributeExpression("my_int"), "my_int"); + assertVerify(adapter, null, exp); + assertVerify(adapter, DataType.INT, exp); + assertVerify(adapter, DataType.STRING, exp); + + assertVerifyThrows(adapter, newSelectInput(new AttributeExpression("my_int"), "my_str"), + "Can not assign string to field 'my_int' which is int."); + assertVerifyThrows(adapter, newSelectInput(new AttributeExpression("my_int"), "my_unknown"), + "Field 'my_unknown' not found."); + } + + @Test + public void requireThatSelectedExpressionIsRun() { + assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("foo"), "foo"); + assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("bar"), "bar"); + assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("foo", "bar"), "foo"); + assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("bar", "baz"), "bar"); + assertSelect(Arrays.asList("foo", "bar"), Arrays.asList("baz", "cox"), null); + } + + private static void assertVerify(FieldTypeAdapter adapter, DataType value, Expression exp) { + assertEquals(value, exp.verify(new VerificationContext(adapter).setValue(value))); + } + + private static void assertVerifyThrows(FieldTypeAdapter adapter, Expression exp, String expectedException) { + try { + exp.verify(new VerificationContext(adapter)); + fail(); + } catch (VerificationException e) { + assertEquals(expectedException, e.getMessage()); + } + } + + private static SelectInputExpression newSelectInput(Expression exp, String... fieldNames) { + List<Pair<String, Expression>> cases = new LinkedList<>(); + for (String fieldName : fieldNames) { + cases.add(new Pair<>(fieldName, exp)); + } + return new SelectInputExpression(cases); + } + + private static void assertSelect(List<String> inputField, List<String> availableFields, String expected) { + SimpleTestAdapter adapter = new SimpleTestAdapter(); + ExecutionContext ctx = new ExecutionContext(adapter); + for (String fieldName : availableFields) { + adapter.createField(new Field(fieldName, DataType.STRING)); + ctx.setOutputValue(null, fieldName, new StringFieldValue(fieldName)); + } + List<Pair<String, Expression>> cases = new LinkedList<>(); + for (String fieldName : inputField) { + cases.add(new Pair<String, Expression>(fieldName, new SetVarExpression("out"))); + } + new SelectInputExpression(cases).execute(ctx); + assertEquals(expected != null ? new StringFieldValue(expected) : null, ctx.getVariable("out")); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageTestCase.java new file mode 100644 index 00000000000..782cedcc571 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageTestCase.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.language.Language; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SetLanguageTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new SetLanguageExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new SetLanguageExpression()); + assertEquals(exp.hashCode(), new SetLanguageExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new SetLanguageExpression(); + assertVerify(DataType.STRING, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + } + + @Test + public void requireThatLanguageIsSet() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("en")); + new SetLanguageExpression().execute(ctx); + assertEquals(Language.ENGLISH, ctx.getLanguage()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueTestCase.java new file mode 100644 index 00000000000..a6e55076286 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueTestCase.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SetValueTestCase { + + @Test + public void requireThatAccessorsWork() { + FieldValue foo = new StringFieldValue("foo"); + SetValueExpression exp = new SetValueExpression(foo); + assertSame(foo, exp.getValue()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + FieldValue foo = new StringFieldValue("foo"); + Expression exp = new SetValueExpression(foo); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new SetValueExpression(new StringFieldValue("bar")))); + assertEquals(exp, new SetValueExpression(foo)); + assertEquals(exp.hashCode(), new SetValueExpression(foo).hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new SetValueExpression(new StringFieldValue("foo")); + assertVerify(null, exp, DataType.STRING); + assertVerify(DataType.INT, exp, DataType.STRING); + assertVerify(DataType.STRING, exp, DataType.STRING); + } + + @Test + public void requireThatNullValueThrowsException() { + try { + new SetValueExpression(null); + fail(); + } catch (NullPointerException e) { + + } + } + + @Test + public void requireThatValueIsSet() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + new SetValueExpression(new StringFieldValue("69")).execute(ctx); + assertEquals(new StringFieldValue("69"), ctx.getValue()); + } + + @Test + public void requireThatLongFieldValueGetsATrailingL() { + assertEquals("69L", new SetValueExpression(new LongFieldValue(69)).toString()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarTestCase.java new file mode 100644 index 00000000000..2a3d913733b --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarTestCase.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SetVarTestCase { + + @Test + public void requireThatAccessorsWork() { + SetVarExpression exp = new SetVarExpression("foo"); + assertEquals("foo", exp.getVariableName()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new SetVarExpression("foo"); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new SetVarExpression("bar"))); + assertEquals(exp, new SetVarExpression("foo")); + assertEquals(exp.hashCode(), new SetVarExpression("foo").hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new SetVarExpression("foo"); + assertVerify(DataType.INT, exp, DataType.INT); + assertVerify(DataType.STRING, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected any input, got null."); + + try { + new VerificationContext().setVariable("foo", DataType.INT).setValue(DataType.STRING).execute(exp); + fail(); + } catch (VerificationException e) { + assertEquals("Attempting to assign conflicting types to variable 'foo', int vs string.", e.getMessage()); + } + } + + @Test + public void requireThatSymbolIsWritten() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new IntegerFieldValue(69)); + new SetVarExpression("out").execute(ctx); + + FieldValue val = ctx.getVariable("out"); + assertTrue(val instanceof IntegerFieldValue); + assertEquals(69, ((IntegerFieldValue)val).getInteger()); + } + + @Test + public void requireThatVariableTypeCanNotChange() { + VerificationContext ctx = new VerificationContext(new SimpleTestAdapter()); + ctx.setValue(DataType.INT); + new SetVarExpression("out").verify(ctx); + + try { + ctx.setValue(DataType.STRING); + new SetVarExpression("out").verify(ctx); + fail(); + } catch (VerificationException e) { + assertTrue(e.getExpression() instanceof SetVarExpression); + assertEquals("Attempting to assign conflicting types to variable 'out', int vs string.", e.getMessage()); + } + } + + @Test + public void requireThatAccessorWorks() { + SetVarExpression exp = new SetVarExpression("69"); + assertEquals("69", exp.getVariableName()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java new file mode 100644 index 00000000000..11f269b4d01 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.FieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +class SimpleExpression extends Expression { + + private boolean hasExecuteValue = false; + private boolean hasVerifyValue = false; + private FieldValue executeValue; + private DataType verifyValue; + private DataType requiredInput; + private DataType createdOutput; + + public SimpleExpression setExecuteValue(FieldValue executeValue) { + this.hasExecuteValue = true; + this.executeValue = executeValue; + return this; + } + + public SimpleExpression setVerifyValue(DataType verifyValue) { + this.hasVerifyValue = true; + this.verifyValue = verifyValue; + return this; + } + + public SimpleExpression setRequiredInput(DataType requiredInput) { + this.requiredInput = requiredInput; + return this; + } + + public SimpleExpression setCreatedOutput(DataType createdOutput) { + this.createdOutput = createdOutput; + return this; + } + + @Override + protected void doExecute(ExecutionContext ctx) { + if (hasExecuteValue) { + ctx.setValue(executeValue); + } + } + + @Override + protected void doVerify(VerificationContext ctx) { + if (hasVerifyValue) { + ctx.setValue(verifyValue); + } + } + + @Override + public DataType requiredInputType() { + return requiredInput; + } + + @Override + public DataType createdOutputType() { + return createdOutput; + } + + @Override + public int hashCode() { + return hashCode(executeValue) + hashCode(verifyValue) + hashCode(requiredInput) + hashCode(createdOutput); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof SimpleExpression)) { + return false; + } + SimpleExpression rhs = (SimpleExpression)obj; + if (hasExecuteValue != rhs.hasExecuteValue) { + return false; + } + if (!equals(executeValue, rhs.executeValue)) { + return false; + } + if (hasVerifyValue != rhs.hasVerifyValue) { + return false; + } + if (!equals(verifyValue, rhs.verifyValue)) { + return false; + } + if (!equals(requiredInput, rhs.requiredInput)) { + return false; + } + if (!equals(createdOutput, rhs.createdOutput)) { + return false; + } + return true; + } + + public static SimpleExpression newOutput(DataType createdOutput) { + return new SimpleExpression().setCreatedOutput(createdOutput) + .setVerifyValue(createdOutput); + } + + public static SimpleExpression newRequired(DataType requiredInput) { + return new SimpleExpression().setRequiredInput(requiredInput); + } + + public static SimpleExpression newConversion(DataType requiredInput, DataType createdOutput) { + return new SimpleExpression().setRequiredInput(requiredInput) + .setCreatedOutput(createdOutput) + .setExecuteValue(createdOutput.createFieldValue()) + .setVerifyValue(createdOutput); + } + + private static int hashCode(Object obj) { + return obj != null ? obj.hashCode() : 0; + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpressionTestCase.java new file mode 100644 index 00000000000..a44891cc55f --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpressionTestCase.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.IntegerFieldValue; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class SimpleExpressionTestCase { + + @Test + public void requireThatAccessorsWork() { + SimpleExpression exp = new SimpleExpression(); + assertNull(exp.requiredInputType()); + assertNull(exp.createdOutputType()); + assertNull(exp.execute()); + assertNull(exp.verify()); + + assertEquals(DataType.INT, new SimpleExpression().setRequiredInput(DataType.INT).requiredInputType()); + assertEquals(DataType.INT, new SimpleExpression().setCreatedOutput(DataType.INT).createdOutputType()); + assertEquals(DataType.INT, new SimpleExpression().setVerifyValue(DataType.INT).verify()); + assertEquals(new IntegerFieldValue(69), + new SimpleExpression().setExecuteValue(new IntegerFieldValue(69)).execute()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new SimpleExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new SimpleExpression()); + assertEquals(exp.hashCode(), new SimpleExpression().hashCode()); + + exp = new SimpleExpression().setExecuteValue(null); + assertFalse(exp.equals(new SimpleExpression())); + assertEquals(exp, new SimpleExpression().setExecuteValue(null)); + + exp = new SimpleExpression().setExecuteValue(new IntegerFieldValue(69)); + assertFalse(exp.equals(new SimpleExpression().setExecuteValue(new IntegerFieldValue(96)))); + assertEquals(exp, new SimpleExpression().setExecuteValue(new IntegerFieldValue(69))); + + exp = new SimpleExpression().setVerifyValue(null); + assertFalse(exp.equals(new SimpleExpression())); + assertEquals(exp, new SimpleExpression().setVerifyValue(null)); + + exp = new SimpleExpression().setVerifyValue(DataType.INT); + assertFalse(exp.equals(new SimpleExpression().setVerifyValue(DataType.STRING))); + assertEquals(exp, new SimpleExpression().setVerifyValue(DataType.INT)); + + exp = new SimpleExpression().setRequiredInput(DataType.INT); + assertFalse(exp.equals(new SimpleExpression().setRequiredInput(DataType.STRING))); + assertEquals(exp, new SimpleExpression().setRequiredInput(DataType.INT)); + + exp = new SimpleExpression().setCreatedOutput(DataType.INT); + assertFalse(exp.equals(new SimpleExpression().setCreatedOutput(DataType.STRING))); + assertEquals(exp, new SimpleExpression().setCreatedOutput(DataType.INT)); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SplitTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SplitTestCase.java new file mode 100644 index 00000000000..4abab5c7c7d --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SplitTestCase.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SplitTestCase { + + @Test + public void requireThatAccessorsWork() { + SplitExpression exp = new SplitExpression("foo"); + assertEquals("foo", exp.getSplitPattern().toString()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new SplitExpression("foo"); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new SplitExpression("bar"))); + assertEquals(exp, new SplitExpression("foo")); + assertEquals(exp.hashCode(), new SplitExpression("foo").hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new SplitExpression(";"); + assertVerify(DataType.STRING, exp, DataType.getArray(DataType.STRING)); + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + } + + @Test + public void requireThatValueIsSplit() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("6;9")); + new SplitExpression(";").execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val.getDataType().equals(DataType.getArray(DataType.STRING))); + assertTrue(val instanceof Array); + + Array arr = (Array)val; + assertEquals(new StringFieldValue("6"), arr.get(0)); + assertEquals(new StringFieldValue("9"), arr.get(1)); + } + + @Test + public void requireThatNullInputProducesNullOutput() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + new SplitExpression(";").execute(ctx); + + assertNull(ctx.getValue()); + } + + @Test + public void requireThatEmptyInputProducesEmptyArray() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("")); + new SplitExpression(";").execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val.getDataType().equals(DataType.getArray(DataType.STRING))); + assertTrue(val instanceof Array); + assertEquals(0, ((Array)val).size()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java new file mode 100644 index 00000000000..cc496126c01 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java @@ -0,0 +1,120 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import java.util.Arrays; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class StatementTestCase { + + @Test + public void requireThatAccessorsWork() { + StatementExpression exp = newStatement(); + assertTrue(exp.isEmpty()); + assertEquals(0, exp.size()); + + Expression foo = new AttributeExpression("foo"); + Expression bar = new AttributeExpression("bar"); + exp = newStatement(foo, bar); + assertFalse(exp.isEmpty()); + assertEquals(2, exp.size()); + assertSame(foo, exp.get(0)); + assertSame(bar, exp.get(1)); + assertEquals(Arrays.asList(foo, bar), exp.asList()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression foo = new AttributeExpression("foo"); + Expression bar = new AttributeExpression("bar"); + Expression exp = newStatement(foo, bar); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new CatExpression(foo, bar))); + assertFalse(exp.equals(newStatement(new IndexExpression("foo")))); + assertFalse(exp.equals(newStatement(new IndexExpression("foo"), + new IndexExpression("bar")))); + assertFalse(exp.equals(newStatement(foo, + new IndexExpression("bar")))); + assertEquals(exp, newStatement(foo, bar)); + assertEquals(exp.hashCode(), newStatement(foo, bar).hashCode()); + } + + @Test + public void requireThatStatementIsFlattened() { + Expression foo = SimpleExpression.newConversion(DataType.INT, DataType.STRING); + Expression bar = SimpleExpression.newConversion(DataType.STRING, DataType.INT); + assertEquals(newStatement(foo, bar), newStatement(foo, bar)); + assertEquals(newStatement(foo, bar), newStatement(null, foo, bar)); + assertEquals(newStatement(foo, bar), newStatement(foo, null, bar)); + assertEquals(newStatement(foo, bar), newStatement(foo, bar, null)); + assertEquals(newStatement(foo, bar), newStatement(newStatement(foo), bar)); + assertEquals(newStatement(foo, bar), newStatement(newStatement(foo), newStatement(bar))); + assertEquals(newStatement(foo, bar), newStatement(foo, newStatement(bar))); + assertEquals(newStatement(foo, bar), newStatement(newStatement(foo, bar))); + } + + @Test + public void requireThatRequiredInputIsDeterminedByFirstNonNullRequiredInput() { + assertEquals(DataType.INT, newStatement(SimpleExpression.newRequired(DataType.INT)).requiredInputType()); + assertEquals(DataType.INT, newStatement(new SimpleExpression(), + SimpleExpression.newRequired(DataType.INT)).requiredInputType()); + assertEquals(DataType.INT, newStatement(SimpleExpression.newRequired(DataType.INT), + SimpleExpression.newRequired(DataType.INT)).requiredInputType()); + } + + @Test + public void requireThatRequiredInputIsNullIfAnyOutputIsCreatedFirst() { + assertNull(newStatement(new SimpleExpression().setCreatedOutput(DataType.INT), + new SimpleExpression().setRequiredInput(DataType.INT)).requiredInputType()); + } + + @Test + public void requireThatCreatedOutputIsDeterminedByLastNonNullCreatedOutput() { + assertEquals(DataType.STRING, newStatement(SimpleExpression.newOutput(DataType.STRING)).createdOutputType()); + assertEquals(DataType.STRING, newStatement(SimpleExpression.newOutput(DataType.INT), + SimpleExpression.newOutput(DataType.STRING)).createdOutputType()); + assertEquals(DataType.STRING, newStatement(SimpleExpression.newOutput(DataType.STRING), + new SimpleExpression()).createdOutputType()); + } + + @Test + public void requireThatInternalVerificationIsPerformed() { + Expression exp = newStatement(SimpleExpression.newOutput(DataType.STRING), + SimpleExpression.newConversion(DataType.INT, DataType.STRING)); + assertVerifyThrows(null, exp, "Expected int input, got string."); + assertVerifyThrows(DataType.INT, exp, "Expected int input, got string."); + assertVerifyThrows(DataType.STRING, exp, "Expected int input, got string."); + + exp = newStatement(SimpleExpression.newOutput(DataType.INT), + SimpleExpression.newConversion(DataType.INT, DataType.STRING)); + assertVerify(null, exp, DataType.STRING); + assertVerify(DataType.INT, exp, DataType.STRING); + assertVerify(DataType.STRING, exp, DataType.STRING); + } + + @Test + public void requireThatStatementIsExecuted() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + StatementExpression statement = newStatement(new SetValueExpression(new IntegerFieldValue(69))); + newStatement(statement).execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof IntegerFieldValue); + assertEquals(69, ((IntegerFieldValue)val).getInteger()); + } + + private static StatementExpression newStatement(Expression... args) { + return new StatementExpression(args); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringTestCase.java new file mode 100644 index 00000000000..04cf2ba5ef6 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringTestCase.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SubstringTestCase { + + @Test + public void requireThatAccessorsWork() { + SubstringExpression exp = new SubstringExpression(6, 9); + assertEquals(6, exp.getFrom()); + assertEquals(9, exp.getTo()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new SubstringExpression(6, 9); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new SubstringExpression(66, 99))); + assertFalse(exp.equals(new SubstringExpression(6, 99))); + assertEquals(exp, new SubstringExpression(6, 9)); + assertEquals(exp.hashCode(), new SubstringExpression(6, 9).hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new SubstringExpression(6, 9); + assertVerify(DataType.STRING, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + } + + @Test + public void requireThatRangeIsCappedToInput() { + assertEquals(new StringFieldValue(""), new SubstringExpression(6, 9).execute(new StringFieldValue("012345"))); + assertEquals(new StringFieldValue("345"), new SubstringExpression(3, 9).execute(new StringFieldValue("012345"))); + } + + @Test + public void requireThatIllegalRangeThrowsException() { + assertIndexOutOfBounds(-1, 69); + assertIndexOutOfBounds(69, -1); + assertIndexOutOfBounds(-9, -6); + assertIndexOutOfBounds(9, 6); + } + + private static void assertIndexOutOfBounds(int from, int to) { + try { + new SubstringExpression(from, to); + fail(); + } catch (IndexOutOfBoundsException e) { + + } + } + + @Test + public void requireThatStringIsSliced() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("666999")); + new SubstringExpression(2, 4).execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof StringFieldValue); + assertEquals("69", ((StringFieldValue)val).getString()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SummaryExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SummaryExpressionTestCase.java new file mode 100644 index 00000000000..8256f3ed730 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SummaryExpressionTestCase.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertExecute; +import static com.yahoo.vespa.indexinglanguage.expressions.OutputAssert.assertVerify; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class SummaryExpressionTestCase { + + @Test + public void requireThatAccessorsWork() { + SummaryExpression exp = new SummaryExpression("foo"); + assertEquals("foo", exp.getFieldName()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new SummaryExpression("foo"); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new AttributeExpression("foo"))); + assertFalse(exp.equals(new SummaryExpression("bar"))); + assertEquals(exp, new SummaryExpression("foo")); + assertEquals(exp.hashCode(), new SummaryExpression("foo").hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + assertVerify(new SummaryExpression("foo")); + } + + @Test + public void requireThatExpressionCanBeExecuted() { + assertExecute(new SummaryExpression("foo")); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java new file mode 100644 index 00000000000..ef1198b238e --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java @@ -0,0 +1,133 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SwitchTestCase { + + @Test + public void requireThatAccessorsWork() { + Map<String, Expression> cases = new HashMap<>(); + Expression foo = new AttributeExpression("foo"); + Expression bar = new AttributeExpression("bar"); + Expression baz = new AttributeExpression("baz"); + cases.put("foo", foo); + cases.put("bar", bar); + SwitchExpression exp = new SwitchExpression(cases, baz); + assertEquals(2, exp.getCases().size()); + assertSame(foo, exp.getCases().get("foo")); + assertSame(bar, exp.getCases().get("bar")); + assertSame(baz, exp.getDefaultExpression()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Map<String, Expression> cases = new HashMap<>(); + Expression foo = new AttributeExpression("foo"); + Expression bar = new AttributeExpression("bar"); + Expression baz = new AttributeExpression("baz"); + cases.put("foo", foo); + cases.put("bar", bar); + SwitchExpression exp = new SwitchExpression(cases, baz); + + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new SwitchExpression(Collections.singletonMap("foo", foo)))); + assertFalse(exp.equals(new SwitchExpression(Collections.singletonMap("foo", foo), baz))); + assertFalse(exp.equals(new SwitchExpression(cases))); + assertEquals(exp, new SwitchExpression(cases, baz)); + assertEquals(exp.hashCode(), new SwitchExpression(cases, baz).hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression foo = SimpleExpression.newConversion(DataType.STRING, DataType.INT); + Expression exp = new SwitchExpression(Collections.singletonMap("foo", foo)); + assertVerify(DataType.STRING, exp, DataType.STRING); // does not touch output + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + } + + @Test + public void requireThatCasesAreVerified() { + Map<String, Expression> cases = new HashMap<>(); + cases.put("foo", SimpleExpression.newRequired(DataType.INT)); + assertVerifyThrows(DataType.STRING, new SwitchExpression(cases), + "Expected int input, got string."); + assertVerifyThrows(DataType.STRING, new SwitchExpression(Collections.<String, Expression>emptyMap(), + SimpleExpression.newRequired(DataType.INT)), + "Expected int input, got string."); + } + + @Test + public void requireThatIllegalArgumentThrows() { + try { + new SwitchExpression(Collections.<String, Expression>emptyMap()).execute(new IntegerFieldValue(69)); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Expected string input, got int.", e.getMessage()); + } + } + + @Test + public void requireThatDefaultExpressionIsNullIfNotGiven() { + assertNull(new SwitchExpression(Collections.<String, Expression>emptyMap()).getDefaultExpression()); + } + + @Test + public void requireThatIsEmptyReflectsOnBothCasesAndDefault() { + assertTrue(new SwitchExpression(Collections.<String, Expression>emptyMap()).isEmpty()); + assertTrue(new SwitchExpression(Collections.<String, Expression>emptyMap(), null).isEmpty()); + assertFalse(new SwitchExpression(Collections.<String, Expression>emptyMap(), + new AttributeExpression("foo")).isEmpty()); + assertFalse(new SwitchExpression(Collections.singletonMap("foo", new AttributeExpression("foo")), + null).isEmpty()); + assertFalse(new SwitchExpression(Collections.singletonMap("foo", new AttributeExpression("foo")), + new AttributeExpression("foo")).isEmpty()); + + } + + @Test + public void requireThatCorrectExpressionIsExecuted() { + Map<String, Expression> cases = new HashMap<>(); + cases.put("foo", new StatementExpression(new SetValueExpression(new StringFieldValue("bar")), + new SetVarExpression("out"))); + cases.put("baz", new StatementExpression(new SetValueExpression(new StringFieldValue("cox")), + new SetVarExpression("out"))); + Expression exp = new SwitchExpression(cases); + assertEvaluate(new StringFieldValue("foo"), exp, new StringFieldValue("bar")); + assertEvaluate(new StringFieldValue("baz"), exp, new StringFieldValue("cox")); + assertEvaluate(new StringFieldValue("???"), exp, null); + } + + @Test + public void requireThatDefaultExpressionIsExecuted() { + Map<String, Expression> cases = new HashMap<>(); + cases.put("foo", new StatementExpression(new SetValueExpression(new StringFieldValue("bar")), + new SetVarExpression("out"))); + Expression defaultExp = new StatementExpression(new SetValueExpression(new StringFieldValue("cox")), + new SetVarExpression("out")); + Expression exp = new SwitchExpression(cases, defaultExp); + assertEvaluate(new StringFieldValue("foo"), exp, new StringFieldValue("bar")); + assertEvaluate(new StringFieldValue("baz"), exp, new StringFieldValue("cox")); + assertEvaluate(null, exp, new StringFieldValue("cox")); + } + + private static void assertEvaluate(FieldValue input, Expression exp, FieldValue expectedOutVar) { + assertEquals(expectedOutVar, new ExecutionContext().setValue(input).execute(exp).getVariable("out")); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ThisTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ThisTestCase.java new file mode 100644 index 00000000000..344a1b0b414 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ThisTestCase.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ThisTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new ThisExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new ThisExpression()); + assertEquals(exp.hashCode(), new ThisExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ThisExpression(); + assertVerify(DataType.INT, exp, DataType.INT); + assertVerify(DataType.STRING, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected any input, got null."); + } + + @Test + public void requireThatValueIsPreserved() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("69")); + new ThisExpression().execute(ctx); + + assertEquals(new StringFieldValue("69"), ctx.getValue()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayTestCase.java new file mode 100644 index 00000000000..193b8748d49 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayTestCase.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToArrayTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new ToArrayExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new ToArrayExpression()); + assertEquals(exp.hashCode(), new ToArrayExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ToArrayExpression(); + assertVerify(DataType.INT, exp, DataType.getArray(DataType.INT)); + assertVerify(DataType.STRING, exp, DataType.getArray(DataType.STRING)); + assertVerifyThrows(null, exp, "Expected any input, got null."); + } + + @Test + public void requireThatValueIsConverted() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("69")).execute(new ToArrayExpression()); + + FieldValue val = ctx.getValue(); + assertEquals(Array.class, val.getClass()); + + Array arr = (Array)val; + ArrayDataType type = arr.getDataType(); + assertEquals(DataType.STRING, type.getNestedType()); + assertEquals(1, arr.size()); + assertEquals(new StringFieldValue("69"), arr.get(0)); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteTestCase.java new file mode 100644 index 00000000000..baa5ed5be93 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteTestCase.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToByteTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new ToByteExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new ToByteExpression()); + assertEquals(exp.hashCode(), new ToByteExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ToByteExpression(); + assertVerify(DataType.INT, exp, DataType.BYTE); + assertVerify(DataType.STRING, exp, DataType.BYTE); + assertVerifyThrows(null, exp, "Expected any input, got null."); + } + + @Test + public void requireThatValueIsConverted() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("69")).execute(new ToByteExpression()); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof ByteFieldValue); + assertEquals(69, ((ByteFieldValue)val).getByte()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleTestCase.java new file mode 100644 index 00000000000..abd4f397b3b --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleTestCase.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToDoubleTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new ToDoubleExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new ToDoubleExpression()); + assertEquals(exp.hashCode(), new ToDoubleExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ToDoubleExpression(); + assertVerify(DataType.INT, exp, DataType.DOUBLE); + assertVerify(DataType.STRING, exp, DataType.DOUBLE); + assertVerifyThrows(null, exp, "Expected any input, got null."); + } + + @Test + public void requireThatValueIsConverted() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("6.9")).execute(new ToDoubleExpression()); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof DoubleFieldValue); + assertEquals(6.9, ((DoubleFieldValue)val).getDouble(), 1e-6); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatTestCase.java new file mode 100644 index 00000000000..b8b3f4ee51e --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatTestCase.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToFloatTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new ToFloatExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new ToFloatExpression()); + assertEquals(exp.hashCode(), new ToFloatExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ToFloatExpression(); + assertVerify(DataType.INT, exp, DataType.FLOAT); + assertVerify(DataType.STRING, exp, DataType.FLOAT); + assertVerifyThrows(null, exp, "Expected any input, got null."); + } + + @Test + public void requireThatValueIsConverted() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("6.9f")).execute(new ToFloatExpression()); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof FloatFieldValue); + assertEquals(6.9f, ((FloatFieldValue)val).getFloat(), 1e-6); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerTestCase.java new file mode 100644 index 00000000000..c3d863de645 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerTestCase.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToIntegerTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new ToIntegerExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new ToIntegerExpression()); + assertEquals(exp.hashCode(), new ToIntegerExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ToIntegerExpression(); + assertVerify(DataType.INT, exp, DataType.INT); + assertVerify(DataType.STRING, exp, DataType.INT); + assertVerifyThrows(null, exp, "Expected any input, got null."); + } + + @Test + public void requireThatValueIsConverted() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("69")).execute(new ToIntegerExpression()); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof IntegerFieldValue); + assertEquals(69, ((IntegerFieldValue)val).getInteger()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongTestCase.java new file mode 100644 index 00000000000..b4fc310db48 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongTestCase.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToLongTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new ToLongExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new ToLongExpression()); + assertEquals(exp.hashCode(), new ToLongExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ToLongExpression(); + assertVerify(DataType.INT, exp, DataType.LONG); + assertVerify(DataType.STRING, exp, DataType.LONG); + assertVerifyThrows(null, exp, "Expected any input, got null."); + } + + @Test + public void requireThatValueIsConverted() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("69")).execute(new ToLongExpression()); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof LongFieldValue); + assertEquals(69L, ((LongFieldValue)val).getLong()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionTestCase.java new file mode 100644 index 00000000000..fb370405643 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionTestCase.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToPositionTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new ToPositionExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new ToPositionExpression()); + assertEquals(exp.hashCode(), new ToPositionExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ToPositionExpression(); + assertVerify(DataType.STRING, exp, PositionDataType.INSTANCE); + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + } + + @Test + public void requireThatPositionIsParsed() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("6;9")).execute(new ToPositionExpression()); + + FieldValue out = ctx.getValue(); + assertTrue(out instanceof StructuredFieldValue); + assertEquals(PositionDataType.INSTANCE, out.getDataType()); + + FieldValue val = ((StructuredFieldValue)out).getFieldValue("x"); + assertTrue(val instanceof IntegerFieldValue); + assertEquals(6, ((IntegerFieldValue)val).getInteger()); + + val = ((StructuredFieldValue)out).getFieldValue("y"); + assertTrue(val instanceof IntegerFieldValue); + assertEquals(9, ((IntegerFieldValue)val).getInteger()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringTestCase.java new file mode 100644 index 00000000000..d20d75ee5dd --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringTestCase.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToStringTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new ToStringExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new ToStringExpression()); + assertEquals(exp.hashCode(), new ToStringExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ToStringExpression(); + assertVerify(DataType.INT, exp, DataType.STRING); + assertVerify(DataType.STRING, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected any input, got null."); + } + + @Test + public void requireThatValueIsConverted() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new IntegerFieldValue(69)).execute(new ToStringExpression()); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof StringFieldValue); + assertEquals("69", ((StringFieldValue)val).getString()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetTestCase.java new file mode 100644 index 00000000000..e761e4332dc --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetTestCase.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ToWsetTestCase { + + @Test + public void requireThatAccessorsWork() { + assertAccessors(false, false); + assertAccessors(false, true); + assertAccessors(true, false); + assertAccessors(true, true); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new ToWsetExpression(true, true); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new ToWsetExpression(false, false))); + assertFalse(exp.equals(new ToWsetExpression(true, false))); + assertFalse(exp.equals(new ToWsetExpression(false, true))); + assertEquals(exp, new ToWsetExpression(true, true)); + assertEquals(exp.hashCode(), new ToWsetExpression(true, true).hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + assertVerify(false, false); + assertVerify(false, true); + assertVerify(true, false); + assertVerify(true, true); + } + + @Test + public void requireThatValueIsConverted() { + assertConvert(false, false); + assertConvert(false, true); + assertConvert(true, false); + assertConvert(true, true); + } + + private static void assertVerify(boolean createIfNonExistent, boolean removeIfZero) { + Expression exp = new ToWsetExpression(createIfNonExistent, removeIfZero); + ExpressionAssert.assertVerify(DataType.INT, exp, + DataType.getWeightedSet(DataType.INT, createIfNonExistent, removeIfZero)); + ExpressionAssert.assertVerify(DataType.STRING, exp, + DataType.getWeightedSet(DataType.STRING, createIfNonExistent, removeIfZero)); + assertVerifyThrows(null, exp, "Expected any input, got null."); + } + + private static void assertConvert(boolean createIfNonExistent, boolean removeIfZero) { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("69")).execute(new ToWsetExpression(createIfNonExistent, removeIfZero)); + + FieldValue val = ctx.getValue(); + assertEquals(WeightedSet.class, val.getClass()); + + WeightedSet wset = (WeightedSet)val; + WeightedSetDataType type = wset.getDataType(); + assertEquals(DataType.STRING, type.getNestedType()); + assertEquals(createIfNonExistent, type.createIfNonExistent()); + assertEquals(removeIfZero, type.removeIfZero()); + + assertEquals(1, wset.size()); + assertEquals(Integer.valueOf(1), wset.get(new StringFieldValue("69"))); + } + + private static void assertAccessors(boolean createIfNonExistent, boolean removeIfZero) { + ToWsetExpression exp = new ToWsetExpression(createIfNonExistent, removeIfZero); + assertEquals(createIfNonExistent, exp.getCreateIfNonExistent()); + assertEquals(removeIfZero, exp.getRemoveIfZero()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java new file mode 100644 index 00000000000..9fdab3b978f --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.annotation.SpanTrees; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; +import org.junit.Test; +import org.mockito.Mockito; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class TokenizeTestCase { + + @Test + public void requireThatAccessorsWork() { + Linguistics linguistics = new SimpleLinguistics(); + AnnotatorConfig config = new AnnotatorConfig(); + TokenizeExpression exp = new TokenizeExpression(linguistics, config); + assertSame(linguistics, exp.getLinguistics()); + assertSame(config, exp.getConfig()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + AnnotatorConfig config = new AnnotatorConfig().setLanguage(Language.ARABIC); + Expression exp = new TokenizeExpression(new SimpleLinguistics(), config); + assertFalse(exp.equals(new Object())); + assertFalse(exp.equals(new TokenizeExpression(Mockito.mock(Linguistics.class), + new AnnotatorConfig().setLanguage(Language.SPANISH)))); + assertFalse(exp.equals(new TokenizeExpression(new SimpleLinguistics(), + new AnnotatorConfig().setLanguage(Language.SPANISH)))); + assertEquals(exp, new TokenizeExpression(new SimpleLinguistics(), config)); + assertEquals(exp.hashCode(), new TokenizeExpression(new SimpleLinguistics(), config).hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new TokenizeExpression(new SimpleLinguistics(), new AnnotatorConfig()); + assertVerify(DataType.STRING, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + } + + @Test + public void requireThatValueIsAnnotated() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("foo")); + new TokenizeExpression(new SimpleLinguistics(), new AnnotatorConfig()).execute(ctx); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof StringFieldValue); + assertNotNull(((StringFieldValue)val).getSpanTree(SpanTrees.LINGUISTICS)); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TrimTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TrimTestCase.java new file mode 100644 index 00000000000..6057cf2f211 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TrimTestCase.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class TrimTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new TrimExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new TrimExpression()); + assertEquals(exp.hashCode(), new TrimExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new TrimExpression(); + assertVerify(DataType.STRING, exp, DataType.STRING); + assertVerifyThrows(null, exp, "Expected string input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + } + + @Test + public void requireThatStringIsTrimmed() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue(" 69 ")).execute(new TrimExpression()); + + FieldValue val = ctx.getValue(); + assertTrue(val instanceof StringFieldValue); + assertEquals("69", ((StringFieldValue)val).getString()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedDataTypeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedDataTypeTestCase.java new file mode 100644 index 00000000000..a1e549c3830 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedDataTypeTestCase.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlStream; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class UnresolvedDataTypeTestCase { + + @Test + public void requireThatFieldValueIsUnresolved() { + assertEquals(UnresolvedFieldValue.class, UnresolvedDataType.INSTANCE.createFieldValue().getClass()); + } + + @Test + public void requireThatTypeIsCompatibleWithAnything() { + assertFalse(UnresolvedDataType.INSTANCE.isValueCompatible(null)); + assertTrue(UnresolvedDataType.INSTANCE.isValueCompatible(new MyFieldValue())); + } + + private static class MyFieldValue extends FieldValue { + + @Override + public DataType getDataType() { + return null; + } + + @Override + public void printXml(XmlStream xml) { + + } + + @Override + public void clear() { + + } + + @Override + public void assign(Object o) { + + } + + @Override + public void serialize(Field field, FieldWriter writer) { + + } + + @Override + public void deserialize(Field field, FieldReader reader) { + + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValueTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValueTestCase.java new file mode 100644 index 00000000000..74ea3b35954 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValueTestCase.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class UnresolvedFieldValueTestCase { + + @Test + public void requireThatDataTypeIsUnresolved() { + assertEquals(UnresolvedDataType.INSTANCE, new UnresolvedFieldValue().getDataType()); + } + + @Test + public void requireThatNoMethodsAreSupported() { + UnresolvedFieldValue val = new UnresolvedFieldValue(); + try { + val.printXml(null); + fail(); + } catch (UnsupportedOperationException e) { + + } + try { + val.clear(); + fail(); + } catch (UnsupportedOperationException e) { + + } + try { + val.assign(new UnresolvedFieldValue()); + fail(); + } catch (UnsupportedOperationException e) { + + } + try { + val.serialize(null, null); + fail(); + } catch (UnsupportedOperationException e) { + + } + try { + val.deserialize(null, null); + fail(); + } catch (UnsupportedOperationException e) { + + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContextTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContextTestCase.java new file mode 100644 index 00000000000..10edcb546fe --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContextTestCase.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import org.junit.Test; + +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class VerificationContextTestCase { + + @Test + public void requireThatValueCanBeSet() { + VerificationContext ctx = new VerificationContext(); + DataType val = DataType.STRING; + ctx.setValue(val); + assertSame(val, ctx.getValue()); + } + + @Test + public void requireThatVariablesCanBeSet() { + VerificationContext ctx = new VerificationContext(); + DataType val = DataType.STRING; + ctx.setVariable("foo", val); + assertSame(val, ctx.getVariable("foo")); + } + + @Test + public void requireThatClearRemovesValue() { + VerificationContext ctx = new VerificationContext(); + ctx.setValue(DataType.STRING); + ctx.clear(); + assertNull(ctx.getValue()); + } + + @Test + public void requireThatClearRemovesVariables() { + VerificationContext ctx = new VerificationContext(); + ctx.setVariable("foo", DataType.STRING); + ctx.clear(); + assertNull(ctx.getVariable("foo")); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationExceptionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationExceptionTestCase.java new file mode 100644 index 00000000000..412da319016 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationExceptionTestCase.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class VerificationExceptionTestCase { + + @Test + public void requireThatAccessorsWork() { + Expression exp = new SimpleExpression(); + VerificationException e = new VerificationException(exp, "foo"); + assertSame(exp, e.getExpression()); + assertEquals("foo", e.getMessage()); + assertTrue(e.toString().contains(exp.toString())); + assertTrue(e.toString().contains(e.getMessage())); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveTestCase.java new file mode 100644 index 00000000000..503aacdd551 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveTestCase.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.geo.ZCurve; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertFalse; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ZCurveTestCase { + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new ZCurveExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new ZCurveExpression()); + assertEquals(exp.hashCode(), new ZCurveExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ZCurveExpression(); + assertVerify(PositionDataType.INSTANCE, exp, DataType.LONG); + assertVerifyThrows(null, exp, "Expected position input, got null."); + assertVerifyThrows(DataType.INT, exp, "Expected position input, got int."); + } + + @Test + public void requireThatInputIsEncoded() { + assertEquals(new LongFieldValue(ZCurve.encode(6, 9)), + new ZCurveExpression().execute(PositionDataType.valueOf(6, 9))); + } + + @Test + public void requireThatMissingFieldEvaluatesToNull() { + assertNull(new ZCurveExpression().execute(PositionDataType.valueOf(null, 9))); + assertNull(new ZCurveExpression().execute(PositionDataType.valueOf(6, null))); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfigTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfigTestCase.java new file mode 100644 index 00000000000..163e4bff4a8 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfigTestCase.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.linguistics; + +import com.yahoo.language.Language; +import com.yahoo.language.process.StemMode; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class AnnotatorConfigTestCase { + + @Test + public void requireThatAccessorsWork() { + AnnotatorConfig config = new AnnotatorConfig(); + for (Language language : Language.values()) { + config.setLanguage(language); + assertEquals(language, config.getLanguage()); + } + for (StemMode mode : StemMode.values()) { + config.setStemMode(mode); + assertEquals(mode, config.getStemMode()); + } + config.setRemoveAccents(true); + assertTrue(config.getRemoveAccents()); + config.setRemoveAccents(false); + assertFalse(config.getRemoveAccents()); + } + + @Test + public void requireThatCopyConstructorIsImplemented() { + AnnotatorConfig config = new AnnotatorConfig(); + config.setLanguage(Language.ARABIC); + config.setStemMode(StemMode.SHORTEST); + config.setRemoveAccents(!config.getRemoveAccents()); + + AnnotatorConfig other = new AnnotatorConfig(config); + assertEquals(config.getLanguage(), other.getLanguage()); + assertEquals(config.getStemMode(), other.getStemMode()); + assertEquals(config.getRemoveAccents(), other.getRemoveAccents()); + } + + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + AnnotatorConfig config = newConfig(Language.DUTCH, StemMode.NONE, true); + assertFalse(config.equals(new Object())); + assertFalse(config.equals(newConfig(Language.SPANISH, StemMode.SHORTEST, false))); + assertFalse(config.equals(newConfig(Language.DUTCH, StemMode.SHORTEST, false))); + assertFalse(config.equals(newConfig(Language.DUTCH, StemMode.NONE, false))); + assertEquals(config, newConfig(Language.DUTCH, StemMode.NONE, true)); + assertEquals(config.hashCode(), newConfig(Language.DUTCH, StemMode.NONE, true).hashCode()); + } + + private static AnnotatorConfig newConfig(Language language, StemMode stemMode, + boolean removeAccents) { + AnnotatorConfig config = new AnnotatorConfig(); + config.setLanguage(language); + config.setStemMode(stemMode); + config.setRemoveAccents(removeAccents); + return config; + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java new file mode 100644 index 00000000000..805bdc96904 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java @@ -0,0 +1,250 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.linguistics;
+
+import com.yahoo.document.annotation.Annotation;
+import com.yahoo.document.annotation.AnnotationTypes;
+import com.yahoo.document.annotation.SpanTree;
+import com.yahoo.document.annotation.SpanTrees;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.language.Language;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.process.StemMode;
+import com.yahoo.language.process.Token;
+import com.yahoo.language.process.TokenType;
+import com.yahoo.language.process.Tokenizer;
+import com.yahoo.language.simple.SimpleToken;
+
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.*;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LinguisticsAnnotatorTestCase {
+
+ private static final AnnotatorConfig CONFIG = new AnnotatorConfig();
+
+ // --------------------------------------------------------------------------------
+ //
+ // Tests
+ //
+ // --------------------------------------------------------------------------------
+
+ @Test
+ public void requireThatAnnotateFailsWithZeroTokens() {
+ assertAnnotations(null, "foo");
+ }
+
+ @Test
+ public void requireThatAnnotateFailsWithoutIndexableTokenString() {
+ for (TokenType type : TokenType.values()) {
+ if (type.isIndexable()) {
+ continue;
+ }
+ assertAnnotations(null, "foo", newToken("foo", "bar", type));
+ }
+ }
+
+ @Test
+ public void requireThatIndexableTokenStringsAreAnnotated() {
+ SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
+ expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));
+ for (TokenType type : TokenType.values()) {
+ if (!type.isIndexable()) {
+ continue;
+ }
+ assertAnnotations(expected, "foo", newToken("foo", "bar", type));
+ }
+ }
+
+ @Test
+ public void requireThatSpecialTokenStringsAreAnnotatedRegardlessOfType() {
+ SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
+ expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));
+ for (TokenType type : TokenType.values()) {
+ assertAnnotations(expected, "foo", newToken("foo", "bar", type, true));
+ }
+ }
+
+ @Test
+ public void requireThatTermAnnotationsAreEmptyIfOrigIsLowerCase() {
+ SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
+ expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM));
+ for (boolean specialToken : Arrays.asList(true, false)) {
+ for (TokenType type : TokenType.values()) {
+ if (!specialToken && !type.isIndexable()) {
+ continue;
+ }
+ assertAnnotations(expected, "foo", newToken("foo", "foo", type, specialToken));
+ }
+ }
+ }
+
+ @Test
+ public void requireThatTermAnnotationsAreLowerCased() {
+ SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
+ expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));
+ for (boolean specialToken : Arrays.asList(true, false)) {
+ for (TokenType type : TokenType.values()) {
+ if (!specialToken && !type.isIndexable()) {
+ continue;
+ }
+ assertAnnotations(expected, "foo", newToken("foo", "BAR", type, specialToken));
+ }
+ }
+ }
+
+ @Test
+ public void requireThatCompositeTokensAreFlattened() {
+ SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
+ expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("foo")));
+ expected.spanList().span(3, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));
+ expected.spanList().span(6, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("baz")));
+
+ SimpleToken token = newToken("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC)
+ .addComponent(newToken("FOO", "foo", TokenType.ALPHABETIC).setOffset(0))
+ .addComponent(newToken("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3)
+ .addComponent(newToken("BAR", "bar", TokenType.ALPHABETIC).setOffset(3))
+ .addComponent(newToken("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6)));
+ assertAnnotations(expected, "foobarbaz", token);
+ }
+
+ @Test
+ public void requireThatCompositeSpecialTokensAreNotFlattened() {
+ SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
+ expected.spanList().span(0, 9).annotate(new Annotation(AnnotationTypes.TERM,
+ new StringFieldValue("foobarbaz")));
+
+ SimpleToken token = newToken("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC).setSpecialToken(true)
+ .addComponent(newToken("FOO", "foo", TokenType.ALPHABETIC).setOffset(0))
+ .addComponent(newToken("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3)
+ .addComponent(newToken("BAR", "bar", TokenType.ALPHABETIC).setOffset(3))
+ .addComponent(newToken("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6)));
+ assertAnnotations(expected, "foobarbaz", token);
+ }
+
+ @Test
+ public void requireThatErrorTokensAreSkipped() {
+ assertAnnotations(null, "foo", new SimpleToken("foo").setType(TokenType.ALPHABETIC)
+ .setOffset(-1));
+ }
+
+ @Test
+ public void requireThatTermReplacementsAreApplied() {
+ SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
+ expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar")));
+ for (boolean specialToken : Arrays.asList(true, false)) {
+ for (TokenType type : TokenType.values()) {
+ if (!specialToken && !type.isIndexable()) {
+ continue;
+ }
+ assertAnnotations(expected, "foo",
+ newLinguistics(Arrays.asList(newToken("foo", "foo", type, specialToken)),
+ Collections.singletonMap("foo", "bar")));
+ }
+ }
+ }
+
+ @Test
+ public void requireThatExistingAnnotationsAreKept() {
+ SpanTree spanTree = new SpanTree(SpanTrees.LINGUISTICS);
+ spanTree.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("baz")));
+
+ StringFieldValue val = new StringFieldValue("foo");
+ val.setSpanTree(spanTree);
+
+ Linguistics linguistics = newLinguistics(Arrays.asList(newToken("foo", "bar", TokenType.ALPHABETIC, false)),
+ Collections.<String, String>emptyMap());
+ new LinguisticsAnnotator(linguistics, CONFIG).annotate(val);
+
+ assertTrue(new LinguisticsAnnotator(linguistics, CONFIG).annotate(val));
+ assertEquals(spanTree, val.getSpanTree(SpanTrees.LINGUISTICS));
+ }
+
+ @Test
+ public void requireThatMaxTermOccurencesIsHonored() {
+ final String inputTerm = "foo";
+ final String stemmedInputTerm = "bar"; // completely different from
+ // inputTerm for safer test
+ final String paddedInputTerm = inputTerm + " ";
+ final SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS);
+ final int inputTermOccurence = AnnotatorConfig.DEFAULT_MAX_TERM_OCCURRENCES * 2;
+ for (int i = 0; i < AnnotatorConfig.DEFAULT_MAX_TERM_OCCURRENCES; ++i) {
+ expected.spanList().span(i * paddedInputTerm.length(), inputTerm.length())
+ .annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue(stemmedInputTerm)));
+ }
+ for (TokenType type : TokenType.values()) {
+ if (!type.isIndexable()) {
+ continue;
+ }
+ StringBuilder input = new StringBuilder();
+ Token[] tokens = new Token[inputTermOccurence];
+ for (int i = 0; i < inputTermOccurence; ++i) {
+ SimpleToken t = newToken(inputTerm, stemmedInputTerm, type);
+ t.setOffset(i * paddedInputTerm.length());
+ tokens[i] = t;
+ input.append(paddedInputTerm);
+ }
+ assertAnnotations(expected, input.toString(), tokens);
+ }
+ }
+
+ // --------------------------------------------------------------------------------
+ //
+ // Utilities
+ //
+ // --------------------------------------------------------------------------------
+
+ private static SimpleToken newToken(String orig, String stem, TokenType type) {
+ return newToken(orig, stem, type, false);
+ }
+
+ private static SimpleToken newToken(String orig, String stem, TokenType type, boolean specialToken) {
+ return new SimpleToken(orig).setTokenString(stem)
+ .setType(type)
+ .setSpecialToken(specialToken);
+ }
+
+ private static void assertAnnotations(SpanTree expected, String value, Token... tokens) {
+ assertAnnotations(expected, value, newLinguistics(Arrays.asList(tokens), Collections.<String, String>emptyMap()));
+ }
+
+ private static void assertAnnotations(SpanTree expected, String str, Linguistics linguistics) {
+ StringFieldValue val = new StringFieldValue(str);
+ assertEquals(expected != null, new LinguisticsAnnotator(linguistics, CONFIG).annotate(val));
+ assertEquals(expected, val.getSpanTree(SpanTrees.LINGUISTICS));
+ }
+
+ private static Linguistics newLinguistics(List<? extends Token> tokens, Map<String, String> replacementTerms) {
+ Linguistics linguistics = Mockito.mock(Linguistics.class);
+ Mockito.when(linguistics.getTokenizer()).thenReturn(new MyTokenizer(tokens, replacementTerms));
+ return linguistics;
+ }
+
+ private static class MyTokenizer implements Tokenizer {
+
+ final List<Token> tokens;
+ final Map<String, String> replacementTerms;
+
+ public MyTokenizer(List<? extends Token> tokens, Map<String, String> replacementTerms) {
+ this.tokens = new ArrayList<>(tokens);
+ this.replacementTerms = replacementTerms;
+ }
+
+ @Override
+ public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) {
+ return tokens;
+ }
+
+ @Override
+ public String getReplacementTerm(String term) {
+ String replacement = replacementTerms.get(term);
+ return replacement != null ? replacement : term;
+ }
+ }
+}
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/DefaultFieldNameTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/DefaultFieldNameTestCase.java new file mode 100644 index 00000000000..8d279f14b5b --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/DefaultFieldNameTestCase.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.parser; + +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.ScriptParserContext; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class DefaultFieldNameTestCase { + + @Test + public void requireThatDefaultFieldNameIsAppliedWhenArgumentIsMissing() throws ParseException { + IndexingInput input = new IndexingInput("input"); + InputExpression exp = (InputExpression)Expression.newInstance(new ScriptParserContext(new SimpleLinguistics()) + .setInputStream(input) + .setDefaultFieldName("foo")); + assertEquals("foo", exp.getFieldName()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java new file mode 100644 index 00000000000..a70de62c1a7 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ExpressionTestCase.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.parser; + +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ExpressionTestCase { + + @Test + public void requireThatAllExpressionTypesAreParsed() throws ParseException { + assertExpression(ArithmeticExpression.class, "1 + 2"); + assertExpression(AttributeExpression.class, "attribute"); + assertExpression(Base64DecodeExpression.class, "base64decode"); + assertExpression(Base64EncodeExpression.class, "base64encode"); + assertExpression(CatExpression.class, "1 . 2"); + assertExpression(ClearStateExpression.class, "clear_state"); + assertExpression(EchoExpression.class, "echo"); + assertExpression(ExactExpression.class, "exact"); + assertExpression(FlattenExpression.class, "flatten"); + assertExpression(ForEachExpression.class, "for_each { 1 }"); + assertExpression(GetFieldExpression.class, "get_field field1"); + assertExpression(GetVarExpression.class, "get_var myvar1"); + assertExpression(GuardExpression.class, "guard { 1 }"); + assertExpression(GuardExpression.class, "guard { 1; 2 }"); + assertExpression(HexDecodeExpression.class, "hexdecode"); + assertExpression(HexEncodeExpression.class, "hexencode"); + assertExpression(HostNameExpression.class, "hostname"); + assertExpression(IfThenExpression.class, "if (1 < 2) { 3 }"); + assertExpression(IfThenExpression.class, "if (1 < 2) { 3 } else { 4 }"); + assertExpression(IndexExpression.class, "index"); + assertExpression(InputExpression.class, "input"); + assertExpression(InputExpression.class, "input field1"); + assertExpression(JoinExpression.class, "join '1'"); + assertExpression(LowerCaseExpression.class, "lowercase"); + assertExpression(NGramExpression.class, "ngram 1"); + assertExpression(NormalizeExpression.class, "normalize"); + assertExpression(NowExpression.class, "now"); + assertExpression(OptimizePredicateExpression.class, "optimize_predicate"); + assertExpression(ParenthesisExpression.class, "(1)"); + assertExpression(RandomExpression.class, "random"); + assertExpression(RandomExpression.class, "random 1"); + assertExpression(ScriptExpression.class, "{ 1; 2 }"); + assertExpression(SelectInputExpression.class, "select_input { field1: 2; }"); + assertExpression(SetLanguageExpression.class, "set_language"); + assertExpression(SetValueExpression.class, "1"); + assertExpression(SetVarExpression.class, "set_var myvar1"); + assertExpression(SplitExpression.class, "split '1'"); + assertExpression(StatementExpression.class, "1 | 2"); + assertExpression(SubstringExpression.class, "substring 1 2"); + assertExpression(SummaryExpression.class, "summary"); + assertExpression(SwitchExpression.class, "switch { case '1': 2; }"); + assertExpression(ThisExpression.class, "this"); + assertExpression(ToArrayExpression.class, "to_array"); + assertExpression(ToByteExpression.class, "to_byte"); + assertExpression(ToDoubleExpression.class, "to_double"); + assertExpression(ToFloatExpression.class, "to_float"); + assertExpression(ToIntegerExpression.class, "to_int"); + assertExpression(TokenizeExpression.class, "tokenize"); + assertExpression(TokenizeExpression.class, "tokenize stem"); + assertExpression(TokenizeExpression.class, "tokenize stem normalize"); + assertExpression(TokenizeExpression.class, "tokenize stem:\"ALL\" normalize"); + assertExpression(TokenizeExpression.class, "tokenize stem:\"ALL\""); + assertExpression(TokenizeExpression.class, "tokenize normalize"); + assertExpression(ToLongExpression.class, "to_long"); + assertExpression(ToPositionExpression.class, "to_pos"); + assertExpression(ToStringExpression.class, "to_string"); + assertExpression(ToWsetExpression.class, "to_wset"); + assertExpression(ToWsetExpression.class, "to_wset create_if_non_existent"); + assertExpression(ToWsetExpression.class, "to_wset remove_if_zero"); + assertExpression(ToWsetExpression.class, "to_wset create_if_non_existent remove_if_zero"); + assertExpression(ToWsetExpression.class, "to_wset remove_if_zero create_if_non_existent"); + assertExpression(TrimExpression.class, "trim"); + assertExpression(ZCurveExpression.class, "zcurve"); + } + + private static void assertExpression(Class expectedClass, String str) throws ParseException { + Linguistics linguistics = new SimpleLinguistics(); + Expression foo = Expression.fromString(str, linguistics); + assertEquals(expectedClass, foo.getClass()); + Expression bar = Expression.fromString(foo.toString(), linguistics); + assertEquals(foo.hashCode(), bar.hashCode()); + assertEquals(foo, bar); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/FieldNameTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/FieldNameTestCase.java new file mode 100644 index 00000000000..9cce117bec3 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/FieldNameTestCase.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.parser; + +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class FieldNameTestCase { + + @Test + public void requireThatDotCanBeUsedInFieldName() throws ParseException { + assertEquals(new AttributeExpression("foo.bar"), Expression.fromString("attribute foo . bar")); + assertEquals(new IndexExpression("foo.bar"), Expression.fromString("index foo . bar")); + assertEquals(new SummaryExpression("foo.bar"), Expression.fromString("summary foo . bar")); + } + + @Test + public void requireThatCatDotIsNotConfusedWithFieldName() throws ParseException { + assertEquals(new CatExpression(new InputExpression("foo"), new InputExpression("bar")), + Expression.fromString("input foo . input bar")); + assertEquals(new CatExpression(new InputExpression("foo"), new SetValueExpression(new StringFieldValue("bar"))), + Expression.fromString("input foo . 'bar'")); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/IdentifierTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/IdentifierTestCase.java new file mode 100644 index 00000000000..69840bff19e --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/IdentifierTestCase.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.parser; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class IdentifierTestCase { + + @Test + public void requireThatThereAreNoReservedWords() throws ParseException { + List<String> tokens = Arrays.asList("attribute", + "base64_decode", + "base64_encode", + "clear_state", + "compact_phrase", + "create_if_non_existent", + "echo", + "exact", + "flatten", + "for_each", + "get_field", + "get_var", + "guard", + "hex_decode", + "hex_encode", + "host_name", + "if", + "index", + "join", + "linguistics", + "lower_case", + "ngram", + "normalize", + "now", + "optimize_predicate", + "predicate_to_raw", + "put_symbol", + "random", + "raw_to_predicate", + "remove_ctrl_chars", + "remove_if_zero", + "remove_so_si", + "select_field", + "set_language", + "set_var", + "split", + "substring", + "summary", + "switch", + "this", + "tokenize", + "to_array", + "to_double", + "to_float", + "to_int", + "to_long", + "to_pos", + "to_string", + "to_wset", + "trim", + "zcurve"); + for (String str : tokens) { + IndexingParser parser = new IndexingParser(new IndexingInput(str)); + assertEquals(str, parser.identifier()); + } + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/MathTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/MathTestCase.java new file mode 100644 index 00000000000..fee4d6ed1a5 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/MathTestCase.java @@ -0,0 +1,92 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.parser; + +import com.yahoo.document.datatypes.*; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class MathTestCase { + + private static final double EPS = 1e-6; + + @Test + public void requireThatNumericTypeIsPreserved() throws ParseException { + assertEquals(1, evaluateDouble("1.0"), EPS); + assertEquals(3, evaluateDouble("1.0 + 2"), EPS); + assertEquals(-1, evaluateDouble("1.0 - 2"), EPS); + assertEquals(2, evaluateDouble("1.0 * 2"), EPS); + assertEquals(0.5, evaluateDouble("1.0 / 2"), EPS); + assertEquals(1, evaluateDouble("1.0 % 2"), EPS); + + assertEquals(1, evaluateFloat("1.0f"), EPS); + assertEquals(3, evaluateFloat("1.0f + 2"), EPS); + assertEquals(-1, evaluateFloat("1.0f - 2"), EPS); + assertEquals(2, evaluateFloat("1.0f * 2"), EPS); + assertEquals(0.5, evaluateFloat("1.0f / 2"), EPS); + assertEquals(1, evaluateFloat("1.0f % 2"), EPS); + + assertEquals(1, evaluateInteger("1")); + assertEquals(3, evaluateInteger("1 + 2")); + assertEquals(-1, evaluateInteger("1 - 2")); + assertEquals(2, evaluateInteger("1 * 2")); + assertEquals(0, evaluateInteger("1 / 2")); + assertEquals(1, evaluateInteger("1 % 2")); + + assertEquals(1, evaluateLong("1L")); + assertEquals(3, evaluateLong("1L + 2")); + assertEquals(-1, evaluateLong("1L - 2")); + assertEquals(2, evaluateLong("1L * 2")); + assertEquals(0, evaluateLong("1L / 2")); + assertEquals(1, evaluateLong("1L % 2")); + + assertEquals(3, evaluateDouble("1.0 + 2"), EPS); + assertEquals(3, evaluateDouble("1.0 + 2L"), EPS); + assertEquals(3, evaluateDouble("1.0 + 2.0f"), EPS); + assertEquals(3, evaluateFloat("1.0f + 2"), EPS); + assertEquals(3, evaluateFloat("1.0f + 2L"), EPS); + assertEquals(3, evaluateLong("1L + 2")); + } + + @Test + public void requireThatParenthesisControlsPrecedence() throws ParseException { + assertEquals(2, evaluateInteger("1 - 2 + 3")); + assertEquals(2, evaluateInteger("(1 - 2) + 3")); + assertEquals(-4, evaluateInteger("1 - (2 + 3)")); + assertEquals(2, evaluateInteger("(1 - 2 + 3)")); + } + + private static double evaluateDouble(String script) throws ParseException { + FieldValue val = evaluateMath(script); + assertTrue(val instanceof DoubleFieldValue); + return ((DoubleFieldValue)val).getDouble(); + } + + private static double evaluateFloat(String script) throws ParseException { + FieldValue val = evaluateMath(script); + assertTrue(val instanceof FloatFieldValue); + return ((FloatFieldValue)val).getFloat(); + } + + private static long evaluateInteger(String script) throws ParseException { + FieldValue val = evaluateMath(script); + assertTrue(val instanceof IntegerFieldValue); + return ((IntegerFieldValue)val).getInteger(); + } + + private static long evaluateLong(String script) throws ParseException { + FieldValue val = evaluateMath(script); + assertTrue(val instanceof LongFieldValue); + return ((LongFieldValue)val).getLong(); + } + + private static FieldValue evaluateMath(String script) throws ParseException { + return Expression.fromString(script).execute(); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/NumberTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/NumberTestCase.java new file mode 100644 index 00000000000..3ff4e957fc6 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/NumberTestCase.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.parser; + +import com.yahoo.document.datatypes.*; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class NumberTestCase { + + @Test + public void requireThatCorrectNumberTypeIsParsed() throws ParseException { + assertTrue(Expression.fromString("6.9").execute() instanceof DoubleFieldValue); + assertTrue(Expression.fromString("6.9f").execute() instanceof FloatFieldValue); + assertTrue(Expression.fromString("6.9F").execute() instanceof FloatFieldValue); + assertTrue(Expression.fromString("69").execute() instanceof IntegerFieldValue); + assertTrue(Expression.fromString("69l").execute() instanceof LongFieldValue); + assertTrue(Expression.fromString("69L").execute() instanceof LongFieldValue); + assertTrue(Expression.fromString("0x69").execute() instanceof IntegerFieldValue); + assertTrue(Expression.fromString("0x69l").execute() instanceof LongFieldValue); + assertTrue(Expression.fromString("0x69L").execute() instanceof LongFieldValue); + assertTrue(Expression.fromString("'69'").execute() instanceof StringFieldValue); + assertTrue(Expression.fromString("\"69\"").execute() instanceof StringFieldValue); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/PrecedenceTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/PrecedenceTestCase.java new file mode 100644 index 00000000000..418e307c539 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/PrecedenceTestCase.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.parser; + +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class PrecedenceTestCase { + + @Test + public void requireThatMathPrecedesConcat() throws ParseException { + assertEquals("15", evaluate("1 . 2 + 3")); + assertEquals("33", evaluate("1 + 2 . 3")); + } + + private static String evaluate(String script) throws ParseException { + return String.valueOf(Expression.fromString(script).execute()); + } +} diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ScriptTestCase.java new file mode 100644 index 00000000000..ff2def03c09 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/ScriptTestCase.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.parser; + +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetValueExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ScriptTestCase { + + @Test + public void requireThatRootProductionIsFlexible() throws ParseException { + assertRoot(SetValueExpression.class, "1"); + assertRoot(StatementExpression.class, "1 | echo"); + assertRoot(StatementExpression.class, "{ 1 | echo }"); + assertRoot(StatementExpression.class, "{ 1 | echo; }"); + assertRoot(ScriptExpression.class, "{ 1 | echo; 2 | echo }"); + } + + @Test + public void requireThatNewlineIsAllowedAfterStatement() throws ParseException { + assertScript("{ 1 }"); + assertScript("{\n 1 }"); + assertScript("{ 1\n }"); + assertScript("{\n 1\n }"); + + assertScript("{ 1; }"); + assertScript("{\n 1; }"); + assertScript("{ 1;\n }"); + assertScript("{\n 1;\n }"); + + assertScript("{ 1; 2 }"); + assertScript("{\n 1; 2 }"); + assertScript("{\n 1;\n 2 }"); + assertScript("{\n 1; 2\n }"); + assertScript("{ 1;\n 2\n }"); + assertScript("{\n 1;\n 2\n }"); + } + + @Test + public void requireThatNewlineIsAllowedWithinStatement() throws ParseException { + assertStatement("1 |\n 2"); + } + + private void assertRoot(Class expectedClass, String input) throws ParseException { + assertEquals(expectedClass, new IndexingParser(input).root().getClass()); + } + + private static void assertScript(String input) throws ParseException { + assertNotNull(new IndexingParser(input).script()); + } + + private static void assertStatement(String input) throws ParseException { + assertNotNull(new IndexingParser(input).statement()); + } +} |