Support missing values in expression evaluation in Java

9 files changed, 95 insertions, 33 deletions

diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/AbstractArrayContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/AbstractArrayContext.java
index 41bf827748a..16549b3ee1c 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/AbstractArrayContext.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/AbstractArrayContext.java
@@ -7,6 +7,7 @@ import com.yahoo.searchlib.rankingexpression.rule.CompositeNode;
 import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
 import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
 
+import java.util.BitSet;
 import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Set;
@@ -33,7 +34,11 @@ public abstract class AbstractArrayContext extends Context implements Cloneable,
      * This will fail if unknown values are attempted added.
      */
     protected AbstractArrayContext(RankingExpression expression) {
-        this(expression, false);
+        this(expression, false, defaultMissingValue);
+    }
+
+    protected AbstractArrayContext(RankingExpression expression, boolean ignoreUnknownValues) {
+        this(expression, ignoreUnknownValues, defaultMissingValue);
     }
 
     /**
@@ -44,10 +49,11 @@ public abstract class AbstractArrayContext extends Context implements Cloneable,
      * @param ignoreUnknownValues whether attempts to put values not present in this expression
      *                            should fail (false - the default), or be ignored (true)
      */
-    protected AbstractArrayContext(RankingExpression expression, boolean ignoreUnknownValues) {
+    protected AbstractArrayContext(RankingExpression expression, boolean ignoreUnknownValues, Value missingValue) {
+        this.missingValue = missingValue.freeze();
         this.ignoreUnknownValues = ignoreUnknownValues;
         this.rankingExpressionName = expression.getName();
-        this.indexedBindings = new IndexedBindings(expression);
+        this.indexedBindings = new IndexedBindings(expression, this.missingValue);
     }
 
     protected final Map<String, Integer> nameToIndex() { return indexedBindings.nameToIndex(); }
@@ -77,6 +83,14 @@ public abstract class AbstractArrayContext extends Context implements Cloneable,
         return indexedBindings.getDouble(index);
     }
 
+    final boolean isMissing(int index) {
+        return indexedBindings.isMissing(index);
+    }
+
+    final void clearMissing(int index) {
+        indexedBindings.clearMissing(index);
+    }
+
     @Override
     public String toString() {
         return "fast lookup context for ranking expression '" + rankingExpressionName +
@@ -107,11 +121,22 @@ public abstract class AbstractArrayContext extends Context implements Cloneable,
         /** The current values set, pre-converted to doubles */
         private double[] doubleValues;
 
-        public IndexedBindings(RankingExpression expression) {
+        /** Which values actually are set */
+        private BitSet setValues;
+
+        /** Value to return if value is missing. */
+        private double missingValue;
+
+        public IndexedBindings(RankingExpression expression, Value missingValue) {
             Set<String> bindTargets = new LinkedHashSet<>();
             extractBindTargets(expression.getRoot(), bindTargets);
 
+            this.missingValue = missingValue.asDouble();
+            setValues = new BitSet(bindTargets.size());
             doubleValues = new double[bindTargets.size()];
+            for (int i = 0; i < bindTargets.size(); ++i) {
+                doubleValues[i] = this.missingValue;
+            }
 
             int i = 0;
             ImmutableMap.Builder<String, Integer> nameToIndexBuilder = new ImmutableMap.Builder<>();
@@ -136,10 +161,13 @@ public abstract class AbstractArrayContext extends Context implements Cloneable,
 
         public Map<String, Integer> nameToIndex() { return nameToIndex; }
         public double[] doubleValues() { return doubleValues; }
+
         public Set<String> names() { return nameToIndex.keySet(); }
         public int getIndex(String name) { return nameToIndex.get(name); }
         public int size() { return doubleValues.length; }
         public double getDouble(int index) { return doubleValues[index]; }
+        public boolean isMissing(int index) { return ! setValues.get(index); }
+        public void clearMissing(int index) { setValues.set(index); }
 
         /**
          * Creates a clone of this context suitable for evaluating against the same ranking expression
@@ -149,7 +177,11 @@ public abstract class AbstractArrayContext extends Context implements Cloneable,
         public IndexedBindings clone() {
             try {
                 IndexedBindings clone = (IndexedBindings)super.clone();
+                clone.setValues = new BitSet(nameToIndex.size());
                 clone.doubleValues = new double[nameToIndex.size()];
+                for (int i = 0; i < nameToIndex.size(); ++i) {
+                    clone.doubleValues[i] = missingValue;
+                }
                 return clone;
             }
             catch (CloneNotSupportedException e) {
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java
index 047d9d761ce..82243fc493d 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java
@@ -19,8 +19,6 @@ public class ArrayContext extends AbstractArrayContext implements Cloneable {
     /** The current values set */
     private Value[] values;
 
-    private static DoubleValue constantZero = DoubleValue.frozen(0);
-
     /**
      * Create a fast lookup context for an expression.
      * This instance should be reused indefinitely by a single thread.
@@ -30,6 +28,14 @@ public class ArrayContext extends AbstractArrayContext implements Cloneable {
         this(expression, false);
     }
 
+    public ArrayContext(RankingExpression expression, boolean ignoreUnknownValues) {
+        this(expression, ignoreUnknownValues, defaultMissingValue);
+    }
+
+    public ArrayContext(RankingExpression expression, Value defaultValue) {
+        this(expression, false, defaultValue);
+    }
+
     /**
      * Create a fast lookup context for an expression.
      * This instance should be reused indefinitely by a single thread.
@@ -37,11 +43,12 @@ public class ArrayContext extends AbstractArrayContext implements Cloneable {
      * @param expression the expression to create a context for
      * @param ignoreUnknownValues whether attempts to put values not present in this expression
      *                            should fail (false - the default), or be ignored (true)
+     * @param missingValue the value to return if not set.
      */
-    public ArrayContext(RankingExpression expression, boolean ignoreUnknownValues) {
-        super(expression, ignoreUnknownValues);
+    public ArrayContext(RankingExpression expression, boolean ignoreUnknownValues, Value missingValue) {
+        super(expression, ignoreUnknownValues, missingValue);
         values = new Value[doubleValues().length];
-        Arrays.fill(values, DoubleValue.zero);
+        Arrays.fill(values, this.missingValue);
     }
 
     /**
@@ -74,6 +81,7 @@ public class ArrayContext extends AbstractArrayContext implements Cloneable {
      */
     public final void put(int index, Value value) {
         values[index] = value.freeze();
+        clearMissing(index);
         try {
             doubleValues()[index] = value.asDouble();
         }
@@ -93,7 +101,7 @@ public class ArrayContext extends AbstractArrayContext implements Cloneable {
     @Override
     public Value get(String name) {
         Integer index = nameToIndex().get(name);
-        if (index == null) return DoubleValue.zero;
+        if (index == null) return missingValue;
         return values[index];
     }
 
@@ -107,7 +115,7 @@ public class ArrayContext extends AbstractArrayContext implements Cloneable {
     @Override
     public final double getDouble(int index) {
         double value = doubleValues()[index];
-        if (Double.isNaN(value))
+        if (Double.isNaN(value) && ! isMissing(index))  // NaN is valid as a missing value
             throw new UnsupportedOperationException("Value at " + index + " has no double representation");
         return value;
     }
@@ -119,7 +127,7 @@ public class ArrayContext extends AbstractArrayContext implements Cloneable {
     public ArrayContext clone() {
         ArrayContext clone = (ArrayContext)super.clone();
         clone.values = new Value[nameToIndex().size()];
-        Arrays.fill(clone.values, constantZero);
+        Arrays.fill(clone.values, missingValue);
         return clone;
     }
 
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/BooleanValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/BooleanValue.java
index 8ac9a6787da..0e187dfc87c 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/BooleanValue.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/BooleanValue.java
@@ -49,8 +49,9 @@ public class BooleanValue extends DoubleCompatibleValue {
     @Override
     public boolean equals(Object other) {
         if (this==other) return true;
-        if ( ! (other instanceof BooleanValue)) return false;
-        return ((BooleanValue)other).value==this.value;
+        if ( ! (other instanceof Value)) return false;
+        if ( ! ((Value) other).hasDouble()) return false;
+        return this.value == ((Value) other).asBoolean();
     }
 
     @Override
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Context.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Context.java
index 4e046df11ca..d68f8c85ad1 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Context.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Context.java
@@ -18,6 +18,12 @@ import java.util.stream.Collectors;
  */
 public abstract class Context implements EvaluationContext<Reference> {
 
+    /** The default value to return if the value has not been set  */
+    static Value defaultMissingValue = DoubleValue.zero;
+
+    /** The value to return if the value has not been set  */
+    Value missingValue;
+
     /**
      * Returns the value of a simple variable name.
      *
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleOnlyArrayContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleOnlyArrayContext.java
index 0004036da4b..257b344f025 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleOnlyArrayContext.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleOnlyArrayContext.java
@@ -19,7 +19,11 @@ public class DoubleOnlyArrayContext extends AbstractArrayContext {
      * This will fail if unknown values are attempted added.
      */
     public DoubleOnlyArrayContext(RankingExpression expression) {
-        this(expression, false);
+        this(expression, false, defaultMissingValue);
+    }
+
+    public DoubleOnlyArrayContext(RankingExpression expression, boolean ignoreUnknownValues) {
+        this(expression, ignoreUnknownValues, defaultMissingValue);
     }
 
     /**
@@ -29,9 +33,10 @@ public class DoubleOnlyArrayContext extends AbstractArrayContext {
      * @param expression the expression to create a context for
      * @param ignoreUnknownValues whether attempts to put values not present in this expression
      *                            should fail (false - the default), or be ignored (true)
+     * @param missingValue the value to return if not set.
      */
-    public DoubleOnlyArrayContext(RankingExpression expression, boolean ignoreUnknownValues) {
-        super(expression, ignoreUnknownValues);
+    public DoubleOnlyArrayContext(RankingExpression expression, boolean ignoreUnknownValues, Value missingValue) {
+        super(expression, ignoreUnknownValues, missingValue);
     }
 
     /**
@@ -56,6 +61,7 @@ public class DoubleOnlyArrayContext extends AbstractArrayContext {
     /** Same as put(index,DoubleValue.frozen(value)) */
     public final void put(int index, double value) {
         doubleValues()[index] = value;
+        clearMissing(index);
     }
 
     /** Puts a value by index. */
@@ -77,7 +83,7 @@ public class DoubleOnlyArrayContext extends AbstractArrayContext {
     @Override
     public Value get(String name) {
         Integer index = nameToIndex().get(name);
-        if (index==null) return DoubleValue.zero;
+        if (index==null) return missingValue;
         return new DoubleValue(getDouble(index));
     }
 
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleValue.java
index 8aa7446cae7..06ab4cba98f 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleValue.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/DoubleValue.java
@@ -20,6 +20,9 @@ public final class DoubleValue extends DoubleCompatibleValue {
     /** The double value instance for 0 */
     public final static DoubleValue zero = DoubleValue.frozen(0);
 
+    /** The double value instance for NaN */
+    public final static DoubleValue NaN = DoubleValue.frozen(Double.NaN);
+
     public DoubleValue(double value) {
         this.value = value;
     }
@@ -146,8 +149,9 @@ public final class DoubleValue extends DoubleCompatibleValue {
     @Override
     public boolean equals(Object other) {
         if (this==other) return true;
-        if ( ! (other instanceof DoubleValue)) return false;
-        return ((DoubleValue)other).value==this.value;
+        if ( ! (other instanceof Value)) return false;
+        if ( ! ((Value) other).hasDouble()) return false;
+        return this.asDouble() == ((Value) other).asDouble();
     }
 
     @Override
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java
index 4ef24d60bba..f531d77762d 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java
@@ -21,13 +21,23 @@ public class MapContext extends Context {
     private boolean frozen = false;
 
     public MapContext() {
+        this(defaultMissingValue);
+    }
+
+    public MapContext(Value missingValue) {
+        this.missingValue = missingValue.freeze();
+    }
+
+    public MapContext(Map<String,Value> bindings) {
+        this(bindings, defaultMissingValue);
     }
 
     /**
      * Creates a map context from a map.
      * All the Values of the map will be frozen.
      */
-    public MapContext(Map<String,Value> bindings) {
+    public MapContext(Map<String,Value> bindings, Value missingValue) {
+        this.missingValue = missingValue.freeze();
         bindings.forEach((k, v) -> this.bindings.put(k, v.freeze()));
     }
 
@@ -52,7 +62,7 @@ public class MapContext extends Context {
     /** Returns the value of a key. 0 is returned if the given key is not bound in this. */
     @Override
     public Value get(String key) {
-        return bindings.getOrDefault(key, DoubleValue.zero);
+        return bindings.getOrDefault(key, missingValue);
     }
 
     /**
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java
index ee66dcc5a03..b109e6503e3 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java
@@ -41,7 +41,9 @@ public class TensorValue extends Value {
 
     @Override
     public boolean asBoolean() {
-        throw new UnsupportedOperationException("A tensor does not have a boolean value");
+        if (hasDouble())
+            return asDouble() != 0.0;
+        throw new UnsupportedOperationException("Tensor does not have a value that can be converted to a boolean");
     }
 
     @Override
@@ -118,18 +120,11 @@ public class TensorValue extends Value {
             return new TensorValue(value.map((value) -> Math.pow(value, argument.asDouble())));
     }
 
-    private Tensor asTensor(Value value, String operationName) {
-        if ( ! (value instanceof TensorValue))
-            throw new UnsupportedOperationException("Could not perform " + operationName +
-                                                    ": The second argument must be a tensor but was " + value);
-        return ((TensorValue)value).value;
-    }
-
     public Tensor asTensor() { return value; }
 
     @Override
     public Value compare(TruthOperator operator, Value argument) {
-        return new TensorValue(compareTensor(operator, asTensor(argument, operator.toString())));
+        return new TensorValue(compareTensor(operator, argument.asTensor()));
     }
 
     private Tensor compareTensor(TruthOperator operator, Tensor argument) {
@@ -148,7 +143,7 @@ public class TensorValue extends Value {
     @Override
     public Value function(Function function, Value arg) {
         if (arg instanceof TensorValue)
-            return new TensorValue(functionOnTensor(function, asTensor(arg, function.toString())));
+            return new TensorValue(functionOnTensor(function, arg.asTensor()));
         else
             return new TensorValue(value.map((value) -> function.evaluate(value, arg.asDouble())));
     }
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java
index 7809cdd4e1b..39e408d27ca 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java
@@ -68,7 +68,7 @@ public abstract class Value {
     public abstract Value compare(TruthOperator operator, Value value);
 
     /** Perform the given binary function on this value and the given value */
-    public abstract Value function(Function function,Value value);
+    public abstract Value function(Function function, Value value);
 
     /**
      * Irreversibly makes this immutable. Overriders must always call super.freeze() and return this