Merge pull request #9955 from vespa-engine/bratseth/tensor-accessors

Bratseth/tensor accessors

10 files changed, 119 insertions, 13 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java
index 22069d0270c..8fdc093122e 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ByteField.java
@@ -24,7 +24,7 @@ public class ByteField extends DocsumField {
         if (value == EMPTY_VALUE) {
             return NanNumber.NaN;
         } else {
-            return Byte.valueOf(value);
+            return value;
         }
     }
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java
index a0c9b10c519..2e9d9d8cad9 100644
--- a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java
+++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java
@@ -8,27 +8,27 @@ package com.yahoo.prelude.hitfield;
  */
 public final class RawData {
 
-    private byte[] content;
+    private final byte[] content;
 
     /**
-     * Constructor, takes ownership
+     * Constructor, takes ownership of the given byte array.
+     *
      * @param content some bytes, handover
      */
     public RawData(byte[] content) {
         this.content = content;
     }
 
-    /**
-     * @return internal byte array containing the actual data received
-     **/
+    /** Returns the internal byte array containing the actual data received */
     public byte[] getInternalData() {
         return content;
     }
 
     /**
-     * an ascii string; non-ascii data is escaped with hex notation
-     * NB: not always uniquely reversible
-     **/
+     * An ascii string; non-ascii data is escaped with hex notation.
+     * NB: not always uniquely reversible.
+     */
+    @Override
     public String toString() {
         StringBuilder buf = new StringBuilder();
         for (byte b : content) {
@@ -46,7 +46,7 @@ public final class RawData {
             } else {
                 // XXX maybe we should only do this? creates possibly-invalid XML though.
                 buf.append("&");
-                buf.append(Integer.toString(i));
+                buf.append(i);
                 buf.append(";");
             }
         }
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
index 54dfbfe1a85..af453983f89 100644
--- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
+++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
@@ -776,7 +776,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
             } else {
                 renderInspectorDirect(data);
             }
-
         }
 
         private void renderInspectorDirect(Inspector data) throws IOException {
diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
index a245d61bafb..1a68f14af06 100644
--- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
@@ -24,6 +24,7 @@ import com.yahoo.prelude.IndexModel;
 import com.yahoo.prelude.SearchDefinition;
 import com.yahoo.prelude.fastsearch.FastHit;
 import com.yahoo.prelude.hitfield.JSONString;
+import com.yahoo.prelude.hitfield.RawData;
 import com.yahoo.prelude.searcher.JuniperSearcher;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
@@ -65,6 +66,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -147,7 +149,8 @@ public class JsonRendererTestCase {
                 + "                        \"scalar2\":2.5,"
                 + "                        \"tensor1\":{\"type\":\"tensor(x[3])\",\"cells\":[{\"address\":{\"x\":\"0\"},\"value\":1.5},{\"address\":{\"x\":\"1\"},\"value\":2.0},{\"address\":{\"x\":\"2\"},\"value\":2.5}]},"
                 + "                        \"tensor2\":{\"type\":\"tensor()\",\"cells\":[{\"address\":{},\"value\":0.5}]}"
-                + "                    }"
+                + "                    },"
+                + "                    \"data\": \"Data \\\\xc3\\\\xa6 \\\\xc3\\\\xa5\""
                 + "                },"
                 + "                \"id\": \"datatypestuff\","
                 + "                \"relevance\": 1.0"
@@ -175,6 +178,7 @@ public class JsonRendererTestCase {
         h.setField("tensor3", Tensor.from("{ {x:a, y:0}: 2.0, {x:a, y:1}: -1 }"));
         h.setField("object", new Thingie());
         h.setField("summaryfeatures", createSummaryFeatures());
+        h.setField("data", new RawData("Data æ å".getBytes(StandardCharsets.UTF_8)));
         r.hits().add(h);
         r.setTotalHitCount(1L);
         String summary = render(r);
diff --git a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java
index 89a5134943a..5881267c252 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java
@@ -18,6 +18,7 @@ import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 public class SingleValueReader {
+
     public static final String UPDATE_ASSIGN = "assign";
     public static final String UPDATE_INCREMENT = "increment";
     public static final String UPDATE_DECREMENT = "decrement";
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java
index 764029f18e0..0b392ebfa03 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java
@@ -13,6 +13,7 @@ public final class Base64DecodeExpression extends Expression {
     public Base64DecodeExpression() {
         super(DataType.STRING);
     }
+
     @Override
     protected void doExecute(ExecutionContext ctx) {
         String input = String.valueOf(ctx.getValue());
diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json
index 3b733105d2e..a16127931e9 100644
--- a/vespajlib/abi-spec.json
+++ b/vespajlib/abi-spec.json
@@ -1099,6 +1099,8 @@
       "public java.lang.Double setValue(java.lang.Double)",
       "public boolean equals(java.lang.Object)",
       "public int hashCode()",
+      "public java.lang.String toString(com.yahoo.tensor.TensorType)",
+      "public com.yahoo.tensor.Tensor$Cell detach()",
       "public bridge synthetic java.lang.Object setValue(java.lang.Object)",
       "public bridge synthetic java.lang.Object getValue()",
       "public bridge synthetic java.lang.Object getKey()"
@@ -1180,6 +1182,8 @@
       "public com.yahoo.tensor.Tensor sum()",
       "public com.yahoo.tensor.Tensor sum(java.lang.String)",
       "public com.yahoo.tensor.Tensor sum(java.util.List)",
+      "public java.util.List largest()",
+      "public java.util.List smallest()",
       "public abstract java.lang.String toString()",
       "public static java.lang.String toStandardString(com.yahoo.tensor.Tensor)",
       "public static java.lang.String contentToString(com.yahoo.tensor.Tensor)",
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java
index 02f54b5790a..1da013de012 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java
@@ -739,6 +739,11 @@ public abstract class IndexedTensor implements Tensor {
         @Override
         public Double getValue() { return value; }
 
+        @Override
+        public Cell detach() {
+            return new Cell(getKey(), value);
+        }
+
     }
 
     // TODO: Make dimensionSizes a class
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java
index 806d27ce70e..8b0aaa64551 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java
@@ -63,7 +63,11 @@ public interface Tensor {
     /** Returns the value of a cell, or NaN if this cell does not exist/have no value */
     double get(TensorAddress address);
 
-    /** Returns the cell of this in some undefined order */
+    /**
+     * Returns the cell of this in some undefined order.
+     * A cell instances is only valid until next() is called.
+     * Call detach() on the cell to obtain a long-lived instance.
+     */
     Iterator<Cell> cellIterator();
 
     /** Returns the values of this in some undefined order */
@@ -250,6 +254,44 @@ public interface Tensor {
     default Tensor sum(String dimension) { return sum(Collections.singletonList(dimension)); }
     default Tensor sum(List<String> dimensions) { return reduce(Reduce.Aggregator.sum, dimensions); }
 
+    // ----------------- non-math query methods (that is, computations not returning a tensor)
+
+    /** Returns the cell(s) of this tensor having the highest value */
+    default List<Cell> largest() {
+        List<Cell> cells = new ArrayList<>(1);
+        double maxValue = Double.MIN_VALUE;
+        for (Iterator<Cell> i = cellIterator(); i.hasNext(); ) {
+            Cell cell = i.next();
+            if (cell.getValue() > maxValue) {
+                cells.clear();
+                cells.add(cell.detach());
+                maxValue = cell.getDoubleValue();
+            }
+            else if (cell.getValue() == maxValue) {
+                cells.add(cell.detach());
+            }
+        }
+        return cells;
+    }
+
+    /** Returns the cell(s) of this tensor having the lowest value */
+    default List<Cell> smallest() {
+        List<Cell> cells = new ArrayList<>(1);
+        double minValue = Double.MAX_VALUE;
+        for (Iterator<Cell> i = cellIterator(); i.hasNext(); ) {
+            Cell cell = i.next();
+            if (cell.getValue() < minValue) {
+                cells.clear();
+                cells.add(cell.detach());
+                minValue = cell.getDoubleValue();
+            }
+            else if (cell.getValue() == minValue) {
+                cells.add(cell.detach());
+            }
+        }
+        return cells;
+    }
+
     // ----------------- serialization
 
     /**
@@ -422,6 +464,13 @@ public interface Tensor {
             return getKey().hashCode() ^ getValue().hashCode(); // by Map.Entry spec
         }
 
+        public String toString(TensorType type) { return address.toString(type) + ":" + value; }
+
+        /**
+         * Return a copy of this tensor cell which is valid beyond the lifetime of any iterator state which supplied it.
+         */
+        public Cell detach() { return this; }
+
     }
 
     interface Builder {
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
index 3d5d8d1f5ae..c6fbb9c009d 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
@@ -15,6 +15,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 import java.util.function.DoubleBinaryOperator;
+import java.util.stream.Collectors;
 
 import static com.yahoo.tensor.TensorType.Dimension.Type;
 import static org.junit.Assert.assertEquals;
@@ -248,6 +249,48 @@ public class TensorTestCase {
                 Tensor.from("tensor(x{},y[3])", "{{x:0,y:0}:2,{x:0,y:1}:3}"));
     }
 
+    @Test
+    public void testLargest() {
+        assertLargest("{d1:l1,d2:l2}:6.0",
+                     "tensor(d1{},d2{}):{{d1:l1,d2:l1}:5.0,{d1:l1,d2:l2}:6.0}");
+        assertLargest("{d1:l1,d2:l1}:6.0, {d1:l1,d2:l2}:6.0",
+                      "tensor(d1{},d2{}):{{d1:l1,d2:l1}:6.0,{d1:l1,d2:l2}:6.0}");
+        assertLargest("{d1:l1,d2:l1}:6.0, {d1:l1,d2:l2}:6.0",
+                      "tensor(d1{},d2{}):{{d1:l1,d2:l1}:6.0,{d1:l1,d2:l3}:5.0,{d1:l1,d2:l2}:6.0}");
+        assertLargest("{x:1,y:1}:4.0",
+                      "tensor(x[2],y[2]):[[1,2],[3,4]");
+        assertLargest("{x:0,y:0}:4.0, {x:1,y:1}:4.0",
+                      "tensor(x[2],y[2]):[[4,2],[3,4]");
+    }
+
+    @Test
+    public void testSmallest() {
+        assertSmallest("{d1:l1,d2:l1}:5.0",
+                       "tensor(d1{},d2{}):{{d1:l1,d2:l1}:5.0,{d1:l1,d2:l2}:6.0}");
+        assertSmallest("{d1:l1,d2:l1}:6.0, {d1:l1,d2:l2}:6.0",
+                       "tensor(d1{},d2{}):{{d1:l1,d2:l1}:6.0,{d1:l1,d2:l2}:6.0}");
+        assertSmallest("{d1:l1,d2:l1}:5.0, {d1:l1,d2:l2}:5.0",
+                       "tensor(d1{},d2{}):{{d1:l1,d2:l1}:5.0,{d1:l1,d2:l3}:6.0,{d1:l1,d2:l2}:5.0}");
+        assertSmallest("{x:0,y:0}:1.0",
+                       "tensor(x[2],y[2]):[[1,2],[3,4]");
+        assertSmallest("{x:0,y:1}:2.0",
+                       "tensor(x[2],y[2]):[[4,2],[3,4]");
+    }
+
+    private void assertLargest(String expectedCells, String tensorString) {
+        Tensor tensor = Tensor.from(tensorString);
+        assertEquals(expectedCells, asString(tensor.largest(), tensor.type()));
+    }
+
+    private void assertSmallest(String expectedCells, String tensorString) {
+        Tensor tensor = Tensor.from(tensorString);
+        assertEquals(expectedCells, asString(tensor.smallest(), tensor.type()));
+    }
+
+    private String asString(List<Tensor.Cell> cells, TensorType type) {
+        return cells.stream().map(cell -> cell.toString(type)).collect(Collectors.joining(", "));
+    }
+
     private void assertTensorModify(DoubleBinaryOperator op, Tensor init, Tensor update, Tensor expected) {
         assertEquals(expected, init.modify(op, update.cells()));
     }