diff options
6 files changed, 135 insertions, 9 deletions
diff --git a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java index bbd9962be77..e7e453d8fe3 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java +++ b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.OutputStream; import java.net.URI; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Map; import java.util.Optional; @@ -90,8 +91,11 @@ public class ModelsEvaluationHandler extends ThreadedHttpRequestHandler { Tensor result = evaluator.evaluate(); Optional<String> format = property(request, "format"); - if (format.isPresent() && format.get().equalsIgnoreCase("short") && result instanceof IndexedTensor) { - return new Response(200, JsonFormat.encodeShortForm((IndexedTensor) result)); + if (format.isPresent() && format.get().equalsIgnoreCase("short")) { + return new Response(200, JsonFormat.encodeShortForm(result)); + } + else if (format.isPresent() && format.get().equalsIgnoreCase("literal")) { + return new Response(200, result.toString().getBytes(StandardCharsets.UTF_8)); } return new Response(200, JsonFormat.encode(result)); } diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java index 8034be6bb22..3a900b0e815 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -48,7 +48,7 @@ public class ModelsEvaluationHandlerTest { public void testListModels() { String url = "http://localhost/model-evaluation/v1"; String expected = - "{\"mnist_softmax\":\"http://localhost/model-evaluation/v1/mnist_softmax\",\"mnist_saved\":\"http://localhost/model-evaluation/v1/mnist_saved\",\"mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/mnist_softmax_saved\",\"xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_2_2\",\"lightgbm_regression\":\"http://localhost/model-evaluation/v1/lightgbm_regression\"}"; + "{\"mnist_softmax\":\"http://localhost/model-evaluation/v1/mnist_softmax\",\"mnist_saved\":\"http://localhost/model-evaluation/v1/mnist_saved\",\"mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/mnist_softmax_saved\",\"vespa_model\":\"http://localhost/model-evaluation/v1/vespa_model\",\"xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_2_2\",\"lightgbm_regression\":\"http://localhost/model-evaluation/v1/lightgbm_regression\"}"; handler.assertResponse(url, 200, expected); } @@ -56,7 +56,7 @@ public class ModelsEvaluationHandlerTest { public void testListModelsWithDifferentHost() { String url = "http://localhost/model-evaluation/v1"; String expected = - "{\"mnist_softmax\":\"http://localhost:8088/model-evaluation/v1/mnist_softmax\",\"mnist_saved\":\"http://localhost:8088/model-evaluation/v1/mnist_saved\",\"mnist_softmax_saved\":\"http://localhost:8088/model-evaluation/v1/mnist_softmax_saved\",\"xgboost_2_2\":\"http://localhost:8088/model-evaluation/v1/xgboost_2_2\",\"lightgbm_regression\":\"http://localhost:8088/model-evaluation/v1/lightgbm_regression\"}"; + "{\"mnist_softmax\":\"http://localhost:8088/model-evaluation/v1/mnist_softmax\",\"mnist_saved\":\"http://localhost:8088/model-evaluation/v1/mnist_saved\",\"mnist_softmax_saved\":\"http://localhost:8088/model-evaluation/v1/mnist_softmax_saved\",\"vespa_model\":\"http://localhost:8088/model-evaluation/v1/vespa_model\",\"xgboost_2_2\":\"http://localhost:8088/model-evaluation/v1/xgboost_2_2\",\"lightgbm_regression\":\"http://localhost:8088/model-evaluation/v1/lightgbm_regression\"}"; handler.assertResponse(url, 200, expected, Map.of("Host", "localhost:8088")); } @@ -214,6 +214,36 @@ public class ModelsEvaluationHandlerTest { } @Test + public void testVespaModelShortOutput() { + Map<String, String> properties = new HashMap<>(); + properties.put("format", "short"); + String url = "http://localhost/model-evaluation/v1/vespa_model/"; + handler.assertResponse(url + "test_mapped/eval", properties, 200, + "{\"type\":\"tensor(d0{})\",\"value\":{\"a\":1.0,\"b\":2.0}}"); + handler.assertResponse(url + "test_indexed/eval", properties, 200, + "{\"type\":\"tensor(d0[2],d1[3])\",\"value\":[[1.0,2.0,3.0],[4.0,5.0,6.0]]}"); + handler.assertResponse(url + "test_mixed/eval", properties, 200, + "{\"type\":\"tensor(x{},y[3])\",\"value\":{\"a\":[1.0,2.0,3.0],\"b\":[4.0,5.0,6.0]}}"); + handler.assertResponse(url + "test_mixed_2/eval", properties, 200, + "{\"type\":\"tensor(a[2],b[2],c{},d[2])\",\"value\":{\"a\":[[[1.0,2.0],[3.0,4.0]],[[5.0,6.0],[7.0,8.0]]],\"b\":[[[1.0,2.0],[3.0,4.0]],[[5.0,6.0],[7.0,8.0]]]}}"); + } + + @Test + public void testVespaModelLiteralOutput() { + Map<String, String> properties = new HashMap<>(); + properties.put("format", "literal"); + String url = "http://localhost/model-evaluation/v1/vespa_model/"; + handler.assertResponse(url + "test_mapped/eval", properties, 200, + "tensor(d0{}):{a:1.0,b:2.0}"); + handler.assertResponse(url + "test_indexed/eval", properties, 200, + "tensor(d0[2],d1[3]):[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]"); + handler.assertResponse(url + "test_mixed/eval", properties, 200, + "tensor(x{},y[3]):{a:[1.0, 2.0, 3.0],b:[4.0, 5.0, 6.0]}"); + handler.assertResponse(url + "test_mixed_2/eval", properties, 200, + "tensor(a[2],b[2],c{},d[2]):{a:[[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]],b:[[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]}"); + } + + @Test public void testMnistSavedEvaluateSpecificFunction() { Map<String, String> properties = new HashMap<>(); properties.put("input", inputTensor()); diff --git a/model-evaluation/src/test/resources/config/models/rank-profiles.cfg b/model-evaluation/src/test/resources/config/models/rank-profiles.cfg index 385115b7cd4..4877a24f171 100644 --- a/model-evaluation/src/test/resources/config/models/rank-profiles.cfg +++ b/model-evaluation/src/test/resources/config/models/rank-profiles.cfg @@ -29,3 +29,12 @@ rankprofile[3].fef.property[4].value "tensor(d1[10])" rankprofile[4].name "lightgbm_regression" rankprofile[4].fef.property[0].name "rankingExpression(lightgbm_regression).rankingScript" rankprofile[4].fef.property[0].value "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in ["k", "l", "m"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in ["d", "e"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in ["a", "b"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in ["k", "l", "m"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)" +rankprofile[5].name "vespa_model" +rankprofile[5].fef.property[0].name "rankingExpression(test_mapped).rankingScript" +rankprofile[5].fef.property[0].value "tensor(d0{}):{a:1, b:2}" +rankprofile[5].fef.property[1].name "rankingExpression(test_indexed).rankingScript" +rankprofile[5].fef.property[1].value "tensor(d0[2],d1[3]):[[1,2,3],[4,5,6]]" +rankprofile[5].fef.property[2].name "rankingExpression(test_mixed).rankingScript" +rankprofile[5].fef.property[2].value "tensor(x{},y[3]):{a:[1,2,3], b:[4,5,6]}" +rankprofile[5].fef.property[3].name "rankingExpression(test_mixed_2).rankingScript" +rankprofile[5].fef.property[3].value "tensor(a[2],b[2],c{},d[2]):{a:[[[1,2], [3,4]], [[5,6], [7,8]]], b:[[[1,2], [3,4]], [[5,6], [7,8]]] }" diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java index 71ed347219e..33dcd458980 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -91,7 +91,7 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { return b.toString(); } - /** Returns a label as a string with approriate quoting/escaping when necessary */ + /** Returns a label as a string with appropriate quoting/escaping when necessary */ public static String labelToString(String label) { if (TensorType.labelMatcher.matches(label)) return label; // no quoting if (label.contains("'")) return "\"" + label + "\""; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java index cb7539d8565..bebd706f815 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java @@ -11,12 +11,19 @@ import com.yahoo.slime.Slime; import com.yahoo.slime.Type; import com.yahoo.tensor.DimensionSizes; import com.yahoo.tensor.IndexedTensor; +import com.yahoo.tensor.MappedTensor; import com.yahoo.tensor.MixedTensor; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.functions.ConstantTensor; +import com.yahoo.tensor.functions.Slice; +import java.util.HashSet; import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; /** * Writes tensors on the JSON format used in Vespa tensor document fields: @@ -46,12 +53,33 @@ public class JsonFormat { } /** Serializes the given tensor type and value into a short-form JSON format */ - public static byte[] encodeShortForm(IndexedTensor tensor) { + public static byte[] encodeShortForm(Tensor tensor) { Slime slime = new Slime(); Cursor root = slime.setObject(); root.setString("type", tensor.type().toString()); - Cursor value = root.setArray("value"); - encodeList(tensor, value, new long[tensor.dimensionSizes().dimensions()], 0); + + // Encode as nested lists if indexed tensor + if (tensor instanceof IndexedTensor) { + IndexedTensor denseTensor = (IndexedTensor) tensor; + encodeList(denseTensor, root.setArray("value"), new long[denseTensor.dimensionSizes().dimensions()], 0); + } + + // Short form for a single mapped dimension + else if (tensor instanceof MappedTensor && tensor.type().dimensions().size() == 1) { + encodeMap((MappedTensor) tensor, root.setObject("value")); + } + + // Short form for a mixed tensor with a single mapped dimension + else if (tensor instanceof MixedTensor && + tensor.type().dimensions().stream().filter(TensorType.Dimension::isMapped).count() == 1) { + encodeMapBlocks((MixedTensor) tensor, root.setObject("value")); + } + + // No other short forms exist: default to standard cell address output + else { + encodeCells(tensor, root.setObject("value")); + } + return com.yahoo.slime.JsonFormat.toJsonBytes(slime); } @@ -81,6 +109,33 @@ public class JsonFormat { } } + private static void encodeMap(MappedTensor tensor, Cursor cursor) { + if (tensor.type().dimensions().size() > 1) + throw new IllegalStateException("JSON encode of mapped tensor can only contain a single dimension"); + tensor.cells().forEach((k,v) -> cursor.setDouble(k.label(0), v)); + } + + private static void encodeMapBlocks(MixedTensor tensor, Cursor cursor) { + var mappedDimensions = tensor.type().dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList()); + if (mappedDimensions.size() != 1) { + throw new IllegalArgumentException("Should be ensured by caller"); + } + String mappedDimensionName = mappedDimensions.get(0).name(); + int mappedDimensionIndex = tensor.type().indexOfDimension(mappedDimensionName). + orElseThrow(() -> new IllegalStateException("Could not find mapped dimension index")); + + // Find all unique indices for the mapped dimension + Set<String> mappedIndices = new HashSet<>(); + tensor.cellIterator().forEachRemaining((cell) -> mappedIndices.add(cell.getKey().label(mappedDimensionIndex))); + + // Slice out dense subspace of each and encode dense subspace as a list + for (String mappedIndex : mappedIndices) { + IndexedTensor denseSubspace = (IndexedTensor) new Slice<>(new ConstantTensor<>(tensor), + List.of(new Slice.DimensionValue<>(mappedDimensionName, mappedIndex))).evaluate(); + encodeList(denseSubspace, cursor.setArray(mappedIndex), new long[denseSubspace.dimensionSizes().dimensions()], 0); + } + } + /** Deserializes the given tensor from JSON format */ // NOTE: This must be kept in sync with com.yahoo.document.json.readers.TensorReader in the document module public static Tensor decode(TensorType type, byte[] jsonTensorValue) { diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java index 87796501917..15017dc95ca 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java @@ -122,6 +122,34 @@ public class JsonFormatTestCase { } @Test + public void testSingleDimensionSparseTensorShortForm() { + assertEncodeShortForm("tensor(x{}):{a:1, b:2}", + "{\"type\":\"tensor(x{})\",\"value\":{\"a\":1.0,\"b\":2.0}}"); + + // Multiple mapped dimensions: no short form available + assertEncodeShortForm("tensor(x{},y{}):{{x:a,y:b}:1,{x:c,y:d}:2}", + "{\"type\":\"tensor(x{},y{})\",\"value\":{\"cells\":[{\"address\":{\"x\":\"a\",\"y\":\"b\"},\"value\":1.0},{\"address\":{\"x\":\"c\",\"y\":\"d\"},\"value\":2.0}]}}"); + } + + @Test + public void testSingleMappedDimensionMixedTensorShortForm() { + assertEncodeShortForm("tensor(x{},y[2]):{a:[1,2], b:[3,4] }", + "{\"type\":\"tensor(x{},y[2])\",\"value\":{\"a\":[1.0,2.0],\"b\":[3.0,4.0]}}"); + assertEncodeShortForm("tensor(x[2],y{}):{a:[1,2], b:[3,4] }", + "{\"type\":\"tensor(x[2],y{})\",\"value\":{\"a\":[1.0,2.0],\"b\":[3.0,4.0]}}"); + assertEncodeShortForm("tensor(x{},y[2],z[2]):{a:[[1,2],[3,4]], b:[[5,6],[7,8]] }", + "{\"type\":\"tensor(x{},y[2],z[2])\",\"value\":{\"a\":[[1.0,2.0],[3.0,4.0]],\"b\":[[5.0,6.0],[7.0,8.0]]}}"); + assertEncodeShortForm("tensor(x[1],y{},z[4]):{a:[[1,2,3,4]], b:[[5,6,7,8]] }", + "{\"type\":\"tensor(x[1],y{},z[4])\",\"value\":{\"a\":[[1.0,2.0,3.0,4.0]],\"b\":[[5.0,6.0,7.0,8.0]]}}"); + assertEncodeShortForm("tensor(x[4],y[1],z{}):{a:[[1],[2],[3],[4]], b:[[5],[6],[7],[8]] }", + "{\"type\":\"tensor(x[4],y[1],z{})\",\"value\":{\"a\":[[1.0],[2.0],[3.0],[4.0]],\"b\":[[5.0],[6.0],[7.0],[8.0]]}}"); + assertEncodeShortForm("tensor(a[2],b[2],c{},d[2]):{a:[[[1,2], [3,4]], [[5,6], [7,8]]], b:[[[1,2], [3,4]], [[5,6], [7,8]]] }", + "{\"type\":\"tensor(a[2],b[2],c{},d[2])\",\"value\":{" + + "\"a\":[[[1.0,2.0],[3.0,4.0]],[[5.0,6.0],[7.0,8.0]]]," + + "\"b\":[[[1.0,2.0],[3.0,4.0]],[[5.0,6.0],[7.0,8.0]]]}}"); + } + + @Test public void testInt8VectorInHexForm() { Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor<int8>(x[2],y[3])")); builder.cell().label("x", 0).label("y", 0).value(2.0); @@ -315,7 +343,7 @@ public class JsonFormatTestCase { } private void assertEncodeShortForm(String tensor, String expected) { - byte[] json = JsonFormat.encodeShortForm((IndexedTensor) Tensor.from(tensor)); + byte[] json = JsonFormat.encodeShortForm(Tensor.from(tensor)); assertEquals(expected, new String(json, StandardCharsets.UTF_8)); } |