diff options
9 files changed, 36 insertions, 54 deletions
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java index 48f2bf43e81..5c96635fd8f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java @@ -27,7 +27,7 @@ public class RankingExpressionWithTensorTestCase { " }"); f.compileRankProfile("my_profile"); f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); - f.assertRankProperty("{{x:1,y:2}:1.0,{x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{},y{}):{{x:1,y:2}:1.0,{x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); } @@ -50,7 +50,7 @@ public class RankingExpressionWithTensorTestCase { " }"); f.compileRankProfile("my_profile"); f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); - f.assertRankProperty("{{x:1,y:2}:1.0,{x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{},y{}):{{x:1,y:2}:1.0,{x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); } @@ -69,7 +69,7 @@ public class RankingExpressionWithTensorTestCase { " }"); f.compileRankProfile("my_profile"); f.assertSecondPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); - f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{}):{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); } @@ -90,7 +90,7 @@ public class RankingExpressionWithTensorTestCase { " }"); f.compileRankProfile("my_profile"); f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); - f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{}):{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); } @@ -113,7 +113,7 @@ public class RankingExpressionWithTensorTestCase { f.compileRankProfile("my_profile"); f.assertFirstPhaseExpression("5.0 + my_macro", "my_profile"); f.assertFunction("reduce(constant(my_tensor), sum)", "my_macro", "my_profile"); - f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{}):{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); } @@ -134,7 +134,7 @@ public class RankingExpressionWithTensorTestCase { " }"); f.compileRankProfile("my_profile"); f.assertFirstPhaseExpression("3.0 + reduce(constant(my_tensor), sum) + 5.0", "my_profile"); - f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{}):{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); } diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index cc597cbc859..60d135f6440 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -926,7 +926,7 @@ public class DocumentGenPluginTest { Book book = new Book(new DocumentId("doc:book:0")); assertNull(book.getVector()); book.setVector(Tensor.from("{{x:0}:1.0, {x:1}:2.0, {x:2}:3.0}")); - assertEquals("{{x:0}:1.0,{x:1}:2.0,{x:2}:3.0}", book.getVector().toString()); + assertEquals("tensor(x{}):{{x:0}:1.0,{x:1}:2.0,{x:2}:3.0}", book.getVector().toString()); } } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/ImportedModel.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/ImportedModel.java index ce91b8bb141..d7ac8bc90b2 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/ImportedModel.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/ImportedModel.java @@ -72,7 +72,7 @@ public class ImportedModel implements ImportedMlModel { * These should have sizes up to a few kb at most, and correspond to constant values given in the source model. */ @Override - public Map<String, String> smallConstants() { return asTensorStrings(smallConstants); } + public Map<String, String> smallConstants() { return asStrings(smallConstants); } boolean hasSmallConstant(String name) { return smallConstants.containsKey(name); } @@ -82,7 +82,7 @@ public class ImportedModel implements ImportedMlModel { * For TensorFlow this corresponds to Variable files stored separately. */ @Override - public Map<String, String> largeConstants() { return asTensorStrings(largeConstants); } + public Map<String, String> largeConstants() { return asStrings(largeConstants); } boolean hasLargeConstant(String name) { return largeConstants.containsKey(name); } @@ -133,7 +133,7 @@ public class ImportedModel implements ImportedMlModel { functions.add(new ImportedMlFunction(signatureEntry.getKey(), new ArrayList<>(signatureEntry.getValue().inputs().values()), expressions().get(signatureEntry.getKey()).getRoot().toString(), - asTensorTypeStrings(signatureEntry.getValue().inputMap()), + asStrings(signatureEntry.getValue().inputMap()), Optional.empty())); } if (signatures().isEmpty()) { // fallback for models without signatures @@ -142,7 +142,7 @@ public class ImportedModel implements ImportedMlModel { functions.add(new ImportedMlFunction(singleEntry.getKey(), new ArrayList<>(inputs.keySet()), singleEntry.getValue().getRoot().toString(), - asTensorTypeStrings(inputs), + asStrings(inputs), Optional.empty())); } else { @@ -150,7 +150,7 @@ public class ImportedModel implements ImportedMlModel { functions.add(new ImportedMlFunction(expressionEntry.getKey(), new ArrayList<>(inputs.keySet()), expressionEntry.getValue().getRoot().toString(), - asTensorTypeStrings(inputs), + asStrings(inputs), Optional.empty())); } } @@ -158,26 +158,13 @@ public class ImportedModel implements ImportedMlModel { return functions; } - private Map<String, String> asTensorStrings(Map<String, Tensor> map) { + private Map<String, String> asStrings(Map<String, ?> map) { HashMap<String, String> values = new HashMap<>(); - for (Map.Entry<String, Tensor> entry : map.entrySet()) { - Tensor tensor = entry.getValue(); - // TODO: See Tensor.toStandardString - if (tensor.isEmpty() && ! tensor.type().dimensions().isEmpty()) - values.put(entry.getKey(), tensor.toString()); - else - values.put(entry.getKey(), tensor.type() + ":" + tensor); - } + for (Map.Entry<String, ?> entry : map.entrySet()) + values.put(entry.getKey(), entry.getValue().toString()); return values; } - private static Map<String, String> asTensorTypeStrings(Map<String, TensorType> map) { - Map<String, String> stringMap = new HashMap<>(); - for (Map.Entry<String, TensorType> entry : map.entrySet()) - stringMap.put(entry.getKey(), entry.getValue().toString()); - return stringMap; - } - private Map<String, String> asExpressionStrings(Map<String, RankingExpression> map) { HashMap<String, String> values = new HashMap<>(); for (Map.Entry<String, RankingExpression> entry : map.entrySet()) @@ -237,9 +224,7 @@ public class ImportedModel implements ImportedMlModel { */ public Map<String, String> skippedOutputs() { return Collections.unmodifiableMap(skippedOutputs); } - /** - * Returns an immutable list of possibly non-fatal warnings encountered during import. - */ + /** Returns an immutable list of possibly non-fatal warnings encountered during import. */ public List<String> importWarnings() { return Collections.unmodifiableList(importWarnings); } /** Returns the expression this output references as an imported function */ @@ -247,7 +232,7 @@ public class ImportedModel implements ImportedMlModel { return new ImportedMlFunction(functionName, new ArrayList<>(inputs.values()), owner().expressions().get(outputs.get(outputName)).getRoot().toString(), - asTensorTypeStrings(inputMap()), + asStrings(inputMap()), Optional.empty()); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 1ee22c69c23..7b8ff369002 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -230,10 +230,7 @@ public interface Tensor { * @return the tensor on the standard string format */ static String toStandardString(Tensor tensor) { - if (tensor.isEmpty() && ! tensor.type().dimensions().isEmpty()) // explicitly output type TODO: Always do that - return tensor.type() + ":" + contentToString(tensor); - else - return contentToString(tensor); + return tensor.type() + ":" + contentToString(tensor); } static String contentToString(Tensor tensor) { diff --git a/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java index c986fe40931..a5fc3d5a5d8 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java @@ -35,9 +35,9 @@ public class IndexedTensorTestCase { public void testSingleValue() { Tensor singleValue = Tensor.Builder.of(TensorType.empty).cell(TensorAddress.of(), 3.5).build(); assertTrue(singleValue instanceof IndexedTensor); - assertEquals("{3.5}", singleValue.toString()); + assertEquals("tensor():{3.5}", singleValue.toString()); Tensor singleValueFromString = Tensor.from(TensorType.empty, "{3.5}"); - assertEquals("{3.5}", singleValueFromString.toString()); + assertEquals("tensor():{3.5}", singleValueFromString.toString()); assertTrue(singleValueFromString instanceof IndexedTensor); assertEquals(singleValue, singleValueFromString); } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java index e312560d04c..502f0270831 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java @@ -35,7 +35,7 @@ public class MappedTensorTestCase { cell().label("x", "0").value(1). cell().label("x", "1").value(2).build(); assertEquals(Sets.newHashSet("x"), tensor.type().dimensionNames()); - assertEquals("{{x:0}:1.0,{x:1}:2.0}", tensor.toString()); + assertEquals("tensor(x{}):{{x:0}:1.0,{x:1}:2.0}", tensor.toString()); } @Test @@ -45,7 +45,7 @@ public class MappedTensorTestCase { cell().label("x", "0").label("y", "0").value(1). cell().label("x", "1").label("y", "0").value(2).build(); assertEquals(Sets.newHashSet("x", "y"), tensor.type().dimensionNames()); - assertEquals("{{x:0,y:0}:1.0,{x:1,y:0}:2.0}", tensor.toString()); + assertEquals("tensor(x{},y{}):{{x:0,y:0}:1.0,{x:1,y:0}:2.0}", tensor.toString()); } } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java index fef8f05f4e1..22b97bff52a 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java @@ -41,7 +41,7 @@ public class MixedTensorTestCase { // {y:2} should be 0.0 and non NaN since we specify indexed size build(); assertEquals(Sets.newHashSet("y"), tensor.type().dimensionNames()); - assertEquals("{{y:0}:1.0,{y:1}:2.0,{y:2}:0.0}", + assertEquals("tensor(y[3]):{{y:0}:1.0,{y:1}:2.0,{y:2}:0.0}", tensor.toString()); } @@ -57,7 +57,7 @@ public class MixedTensorTestCase { cell().label("x", 1).label("y", 2).value(6). build(); assertEquals(Sets.newHashSet("x", "y"), tensor.type().dimensionNames()); - assertEquals("{{x:0,y:0}:1.0,{x:0,y:1}:2.0,{x:0,y:2}:0.0,{x:1,y:0}:4.0,{x:1,y:1}:5.0,{x:1,y:2}:6.0}", + assertEquals("tensor(x[2],y[3]):{{x:0,y:0}:1.0,{x:0,y:1}:2.0,{x:0,y:2}:0.0,{x:1,y:0}:4.0,{x:1,y:1}:5.0,{x:1,y:2}:6.0}", tensor.toString()); } @@ -69,7 +69,7 @@ public class MixedTensorTestCase { cell().label("x", "1").value(2). build(); assertEquals(Sets.newHashSet("x"), tensor.type().dimensionNames()); - assertEquals("{{x:0}:1.0,{x:1}:2.0}", + assertEquals("tensor(x{}):{{x:0}:1.0,{x:1}:2.0}", tensor.toString()); } @@ -84,7 +84,7 @@ public class MixedTensorTestCase { cell().label("x", "1").label("y", "2").value(6). build(); assertEquals(Sets.newHashSet("x", "y"), tensor.type().dimensionNames()); - assertEquals("{{x:0,y:0}:1.0,{x:0,y:1}:2.0,{x:1,y:0}:4.0,{x:1,y:1}:5.0,{x:1,y:2}:6.0}", + assertEquals("tensor(x{},y{}):{{x:0,y:0}:1.0,{x:0,y:1}:2.0,{x:1,y:0}:4.0,{x:1,y:1}:5.0,{x:1,y:2}:6.0}", tensor.toString()); } @@ -100,7 +100,7 @@ public class MixedTensorTestCase { cell().label("x", "2").label("y", 2).value(6). build(); assertEquals(Sets.newHashSet("x", "y"), tensor.type().dimensionNames()); - assertEquals("{{x:1,y:0}:1.0,{x:1,y:1}:2.0,{x:1,y:2}:0.0,{x:2,y:0}:4.0,{x:2,y:1}:5.0,{x:2,y:2}:6.0}", + assertEquals("tensor(x{},y[3]):{{x:1,y:0}:1.0,{x:1,y:1}:2.0,{x:1,y:2}:0.0,{x:2,y:0}:4.0,{x:2,y:1}:5.0,{x:2,y:2}:6.0}", tensor.toString()); } @@ -122,7 +122,7 @@ public class MixedTensorTestCase { cell().label("x", "x2").label("y", 2).label("z","z2").value(16). build(); assertEquals(Sets.newHashSet("x", "y", "z"), tensor.type().dimensionNames()); - assertEquals("{{x:x1,y:0,z:z1}:1.0,{x:x1,y:0,z:z2}:2.0,{x:x1,y:1,z:z1}:3.0,{x:x1,y:1,z:z2}:4.0,{x:x1,y:2,z:z1}:5.0,{x:x1,y:2,z:z2}:6.0,{x:x2,y:0,z:z1}:11.0,{x:x2,y:0,z:z2}:12.0,{x:x2,y:1,z:z1}:13.0,{x:x2,y:1,z:z2}:14.0,{x:x2,y:2,z:z1}:15.0,{x:x2,y:2,z:z2}:16.0}", + assertEquals("tensor(x{},y[3],z{}):{{x:x1,y:0,z:z1}:1.0,{x:x1,y:0,z:z2}:2.0,{x:x1,y:1,z:z1}:3.0,{x:x1,y:1,z:z2}:4.0,{x:x1,y:2,z:z1}:5.0,{x:x1,y:2,z:z2}:6.0,{x:x2,y:0,z:z1}:11.0,{x:x2,y:0,z:z2}:12.0,{x:x2,y:1,z:z1}:13.0,{x:x2,y:1,z:z2}:14.0,{x:x2,y:2,z:z1}:15.0,{x:x2,y:2,z:z2}:16.0}", tensor.toString()); } @@ -148,7 +148,7 @@ public class MixedTensorTestCase { cell().label("i", "b").label("k","d").label("j",1).label("l",1).value(16). build(); assertEquals(Sets.newHashSet("i", "j", "k", "l"), tensor.type().dimensionNames()); - assertEquals("{{i:a,j:0,k:c,l:0}:1.0,{i:a,j:0,k:c,l:1}:2.0,{i:a,j:0,k:d,l:0}:5.0,{i:a,j:0,k:d,l:1}:6.0,{i:a,j:1,k:c,l:0}:3.0,{i:a,j:1,k:c,l:1}:4.0,{i:a,j:1,k:d,l:0}:7.0,{i:a,j:1,k:d,l:1}:8.0,{i:b,j:0,k:c,l:0}:9.0,{i:b,j:0,k:c,l:1}:10.0,{i:b,j:0,k:d,l:0}:13.0,{i:b,j:0,k:d,l:1}:14.0,{i:b,j:1,k:c,l:0}:11.0,{i:b,j:1,k:c,l:1}:12.0,{i:b,j:1,k:d,l:0}:15.0,{i:b,j:1,k:d,l:1}:16.0}", + assertEquals("tensor(i{},j[2],k{},l[2]):{{i:a,j:0,k:c,l:0}:1.0,{i:a,j:0,k:c,l:1}:2.0,{i:a,j:0,k:d,l:0}:5.0,{i:a,j:0,k:d,l:1}:6.0,{i:a,j:1,k:c,l:0}:3.0,{i:a,j:1,k:c,l:1}:4.0,{i:a,j:1,k:d,l:0}:7.0,{i:a,j:1,k:d,l:1}:8.0,{i:b,j:0,k:c,l:0}:9.0,{i:b,j:0,k:c,l:1}:10.0,{i:b,j:0,k:d,l:0}:13.0,{i:b,j:0,k:d,l:1}:14.0,{i:b,j:1,k:c,l:0}:11.0,{i:b,j:1,k:c,l:1}:12.0,{i:b,j:1,k:d,l:0}:15.0,{i:b,j:1,k:d,l:1}:16.0}", tensor.toString()); } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java index 122b6019884..05fbb0dbdd9 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java @@ -29,10 +29,10 @@ public class TensorTestCase { @Test public void testStringForm() { - assertEquals("{5.7}", Tensor.from("{5.7}").toString()); - assertTrue(Tensor.from("{5.7}") instanceof IndexedTensor); - assertEquals("{{d1:l1,d2:l1}:5.0,{d1:l1,d2:l2}:6.0}", Tensor.from("{ {d1:l1,d2:l1}: 5, {d2:l2, d1:l1}:6.0} ").toString()); - assertEquals("{{d1:l1,d2:l1}:-5.3,{d1:l1,d2:l2}:0.0}", Tensor.from("{ {d1:l1,d2:l1}:-5.3, {d2:l2, d1:l1}:0}").toString()); + assertEquals("tensor():{5.7}", Tensor.from("{5.7}").toString()); + assertTrue(Tensor.from("tensor():{5.7}") instanceof IndexedTensor); + assertEquals("tensor(d1{},d2{}):{{d1:l1,d2:l1}:5.0,{d1:l1,d2:l2}:6.0}", Tensor.from("{ {d1:l1,d2:l1}: 5, {d2:l2, d1:l1}:6.0} ").toString()); + assertEquals("tensor(d1{},d2{}):{{d1:l1,d2:l1}:-5.3,{d1:l1,d2:l2}:0.0}", Tensor.from("{ {d1:l1,d2:l1}:-5.3, {d2:l2, d1:l1}:0}").toString()); } @Test @@ -79,7 +79,7 @@ public class TensorTestCase { public void testCombineInDimensionIndexed() { Tensor input = Tensor.from("tensor(input[]):{{input:0}:3, {input:1}:7}"); Tensor result = input.concat(11, "input"); - assertEquals("{{input:0}:3.0,{input:1}:7.0,{input:2}:11.0}", result.toString()); + assertEquals("tensor(input[]):{{input:0}:3.0,{input:1}:7.0,{input:2}:11.0}", result.toString()); } /** All functions are more throughly tested in searchlib EvaluationTestCase */ diff --git a/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java index 55069eaced7..e37bee2d990 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java @@ -14,11 +14,11 @@ public class TensorFunctionTestCase { @Test public void testTranslation() { - assertTranslated("join({{x:1}:1.0}, reduce({{x:1}:1.0}, sum, x), f(a,b)(a / b))", + assertTranslated("join(tensor(x{}):{{x:1}:1.0}, reduce(tensor(x{}):{{x:1}:1.0}, sum, x), f(a,b)(a / b))", new L1Normalize(new ConstantTensor("{{x:1}:1.0}"), "x")); assertTranslated("tensor(x[2],y[3],z[4])((x==y)*(y==z))", new Diag(new TensorType.Builder().indexed("y",3).indexed("x",2).indexed("z",4).build())); - assertTranslated("join({{x:1}:1.0,{x:3}:5.0,{x:9}:3.0}, reduce({{x:1}:1.0,{x:3}:5.0,{x:9}:3.0}, max, x), f(a,b)(a==b))", + assertTranslated("join(tensor(x{}):{{x:1}:1.0,{x:3}:5.0,{x:9}:3.0}, reduce(tensor(x{}):{{x:1}:1.0,{x:3}:5.0,{x:9}:3.0}, max, x), f(a,b)(a==b))", new Argmax(new ConstantTensor("{ {x:1}:1, {x:3}:5, {x:9}:3 }"), "x")); } |