From f9e6262cd5f8b919db33f8a43cad45c25546d2e4 Mon Sep 17 00:00:00 2001 From: Lester Solbakken Date: Fri, 10 Jan 2020 13:07:54 +0100 Subject: Revert "Revert "Require equal sizes in join"" This reverts commit d78f8b089753025421524539e86ca96b7bf3369c. --- .../src/test/derived/tensor/attributes.cfg | 2 +- .../src/test/derived/tensor/documenttypes.cfg | 2 +- .../src/test/derived/tensor/rank-profiles.cfg | 28 +++++++++++----------- config-model/src/test/derived/tensor/tensor.sd | 4 ++-- .../TensorOptimizerTestCase.java | 9 ++++--- vespajlib/abi-spec.json | 1 + .../src/main/java/com/yahoo/tensor/TensorType.java | 24 ++++++++++++------- .../main/java/com/yahoo/tensor/functions/Join.java | 11 ++++++--- .../test/java/com/yahoo/tensor/TensorTestCase.java | 6 ++--- 9 files changed, 49 insertions(+), 38 deletions(-) diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg index 2e0a207d249..4634e120a3a 100644 --- a/config-model/src/test/derived/tensor/attributes.cfg +++ b/config-model/src/test/derived/tensor/attributes.cfg @@ -59,7 +59,7 @@ attribute[].arity 8 attribute[].lowerbound -9223372036854775808 attribute[].upperbound 9223372036854775807 attribute[].densepostinglistthreshold 0.4 -attribute[].tensortype "tensor(x[10],y[20])" +attribute[].tensortype "tensor(x[10],y[10])" attribute[].imported false attribute[].name "f5" attribute[].datatype TENSOR diff --git a/config-model/src/test/derived/tensor/documenttypes.cfg b/config-model/src/test/derived/tensor/documenttypes.cfg index 72fae572b76..68bd394c9d6 100644 --- a/config-model/src/test/derived/tensor/documenttypes.cfg +++ b/config-model/src/test/derived/tensor/documenttypes.cfg @@ -35,7 +35,7 @@ documenttype[].datatype[].sstruct.field[].detailedtype "tensor(x{})" documenttype[].datatype[].sstruct.field[].name "f4" documenttype[].datatype[].sstruct.field[].id 1224191509 documenttype[].datatype[].sstruct.field[].datatype 21 -documenttype[].datatype[].sstruct.field[].detailedtype "tensor(x[10],y[20])" +documenttype[].datatype[].sstruct.field[].detailedtype "tensor(x[10],y[10])" documenttype[].datatype[].sstruct.field[].name "f5" documenttype[].datatype[].sstruct.field[].id 329055840 documenttype[].datatype[].sstruct.field[].datatype 21 diff --git a/config-model/src/test/derived/tensor/rank-profiles.cfg b/config-model/src/test/derived/tensor/rank-profiles.cfg index 29dc39b01ce..7970e05b790 100644 --- a/config-model/src/test/derived/tensor/rank-profiles.cfg +++ b/config-model/src/test/derived/tensor/rank-profiles.cfg @@ -4,7 +4,7 @@ rankprofile[].fef.property[].value "tensor(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor(x[10])" rankprofile[].name "unranked" @@ -21,7 +21,7 @@ rankprofile[].fef.property[].value "tensor(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor(x[10])" rankprofile[].name "profile1" @@ -34,27 +34,27 @@ rankprofile[].fef.property[].value "tensor(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor(x[10])" rankprofile[].name "profile2" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" -rankprofile[].fef.property[].value "reduce(reduce(join(attribute(f4), tensor(x[2],y[2],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x), sum)" +rankprofile[].fef.property[].value "reduce(reduce(join(attribute(f4), tensor(x[10],y[10],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x), sum)" rankprofile[].fef.property[].name "vespa.type.attribute.f2" rankprofile[].fef.property[].value "tensor(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor(x[10])" rankprofile[].name "profile3" rankprofile[].fef.property[].name "rankingExpression(joinedtensors).rankingScript" rankprofile[].fef.property[].value "tensor(i[10])(i) * attribute(f4)" rankprofile[].fef.property[].name "rankingExpression(joinedtensors).type" -rankprofile[].fef.property[].value "tensor(i[10],x[10],y[20])" +rankprofile[].fef.property[].value "tensor(i[10],x[10],y[10])" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" @@ -64,7 +64,7 @@ rankprofile[].fef.property[].value "tensor(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor(x[10])" rankprofile[].name "profile4" @@ -77,7 +77,7 @@ rankprofile[].fef.property[].value "tensor(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor(x[10])" rankprofile[].name "profile5" @@ -90,14 +90,14 @@ rankprofile[].fef.property[].value "tensor(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor(x[10])" rankprofile[].name "profile6" rankprofile[].fef.property[].name "rankingExpression(joinedtensors).rankingScript" rankprofile[].fef.property[].value "tensor(i[10])(i) * attribute(f4)" rankprofile[].fef.property[].name "rankingExpression(joinedtensors).type" -rankprofile[].fef.property[].value "tensor(i[10],x[10],y[20])" +rankprofile[].fef.property[].value "tensor(i[10],x[10],y[10])" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" @@ -107,7 +107,7 @@ rankprofile[].fef.property[].value "tensor(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor(x[10])" rankprofile[].name "profile7" @@ -128,7 +128,7 @@ rankprofile[].fef.property[].value "tensor(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor(x[10])" rankprofile[].name "profile8" @@ -143,7 +143,7 @@ rankprofile[].fef.property[].value "tensor(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor(x[10])" rankprofile[].name "profile9" @@ -158,7 +158,7 @@ rankprofile[].fef.property[].value "tensor(x[2],y[1])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" -rankprofile[].fef.property[].value "tensor(x[10],y[20])" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" rankprofile[].fef.property[].name "vespa.type.attribute.f5" rankprofile[].fef.property[].value "tensor(x[10])" diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd index aa33684a979..a7248fe3200 100644 --- a/config-model/src/test/derived/tensor/tensor.sd +++ b/config-model/src/test/derived/tensor/tensor.sd @@ -11,7 +11,7 @@ search tensor { field f3 type tensor(x{}) { indexing: attribute | summary } - field f4 type tensor(x[10],y[20]) { + field f4 type tensor(x[10],y[10]) { indexing: attribute | summary } field f5 type tensor(x[10]) { @@ -33,7 +33,7 @@ search tensor { rank-profile profile2 { first-phase { - expression: sum(matmul(attribute(f4), diag(x[2],y[2],z[3]), x)) + expression: sum(matmul(attribute(f4), diag(x[10],y[10],z[3]), x)) } } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizerTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizerTestCase.java index 2cfc8cb575e..50f37486b90 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizerTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizerTestCase.java @@ -26,7 +26,6 @@ public class TensorOptimizerTestCase { assertWillOptimize("d0[3]", "d0[3]"); assertWillOptimize("d0[1]", "d0[1]", "d0"); assertWillOptimize("d0[2]", "d0[2]", "d0"); - assertWillOptimize("d0[1]", "d0[3]", "d0"); assertWillOptimize("d0[3]", "d0[3]", "d0"); assertWillOptimize("d0[3]", "d0[3],d1[2]", "d0"); assertWillOptimize("d0[3],d1[2]", "d0[3]", "d0"); @@ -34,10 +33,10 @@ public class TensorOptimizerTestCase { assertWillOptimize("d0[2],d1[3]", "d1[3]", "d1"); assertWillOptimize("d0[2],d2[2]", "d1[3],d2[2]", "d2"); assertWillOptimize("d1[2],d2[2]", "d0[3],d2[2]", "d2"); - assertWillOptimize("d0[1],d2[2]", "d1[3],d2[4]", "d2"); - assertWillOptimize("d0[2],d2[2]", "d1[3],d2[4]", "d2"); - assertWillOptimize("d0[1],d1[2]", "d0[2],d1[3]"); - assertWillOptimize("d0[1],d1[2]", "d0[2],d1[3]", "d0,d1"); + assertWillOptimize("d0[1],d2[4]", "d1[3],d2[4]", "d2"); + assertWillOptimize("d0[2],d2[4]", "d1[3],d2[4]", "d2"); + assertWillOptimize("d0[2],d1[3]", "d0[2],d1[3]"); + assertWillOptimize("d0[2],d1[3]", "d0[2],d1[3]", "d0,d1"); assertWillOptimize("d2[3],d3[4]", "d1[2],d2[3],d3[4]", "d2,d3"); assertWillOptimize("d0[1],d2[3],d3[4]", "d1[2],d2[3],d3[4]", "d2,d3"); assertWillOptimize("d0[1],d1[2],d2[3]", "d2[3],d3[4],d4[5]", "d2"); diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index f631b3e1c58..d0c1abe061a 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1278,6 +1278,7 @@ "public void ()", "public void (com.yahoo.tensor.TensorType$Value)", "public varargs void (com.yahoo.tensor.TensorType[])", + "public varargs void (boolean, com.yahoo.tensor.TensorType[])", "public void (java.lang.Iterable)", "public void (com.yahoo.tensor.TensorType$Value, java.lang.Iterable)", "public int rank()", diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index aeed8c33093..d8959147ee0 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -323,7 +323,7 @@ public class TensorType { * [N] + [] = [] * [] + {} = {} */ - Dimension combineWith(Optional other) { + Dimension combineWith(Optional other, boolean allowDifferentSizes) { if ( ! other.isPresent()) return this; if (this instanceof MappedDimension) return this; if (other.get() instanceof MappedDimension) return other.get(); @@ -333,7 +333,11 @@ public class TensorType { // both are indexed bound IndexedBoundDimension thisIb = (IndexedBoundDimension)this; IndexedBoundDimension otherIb = (IndexedBoundDimension)other.get(); - return thisIb.size().get() < otherIb.size().get() ? thisIb : otherIb; + if (allowDifferentSizes) + return thisIb.size().get() < otherIb.size().get() ? thisIb : otherIb; + if ( ! thisIb.size().equals(otherIb.size())) + throw new IllegalArgumentException("Unequal dimension sizes in " + thisIb + " and " + otherIb); + return thisIb; } @Override @@ -494,9 +498,13 @@ public class TensorType { * The value type will be the largest of the value types of the input types */ public Builder(TensorType ... types) { + this(true, types); + } + + public Builder(boolean allowDifferentSizes, TensorType ... types) { this.valueType = TensorType.combinedValueType(types); for (TensorType type : types) - addDimensionsOf(type); + addDimensionsOf(type, allowDifferentSizes); } /** Creates a builder from the given dimensions, having double as the value type */ @@ -514,17 +522,17 @@ public class TensorType { private static final boolean supportsMixedTypes = false; - private void addDimensionsOf(TensorType type) { + private void addDimensionsOf(TensorType type, boolean allowDifferentSizes) { if ( ! supportsMixedTypes) { // TODO: Support it - addDimensionsOfAndDisallowMixedDimensions(type); + addDimensionsOfAndDisallowMixedDimensions(type, allowDifferentSizes); } else { for (Dimension dimension : type.dimensions) - set(dimension.combineWith(Optional.ofNullable(dimensions.get(dimension.name())))); + set(dimension.combineWith(Optional.ofNullable(dimensions.get(dimension.name())), allowDifferentSizes)); } } - private void addDimensionsOfAndDisallowMixedDimensions(TensorType type) { + private void addDimensionsOfAndDisallowMixedDimensions(TensorType type, boolean allowDifferentSizes) { boolean containsMapped = dimensions.values().stream().anyMatch(d -> ! d.isIndexed()); containsMapped = containsMapped || type.dimensions().stream().anyMatch(d -> ! d.isIndexed()); @@ -532,7 +540,7 @@ public class TensorType { if (containsMapped) dimension = new MappedDimension(dimension.name()); Dimension existing = dimensions.get(dimension.name()); - set(dimension.combineWith(Optional.ofNullable(existing))); + set(dimension.combineWith(Optional.ofNullable(existing), allowDifferentSizes)); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java index 1e0eaa7fad3..5419d04a4fb 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java @@ -48,7 +48,12 @@ public class Join extends PrimitiveTensorFunction extends PrimitiveTensorFunction context) { - return new TensorType.Builder(argumentA.type(context), argumentB.type(context)).build(); + return outputType(argumentA.type(context), argumentB.type(context)); } @Override public Tensor evaluate(EvaluationContext context) { Tensor a = argumentA.evaluate(context); Tensor b = argumentB.evaluate(context); - TensorType joinedType = new TensorType.Builder(a.type(), b.type()).build(); + TensorType joinedType = outputType(a.type(), b.type()); return evaluate(a, b, joinedType, combinator); } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java index b1851b5f120..5bd1bbdba37 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java @@ -130,13 +130,11 @@ public class TensorTestCase { assertEquals("Mapped vector", 42, (int)dotProduct(vector(Type.mapped), vectors(Type.mapped, 2))); assertEquals("Indexed unbound vector", 42, (int)dotProduct(vector(3, Type.indexedUnbound), vectors(5, Type.indexedUnbound, 2))); assertEquals("Indexed unbound vector", 42, (int)dotProduct(vector(5, Type.indexedUnbound), vectors(3, Type.indexedUnbound, 2))); - assertEquals("Indexed bound vector", 42, (int)dotProduct(vector(3, Type.indexedBound), vectors(5, Type.indexedBound, 2))); - assertEquals("Indexed bound vector", 42, (int)dotProduct(vector(5, Type.indexedBound), vectors(3, Type.indexedBound, 2))); + assertEquals("Indexed bound vector", 42, (int)dotProduct(vector(3, Type.indexedBound), vectors(3, Type.indexedBound, 2))); assertEquals("Mapped matrix", 42, (int)dotProduct(vector(Type.mapped), matrix(Type.mapped, 2))); assertEquals("Indexed unbound matrix", 42, (int)dotProduct(vector(3, Type.indexedUnbound), matrix(5, Type.indexedUnbound, 2))); assertEquals("Indexed unbound matrix", 42, (int)dotProduct(vector(5, Type.indexedUnbound), matrix(3, Type.indexedUnbound, 2))); - assertEquals("Indexed bound matrix", 42, (int)dotProduct(vector(3, Type.indexedBound), matrix(5, Type.indexedBound, 2))); - assertEquals("Indexed bound matrix", 42, (int)dotProduct(vector(5, Type.indexedBound), matrix(3, Type.indexedBound, 2))); + assertEquals("Indexed bound matrix", 42, (int)dotProduct(vector(3, Type.indexedBound), matrix(3, Type.indexedBound, 2))); assertEquals("Mixed vector", 42, (int)dotProduct(vector(Type.mapped), vectors(Type.indexedUnbound, 2))); assertEquals("Mixed vector", 42, (int)dotProduct(vector(Type.mapped), vectors(Type.indexedUnbound, 2))); assertEquals("Mixed matrix", 42, (int)dotProduct(vector(Type.mapped), matrix(Type.indexedUnbound, 2))); -- cgit v1.2.3