diff options
21 files changed, 230 insertions, 120 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 9c802075462..e1fe795d2b1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -24,9 +24,9 @@ import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.rule.Arguments; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; -import java.io.File; import java.io.IOException; import java.io.Reader; import java.io.Serializable; @@ -116,7 +116,7 @@ public class RankProfile implements Cloneable { // This cache must be invalidated every time modifications are done to 'functions'. private CachedFunctions allFunctionsCached = null; - private Map<Reference, TensorType> inputs = new LinkedHashMap<>(); + private Map<Reference, Input> inputs = new LinkedHashMap<>(); private Set<String> filterFields = new HashSet<>(); @@ -764,33 +764,31 @@ public class RankProfile implements Cloneable { * All inputs must either be declared through this or in query profile types, * otherwise they are assumes to be scalars. */ - public void addInput(Reference reference, TensorType declaredType) { + public void addInput(Reference reference, Input input) { if (inputs.containsKey(reference)) { - TensorType hadType = inputs().get(reference); - if (! declaredType.equals(hadType)) - throw new IllegalArgumentException("Duplicate input '" + name + "' declared with both type " + - hadType + " and " + declaredType); + Input existing = inputs().get(reference); + if (! input.equals(existing)) + throw new IllegalArgumentException("Duplicate input: Has both " + input + " and existing"); } - inputs.put(reference, declaredType); + inputs.put(reference, input); } /** Returns the inputs of this, which also includes all inputs of the parents of this. */ // This is less restrictive than most other constructs in allowing inputs to be defined in all parent profiles // because inputs are tied closer to functions than the profile itself. - public Map<Reference, TensorType> inputs() { + public Map<Reference, Input> inputs() { if (inputs.isEmpty() && inherited().isEmpty()) return Map.of(); if (inherited().isEmpty()) return Collections.unmodifiableMap(inputs); // Combine - Map<Reference, TensorType> allInputs = new LinkedHashMap<>(); + Map<Reference, Input> allInputs = new LinkedHashMap<>(); for (var inheritedProfile : inherited()) { for (var input : inheritedProfile.inputs().entrySet()) { - TensorType existingType = allInputs.get(input.getKey()); - if (existingType != null && ! existingType.equals(input.getValue())) + Input existing = allInputs.get(input.getKey()); + if (existing != null && ! existing.equals(input.getValue())) throw new IllegalArgumentException(this + " inherits " + inheritedProfile + " which contains " + - input.getValue() + ", with type " + input.getValue() + "" + - " but this input is already defined with type " + existingType + - " in another profile this inherits"); + input.getValue() + ", but this input is already defined as " + + existing + " in another profile this inherits"); allInputs.put(input.getKey(), input.getValue()); } } @@ -1050,7 +1048,8 @@ public class RankProfile implements Cloneable { public MapEvaluationTypeContext typeContext() { return typeContext(new QueryProfileRegistry()); } private Map<Reference, TensorType> featureTypes() { - Map<Reference, TensorType> featureTypes = new HashMap<>(inputs()); + Map<Reference, TensorType> featureTypes = inputs().values().stream() + .collect(Collectors.toMap(input -> input.name(), input -> input.type())); allFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); allImportedFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); return featureTypes; @@ -1070,7 +1069,7 @@ public class RankProfile implements Cloneable { TensorType type = field.getType().asTensorType(); Optional<Reference> feature = Reference.simple(field.getName()); if ( feature.isEmpty() || ! feature.get().name().equals("query")) continue; - if (featureTypes.containsKey(feature)) continue; // Explicit feature types (from inputs) overrides + if (featureTypes.containsKey(feature.get())) continue; // Explicit feature types (from inputs) overrides TensorType existingType = context.getType(feature.get()); if ( ! Objects.equals(existingType, context.defaultTypeOf(feature.get()))) @@ -1392,6 +1391,46 @@ public class RankProfile implements Cloneable { } + public static final class Input { + + private final Reference name; + private final TensorType type; + private final Optional<Tensor> defaultValue; + + public Input(Reference name, TensorType type, Optional<Tensor> defaultValue) { + this.name = name; + this.type = type; + this.defaultValue = defaultValue; + } + + public Reference name() { return name; } + public TensorType type() { return type; } + public Optional<Tensor> defaultValue() { return defaultValue; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof Input)) return false; + Input other = (Input)o; + if ( ! other.name().equals(this.name())) return false; + if ( ! other.type().equals(this.type())) return false; + if ( ! other.defaultValue().equals(this.defaultValue())) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, type, defaultValue); + } + + @Override + public String toString() { + return "input '" + name + "' " + type + + (defaultValue().isPresent() ? ":" + defaultValue.get().toAbbreviatedString() : ""); + } + + } + private static class CachedFunctions { private final Map<String, RankingExpressionFunction> allRankingExpressionFunctions; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 3c14a2b9c63..77245da5ddd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -145,7 +145,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { */ private final NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default"); private final Map<String, String> attributeTypes; - private final Map<Reference, TensorType> inputs; + private final Map<Reference, RankProfile.Input> inputs; private final Set<String> filterFields = new java.util.LinkedHashSet<>(); private final String rankprofileName; @@ -426,10 +426,16 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) { properties.add(new Pair<>("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue())); } - for (Map.Entry<Reference, TensorType> input : inputs.entrySet()) { - if (FeatureNames.isQueryFeature(input.getKey())) - properties.add(new Pair<>("vespa.type.query." + input.getKey().arguments().expressions().get(0), - input.getValue().toString())); + for (var input : inputs.values()) { + if (FeatureNames.isQueryFeature(input.name())) { + properties.add(new Pair<>("vespa.type.query." + input.name().arguments().expressions().get(0), + input.type().toString())); + if (input.defaultValue().isPresent()) + properties.add(new Pair<>(input.name().toString(), + input.type().rank() == 0 ? + String.valueOf(input.defaultValue().get().asDouble()) : + input.defaultValue().get().toString(false, false))); + } } if (properties.size() >= 1000000) throw new IllegalArgumentException("Too many rank properties"); distributeLargeExpressionsAsFiles(properties, largeRankExpressions); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java index eeb3a97eda9..0c1e5a76a89 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java @@ -6,6 +6,7 @@ import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -15,6 +16,7 @@ import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Optional; /** * Information about a schema. @@ -100,7 +102,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc for (var input : rankProfile.inputs().entrySet()) { var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder(); inputConfig.name(input.getKey().toString()); - inputConfig.type(input.getValue().toString()); + inputConfig.type(input.getValue().type().toString()); rankProfileConfig.input(inputConfig); } schemaBuilder.rankprofile(rankProfileConfig); @@ -113,7 +115,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc private final String name; private final boolean hasSummaryFeatures; private final boolean hasRankFeatures; - private final Map<Reference, TensorType> inputs; + private final Map<Reference, RankProfile.Input> inputs; public RankProfileInfo(RankProfile profile) { this.name = profile.name(); @@ -125,7 +127,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc public String name() { return name; } public boolean hasSummaryFeatures() { return hasSummaryFeatures; } public boolean hasRankFeatures() { return hasRankFeatures; } - public Map<Reference, TensorType> inputs() { return inputs; } + public Map<Reference, RankProfile.Input> inputs() { return inputs; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java index 118945369d3..63a120f7c7b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.parser; +import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; import com.yahoo.searchdefinition.RankProfile.MutateOperation; import com.yahoo.searchlib.rankingexpression.FeatureList; @@ -52,7 +53,7 @@ class ParsedRankProfile extends ParsedBlock { private final Map<String, String> fieldsRankType = new LinkedHashMap<>(); private final Map<String, List<String>> rankProperties = new LinkedHashMap<>(); private final Map<String, Value> constants = new LinkedHashMap<>(); - private final Map<Reference, TensorType> inputs = new LinkedHashMap<>(); + private final Map<Reference, RankProfile.Input> inputs = new LinkedHashMap<>(); ParsedRankProfile(String name) { super(name, "rank-profile"); @@ -83,7 +84,7 @@ class ParsedRankProfile extends ParsedBlock { Map<String, String> getFieldsWithRankType() { return Collections.unmodifiableMap(fieldsRankType); } Map<String, List<String>> getRankProperties() { return Collections.unmodifiableMap(rankProperties); } Map<String, Value> getConstants() { return Collections.unmodifiableMap(constants); } - Map<Reference, TensorType> getInputs() { return Collections.unmodifiableMap(inputs); } + Map<Reference, RankProfile.Input> getInputs() { return Collections.unmodifiableMap(inputs); } Optional<String> getInheritedSummaryFeatures() { return Optional.ofNullable(this.inheritedSummaryFeatures); } Optional<String> getSecondPhaseExpression() { return Optional.ofNullable(this.secondPhaseExpression); } @@ -110,9 +111,9 @@ class ParsedRankProfile extends ParsedBlock { constants.put(name, value); } - void addInput(Reference name, TensorType type) { + void addInput(Reference name, RankProfile.Input input) { verifyThat(! inputs.containsKey(name), "already has input", name); - inputs.put(name, type); + inputs.put(name, input); } void addFieldRankFilter(String field, boolean filter) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index 9de6a11ce44..fb7e67f2aab 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -97,7 +97,8 @@ public class RankProfileTypeSettingsProcessor extends Processor { private void addQueryFeatureTypeToRankProfiles(Reference queryFeature, TensorType queryFeatureType) { for (RankProfile profile : rankProfileRegistry.all()) { if (! profile.inputs().containsKey(queryFeature)) // declared inputs have precedence - profile.addInput(queryFeature, queryFeatureType); + profile.addInput(queryFeature, + new RankProfile.Input(queryFeature, queryFeatureType, Optional.empty())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index 8cbd94a8a49..edd269559ed 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -215,7 +215,8 @@ public class ConvertedModel { for (ImportedMlFunction outputFunction : model.outputExpressions()) { ExpressionFunction expression = asExpressionFunction(outputFunction); for (Map.Entry<String, TensorType> input : expression.argumentTypes().entrySet()) { - profile.addInput(Reference.fromIdentifier(input.getKey()), input.getValue()); + Reference name = Reference.fromIdentifier(input.getKey()); + profile.addInput(name, new RankProfile.Input(name, input.getValue(), Optional.empty())); } addExpression(expression, expression.getName(), constantsReplacedByFunctions, store, profile, queryProfiles, expressions); @@ -283,7 +284,8 @@ public class ConvertedModel { String name = output.getFirst(); ExpressionFunction expression = output.getSecond(); for (Map.Entry<String, TensorType> input : expression.argumentTypes().entrySet()) { - profile.addInput(Reference.fromIdentifier(input.getKey()), input.getValue()); + Reference inputName = Reference.fromIdentifier(input.getKey()); + profile.addInput(inputName, new RankProfile.Input(inputName, input.getValue(), Optional.empty())); } TensorType type = expression.getBody().type(profile.typeContext()); if (type != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index d7ce64d1d32..1141af6d79d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -120,7 +120,7 @@ public abstract class SearchCluster extends AbstractConfigProducer<SearchCluster for (var input : rankProfile.inputs().entrySet()) { var inputConfig = new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder(); inputConfig.name(input.getKey().toString()); - inputConfig.type(input.getValue().toString()); + inputConfig.type(input.getValue().type().toString()); rankProfileConfig.input(inputConfig); } docDbBuilder.rankprofile(rankProfileConfig); diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj index fbd0181d307..6ce261ebb8c 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/IntermediateParser.jj @@ -1851,7 +1851,7 @@ void rankProfileItem(ParsedRankProfile profile) : { } } /** - * This rule consumes an inherits statement of a rank-profile. + * Consumes an inherits statement of a rank-profile. * * @param profile the profile to modify */ @@ -2080,11 +2080,7 @@ void input(ParsedRankProfile profile) : } { reference = queryFeature() type = inputType(reference) ( <COLON> (<NL>)* defaultValue = tensorValue(type) )? - { - profile.addInput(reference, type); - if (defaultValue != null) - new TensorValue(defaultValue); - } + { profile.addInput(reference, new RankProfile.Input(reference, type, Optional.ofNullable(defaultValue))); } } TensorType inputType(Reference reference) : @@ -2273,7 +2269,7 @@ void approximateThreshold(ParsedRankProfile profile) : } /** - * This rule consumes a rank-properties block of a rank profile. There + * Consumes a rank-properties block of a rank profile. There * is a little trick within this rule to allow the final rank property * to skip the terminating newline token. * @@ -2286,7 +2282,7 @@ void rankProperties(ParsedRankProfile profile) : { } } /** - * This rule consumes a single rank property pair for a rank profile. + * Consumes a single rank property pair for a rank profile. * * @param profile the rank profile to modify */ @@ -2300,9 +2296,9 @@ void rankProperty(ParsedRankProfile profile) : } /** - * This rule consumes a single rank property for a rank-properties block. + * Consumes a single rank property for a rank-properties block. * - * @return The token image of the consumed item. + * @return the token image of the consumed item */ String rankPropertyItem() : { @@ -2319,9 +2315,9 @@ String rankPropertyItem() : } /** - * This rule consumes a field-weight statement of a rank profile. + * Consumes a field-weight statement of a rank profile. * - * @param profile The rank profile to modify. + * @param profile the rank profile to modify */ void fieldWeight(ParsedRankProfile profile) : { @@ -2334,9 +2330,9 @@ void fieldWeight(ParsedRankProfile profile) : } /** - * This rule consumes a rank-type statement of a rank profile. + * Consumes a rank-type statement of a rank profile. * - * @param profile The rank profile to modify. + * @param profile the rank profile to modify */ void fieldRankType(ParsedRankProfile profile) : { @@ -2349,9 +2345,9 @@ void fieldRankType(ParsedRankProfile profile) : } /** - * This rule consumes a rank filter statement of a rank profile. + * Consumes a rank filter statement of a rank profile. * - * @param profile The rank profile to modify. + * @param profile the rank profile to modify */ void fieldRankFilter(ParsedRankProfile profile) : { @@ -2363,7 +2359,7 @@ void fieldRankFilter(ParsedRankProfile profile) : } /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationBinSize() : { @@ -2376,7 +2372,7 @@ void rankDegradationBinSize() : /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationBinLow() : { @@ -2388,7 +2384,7 @@ void rankDegradationBinLow() : } /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationPosbinSize() : { @@ -2401,7 +2397,7 @@ void rankDegradationPosbinSize() : /** - * This rule consumes part of a rank-degradation statement of a rank profile. + * Consumes part of a rank-degradation statement of a rank profile. */ void rankDegradationItem() : { } { @@ -2411,7 +2407,7 @@ void rankDegradationItem() : { } } /** - * This rule consumes a rank-degradation statement of a rank profile. + * Consumes a rank-degradation statement of a rank profile. */ void rankDegradation() : { diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index ba304f6f9ca..de2ce3d6938 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -2212,7 +2212,7 @@ void input(RankProfile profile) : { reference = queryFeature() type = inputType(reference) { - profile.addInput(reference, type); + profile.addInput(reference, new RankProfile.Input(reference, type, Optional.empty())); } } diff --git a/config-model/src/test/derived/rankproperties/rank-profiles.cfg b/config-model/src/test/derived/rankproperties/rank-profiles.cfg index 47a0438a323..3ca44288282 100644 --- a/config-model/src/test/derived/rankproperties/rank-profiles.cfg +++ b/config-model/src/test/derived/rankproperties/rank-profiles.cfg @@ -1,6 +1,8 @@ rankprofile[].name "default" -rankprofile[].fef.property[].name "$test" +rankprofile[].fef.property[].name "$test1" rankprofile[].fef.property[].value "foo" +rankprofile[].fef.property[].name "query(test2)" +rankprofile[].fef.property[].value "12.3" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "nativeFieldMatch" rankprofile[].fef.property[].name "vespa.rank.secondphase" @@ -21,8 +23,10 @@ rankprofile[].fef.property[].value "0" rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" rankprofile[].fef.property[].value "true" rankprofile[].name "child" -rankprofile[].fef.property[].name "$test" +rankprofile[].fef.property[].name "$test1" rankprofile[].fef.property[].value "foo" +rankprofile[].fef.property[].name "query(test2)" +rankprofile[].fef.property[].value "12.3" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "nativeFieldMatch" rankprofile[].fef.property[].name "vespa.rank.secondphase" diff --git a/config-model/src/test/derived/rankproperties/rankproperties.sd b/config-model/src/test/derived/rankproperties/rankproperties.sd index 24977ae645f..db259f1daec 100644 --- a/config-model/src/test/derived/rankproperties/rankproperties.sd +++ b/config-model/src/test/derived/rankproperties/rankproperties.sd @@ -28,7 +28,8 @@ search rankproperties { expression: match } rank-properties { - $test:"foo" + $test1:"foo" + query(test2): 12.3 #$weight:1 } } diff --git a/config-model/src/test/examples/rankpropvars.sd b/config-model/src/test/examples/rankpropvars.sd index e26358fef5a..339d147dfd0 100644 --- a/config-model/src/test/examples/rankpropvars.sd +++ b/config-model/src/test/examples/rankpropvars.sd @@ -55,13 +55,11 @@ document music { } field artist type string { - ## index-to: a indexing: index | summary } field year type int { indexing: attribute | summary - ## index-to: y } field url type uri {} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java index 86ccb816c10..0f29d2dda40 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java @@ -10,8 +10,10 @@ import java.io.IOException; * @author bratseth */ public class RankPropertiesTestCase extends AbstractExportingTestCase { + @Test public void testRankProperties() throws IOException, ParseException { assertCorrectDeriving("rankproperties"); } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java index a9aa6f4c605..d605c0fdcb3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java @@ -2,12 +2,16 @@ package com.yahoo.vespa.model.search.test; import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.searchdefinition.FeatureNames; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.model.VespaModel; import org.junit.Test; import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; public class SchemaInfoTestCase { @@ -26,7 +30,7 @@ public class SchemaInfoTestCase { " query(myVector) tensor(x[3]):\n\n[1 ,2.0,3]" + " query(myMatrix) tensor(x[2],y[3]):[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]" + " query(myMixed1) tensor(key{},x[2]): { key1:[-1.0, 1.1], key2: [1,2]}" + - " query(myMixed2) tensor(k1{},k2{},x[2]): { {k1:l1,k2:l2}:[-1.0, 1.1], {k1:l1,k2:l2}: [1,2]}" + + " query(myMixed2) tensor(k1{},k2{},x[2]): { {k1:l1,k2:l1}:[-1.0, 1.1], {k1:l1,k2:l2}: [1,2]}" + " }" + " }"; List<String> schemas = List.of("type1", "type2"); @@ -39,6 +43,8 @@ public class SchemaInfoTestCase { private void assertSchemaInfo(String configId, VespaModel model, SchemaTester tester) { { SchemaInfoConfig schemaInfoConfig = model.getConfig(SchemaInfoConfig.class, configId); + RankProfilesConfig rankProfilesConfig = model.getConfig(RankProfilesConfig.class, "test/search/cluster.test/type1"); + assertEquals(2, schemaInfoConfig.schema().size()); { // type1 @@ -52,18 +58,20 @@ public class SchemaInfoTestCase { tester.assertRankProfile(schema, 3, "summaryfeatures", true, false); tester.assertRankProfile(schema, 4, "inheritedsummaryfeatures", true, false); tester.assertRankProfile(schema, 5, "rankfeatures", false, true); - var inputs = tester.assertRankProfile(schema, 6, "inputs", false, false); - assertEquals(9, inputs.input().size()); - assertInput("query(foo)", "tensor<float>(x[10])", inputs.input(0)); - assertInput("query(bar)", "tensor(key{},x[1000])", inputs.input(1)); - assertInput("query(myDouble1)", "tensor()", inputs.input(2)); - assertInput("query(myDouble2)", "tensor()", inputs.input(3)); - assertInput("query(myMap)", "tensor(key{})", inputs.input(4)); - assertInput("query(myVector)", "tensor(x[3])", inputs.input(5)); - assertInput("query(myMatrix)", "tensor(x[2],y[3])", inputs.input(6)); - assertInput("query(myMixed1)", "tensor(key{},x[2])", inputs.input(7)); - assertInput("query(myMixed2)", "tensor(k1{},k2{},x[2])", inputs.input(8)); + var schemaInfoProfile = tester.assertRankProfile(schema, 6, "inputs", false, false); + assertEquals(9, schemaInfoProfile.input().size()); + var rankProfilesProfile = rankProfilesConfig.rankprofile().get(6); + assertEquals("inputs", rankProfilesProfile.name()); + assertInput("query(foo)", "tensor<float>(x[10])", null, 0, schemaInfoProfile, rankProfilesProfile); + assertInput("query(bar)", "tensor(key{},x[1000])", null, 1, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myDouble1)", "tensor()", "0.5", 2, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myDouble2)", "tensor()", null, 3, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMap)", "tensor(key{})", "{{key:label1}:1.0, {key:label2}:2.0, {key:label3}:3.0}", 4, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myVector)", "tensor(x[3])", "{{x:0}:1.0, {x:1}:2.0, {x:2}:3.0}", 5, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMatrix)", "tensor(x[2],y[3])", "{{x:0,y:0}:1.0, {x:0,y:1}:2.0, {x:0,y:2}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0, {x:1,y:2}:6.0}", 6, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMixed1)", "tensor(key{},x[2])", "{{key:key1,x:0}:-1.0, {key:key1,x:1}:1.1, {key:key2,x:0}:1.0, {key:key2,x:1}:2.0}", 7, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMixed2)", "tensor(k1{},k2{},x[2])", "{{k1:l1,k2:l1,x:0}:-1.0, {k1:l1,k2:l1,x:1}:1.1, {k1:l1,k2:l2,x:0}:1.0, {k1:l1,k2:l2,x:1}:2.0}", 8, schemaInfoProfile, rankProfilesProfile); assertEquals(2, schema.summaryclass().size()); assertEquals("default", schema.summaryclass(0).name()); @@ -78,9 +86,23 @@ public class SchemaInfoTestCase { } } - private void assertInput(String name, String type, SchemaInfoConfig.Schema.Rankprofile.Input input) { - assertEquals(name, input.name()); - assertEquals(type, input.type()); + private void assertInput(String name, String type, String defaultValue, + int index, + SchemaInfoConfig.Schema.Rankprofile schemaInfoProfile, + RankProfilesConfig.Rankprofile rankProfilesProfile) { + assertEquals(name, schemaInfoProfile.input(index).name()); + assertEquals(type, schemaInfoProfile.input(index).type()); + if (defaultValue != null) { + boolean found = false; + for (var property : rankProfilesProfile.fef().property()) { + if (property.name().equals(name)) { + assertEquals(defaultValue, property.value()); + found = true; + } + } + if ( ! found) + fail("Missing property " + name); + } } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index 19e0e441359..abd23c1822d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -147,7 +147,7 @@ public class QueryProfileProperties extends Properties { private String toShortString(Object value) { if ( ! (value instanceof Tensor)) return value.toString(); - return ((Tensor)value).toShortString(); + return ((Tensor)value).toAbbreviatedString(); } private Object convertByType(CompoundName name, Object value, Map<String, String> context) { diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 5e6168e71f7..f7be61946ba 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -906,7 +906,8 @@ "public java.util.Map cells()", "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public java.lang.String toString()", - "public java.lang.String toShortString()", + "public java.lang.String toString(boolean, boolean)", + "public java.lang.String toAbbreviatedString()", "public boolean equals(java.lang.Object)", "public bridge synthetic com.yahoo.tensor.Tensor withType(com.yahoo.tensor.TensorType)" ], @@ -957,7 +958,8 @@ "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public int hashCode()", "public java.lang.String toString()", - "public java.lang.String toShortString()", + "public java.lang.String toString(boolean, boolean)", + "public java.lang.String toAbbreviatedString()", "public boolean equals(java.lang.Object)" ], "fields": [] @@ -1049,7 +1051,8 @@ "public com.yahoo.tensor.Tensor remove(java.util.Set)", "public int hashCode()", "public java.lang.String toString()", - "public java.lang.String toShortString()", + "public java.lang.String toString(boolean, boolean)", + "public java.lang.String toAbbreviatedString()", "public boolean equals(java.lang.Object)", "public long denseSubspaceSize()", "public static com.yahoo.tensor.TensorType createPartialType(com.yahoo.tensor.TensorType$Value, java.util.List)" @@ -1237,9 +1240,11 @@ "public java.util.List largest()", "public java.util.List smallest()", "public abstract java.lang.String toString()", - "public abstract java.lang.String toShortString()", - "public static java.lang.String toStandardString(com.yahoo.tensor.Tensor, long)", - "public static java.lang.String contentToString(com.yahoo.tensor.Tensor, long)", + "public abstract java.lang.String toString(boolean, boolean)", + "public abstract java.lang.String toAbbreviatedString()", + "public java.lang.String toShortString()", + "public static java.lang.String toStandardString(com.yahoo.tensor.Tensor, boolean, boolean, long)", + "public static java.lang.String valueToString(com.yahoo.tensor.Tensor, boolean, long)", "public abstract boolean equals(java.lang.Object)", "public abstract int hashCode()", "public static boolean equals(com.yahoo.tensor.Tensor, com.yahoo.tensor.Tensor)", diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index 76629a20b2f..c4316eb334a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -219,21 +219,26 @@ public abstract class IndexedTensor implements Tensor { } @Override - public String toString() { return toString(Long.MAX_VALUE); } + public String toString() { return toString(true, true); } @Override - public String toShortString() { - return toString(Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + public String toString(boolean withType, boolean shortForms) { + return toString(withType, shortForms, Long.MAX_VALUE); } - private String toString(long maxCells) { - if (type.rank() == 0) return Tensor.toStandardString(this, maxCells); - if (type.dimensions().stream().anyMatch(d -> d.size().isEmpty())) - return Tensor.toStandardString(this, maxCells); + @Override + public String toAbbreviatedString() { + return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + } - Indexes indexes = Indexes.of(dimensionSizes); + private String toString(boolean withType, boolean shortForms, long maxCells) { + if (! shortForms || type.rank() == 0 || type.dimensions().stream().anyMatch(d -> d.size().isEmpty())) + return Tensor.toStandardString(this, withType, shortForms, maxCells); - StringBuilder b = new StringBuilder(type.toString()).append(":"); + Indexes indexes = Indexes.of(dimensionSizes); + StringBuilder b = new StringBuilder(); + if (withType) + b.append(type).append(":"); indexedBlockToString(this, indexes, maxCells, b); return b.toString(); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java index ad945ed18bf..946d8fe0f4a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MappedTensor.java @@ -72,11 +72,18 @@ public class MappedTensor implements Tensor { public int hashCode() { return cells.hashCode(); } @Override - public String toString() { return Tensor.toStandardString(this, Long.MAX_VALUE); } + public String toString() { return toString(true, true); } @Override - public String toShortString() { - return Tensor.toStandardString(this, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + public String toString(boolean withType, boolean shortForms) { return toString(withType, shortForms, Long.MAX_VALUE); } + + @Override + public String toAbbreviatedString() { + return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + } + + private String toString(boolean withType, boolean shortForms, long maxCells) { + return Tensor.toStandardString(this, withType, shortForms, maxCells); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java index 56bd94a86e9..d2fed9b96f9 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java @@ -145,23 +145,27 @@ public class MixedTensor implements Tensor { @Override public String toString() { - return toString(Long.MAX_VALUE); + return toString(true, true); } @Override - public String toShortString() { - return toString(Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + public String toString(boolean withType, boolean shortForms) { + return toString(withType, shortForms, Long.MAX_VALUE); } - private String toString(long maxCells) { - if (type.rank() == 0) - return Tensor.toStandardString(this, maxCells); - if (type.rank() > 1 && type.dimensions().stream().filter(d -> d.isIndexed()).anyMatch(d -> d.size().isEmpty())) - return Tensor.toStandardString(this, maxCells); - if (type.dimensions().stream().filter(d -> d.isMapped()).count() > 1) - return Tensor.toStandardString(this, maxCells); + @Override + public String toAbbreviatedString() { + return toString(true, true, Math.max(2, 10 / (type().dimensions().stream().filter(d -> d.isMapped()).count() + 1))); + } + + private String toString(boolean withType, boolean shortForms, long maxCells) { + if (! shortForms + || type.rank() == 0 + || type.rank() > 1 && type.dimensions().stream().filter(d -> d.isIndexed()).anyMatch(d -> d.size().isEmpty()) + || type.dimensions().stream().filter(d -> d.isMapped()).count() > 1) + return Tensor.toStandardString(this, withType, shortForms, maxCells); - return type + ":" + index.contentToString(this, maxCells); + return (withType ? type + ":" : "") + index.contentToString(this, maxCells); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 94b00e7e277..8a84e97fe05 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -32,7 +32,6 @@ import java.util.Set; import java.util.function.DoubleBinaryOperator; import java.util.function.DoubleUnaryOperator; import java.util.function.Function; -import java.util.stream.Collectors; import static com.yahoo.tensor.functions.ScalarFunctions.Hamming; @@ -316,8 +315,22 @@ public interface Tensor { @Override String toString(); + /** + * Returns this tensor on the + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a>. + * + * @param withType whether to prefix the value by the type of this + * @param shortForms whether to use short forms where applicable, or always using the verbose form + */ + String toString(boolean withType, boolean shortForms); + /** Returns an abbreviated string representation of this tensor suitable for human-readable messages */ - String toShortString(); + String toAbbreviatedString(); + + // TODO: Remove on Vespa 8 + /** @deprecated use toAbbreviatedString */ + @Deprecated + default String toShortString() { return toAbbreviatedString(); } /** * Call this from toString in implementations to return this tensor on the @@ -325,15 +338,16 @@ public interface Tensor { * (toString cannot be a default method because default methods cannot override super methods). * * @param tensor the tensor to return the standard string format of + * @param withType whether the type should be prepended to the content * @param maxCells the max number of cells to output, after which just , "..." is output to represent the rest * of the cells * @return the tensor on the standard string format */ - static String toStandardString(Tensor tensor, long maxCells) { - return tensor.type() + ":" + contentToString(tensor, maxCells); + static String toStandardString(Tensor tensor, boolean withType, boolean shortForms, long maxCells) { + return (withType ? tensor.type() + ":" : "") + valueToString(tensor, shortForms, maxCells); } - static String contentToString(Tensor tensor, long maxCells) { + static String valueToString(Tensor tensor, boolean shortForms, long maxCells) { var cellEntries = new ArrayList<>(tensor.cells().entrySet()); cellEntries.sort(Map.Entry.comparingByKey()); if (tensor.type().dimensions().isEmpty()) { @@ -345,7 +359,7 @@ public interface Tensor { for (; i < cellEntries.size() && i < maxCells; i++) { if (i > 0) b.append(", "); - b.append(cellToString(cellEntries.get(i), tensor.type())); + b.append(cellToString(cellEntries.get(i), tensor.type(), shortForms)); } if (i == maxCells && i < tensor.size()) b.append(", ..."); @@ -353,8 +367,9 @@ public interface Tensor { return b.toString(); } - private static String cellToString(Map.Entry<TensorAddress, Double> cell, TensorType type) { - return (type.rank() > 1 ? cell.getKey().toString(type) : TensorAddress.labelToString(cell.getKey().label(0))) + + private static String cellToString(Map.Entry<TensorAddress, Double> cell, TensorType type, boolean shortForms) { + return (shortForms && type.rank() == 1 ? TensorAddress.labelToString(cell.getKey().label(0)) + : cell.getKey().toString(type) ) + ":" + cell.getValue(); } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java index 2067d7a8492..920f8512c53 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java @@ -65,24 +65,24 @@ public class TensorTestCase { @Test public void testToShortString() { assertEquals("tensor(x[10]):[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]", - Tensor.from("tensor(x[10]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]").toShortString()); + Tensor.from("tensor(x[10]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]").toAbbreviatedString()); assertEquals("tensor(x[14]):[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, ...]", - Tensor.from("tensor(x[14]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]").toShortString()); + Tensor.from("tensor(x[14]):[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]").toAbbreviatedString()); assertEquals("tensor(d1{},d2{}):{{d1:l1,d2:l1}:6.0, {d1:l1,d2:l2}:6.0, {d1:l1,d2:l3}:6.0, ...}", Tensor.from("{{d1:l1,d2:l1}:6, {d2:l2,d1:l1}:6, {d2:l3,d1:l1}:6, {d2:l4,d1:l1}:6, {d2:l5,d1:l1}:6," + " {d2:l6,d1:l1}:6, {d2:l7,d1:l1}:6, {d2:l8,d1:l1}:6, {d2:l9,d1:l1}:6, {d2:l2,d1:l2}:6," + - " {d2:l2,d1:l3}:6, {d2:l2,d1:l4}:6}").toShortString()); + " {d2:l2,d1:l3}:6, {d2:l2,d1:l4}:6}").toAbbreviatedString()); assertEquals("tensor(m{},x[3]):{k1:[0.0, 1.0, 2.0], k2:[0.0, 1.0, ...}", - Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toShortString()); + Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toAbbreviatedString()); assertEquals("tensor(m{},x[3]):{k1:[0.0, 1.0, 2.0], k2:[0.0, 1.0, ...}", - Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toShortString()); + Tensor.from("tensor(m{},x[3]):{k1:[0,1,2], k2:[0,1,2], k3:[0,1,2], k4:[0,1,2]}").toAbbreviatedString()); assertEquals("tensor(m{},n{},x[3]):{{m:k1,n:k1,x:0}:0.0, {m:k1,n:k1,x:1}:1.0, {m:k1,n:k1,x:2}:2.0, ...}", Tensor.from("tensor(m{},n{},x[3]):" + "{{m:k1,n:k1,x:0}:0, {m:k1,n:k1,x:1}:1, {m:k1,n:k1,x:2}:2, " + " {m:k2,n:k1,x:0}:0, {m:k2,n:k1,x:1}:1, {m:k2,n:k1,x:2}:2, " + - " {m:k3,n:k1,x:0}:0, {m:k3,n:k1,x:1}:1, {m:k3,n:k1,x:2}:2}").toShortString()); + " {m:k3,n:k1,x:0}:0, {m:k3,n:k1,x:1}:1, {m:k3,n:k1,x:2}:2}").toAbbreviatedString()); assertEquals("tensor(m{},x[2],y[2]):{k1:[[0.0, 1.0], [2.0, 3.0]], k2:[[0.0, ...}", - Tensor.from("tensor(m{},x[2],y[2]):{k1:[[0,1],[2,3]], k2:[[0,1],[2,3]], k3:[[0,1],[2,3]]}").toShortString()); + Tensor.from("tensor(m{},x[2],y[2]):{k1:[[0,1],[2,3]], k2:[[0,1],[2,3]], k3:[[0,1],[2,3]]}").toAbbreviatedString()); } @Test |