diff options
Diffstat (limited to 'config-model')
11 files changed, 79 insertions, 41 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 9b3e236612a..22bf1880cd7 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -17,6 +17,7 @@ import com.yahoo.schema.expressiontransforms.ExpressionTransforms; import com.yahoo.schema.expressiontransforms.RankProfileTransformContext; import com.yahoo.schema.expressiontransforms.InputRecorder; import com.yahoo.schema.parser.ParseException; +import com.yahoo.search.schema.RankProfile.InputType; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.RankingExpression; @@ -841,7 +842,7 @@ public class RankProfile implements Cloneable { if (inputs.containsKey(reference)) { Input existing = inputs().get(reference); if (! input.equals(existing)) - throw new IllegalArgumentException("Duplicate input: Has both " + input + " and existing"); + throw new IllegalArgumentException("Duplicate input: Has both " + input + " and existing " + existing); } inputs.put(reference, input); } @@ -1177,7 +1178,8 @@ public class RankProfile implements Cloneable { private Map<Reference, TensorType> featureTypes() { Map<Reference, TensorType> featureTypes = inputs().values().stream() - .collect(Collectors.toMap(input -> input.name(), input -> input.type())); + .collect(Collectors.toMap(input -> input.name(), + input -> input.type().tensorType())); allFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); allImportedFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); return featureTypes; @@ -1545,17 +1547,21 @@ public class RankProfile implements Cloneable { public static final class Input { private final Reference name; - private final TensorType type; + private final InputType type; private final Optional<Tensor> defaultValue; - public Input(Reference name, TensorType type, Optional<Tensor> defaultValue) { + public Input(Reference name, InputType type, Optional<Tensor> defaultValue) { this.name = name; this.type = type; this.defaultValue = defaultValue; } + public Input(Reference name, TensorType tType, Optional<Tensor> defaultValue) { + this(name, new InputType(tType, false), defaultValue); + } + public Reference name() { return name; } - public TensorType type() { return type; } + public InputType type() { return type; } public Optional<Tensor> defaultValue() { return defaultValue; } @Override diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index 33a363312d6..db76d6397fc 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -519,12 +519,12 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (var input : inputs.values()) { if (FeatureNames.isQueryFeature(input.name())) { - if (input.type().rank() > 0) // Proton does not like representing the double type as a rank 0 tensor + if (input.type().tensorType().rank() > 0) // Proton does not like representing the double type as a rank 0 tensor properties.add(new Pair<>("vespa.type.query." + input.name().arguments().expressions().get(0), input.type().toString())); if (input.defaultValue().isPresent()) { properties.add(new Pair<>(input.name().toString(), - input.type().rank() == 0 ? + input.type().tensorType().rank() == 0 ? String.valueOf(input.defaultValue().get().asDouble()) : input.defaultValue().get().toString(true, false))); } diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DataTypeRecognizer.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DataTypeRecognizer.java index ecacfa43757..0c3b750d876 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DataTypeRecognizer.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DataTypeRecognizer.java @@ -35,12 +35,6 @@ public class DataTypeRecognizer { DataTypeRecognizer() { } - DataTypeRecognizer(DataTypeCollection dtc) { - for (var type : dtc.getTypes()) { - System.err.println("added: "+nameOf(type)); - } - } - String nameOf(Object type) { return toUniqueNames.computeIfAbsent(type, t -> makeUniqueName(t)); } diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index f298293ed57..4be95f33d19 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -34,6 +34,7 @@ import com.yahoo.schema.document.Stemming; import com.yahoo.schema.document.SDField; import com.yahoo.schema.FeatureNames; import com.yahoo.schema.fieldoperation.IndexingOperation; +import com.yahoo.search.schema.RankProfile.InputType; import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; @@ -298,6 +299,8 @@ TOKEN : | < HNSW: "hnsw" > | < MAX_LINKS_PER_NODE: "max-links-per-node" > | < DOUBLE_KEYWORD: "double" > +| < LONG_KEYWORD: "long" > +| < STRING_KEYWORD: "string" > | < DISTANCE_METRIC: "distance-metric" > | < NEIGHBORS_TO_EXPLORE_AT_INSERT: "neighbors-to-explore-at-insert" > | < MULTI_THREADED_INDEXING: "multi-threaded-indexing" > @@ -2005,18 +2008,19 @@ void inputs(ParsedRankProfile profile) : } { <INPUTS> <LBRACE> (<NL>)* - ( input(profile) (<NL>)*) * + (input(profile))? + (<NL> (input(profile))?)* <RBRACE> } void input(ParsedRankProfile profile) : { Reference reference; - TensorType type = TensorType.empty; + InputType type = new InputType(TensorType.empty, false); Tensor defaultValue = null; } { - reference = inputName() ( type = valueType(reference))? ( <COLON> (<NL>)* defaultValue = tensorValue(type) )? + reference = inputName() ( type = valueType(reference))? ( <COLON> (<NL>)* defaultValue = tensorValue(type.tensorType()) )? { profile.addInput(reference, new RankProfile.Input(reference, type, Optional.ofNullable(defaultValue))); } } @@ -2034,18 +2038,25 @@ Reference inputName() : { return FeatureNames.asQueryFeature(name); } } -TensorType valueType(Reference reference) : +InputType valueType(Reference reference) : { TensorType type; - + InputType result; } { ( - ( type = tensorType("Type of " + reference) ) + ( type = tensorType("Type of " + reference) { result = new InputType(type, false); } ) + | + ( <DOUBLE_KEYWORD> { result = new InputType(TensorType.empty, false); } ) | - ( <DOUBLE_KEYWORD> { type = TensorType.empty; } ) + ( <LONG_KEYWORD> { + result = new InputType(TensorType.empty, false); + deployLogger.logApplicationPackage(Level.WARNING, "Input " + reference + ": 'long' is not possible, treated as 'double'"); + } ) + | + ( <STRING_KEYWORD> { result = new InputType(TensorType.empty, true); } ) ) - { return type; } + { return result; } } /** @@ -2329,7 +2340,7 @@ void constants(ParsedSchema schema, ParsedRankProfile profile) : void constant(ParsedSchema schema, ParsedRankProfile profile) : { Reference name = null; - TensorType type = TensorType.empty; + InputType type = new InputType(TensorType.empty, false); Tensor value = null; String valuePath = null; } @@ -2337,12 +2348,12 @@ void constant(ParsedSchema schema, ParsedRankProfile profile) : ( name = constantName() ( - LOOKAHEAD(4) ( ( type = valueType(name) )? <COLON> (<NL>)* ( value = tensorValue(type) | valuePath = fileItem()) + LOOKAHEAD(4) ( ( type = valueType(name) )? <COLON> (<NL>)* ( value = tensorValue(type.tensorType()) | valuePath = fileItem()) { if (value != null) profile.add(new RankProfile.Constant(name, value)); else - profile.add(new RankProfile.Constant(name, type, valuePath)); + profile.add(new RankProfile.Constant(name, type.tensorType(), valuePath)); } ) | // Deprecated forms (TODO: Vespa > 8: Add warning): @@ -2710,6 +2721,8 @@ String identifier() : { } | <DIVERSITY> | <DOCUMENT> | <DOUBLE_KEYWORD> + | <LONG_KEYWORD> + | <STRING_KEYWORD> | <DYNAMIC> | <EXACT> | <FALSE> diff --git a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg index 87882eef273..8b769360053 100644 --- a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg @@ -670,3 +670,14 @@ rankprofile[].fef.property[].name "vespa.feature.rename" rankprofile[].fef.property[].value "useAttr(t1,42)" rankprofile[].fef.property[].name "vespa.type.attribute.t1" rankprofile[].fef.property[].value "tensor(m{},v[3])" +rankprofile[].name "withstringcompare" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(firstphase)" +rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" +rankprofile[].fef.property[].value "if (attribute(surl) == query(myquerystring), 0.75, 0.25)" +rankprofile[].fef.property[].name "vespa.rank.secondphase" +rankprofile[].fef.property[].value "rankingExpression(secondphase)" +rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" +rankprofile[].fef.property[].value "if (attribute(surl) == query(undeclaredinput), 0.75, 0.25)" +rankprofile[].fef.property[].name "vespa.type.attribute.t1" +rankprofile[].fef.property[].value "tensor(m{},v[3])" diff --git a/config-model/src/test/derived/rankingexpression/rankexpression.sd b/config-model/src/test/derived/rankingexpression/rankexpression.sd index 1ccf74bfe17..76672906252 100644 --- a/config-model/src/test/derived/rankingexpression/rankexpression.sd +++ b/config-model/src/test/derived/rankingexpression/rankexpression.sd @@ -20,7 +20,7 @@ schema rankexpression { } field surl type string { - indexing: summary + indexing: summary | attribute } field year type int { @@ -507,4 +507,15 @@ schema rankexpression { } } + rank-profile withstringcompare { + inputs { + query(myquerystring) string + } + first-phase { + expression: if (attribute(surl) == query(myquerystring), 0.75, 0.25) + } + second-phase { + expression: if (attribute(surl) == query(undeclaredinput), 0.75, 0.25) + } + } } diff --git a/config-model/src/test/derived/rankingexpression/summary.cfg b/config-model/src/test/derived/rankingexpression/summary.cfg index b52cb055164..9f96f5ab4a7 100644 --- a/config-model/src/test/derived/rankingexpression/summary.cfg +++ b/config-model/src/test/derived/rankingexpression/summary.cfg @@ -10,8 +10,8 @@ classes[].fields[].name "title" classes[].fields[].command "" classes[].fields[].source "" classes[].fields[].name "surl" -classes[].fields[].command "" -classes[].fields[].source "" +classes[].fields[].command "attribute" +classes[].fields[].source "surl" classes[].fields[].name "year" classes[].fields[].command "attribute" classes[].fields[].source "year" @@ -24,7 +24,7 @@ classes[].fields[].source "" classes[].fields[].name "documentid" classes[].fields[].command "documentid" classes[].fields[].source "" -classes[].id 399614584 +classes[].id 799304810 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "nrtgmp" @@ -33,6 +33,9 @@ classes[].fields[].source "nrtgmp" classes[].fields[].name "glmpfw" classes[].fields[].command "attribute" classes[].fields[].source "glmpfw" +classes[].fields[].name "surl" +classes[].fields[].command "attribute" +classes[].fields[].source "surl" classes[].fields[].name "year" classes[].fields[].command "attribute" classes[].fields[].source "year" diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java index e5eba3d416a..b41d92bd63f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java @@ -534,7 +534,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { " }" + " rank-profile inputs {" + " inputs {" + - " query(foo) tensor<float>(x[10])" + + " query(foo) tensor<float>(x[10])\n" + " query(bar) tensor(key{},x[1000])" + " }" + " }" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 5b501bad876..d1cd6a60821 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -277,7 +277,7 @@ public class DocumentDatabaseTestCase { String inputsProfile = " rank-profile inputs {" + " inputs {" + - " query(foo) tensor<float>(x[10])" + + " query(foo) tensor<float>(x[10])\n" + " query(bar) tensor(key{},x[1000])" + " }" + " }"; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java index 8e14ff9e5e6..8502bfa92f4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java @@ -130,16 +130,16 @@ public class SchemaInfoTestCase { String inputs = " rank-profile inputs {" + " inputs {" + - " query(foo) tensor<float>(x[10])" + - " query(bar) tensor(key{},x[1000])" + - " query(myDouble1) double: 0.5" + - " query(myDouble2) tensor()" + - " query(myMap) tensor(key{}): { label1:1.0,\n \"label2\": 2.0, 'label3': 3.0 }" + - " query(myVector1) tensor(x[3]):\n\n[1 ,2.0,3]" + - " query(myVector2) tensor(x[3]):{{x:0}:1,{x: 1}: 2 , { x:2}:3.0 }" + - " query(myMatrix) tensor(x[2],y[3]):[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]" + - " query(myMixed1) tensor(key{},x[2]): { key1:[-1.0, 1.1], key2: [1,2]}" + - " query(myMixed2) tensor(k1{},k2{},x[2]): { {k1:l1,k2:l1}:[-1.0, 1.1], {k1:l1,k2:l2}: [1,2]}" + + " query(foo) tensor<float>(x[10])\n"+ + " query(bar) tensor(key{},x[1000])\n"+ + " query(myDouble1) double: 0.5\n"+ + " query(myDouble2) tensor()\n"+ + " query(myMap) tensor(key{}): { label1:1.0,\n \"label2\": 2.0, 'label3': 3.0 }\n"+ + " query(myVector1) tensor(x[3]):\n\n[1 ,2.0,3]\n"+ + " query(myVector2) tensor(x[3]):{{x:0}:1,{x: 1}: 2 , { x:2}:3.0 }\n"+ + " query(myMatrix) tensor(x[2],y[3]):[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]\n"+ + " query(myMixed1) tensor(key{},x[2]): { key1:[-1.0, 1.1], key2: [1,2]}\n"+ + " query(myMixed2) tensor(k1{},k2{},x[2]): { {k1:l1,k2:l1}:[-1.0, 1.1], {k1:l1,k2:l2}: [1,2]}\n"+ " }" + " }"; List<String> schemas = List.of("type1", "type2"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java index 1367b7a38ff..b9bc34688b7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java @@ -42,7 +42,7 @@ public class ApplicationPackageUtils { " }" + " rank-profile inputs {" + " inputs {" + - " query(foo) tensor<float>(x[10])" + + " query(foo) tensor<float>(x[10])\n" + " query(bar) tensor(key{},x[1000])" + " }" + " }" + |