From a2eb1feb808a0533b5d7ef2b691a873f91c4c26b Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 6 May 2022 13:50:08 +0200 Subject: Parse input default values --- config-model/src/main/javacc/IntermediateParser.jj | 159 ++++++++++++++++----- config-model/src/main/javacc/SDParser.jj | 16 ++- .../model/search/test/SchemaInfoTestCase.java | 25 ++-- 3 files changed, 148 insertions(+), 52 deletions(-) (limited to 'config-model') diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj index 9b3383f9f41..3fe2b6d3cdd 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/IntermediateParser.jj @@ -1,9 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Duplicate Schema parser - work in progress -// @author arnej27959 -// NOTE: When grammar is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf - options { UNICODE_INPUT = true; CACHE_TOKENS = false; @@ -44,6 +40,7 @@ import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.TensorAddress; import java.util.Optional; import java.util.Map; @@ -55,6 +52,8 @@ import java.util.logging.Level; /** * The schema parser * + * NOTE: When this grammar is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf + * * @author bratseth */ public class IntermediateParser { @@ -62,9 +61,7 @@ public class IntermediateParser { private DeployLogger deployLogger; private ModelContext.Properties properties; - /** - * Creates a parser - */ + /** Creates a parser. */ public IntermediateParser(SimpleCharStream stream, DeployLogger deployLogger, ModelContext.Properties properties) @@ -341,7 +338,8 @@ TOKEN : | < URI: "uri" > | < IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_"])* > | < IDENTIFIER_WITH_DASH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-"])* > -| < QUOTEDSTRING: "\"" ( ~["\""] )* "\"" > +| < DOUBLEQUOTEDSTRING: "\"" ( ~["\""] )* "\"" > +| < SINGLEQUOTEDSTRING: "'" ( ~["'"] )* "'" > | < CONTEXT: ["a"-"z","A"-"Z"] (["a"-"z", "A"-"Z", "0"-"9"])* > | < DOUBLE: ("-")? (["0"-"9"])+ "." (["0"-"9"])+ > | < INTEGER: ("-")? (["0"-"9"])+ > @@ -353,8 +351,8 @@ TOKEN : | < LESSTHAN: "<" > | < GREATERTHAN: ">" > | < VARIABLE: "$" > -| < ONNX_INPUT_SL: "input" (" ")* (|) (" ")* ":" (" ")* (~["\n"])* ("\n")? > -| < ONNX_OUTPUT_SL: "output" (" ")* (|) (" ")* ":" (" ")* (~["\n"])* ("\n")? > +| < ONNX_INPUT_SL: "input" (" ")* (|) (" ")* ":" (" ")* (~["\n"])* ("\n")? > +| < ONNX_OUTPUT_SL: "output" (" ")* (|) (" ")* ":" (" ")* (~["\n"])* ("\n")? > } // Declare a special skip token for comments. @@ -2068,10 +2066,25 @@ void inputs(ParsedRankProfile profile) : } { ()* - ( reference = queryFeature() type = inputType(reference) { profile.addInput(reference, type); } ()*) * + ( input(profile) ()*) * } +void input(ParsedRankProfile profile) : +{ + Reference reference; + TensorType type; + Tensor defaultValue = null; +} +{ + reference = queryFeature() type = inputType(reference) ( ()* defaultValue = tensorValue(type) )? + { + profile.addInput(reference, type); + if (defaultValue != null) + new TensorValue(defaultValue); + } +} + TensorType inputType(Reference reference) : { TensorType type; @@ -2372,7 +2385,6 @@ void rankDegradationBinLow() : { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'min-fullrank-docs' in 'rank-degradation' is deprecated and has no effect."); } } - /** * This rule consumes part of a rank-degradation statement of a rank profile. */ @@ -2439,7 +2451,7 @@ void constantTensor(ParsedRankProfile profile, String name) : } { ()* - (( tensorString = tensorValue() | + (( tensorString = tensorValuePrefixedByValue() | tensorType = tensorTypeWithPrefix(constantTensorErrorMessage(profile.name(), name)) ) ()* )* { if (tensorType != null) { @@ -2455,7 +2467,93 @@ String constantTensorErrorMessage(String rankProfileName, String constantTensorN { return "For constant tensor '" + constantTensorName + "' in rank profile '" + rankProfileName + "'"; } } -String tensorValue() : +/** + * Parses a tensor written in a tensor literal form, + * https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form + */ +Tensor tensorValue(TensorType type) : +{ + Tensor.Builder builder = Tensor.Builder.of(type); +} +{ + ( mappedTensorValue(builder) | indexedTensorValue(builder) ) + { return builder.build(); } +} + +/** A mapped or mixed tensor value. */ +void mappedTensorValue(Tensor.Builder builder) : {} +{ + "{" ( mappedTensorBlock(builder) )* ( ()* mappedTensorBlock(builder) )* "}" +} + +void indexedTensorValue(Tensor.Builder builder) : {} +{ + "[" ( indexedTensorCell(builder) )* ( ()* indexedTensorCell(builder) )* "]" +} + +void mappedTensorBlock(Tensor.Builder builder) : +{ + TensorAddress address; + double value; +} +{ + address = tensorAddress(builder.type()) ()* value = tensorCellValue() + { builder.cell(address, value); } +} + +void indexedTensorCell(Tensor.Builder builder) : +{ + double value; +} +{ + value = tensorCellValue() + { builder.cell(value); } +} + +TensorAddress tensorAddress(TensorType type) : +{ + TensorAddress.Builder builder = new TensorAddress.Builder(type); + String label; +} +{ + ( + label = tensorAddressLabel() { builder.add(label); } + | + ( "{" ( tensorAddressElement(builder) )* ( tensorAddressElement(builder) )* "}" ) + ) + { return builder.build(); } +} + +void tensorAddressElement(TensorAddress.Builder builder) : +{ + String dimension; + String label; +} +{ + dimension = identifier() ()* label = tensorAddressLabel() + { builder.add(dimension, label); } +} + +String tensorAddressLabel() : +{ + String label; +} +{ + ( label = identifier() | label = quotedString() ) + { return label; } +} + +double tensorCellValue() : +{ + Number value; +} +{ + value = consumeNumber() + { return value.doubleValue(); } +} + +/** Undocumented syntax for supplying a tensor constant value by a string prefixed by "value" */ +String tensorValuePrefixedByValue() : { String tensor; } @@ -2701,39 +2799,28 @@ String string() : { } * unescaping of the content, it simply removes the first and last character of the image. However, the token itself can * contain anything but a double quote. * - * @return The unquoted token image. + * @return the unquoted token image */ String quotedString() : { } { - { return token.image.substring(1, token.image.length() - 1); } + ( | ) + { return token.image.substring(1, token.image.length() - 1); } } -/** - * This rule consumes a boolean value. - * - * @return The consumed boolean value. - */ +/** A boolean value. */ Boolean bool() : { } { ( ( | ) { return true; } | ( | ) { return false; } ) } -/** - * This rule consumes an integer token and returns its numeric value. - * - * @return The consumed integer value. - */ +/** Consumes an integer token and returns its numeric value. */ int integer() : { } { { return Integer.parseInt(token.image); } } -/** - * This rule consumes a long or integer token and returns its numeric value. - * - * @return The consumed long value. - */ +/** Consumes a long or integer token and returns its numeric value. */ long consumeLong() : { } { ( { return Long.parseLong(token.image); } | @@ -2741,11 +2828,7 @@ long consumeLong() : { } ) } -/** - * This rule consumes a floating-point token and returns its numeric value. - * - * @return The consumed value. - */ +/** Consumes a floating-point token and returns its numeric value. */ double consumeFloat() : { } { { return Double.valueOf(token.image); } @@ -2759,9 +2842,7 @@ Number consumeNumber() : (num = consumeFloat() | num = consumeLong()) { return num; } } -/** - * This rule consumes an opening brace with leading and trailing newline tokens. - */ +/** Consumes an opening brace with leading and trailing newline tokens. */ void lbrace() : { } { ()* ()* diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 66513a8205a..ba304f6f9ca 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -2197,15 +2197,23 @@ void secondPhaseItem(RankProfile profile) : } /** Consumes an inputs block of a rank profile. */ -void inputs(RankProfile profile) : +void inputs(RankProfile profile) : {} +{ + ()* + ( input(profile) ()*) * + +} + +void input(RankProfile profile) : { Reference reference; TensorType type; } { - ()* - ( reference = queryFeature() type = inputType(reference) { profile.addInput(reference, type); } ()*) * - + reference = queryFeature() type = inputType(reference) + { + profile.addInput(reference, type); + } } TensorType inputType(Reference reference) : diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java index a34be715e23..fb2e736a79f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java @@ -21,6 +21,9 @@ public class SchemaInfoTestCase { " query(bar) tensor(key{},x[1000])" + " query(myDouble1) double" + " query(myDouble2) tensor()" + + " query(myMap) tensor(key{}): { label1:1.0,\n \"label2\": 2.0, 'label3': 3.0 }" + + " query(myVector) tensor(x[3]):\n\n[1 ,2.0,3]" + + // " query(myMixed) tensor(key{},x[2]): { key1:[-1.0, 1.1], key2: [1,2]}" + " }" + " }"; List schemas = List.of("type1", "type2"); @@ -48,15 +51,14 @@ public class SchemaInfoTestCase { tester.assertRankProfile(schema, 5, "rankfeatures", false, true); var inputs = tester.assertRankProfile(schema, 6, "inputs", false, false); - assertEquals(4, inputs.input().size()); - assertEquals("query(foo)", inputs.input(0).name()); - assertEquals("tensor(x[10])", inputs.input(0).type()); - assertEquals("query(bar)", inputs.input(1).name()); - assertEquals("tensor(key{},x[1000])", inputs.input(1).type()); - assertEquals("query(myDouble1)", inputs.input(2).name()); - assertEquals("tensor()", inputs.input(2).type()); - assertEquals("query(myDouble2)", inputs.input(3).name()); - assertEquals("tensor()", inputs.input(3).type()); + assertEquals(6, inputs.input().size()); + assertInput("query(foo)", "tensor(x[10])", inputs.input(0)); + assertInput("query(bar)", "tensor(key{},x[1000])", inputs.input(1)); + assertInput("query(myDouble1)", "tensor()", inputs.input(2)); + assertInput("query(myDouble2)", "tensor()", inputs.input(3)); + assertInput("query(myMap)", "tensor(key{})", inputs.input(4)); + assertInput("query(myVector)", "tensor(x[3])", inputs.input(5)); + // assertInput("query(myMixed)", "tensor(key{},x[2])", inputs.input(5)); assertEquals(2, schema.summaryclass().size()); assertEquals("default", schema.summaryclass(0).name()); @@ -71,4 +73,9 @@ public class SchemaInfoTestCase { } } + private void assertInput(String name, String type, SchemaInfoConfig.Schema.Rankprofile.Input input) { + assertEquals(name, input.name()); + assertEquals(type, input.type()); + } + } -- cgit v1.2.3