diff options
10 files changed, 419 insertions, 84 deletions
diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index 421894e51cd..86a623071b5 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -1667,15 +1667,15 @@ void indexBody(ParsedIndex index) : double threshold; } { - ( <PREFIX> { index.setPrefix(true); } - | <ALIAS> <COLON> str = identifierWithDash() { index.addAlias(str); } - | <STEMMING> <COLON> str = identifierWithDash() { index.setStemming(Stemming.get(str)); } - | <ARITY> <COLON> arity = integer() { index.setArity(arity); } - | <LOWERBOUND> <COLON> num = consumeLong() { index.setLowerBound(num); } - | <UPPERBOUND> <COLON> num = consumeLong() { index.setUpperBound(num); } - | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); } - | <ENABLE_BM25> { index.setEnableBm25(true); } - | hnswIndex(index) { } + ( <PREFIX> { index.setPrefix(true); } + | <ALIAS> <COLON> str = identifierWithDash() { index.addAlias(str); } + | <STEMMING> <COLON> str = identifierWithDash() { index.setStemming(Stemming.get(str)); } + | <ARITY> <COLON> arity = integer() { index.setArity(arity); } + | <LOWERBOUND> <COLON> num = longValue() { index.setLowerBound(num); } + | <UPPERBOUND> <COLON> num = longValue() { index.setUpperBound(num); } + | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = floatValue() { index.setDensePostingListThreshold(threshold); } + | <ENABLE_BM25> { index.setEnableBm25(true); } + | hnswIndex(index) { } ) } @@ -1793,6 +1793,7 @@ String fileItem() : { (<FILE> <COLON> ( <FILE_PATH> | <STRING> | <IDENTIFIER>) { path = com.yahoo.path.Path.fromString(token.image).getRelative(); } { } (<NL>)*) { return path; } } + String uriItem() : { String path; @@ -1905,7 +1906,7 @@ String mutate_expr() : Number constant = null; } { - (("+=" | "-=" | "=") { op = token.image; } constant = consumeNumber()) + (("+=" | "-=" | "=") { op = token.image; } constant = number()) { return constant != null ? (op + constant) : op; } } @@ -1977,9 +1978,9 @@ void matchPhaseItem(MatchPhaseSettings settings) : | <ORDER> <COLON> ( <ASCENDING> { settings.setAscending(true); } | <DESCENDING> { settings.setAscending(false); } ) | <MAXHITS> <COLON> num = integer() { settings.setMaxHits(num); } - | <MAXFILTERCOVERAGE> <COLON> coverage = consumeFloat() { settings.setMaxFilterCoverage(coverage); } - | <EVALUATION_POINT> <COLON> multiplier = consumeFloat() { settings.setEvaluationPoint(multiplier); } - | <PRE_POST_FILTER_TIPPING_POINT> <COLON> multiplier = consumeFloat() { settings.setPrePostFilterTippingPoint(multiplier); } + | <MAXFILTERCOVERAGE> <COLON> coverage = floatValue() { settings.setMaxFilterCoverage(coverage); } + | <EVALUATION_POINT> <COLON> multiplier = floatValue() { settings.setEvaluationPoint(multiplier); } + | <PRE_POST_FILTER_TIPPING_POINT> <COLON> multiplier = floatValue() { settings.setPrePostFilterTippingPoint(multiplier); } ) } @@ -2008,7 +2009,7 @@ void diversityItem(DiversitySettings settings) : { ( <ATTRIBUTE> <COLON> str = identifier() { settings.setAttribute(str); } | <MIN_GROUPS> <COLON> num = integer() { settings.setMinGroups(num); } - | <CUTOFF_FACTOR> <COLON> multiplier = consumeFloat() { settings.setCutoffFactor(multiplier); } + | <CUTOFF_FACTOR> <COLON> multiplier = floatValue() { settings.setCutoffFactor(multiplier); } | <CUTOFF_STRATEGY> <COLON> ( <STRICT> { settings.setCutoffStrategy(Diversity.CutoffStrategy.strict); } | <LOOSE> { settings.setCutoffStrategy(Diversity.CutoffStrategy.loose); } @@ -2038,9 +2039,9 @@ void firstPhaseItem(ParsedRankProfile profile) : double dropLimit; } { - ( expression = expression() { profile.setFirstPhaseRanking(expression); } - | (<KEEPRANKCOUNT> <COLON> keepRankCount = integer()) { profile.setKeepRankCount(keepRankCount); } - | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = consumeFloat()) { profile.setRankScoreDropLimit(dropLimit); } + ( expression = expression() { profile.setFirstPhaseRanking(expression); } + | (<KEEPRANKCOUNT> <COLON> keepRankCount = integer()) { profile.setKeepRankCount(keepRankCount); } + | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = floatValue()) { profile.setRankScoreDropLimit(dropLimit); } ) } @@ -2254,7 +2255,7 @@ void termwiseLimit(ParsedRankProfile profile) : double num; } { - (<TERMWISELIMIT> <COLON> num = consumeFloat()) { profile.setTermwiseLimit(num); } + (<TERMWISELIMIT> <COLON> num = floatValue()) { profile.setTermwiseLimit(num); } } /** @@ -2267,7 +2268,7 @@ void postFilterThreshold(ParsedRankProfile profile) : double threshold; } { - (<POSTFILTERTHRESHOLD> <COLON> threshold = consumeFloat()) { profile.setPostFilterThreshold(threshold); } + (<POSTFILTERTHRESHOLD> <COLON> threshold = floatValue()) { profile.setPostFilterThreshold(threshold); } } /** @@ -2280,7 +2281,7 @@ void approximateThreshold(ParsedRankProfile profile) : double threshold; } { - (<APPROXIMATETHRESHOLD> <COLON> threshold = consumeFloat()) { profile.setApproximateThreshold(threshold); } + (<APPROXIMATETHRESHOLD> <COLON> threshold = floatValue()) { profile.setApproximateThreshold(threshold); } } /** @@ -2381,7 +2382,7 @@ void rankDegradationBinSize() : double freq; } { - <RPBINSIZE> <COLON> freq = consumeFloat() + <RPBINSIZE> <COLON> freq = floatValue() { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'doc-frequency' in 'rank-degradation' is deprecated and has no effect."); } } @@ -2406,7 +2407,7 @@ void rankDegradationPosbinSize() : double avgOcc; } { - <RPPOSBINSIZE> <COLON> avgOcc = consumeFloat() + <RPPOSBINSIZE> <COLON> avgOcc = floatValue() { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'occurrences-per-doc' in 'rank-degradation' is deprecated and has no effect."); } } @@ -2429,7 +2430,7 @@ void rankDegradation() : double freq; } { - ( <RANKDEGRADATIONFREQ> <COLON> freq = consumeFloat() + ( <RANKDEGRADATIONFREQ> <COLON> freq = floatValue() { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'rank-degradation-frequency' in 'rank-profile' is deprecated and has no effect."); } | <RANKDEGRADATION> lbrace() ( rankDegradationItem() (<NL>)*)+ <RBRACE> ) @@ -2526,7 +2527,7 @@ Tensor tensorValue(TensorType type) : Number doubleValue = null; } { - ( mappedTensorValue(builder) | indexedTensorValues(builder) | doubleValue = consumeNumber() ) + ( mappedTensorValue(builder) | indexedTensorValues(builder) | doubleValue = number() ) { if (doubleValue != null) { if (type.rank() > 0) @@ -2540,7 +2541,10 @@ Tensor tensorValue(TensorType type) : /** A mapped or mixed tensor value. */ void mappedTensorValue(Tensor.Builder builder) : {} { - "{" ( mappedTensorBlock(builder) )* ( <COMMA> (<NL>)* mappedTensorBlock(builder) )* "}" + "{" + ( mappedTensorBlock(builder) )* + ( <COMMA> (<NL>)* mappedTensorBlock(builder) )* + "}" } @@ -2549,11 +2553,11 @@ void mappedTensorBlock(Tensor.Builder builder) : TensorAddress mappedAddress; } { - mappedAddress = tensorAddress(builder.type().mappedSubtype()) <COLON> (<NL>)* + mappedAddress = tensorAddress(builder.type()) <COLON> (<NL>)* ( mappedTensorCellValue(mappedAddress, builder) | indexedTensorBlockValues(mappedAddress, builder) ) } -void indexedTensorBlockValues(TensorAddress sparseAddress, Tensor.Builder builder) : +void indexedTensorBlockValues(TensorAddress mappedAddress, Tensor.Builder builder) : { List<Double> values = new ArrayList<Double>(); } @@ -2565,7 +2569,7 @@ void indexedTensorBlockValues(TensorAddress sparseAddress, Tensor.Builder builde for (int i = 0; i < values.size(); i++ ) { arrayValues[i] = values.get(i); } - boundBuilder.block(sparseAddress, arrayValues); + boundBuilder.block(mappedAddress, arrayValues); } } @@ -2598,7 +2602,7 @@ void indexedTensorValue(List<Double> values) : Number value; } { - value = consumeNumber() + value = number() { values.add(value.doubleValue()); } } @@ -2613,7 +2617,7 @@ void mappedTensorCellValue(TensorAddress address, Tensor.Builder builder) : TensorAddress tensorAddress(TensorType type) : { - TensorAddress.Builder builder = new TensorAddress.Builder(type); + TensorAddress.Builder builder = new TensorAddress.PartialBuilder(type); String label; } { @@ -2649,7 +2653,7 @@ double tensorCellValue() : Number value; } { - value = consumeNumber() + value = number() { return value.doubleValue(); } } @@ -2922,7 +2926,7 @@ int integer() : { } } /** Consumes a long or integer token and returns its numeric value. */ -long consumeLong() : { } +long longValue() : { } { ( <INTEGER> { return Long.parseLong(token.image); } | <LONG> { return Long.parseLong(token.image.substring(0, token.image.length()-1)); } @@ -2930,17 +2934,17 @@ long consumeLong() : { } } /** Consumes a floating-point token and returns its numeric value. */ -double consumeFloat() : { } +double floatValue() : { } { <DOUBLE> { return Double.valueOf(token.image); } } -Number consumeNumber() : +Number number() : { Number num; } { - (num = consumeFloat() | num = consumeLong()) { return num; } + ( num = floatValue() | num = longValue() ) { return num; } } /** Consumes an opening brace with leading and trailing newline tokens. */ diff --git a/config-model/src/test/cfg/application/stateless_eval/example.model b/config-model/src/test/cfg/application/stateless_eval/example.model index 1d2db15c3ba..af1c85be4f0 100644 --- a/config-model/src/test/cfg/application/stateless_eval/example.model +++ b/config-model/src/test/cfg/application/stateless_eval/example.model @@ -7,7 +7,6 @@ model example { constants { constant1: tensor(x[3]):{{x:0}:0.5, {x:1}:1.5, {x:2}:2.5} constant2: 3.0 - #constant1asLarge tensor(x[3]): file:constant1asLarge.json } constant constant1asLarge { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/SchemaParserTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/SchemaParserTestCase.java index 17d94639d87..2284acc705c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/SchemaParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/parser/SchemaParserTestCase.java @@ -93,6 +93,7 @@ public class SchemaParserTestCase { assertNotEquals("", schema.name()); } + // TODO: Many (all)? of the files below are parsed from other tests and can be removed from here @Test public void parse_various_old_sdfiles() throws Exception { checkFileParses("src/test/cfg/search/data/travel/schemas/TTData.sd"); @@ -180,7 +181,6 @@ public class SchemaParserTestCase { checkFileParses("src/test/derived/namecollision/collision.sd"); checkFileParses("src/test/derived/namecollision/collisionstruct.sd"); checkFileParses("src/test/derived/nearestneighbor/test.sd"); - checkFileParses("src/test/derived/neuralnet/neuralnet.sd"); checkFileParses("src/test/derived/newrank/newrank.sd"); checkFileParses("src/test/derived/nuwa/newsindex.sd"); checkFileParses("src/test/derived/orderilscripts/orderilscripts.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java index 114486a5ddc..5bf2115c3f7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java @@ -13,7 +13,7 @@ import static org.junit.Assert.fail; public class RankingExpressionWithTensorTestCase { @Test - public void requireThatSingleLineConstantTensorAndTypeCanBeParsed() throws ParseException { + public void requireThatSingleLineConstantMappedTensorCanBeParsed() throws ParseException { RankProfileSearchFixture f = new RankProfileSearchFixture( " rank-profile my_profile {\n" + " first-phase {\n" + @@ -30,6 +30,40 @@ public class RankingExpressionWithTensorTestCase { } @Test + public void requireThatSingleLineConstantIndexedTensorCanBeParsed() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x[3]):{ {x:0}:1, {x:1}:2, {x:2}:3 }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x[3]):[1.0, 2.0, 3.0]", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x[3])", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatSingleLineConstantIndexedTensorShortFormCanBeParsed() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x[3]):[1, 2, 3]\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x[3]):[1.0, 2.0, 3.0]", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x[3])", "constant(my_tensor).type", "my_profile"); + } + + @Test public void requireConstantTensorCanBeReferredViaConstantFeature() throws ParseException { RankProfileSearchFixture f = new RankProfileSearchFixture( " rank-profile my_profile {\n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java index 3deeef7f2a2..9183ed316e2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java @@ -21,7 +21,7 @@ public class ModelsEvaluatorTest { public void testModelsEvaluator() { // Assumption fails but test passes on Intel macs // Assumption fails and test fails on ARM64 - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + //assumeTrue(OnnxEvaluator.isRuntimeAvailable()); ModelsEvaluator modelsEvaluator = ModelsEvaluatorTester.create("src/test/cfg/application/stateless_eval"); assertEquals(3, modelsEvaluator.models().size()); diff --git a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java index 95494190734..6b510aadd3f 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java +++ b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java @@ -63,9 +63,8 @@ public class Coverage { } public void merge(Coverage other) { - if (other == null) { - return; - } + if (other == null) return; + docs += other.getDocs(); nodes += other.getNodes(); nodesTried += other.nodesTried; @@ -77,17 +76,17 @@ public class Coverage { // explicitly incomplete beats doc count beats explicitly full switch (other.fullReason) { - case EXPLICITLY_FULL: - // do nothing - break; - case EXPLICITLY_INCOMPLETE: - fullReason = FullCoverageDefinition.EXPLICITLY_INCOMPLETE; - break; - case DOCUMENT_COUNT: - if (fullReason == FullCoverageDefinition.EXPLICITLY_FULL) { - fullReason = FullCoverageDefinition.DOCUMENT_COUNT; - } - break; + case EXPLICITLY_FULL: + // do nothing + break; + case EXPLICITLY_INCOMPLETE: + fullReason = FullCoverageDefinition.EXPLICITLY_INCOMPLETE; + break; + case DOCUMENT_COUNT: + if (fullReason == FullCoverageDefinition.EXPLICITLY_FULL) { + fullReason = FullCoverageDefinition.DOCUMENT_COUNT; + } + break; } } diff --git a/model-integration/src/main/javacc/ModelParser.jj b/model-integration/src/main/javacc/ModelParser.jj index 6f6f3508beb..668fd017aa9 100644 --- a/model-integration/src/main/javacc/ModelParser.jj +++ b/model-integration/src/main/javacc/ModelParser.jj @@ -30,8 +30,11 @@ import java.util.List; import java.util.ArrayList; import ai.vespa.rankingexpression.importer.ImportedModel; import com.yahoo.io.IOUtils; -import com.yahoo.tensor.TensorType; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.IndexedTensor; +import com.yahoo.tensor.MixedTensor; +import com.yahoo.tensor.TensorAddress; +import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.JsonFormat; import com.yahoo.searchlib.rankingexpression.RankingExpression; @@ -80,8 +83,7 @@ TOKEN : { < NL: "\n" > | < FUNCTION: "function" > -| < TENSOR_TYPE: "tensor(" (~["(",")"])+ ")" > -| < TENSORVALUE: (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? > +| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" > | < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? > | < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? > | < LBRACE: "{" > @@ -89,6 +91,7 @@ TOKEN : | < COLON: ":" > | < DOT: "." > | < COMMA: "," > +| < DOUBLE_KEYWORD: "double" > | < MODEL: "model" > | < TYPE: "type" > | < EXPRESSION_SL: "expression" (" ")* ":" (("{"<BRACE_SL_LEVEL_1>)|<BRACE_SL_CONTENT>)* ("\n")? > @@ -107,8 +110,12 @@ TOKEN : | < FILE: "file" > | < URI: "uri" > | < IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_"])* > +| < DOUBLEQUOTEDSTRING: "\"" ( ~["\""] )* "\"" > +| < SINGLEQUOTEDSTRING: "'" ( ~["'"] )* "'" > | < CONTEXT: ["a"-"z","A"-"Z"] (["a"-"z", "A"-"Z", "0"-"9"])* > | < DOUBLE: ("-")? (["0"-"9"])+ "." (["0"-"9"])+ > +| < INTEGER: ("-")? (["0"-"9"])+ > +| < LONG: ("-")? (["0"-"9"])+"L" > | < STRING: (["a"-"z","A"-"Z","_","0"-"9","."])+ > | < FILE_PATH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-", "/", "."])+ > | < HTTP: ["h","H"] ["t","T"] ["t","T"] ["p","P"] (["s","S"])? > @@ -152,7 +159,12 @@ void modelContent() : { } { - ( <NL> | input() | constants() | largeConstant() | function() )* + ( <NL> | + constants() | + largeConstant() | + function() | + input() + )* } /** Declared input variables (aka features). All non-scalar inputs must be declared. */ @@ -191,36 +203,244 @@ void constants() : } { <CONSTANTS> <LBRACE> (<NL>)* - ( name = identifier() <COLON> ( constantDouble(name) | constantTensor(name) ) (<NL>)* )* + ( constant() (<NL>)* )* <RBRACE> } -void constantDouble(String name) : +String constantTensorErrorMessage(String constantTensorName) : {} +{ + { return "For constant tensor '" + constantTensorName + "' in '" + model + "'"; } +} + +void constant() : +{ + String name = null; + TensorType type = TensorType.empty; + Tensor value = null; + String valuePath = null; +} +{ + ( + name = identifier() (<COLON>)? + ( + LOOKAHEAD(4) ( ( type = valueType(name) )? (<COLON>)? (<NL>)* ( value = tensorValue(type) | valuePath = fileItem()) + { + if (value != null) { + model.smallConstant(name, value); + } + else { + try { + value = JsonFormat.decode(type, IOUtils.readFileBytes(model.relativeFile(valuePath, "constant '" + name + "'"))); + model.largeConstant(name, value); + } + catch (Exception e) { + throw new IllegalArgumentException("Could not read constant '" + name + "'", e); + } + } + } + ) + | // Deprecated forms (TODO: Add warning on Vespa 8): + ( constantValue(name) | constantTensor(name) ) + ) + ) +} + +// Deprecated form +void constantValue(String name) : { Token value; } { - value = <DOUBLE> { model.smallConstant(name, Tensor.from(Double.parseDouble(value.image))); } + <COLON> ( value = <DOUBLE> | value = <INTEGER> | value = <IDENTIFIER> ) + { model.smallConstant(name, Tensor.from(value.image)); } } +// Deprecated form void constantTensor(String name) : { + String tensorString = ""; + TensorType type = null; +} +{ + <LBRACE> (<NL>)* + (( tensorString = tensorValuePrefixedByValue() | + type = tensorTypeWithPrefix(constantTensorErrorMessage(name)) ) (<NL>)* )* <RBRACE> + { model.smallConstant(name, type != null ? Tensor.from(type, tensorString) : Tensor.from(tensorString)); } +} + +TensorType valueType(String name) : +{ TensorType type; - Token value; + +} +{ + ( + ( type = tensorType("Type of " + name) ) + | + ( <DOUBLE_KEYWORD> { type = TensorType.empty; } ) + ) + { return type; } +} + +TensorType tensorType(String errorMessage) : +{ + String tensorTypeString; +} +{ + <TENSOR_TYPE> { tensorTypeString = token.image; } + { + TensorType tensorType; + try { + tensorType = TensorType.fromSpec(tensorTypeString); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(errorMessage + ": Illegal tensor type spec: " + e.getMessage()); + } + return tensorType; + } +} + +/** + * Parses a tensor written in a tensor literal form, + * https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form + */ +Tensor tensorValue(TensorType type) : +{ + Tensor.Builder builder = Tensor.Builder.of(type); + Number doubleValue = null; } { - type = tensorType("constant '" + name + "'") value = <TENSORVALUE> - { - model.smallConstant(name, Tensor.from(type, value.image.substring(1))); - } + ( mappedTensorValue(builder) | indexedTensorValues(builder) | doubleValue = number() ) + { + if (doubleValue != null) { + if (type.rank() > 0) + throw new IllegalArgumentException("A tensor of type " + type + " cannot be a number"); + builder.cell(doubleValue.doubleValue()); + } + return builder.build(); + } } -String constantTensorErrorMessage(String model, String constantTensorName) : {} +/** A mapped or mixed tensor value. */ +void mappedTensorValue(Tensor.Builder builder) : {} { - { return "For constant tensor '" + constantTensorName + "' in model '" + model + "'"; } + "{" + ( mappedTensorBlock(builder) )* + ( <COMMA> (<NL>)* mappedTensorBlock(builder) )* + "}" } -String tensorValue() : + +void mappedTensorBlock(Tensor.Builder builder) : +{ + TensorAddress mappedAddress; +} +{ + mappedAddress = tensorAddress(builder.type()) <COLON> (<NL>)* + ( mappedTensorCellValue(mappedAddress, builder) | indexedTensorBlockValues(mappedAddress, builder) ) +} + +void indexedTensorBlockValues(TensorAddress mappedAddress, Tensor.Builder builder) : +{ + List<Double> values = new ArrayList<Double>(); +} +{ + arrayTensorValues(values) + { + MixedTensor.BoundBuilder boundBuilder = (MixedTensor.BoundBuilder)builder; + double[] arrayValues = new double[values.size()]; + for (int i = 0; i < values.size(); i++ ) { + arrayValues[i] = values.get(i); + } + boundBuilder.block(mappedAddress, arrayValues); + } +} + +void indexedTensorValues(Tensor.Builder builder) : +{ + List<Double> values = new ArrayList<Double>(); +} +{ + arrayTensorValues(values) + { + IndexedTensor.BoundBuilder boundBuilder = (IndexedTensor.BoundBuilder)builder; + double[] arrayValues = new double[values.size()]; + for (int i = 0; i < values.size(); i++ ) { + arrayValues[i] = values.get(i); + } + boundBuilder.fill(arrayValues); + } +} + +/** Tensor array values. Using sub-bracketing for multiple dimensions is optional and therefore ignored here. */ +void arrayTensorValues(List<Double> values) : {} +{ + "[" ( ( indexedTensorValue(values) | arrayTensorValues(values)) )* + ( <COMMA> (<NL>)* ( indexedTensorValue(values) | arrayTensorValues(values)) )* + "]" +} + +void indexedTensorValue(List<Double> values) : +{ + Number value; +} +{ + value = number() + { values.add(value.doubleValue()); } +} + +void mappedTensorCellValue(TensorAddress address, Tensor.Builder builder) : +{ + double value; +} +{ + value = tensorCellValue() + { builder.cell(address, value); } +} + +TensorAddress tensorAddress(TensorType type) : +{ + TensorAddress.Builder builder = new TensorAddress.PartialBuilder(type); + String label; +} +{ + ( + label = tensorAddressLabel() { builder.add(label); } + | + ( "{" ( tensorAddressElement(builder) )* ( <COMMA> tensorAddressElement(builder) )* "}" ) + ) + { return builder.build(); } +} + +void tensorAddressElement(TensorAddress.Builder builder) : +{ + String dimension; + String label; +} +{ + dimension = identifier() <COLON> (<NL>)* label = tensorAddressLabel() + { builder.add(dimension, label); } +} + +String tensorAddressLabel() : +{ + String label; +} +{ + ( label = identifier() | label = quotedString() ) + { return label; } +} + +double tensorCellValue() : +{ + Number value; +} +{ + value = number() + { return value.doubleValue(); } +} + +/** Undocumented syntax for supplying a tensor constant value by a string prefixed by "value" */ +String tensorValuePrefixedByValue() : { String tensor; } @@ -233,7 +453,7 @@ String tensorValue() : } } -TensorType tensorType(String errorMessage) : +TensorType tensorTypeWithPrefix(String errorMessage) : { String tensorTypeString; } @@ -250,7 +470,7 @@ TensorType tensorType(String errorMessage) : } } -/** Consumes a large constant. */ +/** Consumes a large constant. */ // TODO: Remove on Vespa 9 void largeConstant() : { String name; @@ -311,18 +531,64 @@ String expression() : String identifier() : { } { ( - <IDENTIFIER> - | <DOUBLE> - | <FILE> - | <URI> - | <MODEL> - | <TYPE> + <CONSTANT> | + <CONSTANTS> | + <DOUBLE_KEYWORD> | + <FILE> | + <IDENTIFIER> | + <INTEGER> | + <MODEL> | + <TYPE> | + <URI> ) { return token.image; } } +Number number() : +{ + Number num; +} +{ + (num = floatValue() | num = longValue() ) { return num; } +} + +/** Consumes a long or integer token and returns its numeric value. */ +long longValue() : { } +{ + ( <INTEGER> { return Long.parseLong(token.image); } | + <LONG> { return Long.parseLong(token.image.substring(0, token.image.length()-1)); } + ) +} + +/** Consumes a floating-point token and returns its numeric value. */ +double floatValue() : { } +{ + <DOUBLE> { return Double.valueOf(token.image); } +} + /** Consumes an opening brace with leading and trailing newline tokens. */ void lbrace() : { } { (<NL>)* <LBRACE> (<NL>)* } + +String fileItem() : +{ + String path; +} +{ + (<FILE> <COLON> ( <FILE_PATH> | <STRING> | <IDENTIFIER>) { path = com.yahoo.path.Path.fromString(token.image).getRelative(); } { } (<NL>)*) { return path; } +} + +/** + * Consumes a quoted string token and returns the token image minus the quotes. This does not perform + * unescaping of the content, it simply removes the first and last character of the image. However, the token itself can + * contain anything but a double quote. + * + * @return the unquoted token image + */ +String quotedString() : { } +{ + ( <DOUBLEQUOTEDSTRING> | <SINGLEQUOTEDSTRING> ) + { return token.image.substring(1, token.image.length() - 1); } +} diff --git a/model-integration/src/test/models/vespa/example.model b/model-integration/src/test/models/vespa/example.model index 269ed83b695..fd8565f2b92 100644 --- a/model-integration/src/test/models/vespa/example.model +++ b/model-integration/src/test/models/vespa/example.model @@ -7,11 +7,7 @@ model example { constants { constant1: tensor(x[3]):{{x:0}:0.5, {x:1}:1.5, {x:2}:2.5} constant2: 3.0 - } - - constant constant1asLarge { - type: tensor(x[3]) - file: constant1asLarge.json + constant1asLarge tensor(x[3]): file:constant1asLarge.json } function foo1() { diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index f7be61946ba..654042372cf 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1273,6 +1273,18 @@ ], "fields": [] }, + "com.yahoo.tensor.TensorAddress$PartialBuilder": { + "superClass": "com.yahoo.tensor.TensorAddress$Builder", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.tensor.TensorType)", + "public com.yahoo.tensor.TensorAddress$Builder copy()" + ], + "fields": [] + }, "com.yahoo.tensor.TensorAddress": { "superClass": "java.lang.Object", "interfaces": [ diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java index d9ab67d6c5f..bce8aa7c82b 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -171,8 +171,8 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { /** Builder of a tensor address */ public static class Builder { - private final TensorType type; - private final String[] labels; + final TensorType type; + final String[] labels; public Builder(TensorType type) { this(type, new String[type.dimensions().size()]); @@ -218,14 +218,39 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { /** Returns the type of the tensor this address is being built for. */ public TensorType type() { return type; } - public TensorAddress build() { + void validate() { for (int i = 0; i < labels.length; i++) if (labels[i] == null) throw new IllegalArgumentException("Missing a label for dimension " + type.dimensions().get(i).name() + " for " + type); + } + + public TensorAddress build() { + validate(); return TensorAddress.of(labels); } } + /** Builder of an address to a subset of the dimensions of a tensor type */ + public static class PartialBuilder extends Builder { + + public PartialBuilder(TensorType type) { + super(type); + } + + private PartialBuilder(TensorType type, String[] labels) { + super(type, labels); + } + + /** Creates a copy of this which can be modified separately */ + public Builder copy() { + return new PartialBuilder(type, Arrays.copyOf(labels, labels.length)); + } + + @Override + void validate() { } + + } + } |