Revert "Bratseth/model syntax"

6 files changed, 45 insertions, 84 deletions

diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj
index 86a623071b5..421894e51cd 100644
--- a/config-model/src/main/javacc/SchemaParser.jj
+++ b/config-model/src/main/javacc/SchemaParser.jj
@@ -1667,15 +1667,15 @@ void indexBody(ParsedIndex index) :
     double threshold;
 }
 {
-    ( <PREFIX>                                                       { index.setPrefix(true); }
-      | <ALIAS> <COLON> str = identifierWithDash()                   { index.addAlias(str); }
-      | <STEMMING> <COLON> str = identifierWithDash()                { index.setStemming(Stemming.get(str)); }
-      | <ARITY> <COLON> arity = integer()                            { index.setArity(arity); }
-      | <LOWERBOUND> <COLON> num = longValue()                       { index.setLowerBound(num); }
-      | <UPPERBOUND> <COLON> num = longValue()                       { index.setUpperBound(num); }
-      | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = floatValue() { index.setDensePostingListThreshold(threshold); }
-      | <ENABLE_BM25>                                                { index.setEnableBm25(true); }
-      | hnswIndex(index)                                             { }
+    ( <PREFIX>                                           { index.setPrefix(true); }
+      | <ALIAS> <COLON> str = identifierWithDash()       { index.addAlias(str); }
+      | <STEMMING> <COLON> str = identifierWithDash()    { index.setStemming(Stemming.get(str)); }
+      | <ARITY> <COLON> arity = integer()                              { index.setArity(arity); }
+      | <LOWERBOUND> <COLON> num = consumeLong()                       { index.setLowerBound(num); }
+      | <UPPERBOUND> <COLON> num = consumeLong()                       { index.setUpperBound(num); }
+      | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); }
+      | <ENABLE_BM25>                                                  { index.setEnableBm25(true); }
+      | hnswIndex(index)                                               { }
     )
 }
 
@@ -1793,7 +1793,6 @@ String fileItem() :
 {
   (<FILE> <COLON> ( <FILE_PATH> | <STRING> | <IDENTIFIER>) { path = com.yahoo.path.Path.fromString(token.image).getRelative(); } { } (<NL>)*) { return path; }
 }
-
 String uriItem() :
 {
    String path;
@@ -1906,7 +1905,7 @@ String mutate_expr() :
      Number constant = null;
 }
 {
-    (("+=" | "-=" | "=") { op = token.image; } constant = number())
+    (("+=" | "-=" | "=") { op = token.image; } constant = consumeNumber())
     { return constant != null ? (op + constant) : op; }
 }
 
@@ -1978,9 +1977,9 @@ void matchPhaseItem(MatchPhaseSettings settings) :
       | <ORDER> <COLON> ( <ASCENDING> { settings.setAscending(true); } 
                           | <DESCENDING> { settings.setAscending(false); } )
       | <MAXHITS> <COLON> num = integer() { settings.setMaxHits(num); }
-      | <MAXFILTERCOVERAGE> <COLON> coverage = floatValue() { settings.setMaxFilterCoverage(coverage); }
-      | <EVALUATION_POINT> <COLON> multiplier = floatValue() { settings.setEvaluationPoint(multiplier); }
-      | <PRE_POST_FILTER_TIPPING_POINT> <COLON> multiplier = floatValue() { settings.setPrePostFilterTippingPoint(multiplier); }
+      | <MAXFILTERCOVERAGE> <COLON> coverage = consumeFloat() { settings.setMaxFilterCoverage(coverage); }
+      | <EVALUATION_POINT> <COLON> multiplier = consumeFloat() { settings.setEvaluationPoint(multiplier); }
+      | <PRE_POST_FILTER_TIPPING_POINT> <COLON> multiplier = consumeFloat() { settings.setPrePostFilterTippingPoint(multiplier); }
     )
 }
 
@@ -2009,7 +2008,7 @@ void diversityItem(DiversitySettings settings) :
 {
     (   <ATTRIBUTE> <COLON> str = identifier() { settings.setAttribute(str); }
       | <MIN_GROUPS> <COLON> num = integer() { settings.setMinGroups(num); }
-      | <CUTOFF_FACTOR> <COLON> multiplier = floatValue() { settings.setCutoffFactor(multiplier); }
+      | <CUTOFF_FACTOR> <COLON> multiplier = consumeFloat() { settings.setCutoffFactor(multiplier); }
       | <CUTOFF_STRATEGY> <COLON>
         (   <STRICT> { settings.setCutoffStrategy(Diversity.CutoffStrategy.strict); }
           | <LOOSE>  { settings.setCutoffStrategy(Diversity.CutoffStrategy.loose); }
@@ -2039,9 +2038,9 @@ void firstPhaseItem(ParsedRankProfile profile) :
     double dropLimit;
 }
 {
-    ( expression = expression()                               { profile.setFirstPhaseRanking(expression); }
-    | (<KEEPRANKCOUNT> <COLON> keepRankCount = integer())     { profile.setKeepRankCount(keepRankCount); }
-    | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = floatValue()) { profile.setRankScoreDropLimit(dropLimit); }
+    ( expression = expression()                                 { profile.setFirstPhaseRanking(expression); }
+    | (<KEEPRANKCOUNT> <COLON> keepRankCount = integer())       { profile.setKeepRankCount(keepRankCount); }
+    | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = consumeFloat()) { profile.setRankScoreDropLimit(dropLimit); }
     )
 }
 
@@ -2255,7 +2254,7 @@ void termwiseLimit(ParsedRankProfile profile) :
     double num;
 }
 {
-    (<TERMWISELIMIT> <COLON> num = floatValue()) { profile.setTermwiseLimit(num); }
+    (<TERMWISELIMIT> <COLON> num = consumeFloat()) { profile.setTermwiseLimit(num); }
 }
 
 /**
@@ -2268,7 +2267,7 @@ void postFilterThreshold(ParsedRankProfile profile) :
     double threshold;
 }
 {
-    (<POSTFILTERTHRESHOLD> <COLON> threshold = floatValue()) { profile.setPostFilterThreshold(threshold); }
+    (<POSTFILTERTHRESHOLD> <COLON> threshold = consumeFloat()) { profile.setPostFilterThreshold(threshold); }
 }
 
 /**
@@ -2281,7 +2280,7 @@ void approximateThreshold(ParsedRankProfile profile) :
     double threshold;
 }
 {
-    (<APPROXIMATETHRESHOLD> <COLON> threshold = floatValue()) { profile.setApproximateThreshold(threshold); }
+    (<APPROXIMATETHRESHOLD> <COLON> threshold = consumeFloat()) { profile.setApproximateThreshold(threshold); }
 }
 
 /**
@@ -2382,7 +2381,7 @@ void rankDegradationBinSize() :
     double freq;
 }
 {
-    <RPBINSIZE> <COLON> freq = floatValue()
+    <RPBINSIZE> <COLON> freq = consumeFloat()
     { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'doc-frequency' in 'rank-degradation' is deprecated and has no effect."); }
 }
 
@@ -2407,7 +2406,7 @@ void rankDegradationPosbinSize() :
     double avgOcc;
 }
 {
-    <RPPOSBINSIZE> <COLON> avgOcc = floatValue()
+    <RPPOSBINSIZE> <COLON> avgOcc = consumeFloat()
     { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'occurrences-per-doc' in 'rank-degradation' is deprecated and has no effect."); }
 }
 
@@ -2430,7 +2429,7 @@ void rankDegradation() :
     double freq;
 }
 {
-    ( <RANKDEGRADATIONFREQ> <COLON> freq = floatValue()
+    ( <RANKDEGRADATIONFREQ> <COLON> freq = consumeFloat()
     { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'rank-degradation-frequency' in 'rank-profile' is deprecated and has no effect."); }
       | <RANKDEGRADATION> lbrace() ( rankDegradationItem() (<NL>)*)+ <RBRACE>
     )
@@ -2527,7 +2526,7 @@ Tensor tensorValue(TensorType type) :
     Number doubleValue = null;
 }
 {
-    ( mappedTensorValue(builder) | indexedTensorValues(builder) |  doubleValue = number() )
+    ( mappedTensorValue(builder) | indexedTensorValues(builder) |  doubleValue = consumeNumber() )
     {
         if (doubleValue != null) {
             if (type.rank() > 0)
@@ -2541,10 +2540,7 @@ Tensor tensorValue(TensorType type) :
 /** A mapped or mixed tensor value. */
 void mappedTensorValue(Tensor.Builder builder) : {}
 {
-    "{"
-    ( mappedTensorBlock(builder) )*
-    ( <COMMA> (<NL>)* mappedTensorBlock(builder) )*
-    "}"
+    "{" ( mappedTensorBlock(builder) )* ( <COMMA> (<NL>)* mappedTensorBlock(builder) )* "}"
 }
 
 
@@ -2553,11 +2549,11 @@ void mappedTensorBlock(Tensor.Builder builder) :
     TensorAddress mappedAddress;
 }
 {
-    mappedAddress = tensorAddress(builder.type()) <COLON> (<NL>)*
+    mappedAddress = tensorAddress(builder.type().mappedSubtype()) <COLON> (<NL>)*
     ( mappedTensorCellValue(mappedAddress, builder) | indexedTensorBlockValues(mappedAddress, builder) )
 }
 
-void indexedTensorBlockValues(TensorAddress mappedAddress, Tensor.Builder builder) :
+void indexedTensorBlockValues(TensorAddress sparseAddress, Tensor.Builder builder) :
 {
     List<Double> values = new ArrayList<Double>();
 }
@@ -2569,7 +2565,7 @@ void indexedTensorBlockValues(TensorAddress mappedAddress, Tensor.Builder builde
         for (int i = 0; i < values.size(); i++ ) {
             arrayValues[i] = values.get(i);
         }
-        boundBuilder.block(mappedAddress, arrayValues);
+        boundBuilder.block(sparseAddress, arrayValues);
     }
 }
 
@@ -2602,7 +2598,7 @@ void indexedTensorValue(List<Double> values) :
     Number value;
 }
 {
-    value = number()
+    value = consumeNumber()
     { values.add(value.doubleValue()); }
 }
 
@@ -2617,7 +2613,7 @@ void mappedTensorCellValue(TensorAddress address, Tensor.Builder builder) :
 
 TensorAddress tensorAddress(TensorType type) :
 {
-    TensorAddress.Builder builder = new TensorAddress.PartialBuilder(type);
+    TensorAddress.Builder builder = new TensorAddress.Builder(type);
     String label;
 }
 {
@@ -2653,7 +2649,7 @@ double tensorCellValue() :
     Number value;
 }
 {
-    value = number()
+    value = consumeNumber()
     { return value.doubleValue(); }
 }
 
@@ -2926,7 +2922,7 @@ int integer() : { }
 }
 
 /** Consumes a long or integer token and returns its numeric value. */
-long longValue() : { }
+long consumeLong() : { }
 {
     ( <INTEGER> { return Long.parseLong(token.image); } |
       <LONG>    { return Long.parseLong(token.image.substring(0, token.image.length()-1)); }
@@ -2934,17 +2930,17 @@ long longValue() : { }
 }
 
 /** Consumes a floating-point token and returns its numeric value. */
-double floatValue() : { }
+double consumeFloat() : { }
 {
     <DOUBLE> { return Double.valueOf(token.image); }
 }
 
-Number number() :
+Number consumeNumber() :
 {
     Number num;
 }
 {
-    ( num = floatValue() | num = longValue() ) { return num; }
+    (num = consumeFloat() | num = consumeLong()) { return num; }
 }
 
 /** Consumes an opening brace with leading and trailing newline tokens. */
diff --git a/config-model/src/test/cfg/application/stateless_eval/example.model b/config-model/src/test/cfg/application/stateless_eval/example.model
index af1c85be4f0..1d2db15c3ba 100644
--- a/config-model/src/test/cfg/application/stateless_eval/example.model
+++ b/config-model/src/test/cfg/application/stateless_eval/example.model
@@ -7,6 +7,7 @@ model example {
     constants {
         constant1: tensor(x[3]):{{x:0}:0.5, {x:1}:1.5, {x:2}:2.5}
         constant2: 3.0
+        #constant1asLarge tensor(x[3]): file:constant1asLarge.json
     }
 
     constant constant1asLarge {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/SchemaParserTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/SchemaParserTestCase.java
index 2284acc705c..17d94639d87 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/SchemaParserTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/parser/SchemaParserTestCase.java
@@ -93,7 +93,6 @@ public class SchemaParserTestCase {
         assertNotEquals("", schema.name());
     }
 
-    // TODO: Many (all)? of the files below are parsed from other tests and can be removed from here
     @Test
     public void parse_various_old_sdfiles() throws Exception {
         checkFileParses("src/test/cfg/search/data/travel/schemas/TTData.sd");
@@ -181,6 +180,7 @@ public class SchemaParserTestCase {
         checkFileParses("src/test/derived/namecollision/collision.sd");
         checkFileParses("src/test/derived/namecollision/collisionstruct.sd");
         checkFileParses("src/test/derived/nearestneighbor/test.sd");
+        checkFileParses("src/test/derived/neuralnet/neuralnet.sd");
         checkFileParses("src/test/derived/newrank/newrank.sd");
         checkFileParses("src/test/derived/nuwa/newsindex.sd");
         checkFileParses("src/test/derived/orderilscripts/orderilscripts.sd");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java
index 5bf2115c3f7..114486a5ddc 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java
@@ -13,7 +13,7 @@ import static org.junit.Assert.fail;
 public class RankingExpressionWithTensorTestCase {
 
     @Test
-    public void requireThatSingleLineConstantMappedTensorCanBeParsed() throws ParseException {
+    public void requireThatSingleLineConstantTensorAndTypeCanBeParsed() throws ParseException {
         RankProfileSearchFixture f = new RankProfileSearchFixture(
                 "  rank-profile my_profile {\n" +
                 "    first-phase {\n" +
@@ -30,40 +30,6 @@ public class RankingExpressionWithTensorTestCase {
     }
 
     @Test
-    public void requireThatSingleLineConstantIndexedTensorCanBeParsed() throws ParseException {
-        RankProfileSearchFixture f = new RankProfileSearchFixture(
-                "  rank-profile my_profile {\n" +
-                "    first-phase {\n" +
-                "      expression: sum(my_tensor)\n" +
-                "    }\n" +
-                "    constants {\n" +
-                "      my_tensor tensor(x[3]):{ {x:0}:1, {x:1}:2, {x:2}:3 }\n" +
-                "    }\n" +
-                "  }");
-        f.compileRankProfile("my_profile");
-        f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile");
-        f.assertRankProperty("tensor(x[3]):[1.0, 2.0, 3.0]", "constant(my_tensor).value", "my_profile");
-        f.assertRankProperty("tensor(x[3])", "constant(my_tensor).type", "my_profile");
-    }
-
-    @Test
-    public void requireThatSingleLineConstantIndexedTensorShortFormCanBeParsed() throws ParseException {
-        RankProfileSearchFixture f = new RankProfileSearchFixture(
-                "  rank-profile my_profile {\n" +
-                "    first-phase {\n" +
-                "      expression: sum(my_tensor)\n" +
-                "    }\n" +
-                "    constants {\n" +
-                "      my_tensor tensor(x[3]):[1, 2, 3]\n" +
-                "    }\n" +
-                "  }");
-        f.compileRankProfile("my_profile");
-        f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile");
-        f.assertRankProperty("tensor(x[3]):[1.0, 2.0, 3.0]", "constant(my_tensor).value", "my_profile");
-        f.assertRankProperty("tensor(x[3])", "constant(my_tensor).type", "my_profile");
-    }
-
-    @Test
     public void requireConstantTensorCanBeReferredViaConstantFeature() throws ParseException {
         RankProfileSearchFixture f = new RankProfileSearchFixture(
                 "  rank-profile my_profile {\n" +
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java
index 9183ed316e2..3deeef7f2a2 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java
@@ -21,7 +21,7 @@ public class ModelsEvaluatorTest {
     public void testModelsEvaluator() {
         // Assumption fails but test passes on Intel macs
         // Assumption fails and test fails on ARM64
-        //assumeTrue(OnnxEvaluator.isRuntimeAvailable());
+        assumeTrue(OnnxEvaluator.isRuntimeAvailable());
 
         ModelsEvaluator modelsEvaluator = ModelsEvaluatorTester.create("src/test/cfg/application/stateless_eval");
         assertEquals(3, modelsEvaluator.models().size());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java
index b8a14bc763b..34afca703e9 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java
@@ -25,8 +25,7 @@ public class SchemaInfoTestCase {
                 "      query(myDouble1) double: 0.5" +
                 "      query(myDouble2) tensor()" +
                 "      query(myMap) tensor(key{}): { label1:1.0,\n \"label2\": 2.0, 'label3': 3.0 }" +
-                "      query(myVector1) tensor(x[3]):\n\n[1 ,2.0,3]" +
-                "      query(myVector2) tensor(x[3]):{{x:0}:1,{x: 1}: 2 , { x:2}:3.0 }" +
+                "      query(myVector) tensor(x[3]):\n\n[1 ,2.0,3]" +
                 "      query(myMatrix) tensor(x[2],y[3]):[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]" +
                 "      query(myMixed1) tensor(key{},x[2]): { key1:[-1.0, 1.1], key2: [1,2]}" +
                 "      query(myMixed2) tensor(k1{},k2{},x[2]): { {k1:l1,k2:l1}:[-1.0, 1.1], {k1:l1,k2:l2}: [1,2]}" +
@@ -59,7 +58,7 @@ public class SchemaInfoTestCase {
                 tester.assertRankProfile(schema, 5, "rankfeatures", false, true);
 
                 var schemaInfoProfile = tester.assertRankProfile(schema, 6, "inputs", false, false);
-                assertEquals(10, schemaInfoProfile.input().size());
+                assertEquals(9, schemaInfoProfile.input().size());
                 var rankProfilesProfile = rankProfilesConfig.rankprofile().get(6);
                 assertEquals("inputs", rankProfilesProfile.name());
                 assertInput("query(foo)", "tensor<float>(x[10])", null, 0, schemaInfoProfile, rankProfilesProfile);
@@ -67,11 +66,10 @@ public class SchemaInfoTestCase {
                 assertInput("query(myDouble1)", "tensor()", "0.5", 2, schemaInfoProfile, rankProfilesProfile);
                 assertInput("query(myDouble2)", "tensor()", null, 3, schemaInfoProfile, rankProfilesProfile);
                 assertInput("query(myMap)", "tensor(key{})", "tensor(key{}):{{key:label1}:1.0, {key:label2}:2.0, {key:label3}:3.0}", 4, schemaInfoProfile, rankProfilesProfile);
-                assertInput("query(myVector1)", "tensor(x[3])", "tensor(x[3]):{{x:0}:1.0, {x:1}:2.0, {x:2}:3.0}", 5, schemaInfoProfile, rankProfilesProfile);
-                assertInput("query(myVector2)", "tensor(x[3])", "tensor(x[3]):{{x:0}:1.0, {x:1}:2.0, {x:2}:3.0}", 6, schemaInfoProfile, rankProfilesProfile);
-                assertInput("query(myMatrix)", "tensor(x[2],y[3])", "tensor(x[2],y[3]):{{x:0,y:0}:1.0, {x:0,y:1}:2.0, {x:0,y:2}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0, {x:1,y:2}:6.0}", 7, schemaInfoProfile, rankProfilesProfile);
-                assertInput("query(myMixed1)", "tensor(key{},x[2])", "tensor(key{},x[2]):{{key:key1,x:0}:-1.0, {key:key1,x:1}:1.1, {key:key2,x:0}:1.0, {key:key2,x:1}:2.0}", 8, schemaInfoProfile, rankProfilesProfile);
-                assertInput("query(myMixed2)", "tensor(k1{},k2{},x[2])", "tensor(k1{},k2{},x[2]):{{k1:l1,k2:l1,x:0}:-1.0, {k1:l1,k2:l1,x:1}:1.1, {k1:l1,k2:l2,x:0}:1.0, {k1:l1,k2:l2,x:1}:2.0}", 9, schemaInfoProfile, rankProfilesProfile);
+                assertInput("query(myVector)", "tensor(x[3])", "tensor(x[3]):{{x:0}:1.0, {x:1}:2.0, {x:2}:3.0}", 5, schemaInfoProfile, rankProfilesProfile);
+                assertInput("query(myMatrix)", "tensor(x[2],y[3])", "tensor(x[2],y[3]):{{x:0,y:0}:1.0, {x:0,y:1}:2.0, {x:0,y:2}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0, {x:1,y:2}:6.0}", 6, schemaInfoProfile, rankProfilesProfile);
+                assertInput("query(myMixed1)", "tensor(key{},x[2])", "tensor(key{},x[2]):{{key:key1,x:0}:-1.0, {key:key1,x:1}:1.1, {key:key2,x:0}:1.0, {key:key2,x:1}:2.0}", 7, schemaInfoProfile, rankProfilesProfile);
+                assertInput("query(myMixed2)", "tensor(k1{},k2{},x[2])", "tensor(k1{},k2{},x[2]):{{k1:l1,k2:l1,x:0}:-1.0, {k1:l1,k2:l1,x:1}:1.1, {k1:l1,k2:l2,x:0}:1.0, {k1:l1,k2:l2,x:1}:2.0}", 8, schemaInfoProfile, rankProfilesProfile);
 
                 assertEquals(2, schema.summaryclass().size());
                 assertEquals("default", schema.summaryclass(0).name());