Rename linguistics-tokens to tokens.

8 files changed, 16 insertions, 16 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
index 94b456b3f5e..300a55e521a 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
@@ -156,7 +156,7 @@ public class SummaryClass extends Derived {
                 summaryField.getTransform() == SummaryTransform.POSITIONS ||
                 summaryField.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER ||
                 summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER ||
-                summaryField.getTransform() == SummaryTransform.LINGUISTICS_TOKENS)
+                summaryField.getTransform() == SummaryTransform.TOKENS)
         {
             return summaryField.getSingleSource();
         } else if (summaryField.getTransform().isDynamic()) {
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java
index 54a4883fa00..2f60cd8eb06 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java
@@ -92,7 +92,7 @@ public class SummaryClassField {
                 return Type.FEATUREDATA;
             } else if (transform != null && transform.equals(SummaryTransform.SUMMARYFEATURES)) {
                 return Type.FEATUREDATA;
-            } else if (transform != null && transform.equals(SummaryTransform.LINGUISTICS_TOKENS)) {
+            } else if (transform != null && transform.equals(SummaryTransform.TOKENS)) {
                 return Type.JSONSTRING;
             } else {
                 return Type.LONGSTRING;
diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java
index 61f68defe40..ffb86f8ecf2 100644
--- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java
+++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java
@@ -217,8 +217,8 @@ public class ConvertParsedFields {
             transform = SummaryTransform.MATCHED_ELEMENTS_FILTER;
         } else if (parsed.getDynamic()) {
             transform = SummaryTransform.DYNAMICTEASER;
-        } else if (parsed.getLinguisticsTokens()) {
-            transform = SummaryTransform.LINGUISTICS_TOKENS;
+        } else if (parsed.getTokens()) {
+            transform = SummaryTransform.TOKENS;
         }
         if (parsed.getBolded()) {
             transform = transform.bold();
diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java
index 446981f1ba4..8b732c358f5 100644
--- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java
+++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java
@@ -18,7 +18,7 @@ class ParsedSummaryField extends ParsedBlock {
     private boolean isMEO = false;
     private boolean isFull = false;
     private boolean isBold = false;
-    private boolean isLinguisticsTokens = false;
+    private boolean isTokens = false;
     private final List<String> sources = new ArrayList<>();
     private final List<String> destinations = new ArrayList<>();
 
@@ -38,7 +38,7 @@ class ParsedSummaryField extends ParsedBlock {
     boolean getDynamic() { return isDyn; }
     boolean getFull() { return isFull; }
     boolean getMatchedElementsOnly() { return isMEO; }
-    boolean getLinguisticsTokens() { return isLinguisticsTokens; }
+    boolean getTokens() { return isTokens; }
 
     void addDestination(String dst) { destinations.add(dst); }
     void addSource(String src) { sources.add(src); }
@@ -46,7 +46,7 @@ class ParsedSummaryField extends ParsedBlock {
     void setDynamic() { this.isDyn = true; }
     void setFull() { this.isFull = true; }
     void setMatchedElementsOnly() { this.isMEO = true; }
-    void setLinguisticsTokens() { this.isLinguisticsTokens = true; }
+    void setTokens() { this.isTokens = true; }
     void setType(ParsedType value) {
         verifyThat(type == null, "Cannot change type from ", type, "to", value);
         this.type = value;
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
index e54f8d3e881..e4116c3f9d5 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
@@ -79,7 +79,7 @@ public class IndexingOutputs extends Processor {
             }
             dynamicSummary.add(summaryName);
         } else if (summaryTransform != SummaryTransform.ATTRIBUTE &&
-                summaryTransform != SummaryTransform.LINGUISTICS_TOKENS) {
+                summaryTransform != SummaryTransform.TOKENS) {
             staticSummary.add(summaryName);
         }
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
index c7c1606951e..50be01db04b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
@@ -24,7 +24,7 @@ public enum SummaryTransform {
     MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"),
     COPY("copy"),
     DOCUMENT_ID("documentid"),
-    LINGUISTICS_TOKENS("linguistics-tokens");
+    TOKENS("tokens");
 
     private final String name;
 
diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj
index a5238afc86a..f186caacb5f 100644
--- a/config-model/src/main/javacc/SchemaParser.jj
+++ b/config-model/src/main/javacc/SchemaParser.jj
@@ -201,7 +201,7 @@ TOKEN :
 | < FULL: "full" >
 | < STATIC: "static" >
 | < DYNAMIC: "dynamic" >
-| < LINGUISTICS_TOKENS: "linguistics-tokens" >
+| < TOKENS: "tokens" >
 | < MATCHED_ELEMENTS_ONLY: "matched-elements-only" >
 | < SSCONTEXTUAL: "contextual" >
 | < SSOVERRIDE: "override" >
@@ -1129,7 +1129,7 @@ void summaryInFieldShort(ParsedField field) :
     <COLON> ( <DYNAMIC> { psf.setDynamic(); }
               | <MATCHED_ELEMENTS_ONLY> { psf.setMatchedElementsOnly(); }
               | (<FULL> | <STATIC>) { psf.setFull(); }
-              | <LINGUISTICS_TOKENS> { psf.setLinguisticsTokens(); }
+              | <TOKENS> { psf.setTokens(); }
             )
 }
 
@@ -1175,7 +1175,7 @@ void summaryTransform(ParsedSummaryField field) : { }
     ( <DYNAMIC>              { field.setDynamic(); }
      | <MATCHED_ELEMENTS_ONLY> { field.setMatchedElementsOnly(); }
      | (<FULL> | <STATIC>)   { field.setFull(); }
-     | <LINGUISTICS_TOKENS> { field.setLinguisticsTokens(); }
+     | <TOKENS> { field.setTokens(); }
     )
 }
 
@@ -2715,7 +2715,6 @@ String identifier() : { }
       | <INLINE>
       | <INPUTS>
       | <INTEGER>
-      | <LINGUISTICS_TOKENS>
       | <LITERAL>
       | <LOCALE>
       | <LONG>
@@ -2769,6 +2768,7 @@ String identifier() : { }
       | <TERTIARY>
       | <TEXT>
       | <TO>
+      | <TOKENS>
       | <TRUE>
       | <TYPE>
       | <UCA>
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java
index 4128baddcb7..62ddc5d6b14 100644
--- a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java
@@ -227,16 +227,16 @@ public class SummaryTestCase extends AbstractSchemaTestCase {
     }
 
     @Test
-    void linguistics_tokenizer_override() throws ParseException {
+    void tokensr_override() throws ParseException {
         var schema = buildSchema("field foo type string { indexing: summary }",
                 joinLines("document-summary bar {",
                         "    summary baz type string {",
                         "        source: foo ",
-                        "        linguistics-tokens",
+                        "        tokens",
                         "     }",
                         "    from-disk",
                         "}"));
-        assertOverride(schema, "baz", SummaryTransform.LINGUISTICS_TOKENS.getName(), "foo", "bar");
+        assertOverride(schema, "baz", SummaryTransform.TOKENS.getName(), "foo", "bar");
     }
 
     @Test