Rename linguistics-tokens to tokens.

20 files changed, 75 insertions, 75 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
index 94b456b3f5e..300a55e521a 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
@@ -156,7 +156,7 @@ public class SummaryClass extends Derived {
                 summaryField.getTransform() == SummaryTransform.POSITIONS ||
                 summaryField.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER ||
                 summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER ||
-                summaryField.getTransform() == SummaryTransform.LINGUISTICS_TOKENS)
+                summaryField.getTransform() == SummaryTransform.TOKENS)
         {
             return summaryField.getSingleSource();
         } else if (summaryField.getTransform().isDynamic()) {
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java
index 54a4883fa00..2f60cd8eb06 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java
@@ -92,7 +92,7 @@ public class SummaryClassField {
                 return Type.FEATUREDATA;
             } else if (transform != null && transform.equals(SummaryTransform.SUMMARYFEATURES)) {
                 return Type.FEATUREDATA;
-            } else if (transform != null && transform.equals(SummaryTransform.LINGUISTICS_TOKENS)) {
+            } else if (transform != null && transform.equals(SummaryTransform.TOKENS)) {
                 return Type.JSONSTRING;
             } else {
                 return Type.LONGSTRING;
diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java
index 61f68defe40..ffb86f8ecf2 100644
--- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java
+++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java
@@ -217,8 +217,8 @@ public class ConvertParsedFields {
             transform = SummaryTransform.MATCHED_ELEMENTS_FILTER;
         } else if (parsed.getDynamic()) {
             transform = SummaryTransform.DYNAMICTEASER;
-        } else if (parsed.getLinguisticsTokens()) {
-            transform = SummaryTransform.LINGUISTICS_TOKENS;
+        } else if (parsed.getTokens()) {
+            transform = SummaryTransform.TOKENS;
         }
         if (parsed.getBolded()) {
             transform = transform.bold();
diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java
index 446981f1ba4..8b732c358f5 100644
--- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java
+++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java
@@ -18,7 +18,7 @@ class ParsedSummaryField extends ParsedBlock {
     private boolean isMEO = false;
     private boolean isFull = false;
     private boolean isBold = false;
-    private boolean isLinguisticsTokens = false;
+    private boolean isTokens = false;
     private final List<String> sources = new ArrayList<>();
     private final List<String> destinations = new ArrayList<>();
 
@@ -38,7 +38,7 @@ class ParsedSummaryField extends ParsedBlock {
     boolean getDynamic() { return isDyn; }
     boolean getFull() { return isFull; }
     boolean getMatchedElementsOnly() { return isMEO; }
-    boolean getLinguisticsTokens() { return isLinguisticsTokens; }
+    boolean getTokens() { return isTokens; }
 
     void addDestination(String dst) { destinations.add(dst); }
     void addSource(String src) { sources.add(src); }
@@ -46,7 +46,7 @@ class ParsedSummaryField extends ParsedBlock {
     void setDynamic() { this.isDyn = true; }
     void setFull() { this.isFull = true; }
     void setMatchedElementsOnly() { this.isMEO = true; }
-    void setLinguisticsTokens() { this.isLinguisticsTokens = true; }
+    void setTokens() { this.isTokens = true; }
     void setType(ParsedType value) {
         verifyThat(type == null, "Cannot change type from ", type, "to", value);
         this.type = value;
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
index e54f8d3e881..e4116c3f9d5 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
@@ -79,7 +79,7 @@ public class IndexingOutputs extends Processor {
             }
             dynamicSummary.add(summaryName);
         } else if (summaryTransform != SummaryTransform.ATTRIBUTE &&
-                summaryTransform != SummaryTransform.LINGUISTICS_TOKENS) {
+                summaryTransform != SummaryTransform.TOKENS) {
             staticSummary.add(summaryName);
         }
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
index c7c1606951e..50be01db04b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
@@ -24,7 +24,7 @@ public enum SummaryTransform {
     MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"),
     COPY("copy"),
     DOCUMENT_ID("documentid"),
-    LINGUISTICS_TOKENS("linguistics-tokens");
+    TOKENS("tokens");
 
     private final String name;
 
diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj
index a5238afc86a..f186caacb5f 100644
--- a/config-model/src/main/javacc/SchemaParser.jj
+++ b/config-model/src/main/javacc/SchemaParser.jj
@@ -201,7 +201,7 @@ TOKEN :
 | < FULL: "full" >
 | < STATIC: "static" >
 | < DYNAMIC: "dynamic" >
-| < LINGUISTICS_TOKENS: "linguistics-tokens" >
+| < TOKENS: "tokens" >
 | < MATCHED_ELEMENTS_ONLY: "matched-elements-only" >
 | < SSCONTEXTUAL: "contextual" >
 | < SSOVERRIDE: "override" >
@@ -1129,7 +1129,7 @@ void summaryInFieldShort(ParsedField field) :
     <COLON> ( <DYNAMIC> { psf.setDynamic(); }
               | <MATCHED_ELEMENTS_ONLY> { psf.setMatchedElementsOnly(); }
               | (<FULL> | <STATIC>) { psf.setFull(); }
-              | <LINGUISTICS_TOKENS> { psf.setLinguisticsTokens(); }
+              | <TOKENS> { psf.setTokens(); }
             )
 }
 
@@ -1175,7 +1175,7 @@ void summaryTransform(ParsedSummaryField field) : { }
     ( <DYNAMIC>              { field.setDynamic(); }
      | <MATCHED_ELEMENTS_ONLY> { field.setMatchedElementsOnly(); }
      | (<FULL> | <STATIC>)   { field.setFull(); }
-     | <LINGUISTICS_TOKENS> { field.setLinguisticsTokens(); }
+     | <TOKENS> { field.setTokens(); }
     )
 }
 
@@ -2715,7 +2715,6 @@ String identifier() : { }
       | <INLINE>
       | <INPUTS>
       | <INTEGER>
-      | <LINGUISTICS_TOKENS>
       | <LITERAL>
       | <LOCALE>
       | <LONG>
@@ -2769,6 +2768,7 @@ String identifier() : { }
       | <TERTIARY>
       | <TEXT>
       | <TO>
+      | <TOKENS>
       | <TRUE>
       | <TYPE>
       | <UCA>
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java
index 4128baddcb7..62ddc5d6b14 100644
--- a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java
@@ -227,16 +227,16 @@ public class SummaryTestCase extends AbstractSchemaTestCase {
     }
 
     @Test
-    void linguistics_tokenizer_override() throws ParseException {
+    void tokensr_override() throws ParseException {
         var schema = buildSchema("field foo type string { indexing: summary }",
                 joinLines("document-summary bar {",
                         "    summary baz type string {",
                         "        source: foo ",
-                        "        linguistics-tokens",
+                        "        tokens",
                         "     }",
                         "    from-disk",
                         "}"));
-        assertOverride(schema, "baz", SummaryTransform.LINGUISTICS_TOKENS.getName(), "foo", "bar");
+        assertOverride(schema, "baz", SummaryTransform.TOKENS.getName(), "foo", "bar");
     }
 
     @Test
diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt
index a091f8b5358..f771a8e4494 100644
--- a/searchsummary/CMakeLists.txt
+++ b/searchsummary/CMakeLists.txt
@@ -20,7 +20,7 @@ vespa_define_module(
     src/tests/docsummary/attribute_combiner
     src/tests/docsummary/attributedfw
     src/tests/docsummary/document_id_dfw
-    src/tests/docsummary/linguistics_tokens_converter
+    src/tests/docsummary/tokens_converter
     src/tests/docsummary/matched_elements_filter
     src/tests/docsummary/query_term_filter_factory
     src/tests/docsummary/result_class
diff --git a/searchsummary/src/tests/docsummary/linguistics_tokens_converter/CMakeLists.txt b/searchsummary/src/tests/docsummary/linguistics_tokens_converter/CMakeLists.txt
deleted file mode 100644
index d9510c3a2b3..00000000000
--- a/searchsummary/src/tests/docsummary/linguistics_tokens_converter/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(searchsummary_linguistics_tokens_converter_test_app TEST
-    SOURCES
-    linguistics_tokens_converter_test.cpp
-    DEPENDS
-    searchsummary
-    GTest::gtest
-)
-
-vespa_add_test(NAME searchsummary_linguistics_tokens_converter_test_app COMMAND searchsummary_linguistics_tokens_converter_test_app)
diff --git a/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt b/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt
new file mode 100644
index 00000000000..68885a74b1b
--- /dev/null
+++ b/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchsummary_tokens_converter_test_app TEST
+    SOURCES
+    tokens_converter_test.cpp
+    DEPENDS
+    searchsummary
+    GTest::gtest
+)
+
+vespa_add_test(NAME searchsummary_tokens_converter_test_app COMMAND searchsummary_tokens_converter_test_app)
diff --git a/searchsummary/src/tests/docsummary/linguistics_tokens_converter/linguistics_tokens_converter_test.cpp b/searchsummary/src/tests/docsummary/tokens_converter/tokens_converter_test.cpp
index beaa43c7af8..493cbe0ecba 100644
--- a/searchsummary/src/tests/docsummary/linguistics_tokens_converter/linguistics_tokens_converter_test.cpp
+++ b/searchsummary/src/tests/docsummary/tokens_converter/tokens_converter_test.cpp
@@ -10,7 +10,7 @@
 #include <vespa/document/repo/fixedtyperepo.h>
 #include <vespa/searchlib/util/linguisticsannotation.h>
 #include <vespa/searchlib/util/token_extractor.h>
-#include <vespa/searchsummary/docsummary/linguistics_tokens_converter.h>
+#include <vespa/searchsummary/docsummary/tokens_converter.h>
 #include <vespa/vespalib/data/simple_buffer.h>
 #include <vespa/vespalib/data/slime/json_format.h>
 #include <vespa/vespalib/data/slime/slime.h>
@@ -24,7 +24,7 @@ using document::Span;
 using document::SpanList;
 using document::SpanTree;
 using document::StringFieldValue;
-using search::docsummary::LinguisticsTokensConverter;
+using search::docsummary::TokensConverter;
 using search::linguistics::SPANTREE_NAME;
 using search::linguistics::TokenExtractor;
 using vespalib::SimpleBuffer;
@@ -55,7 +55,7 @@ get_document_types_config()
 
 }
 
-class LinguisticsTokensConverterTest : public testing::Test
+class TokensConverterTest : public testing::Test
 {
 protected:
     std::shared_ptr<const DocumentTypeRepo> _repo;
@@ -64,8 +64,8 @@ protected:
     vespalib::string                        _dummy_field_name;
     TokenExtractor                          _token_extractor;
 
-    LinguisticsTokensConverterTest();
-    ~LinguisticsTokensConverterTest() override;
+    TokensConverterTest();
+    ~TokensConverterTest() override;
     void set_span_tree(StringFieldValue& value, std::unique_ptr<SpanTree> tree);
     StringFieldValue make_annotated_string(bool alt_tokens);
     StringFieldValue make_annotated_chinese_string();
@@ -73,7 +73,7 @@ protected:
     vespalib::string convert(const StringFieldValue& fv);
 };
 
-LinguisticsTokensConverterTest::LinguisticsTokensConverterTest()
+TokensConverterTest::TokensConverterTest()
     : testing::Test(),
       _repo(std::make_unique<DocumentTypeRepo>(get_document_types_config())),
       _document_type(_repo->getDocumentType("indexingdocument")),
@@ -83,10 +83,10 @@ LinguisticsTokensConverterTest::LinguisticsTokensConverterTest()
 {
 }
 
-LinguisticsTokensConverterTest::~LinguisticsTokensConverterTest() = default;
+TokensConverterTest::~TokensConverterTest() = default;
 
 void
-LinguisticsTokensConverterTest::set_span_tree(StringFieldValue & value, std::unique_ptr<SpanTree> tree)
+TokensConverterTest::set_span_tree(StringFieldValue & value, std::unique_ptr<SpanTree> tree)
 {
     StringFieldValue::SpanTrees trees;
     trees.push_back(std::move(tree));
@@ -94,7 +94,7 @@ LinguisticsTokensConverterTest::set_span_tree(StringFieldValue & value, std::uni
 }
 
 StringFieldValue
-LinguisticsTokensConverterTest::make_annotated_string(bool alt_tokens)
+TokensConverterTest::make_annotated_string(bool alt_tokens)
 {
     auto span_list_up = std::make_unique<SpanList>();
     auto span_list = span_list_up.get();
@@ -111,7 +111,7 @@ LinguisticsTokensConverterTest::make_annotated_string(bool alt_tokens)
 }
 
 StringFieldValue
-LinguisticsTokensConverterTest::make_annotated_chinese_string()
+TokensConverterTest::make_annotated_chinese_string()
 {
     auto span_list_up = std::make_unique<SpanList>();
     auto span_list = span_list_up.get();
@@ -125,50 +125,50 @@ LinguisticsTokensConverterTest::make_annotated_chinese_string()
 }
 
 vespalib::string
-LinguisticsTokensConverterTest::make_exp_annotated_chinese_string_tokens()
+TokensConverterTest::make_exp_annotated_chinese_string_tokens()
 {
     return R"(["我就是那个","大灰狼"])";
 }
 
 vespalib::string
-LinguisticsTokensConverterTest::convert(const StringFieldValue& fv)
+TokensConverterTest::convert(const StringFieldValue& fv)
 {
-    LinguisticsTokensConverter converter(_token_extractor);
+    TokensConverter converter(_token_extractor);
     Slime slime;
     SlimeInserter inserter(slime);
     converter.convert(fv, inserter);
     return slime_to_string(slime);
 }
 
-TEST_F(LinguisticsTokensConverterTest, convert_empty_string)
+TEST_F(TokensConverterTest, convert_empty_string)
 {
     vespalib::string exp(R"([])");
     StringFieldValue plain_string("");
     EXPECT_EQ(exp, convert(plain_string));
 }
 
-TEST_F(LinguisticsTokensConverterTest, convert_plain_string)
+TEST_F(TokensConverterTest, convert_plain_string)
 {
     vespalib::string exp(R"(["Foo Bar Baz"])");
     StringFieldValue plain_string("Foo Bar Baz");
     EXPECT_EQ(exp, convert(plain_string));
 }
 
-TEST_F(LinguisticsTokensConverterTest, convert_annotated_string)
+TEST_F(TokensConverterTest, convert_annotated_string)
 {
     vespalib::string exp(R"(["foo","baz"])");
     auto annotated_string = make_annotated_string(false);
     EXPECT_EQ(exp, convert(annotated_string));
 }
 
-TEST_F(LinguisticsTokensConverterTest, convert_annotated_string_with_alternatives)
+TEST_F(TokensConverterTest, convert_annotated_string_with_alternatives)
 {
     vespalib::string exp(R"(["foo",["bar","baz"]])");
     auto annotated_string = make_annotated_string(true);
     EXPECT_EQ(exp, convert(annotated_string));
 }
 
-TEST_F(LinguisticsTokensConverterTest, convert_annotated_chinese_string)
+TEST_F(TokensConverterTest, convert_annotated_chinese_string)
 {
     auto exp = make_exp_annotated_chinese_string_tokens();
     auto annotated_chinese_string = make_annotated_chinese_string();
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt
index 57b6004fb61..0287517f830 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt
+++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt
@@ -23,8 +23,6 @@ vespa_add_library(searchsummary_docsummary OBJECT
     juniper_dfw_term_visitor.cpp
     juniper_query_adapter.cpp
     juniperproperties.cpp
-    linguistics_tokens_converter.cpp
-    linguistics_tokens_dfw.cpp
     matched_elements_filter_dfw.cpp
     positionsdfw.cpp
     query_term_filter.cpp
@@ -39,4 +37,6 @@ vespa_add_library(searchsummary_docsummary OBJECT
     struct_fields_resolver.cpp
     struct_map_attribute_combiner_dfw.cpp
     summaryfeaturesdfw.cpp
+    tokens_converter.cpp
+    tokens_dfw.cpp
 )
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp
index c4823f6beeb..2ac5d1babbf 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp
@@ -12,12 +12,12 @@ const vespalib::string documentid("documentid");
 const vespalib::string dynamic_teaser("dynamicteaser");
 const vespalib::string empty("empty");
 const vespalib::string geo_position("geopos");
-const vespalib::string linguistics_tokens("linguistics-tokens");
 const vespalib::string matched_attribute_elements_filter("matchedattributeelementsfilter");
 const vespalib::string matched_elements_filter("matchedelementsfilter");
 const vespalib::string positions("positions");
 const vespalib::string rank_features("rankfeatures");
 const vespalib::string summary_features("summaryfeatures");
+const vespalib::string tokens("tokens");
 
 }
 
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h
index 2d0b8c23855..d53351d8b04 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h
@@ -18,11 +18,11 @@ extern const vespalib::string documentid;
 extern const vespalib::string dynamic_teaser;
 extern const vespalib::string empty;
 extern const vespalib::string geo_position;
-extern const vespalib::string linguistics_tokens;
 extern const vespalib::string matched_attribute_elements_filter;
 extern const vespalib::string matched_elements_filter;
 extern const vespalib::string positions;
 extern const vespalib::string rank_features;
 extern const vespalib::string summary_features;
+extern const vespalib::string tokens;
 
 }
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp
index d19d2994104..2f7d9acdb65 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp
@@ -9,11 +9,11 @@
 #include "geoposdfw.h"
 #include "idocsumenvironment.h"
 #include "juniperdfw.h"
-#include "linguistics_tokens_dfw.h"
 #include "matched_elements_filter_dfw.h"
 #include "positionsdfw.h"
 #include "rankfeaturesdfw.h"
 #include "summaryfeaturesdfw.h"
+#include "tokens_dfw.h"
 #include <vespa/searchlib/common/matching_elements_fields.h>
 #include <vespa/vespalib/util/exceptions.h>
 
@@ -85,9 +85,9 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie
         } else {
             throw_missing_source(command);
         }
-    } else if (command == command::linguistics_tokens) {
+    } else if (command == command::tokens) {
         if (!source.empty()) {
-            fieldWriter = std::make_unique<LinguisticsTokensDFW>(source);
+            fieldWriter = std::make_unique<TokensDFW>(source);
         } else {
             throw_missing_source(command);
         }
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/tokens_converter.cpp
index b9b9d7c4c97..e2849fe793e 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/tokens_converter.cpp
@@ -1,6 +1,6 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include "linguistics_tokens_converter.h"
+#include "tokens_converter.h"
 #include <vespa/document/fieldvalue/stringfieldvalue.h>
 #include <vespa/searchlib/util/token_extractor.h>
 #include <vespa/vespalib/data/slime/slime.h>
@@ -14,18 +14,18 @@ using vespalib::slime::Inserter;
 
 namespace search::docsummary {
 
-LinguisticsTokensConverter::LinguisticsTokensConverter(const TokenExtractor& token_extractor)
+TokensConverter::TokensConverter(const TokenExtractor& token_extractor)
     : IStringFieldConverter(),
       _token_extractor(token_extractor),
       _text()
 {
 }
 
-LinguisticsTokensConverter::~LinguisticsTokensConverter() = default;
+TokensConverter::~TokensConverter() = default;
 
 template <typename ForwardIt>
 void
-LinguisticsTokensConverter::handle_alternative_index_terms(ForwardIt it, ForwardIt last, Inserter& inserter)
+TokensConverter::handle_alternative_index_terms(ForwardIt it, ForwardIt last, Inserter& inserter)
 {
     Cursor& a = inserter.insertArray();
     ArrayInserter ai(a);
@@ -35,13 +35,13 @@ LinguisticsTokensConverter::handle_alternative_index_terms(ForwardIt it, Forward
 }
 
 void
-LinguisticsTokensConverter::handle_index_term(vespalib::stringref word, Inserter& inserter)
+TokensConverter::handle_index_term(vespalib::stringref word, Inserter& inserter)
 {
     inserter.insertString(Memory(word));
 }
 
 void
-LinguisticsTokensConverter::handle_indexing_terms(const StringFieldValue& value, vespalib::slime::Inserter& inserter)
+TokensConverter::handle_indexing_terms(const StringFieldValue& value, vespalib::slime::Inserter& inserter)
 {
     Cursor& a = inserter.insertArray();
     ArrayInserter ai(a);
@@ -63,14 +63,14 @@ LinguisticsTokensConverter::handle_indexing_terms(const StringFieldValue& value,
 }
 
 void
-LinguisticsTokensConverter::convert(const StringFieldValue &input, vespalib::slime::Inserter& inserter)
+TokensConverter::convert(const StringFieldValue &input, vespalib::slime::Inserter& inserter)
 {
     _text = input.getValueRef();
     handle_indexing_terms(input, inserter);
 }
 
 bool
-LinguisticsTokensConverter::render_weighted_set_as_array() const
+TokensConverter::render_weighted_set_as_array() const
 {
     return true;
 }
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.h b/searchsummary/src/vespa/searchsummary/docsummary/tokens_converter.h
index d752fe89ed9..1798abac203 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/tokens_converter.h
@@ -10,10 +10,10 @@ namespace search::docsummary {
 
 /*
  * Class converting a string field value with annotations into an array
- * containing the index terms. Multiple index terms at same position are
+ * containing the tokens. Multiple tokens at same position are
  * placed in a nested array.
  */
-class LinguisticsTokensConverter : public IStringFieldConverter
+class TokensConverter : public IStringFieldConverter
 {
     const linguistics::TokenExtractor& _token_extractor;
     vespalib::stringref                _text;
@@ -23,8 +23,8 @@ class LinguisticsTokensConverter : public IStringFieldConverter
     void handle_index_term(vespalib::stringref word, vespalib::slime::Inserter& inserter);
     void handle_indexing_terms(const document::StringFieldValue& value, vespalib::slime::Inserter& inserter);
 public:
-    LinguisticsTokensConverter(const linguistics::TokenExtractor& token_extractor);
-    ~LinguisticsTokensConverter() override;
+    TokensConverter(const linguistics::TokenExtractor& token_extractor);
+    ~TokensConverter() override;
     void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override;
     bool render_weighted_set_as_array() const override;
 };
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/tokens_dfw.cpp
index 5e94e270c53..0741e5cc352 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_dfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/tokens_dfw.cpp
@@ -1,34 +1,34 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include "linguistics_tokens_dfw.h"
+#include "tokens_dfw.h"
 #include "i_docsum_store_document.h"
-#include "linguistics_tokens_converter.h"
+#include "tokens_converter.h"
 #include <vespa/searchlib/memoryindex/field_inverter.h>
 
 using search::memoryindex::FieldInverter;
 
 namespace search::docsummary {
 
-LinguisticsTokensDFW::LinguisticsTokensDFW(const vespalib::string& input_field_name)
+TokensDFW::TokensDFW(const vespalib::string& input_field_name)
     : DocsumFieldWriter(),
       _input_field_name(input_field_name),
       _token_extractor(_input_field_name, FieldInverter::max_word_len)
 {
 }
 
-LinguisticsTokensDFW::~LinguisticsTokensDFW() = default;
+TokensDFW::~TokensDFW() = default;
 
 bool
-LinguisticsTokensDFW::isGenerated() const
+TokensDFW::isGenerated() const
 {
     return false;
 }
 
 void
-LinguisticsTokensDFW::insertField(uint32_t, const IDocsumStoreDocument* doc, GetDocsumsState&, vespalib::slime::Inserter& target) const
+TokensDFW::insertField(uint32_t, const IDocsumStoreDocument* doc, GetDocsumsState&, vespalib::slime::Inserter& target) const
 {
     if (doc != nullptr) {
-        LinguisticsTokensConverter converter(_token_extractor);
+        TokensConverter converter(_token_extractor);
         doc->insert_summary_field(_input_field_name, target, &converter);
     }
 }
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/tokens_dfw.h
index 9c6955b322e..e9f91ab683a 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_dfw.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/tokens_dfw.h
@@ -10,17 +10,17 @@ namespace search::docsummary {
 
 /*
  * Class for writing annotated string field values from document as
- * arrays containing the indexing terms.
+ * arrays containing the tokens.
  */
-class LinguisticsTokensDFW : public DocsumFieldWriter
+class TokensDFW : public DocsumFieldWriter
 {
 private:
     vespalib::string            _input_field_name;
     linguistics::TokenExtractor _token_extractor;
 
 public:
-    explicit LinguisticsTokensDFW(const vespalib::string& input_field_name);
-    ~LinguisticsTokensDFW() override;
+    explicit TokensDFW(const vespalib::string& input_field_name);
+    ~TokensDFW() override;
     bool isGenerated() const override;
     void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState& state, vespalib::slime::Inserter& target) const override;
 };