1 files changed, 7 insertions, 17 deletions
diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h
index 2178efc31bf..4e3934ba322 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h
@@ -1,10 +1,11 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #pragma once
 
 #include "i_field_index_remove_listener.h"
 #include <vespa/document/annotation/span.h>
 #include <vespa/searchlib/index/docidandfeatures.h>
+#include <vespa/searchlib/util/token_extractor.h>
 #include <vespa/vespalib/stllike/allocator.h>
 #include <vespa/vespalib/stllike/hash_map.h>
 #include <limits>
@@ -172,6 +173,7 @@ private:
     uint32_t                       _oldPosSize;
 
     const index::Schema           &_schema;
+    linguistics::TokenExtractor    _token_extractor;
 
     WordBuffer                     _words;
     ElemInfoVec                    _elems;
@@ -179,9 +181,8 @@ private:
     index::DocIdAndPosOccFeatures  _features;
     UInt32Vector                   _wordRefs;
 
-    using SpanTerm = std::pair<document::Span, const document::FieldValue *>;
-    using SpanTermVector = std::vector<SpanTerm>;
-    SpanTermVector                      _terms;
+    using SpanTerm = linguistics::TokenExtractor::SpanTerm;
+    std::vector<SpanTerm>          _terms;
 
     // Info about aborted and pending documents.
     std::vector<PositionRange>                  _abortedDocs;
@@ -202,12 +203,7 @@ private:
     /**
      * Save the given word in the word buffer and return the word reference.
      */
-    VESPA_DLL_LOCAL uint32_t saveWord(const vespalib::stringref word, const document::Document* doc);
-
-    /**
-     * Save the field value as a word in the word buffer and return the word reference.
-     */
-    VESPA_DLL_LOCAL uint32_t saveWord(const document::FieldValue &fv, const document::Document& doc);
+    VESPA_DLL_LOCAL uint32_t saveWord(vespalib::stringref word);
 
     /**
      * Get pointer to saved word from a word reference.
@@ -326,13 +322,7 @@ public:
 
     void endDoc();
 
-    void addWord(const vespalib::stringref word, const document::Document& doc) {
-        uint32_t wordRef = saveWord(word, &doc);
-        if (wordRef != 0u) {
-            add(wordRef);
-            stepWordPos();
-        }
-    }
+    void addWord(vespalib::stringref word, const document::Document& doc);
 };
 
 }