Extend MemoryIndex to take an instance of IFieldLengthInspector to setup field length calculators per field index.

7 files changed, 44 insertions, 36 deletions

diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp
index d0b19d77181..9a343667fd6 100644
--- a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp
+++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp
@@ -1,27 +1,29 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("feed_and_search_test");
 
 #include <vespa/document/datatype/datatype.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/document/fieldvalue/fieldvalue.h>
-#include <vespa/searchlib/memoryindex/memory_index.h>
+#include <vespa/searchlib/common/documentsummary.h>
+#include <vespa/searchlib/common/sequencedtaskexecutor.h>
 #include <vespa/searchlib/diskindex/diskindex.h>
+#include <vespa/searchlib/diskindex/fusion.h>
 #include <vespa/searchlib/diskindex/indexbuilder.h>
 #include <vespa/searchlib/fef/fef.h>
 #include <vespa/searchlib/index/docbuilder.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/memoryindex/memory_index.h>
+#include <vespa/searchlib/test/index/mock_field_length_inspector.h>
 #include <vespa/searchlib/query/base.h>
 #include <vespa/searchlib/query/tree/simplequery.h>
 #include <vespa/searchlib/queryeval/blueprint.h>
-#include <vespa/searchlib/queryeval/searchiterator.h>
 #include <vespa/searchlib/queryeval/fake_requestcontext.h>
+#include <vespa/searchlib/queryeval/searchiterator.h>
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/util/threadstackexecutor.h>
 #include <sstream>
-#include <vespa/searchlib/diskindex/fusion.h>
-#include <vespa/searchlib/common/documentsummary.h>
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP("feed_and_search_test");
 
 using document::DataType;
 using document::Document;
@@ -32,25 +34,26 @@ using search::TuneFileSearch;
 using search::diskindex::DiskIndex;
 using search::diskindex::IndexBuilder;
 using search::diskindex::SelectorArray;
+using search::docsummary::DocumentSummary;
 using search::fef::FieldPositionsIterator;
 using search::fef::MatchData;
 using search::fef::MatchDataLayout;
 using search::fef::TermFieldHandle;
 using search::fef::TermFieldMatchData;
 using search::index::DocBuilder;
-using search::index::Schema;
 using search::index::DummyFileHeaderContext;
+using search::index::Schema;
+using search::index::test::MockFieldLengthInspector;
 using search::memoryindex::MemoryIndex;
 using search::query::SimpleStringTerm;
 using search::queryeval::Blueprint;
+using search::queryeval::FakeRequestContext;
 using search::queryeval::FieldSpec;
 using search::queryeval::FieldSpecList;
 using search::queryeval::SearchIterator;
 using search::queryeval::Searchable;
-using search::queryeval::FakeRequestContext;
 using std::ostringstream;
 using vespalib::string;
-using search::docsummary::DocumentSummary;
 
 namespace {
 
@@ -148,7 +151,7 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() {
     vespalib::ThreadStackExecutor sharedExecutor(2, 0x10000);
     search::SequencedTaskExecutor indexFieldInverter(2);
     search::SequencedTaskExecutor indexFieldWriter(2);
-    MemoryIndex memory_index(schema, indexFieldInverter, indexFieldWriter);
+    MemoryIndex memory_index(schema, MockFieldLengthInspector(), indexFieldInverter, indexFieldWriter);
     DocBuilder doc_builder(schema);
 
     Document::UP doc = buildDocument(doc_builder, doc_id1, word1);
diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp
index 8f9944e178e..25e7a4ffa6b 100644
--- a/searchcore/src/tests/proton/index/fusionrunner_test.cpp
+++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp
@@ -1,19 +1,19 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-// Unit tests for fusionrunner.
 
+#include <vespa/fastos/file.h>
 #include <vespa/searchcore/proton/index/indexmanager.h>
 #include <vespa/searchcore/proton/server/executorthreadingservice.h>
 #include <vespa/searchcorespi/index/fusionrunner.h>
-#include <vespa/searchlib/memoryindex/memory_index.h>
+#include <vespa/searchlib/common/isequencedtaskexecutor.h>
 #include <vespa/searchlib/diskindex/diskindex.h>
 #include <vespa/searchlib/diskindex/indexbuilder.h>
 #include <vespa/searchlib/fef/matchdatalayout.h>
 #include <vespa/searchlib/index/docbuilder.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/memoryindex/memory_index.h>
 #include <vespa/searchlib/query/tree/simplequery.h>
-#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <vespa/searchlib/test/index/mock_field_length_inspector.h>
 #include <vespa/vespalib/testkit/testapp.h>
-#include <vespa/fastos/file.h>
 #include <set>
 
 using document::Document;
@@ -35,6 +35,7 @@ using search::index::DocBuilder;
 using search::index::DummyFileHeaderContext;
 using search::index::Schema;
 using search::index::schema::DataType;
+using search::index::test::MockFieldLengthInspector;
 using search::memoryindex::MemoryIndex;
 using search::query::SimpleStringTerm;
 using search::queryeval::Blueprint;
@@ -171,7 +172,8 @@ void Test::createIndex(const string &dir, uint32_t id, bool fusion) {
 
     Schema schema = getSchema();
     DocBuilder doc_builder(schema);
-    MemoryIndex memory_index(schema, _threadingService.indexFieldInverter(),
+    MemoryIndex memory_index(schema, MockFieldLengthInspector(),
+                             _threadingService.indexFieldInverter(),
                              _threadingService.indexFieldWriter());
     addDocument(doc_builder, memory_index, *_selector, id, id + 0, term);
     addDocument(doc_builder, memory_index, *_selector, id, id + 1, "bar");
diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp
index 9ceb85981c0..4149d563bf9 100644
--- a/searchcore/src/tests/proton/index/indexmanager_test.cpp
+++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp
@@ -19,6 +19,7 @@
 #include <vespa/searchlib/memoryindex/field_index_collection.h>
 #include <vespa/searchlib/memoryindex/field_inverter.h>
 #include <vespa/searchlib/queryeval/isourceselector.h>
+#include <vespa/searchlib/test/index/mock_field_length_inspector.h>
 #include <vespa/searchlib/util/dirtraverse.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/testkit/testapp.h>
@@ -41,6 +42,7 @@ using search::index::DocBuilder;
 using search::index::DummyFileHeaderContext;
 using search::index::Schema;
 using search::index::schema::DataType;
+using search::index::test::MockFieldLengthInspector;
 using vespalib::makeLambdaTask;
 using search::memoryindex::CompactWordsStore;
 using search::memoryindex::FieldIndexCollection;
@@ -58,8 +60,7 @@ using namespace searchcorespi::index;
 
 namespace {
 
-class IndexManagerDummyReconfigurer : public searchcorespi::IIndexManager::Reconfigurer
-{
+class IndexManagerDummyReconfigurer : public searchcorespi::IIndexManager::Reconfigurer {
     virtual bool
     reconfigure(vespalib::Closure0<bool>::UP closure) override
     {
@@ -360,7 +361,7 @@ TEST_F("requireThatSourceSelectorIsFlushed", Fixture) {
 
 TEST_F("requireThatFlushStatsAreCalculated", Fixture) {
     Schema schema(getSchema());
-    FieldIndexCollection fic(schema);
+    FieldIndexCollection fic(schema, MockFieldLengthInspector());
     SequencedTaskExecutor invertThreads(2);
     SequencedTaskExecutor pushThreads(2);
     search::memoryindex::DocumentInverter inverter(schema, invertThreads,
diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
index b74cc2c603f..8e838414015 100644
--- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
@@ -37,9 +37,11 @@ IndexManager::MaintainerOperations::MaintainerOperations(const FileHeaderContext
 }
 
 IMemoryIndex::SP
-IndexManager::MaintainerOperations::createMemoryIndex(const Schema &schema, SerialNum serialNum)
+IndexManager::MaintainerOperations::createMemoryIndex(const Schema& schema,
+                                                      const IFieldLengthInspector& inspector,
+                                                      SerialNum serialNum)
 {
-    return std::make_shared<MemoryIndexWrapper>(schema, _fileHeaderContext, _tuneFileIndexing,
+    return std::make_shared<MemoryIndexWrapper>(schema, inspector, _fileHeaderContext, _tuneFileIndexing,
                                                 _threadingService, serialNum);
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h
index 4f2d03c81e6..b14912239a3 100644
--- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h
+++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h
@@ -48,7 +48,9 @@ public:
                              size_t cacheSize,
                              searchcorespi::index::IThreadingService &threadingService);
 
-        IMemoryIndex::SP createMemoryIndex(const Schema &schema, SerialNum serialNum) override;
+        IMemoryIndex::SP createMemoryIndex(const Schema& schema,
+                                           const IFieldLengthInspector& inspector,
+                                           SerialNum serialNum) override;
         IDiskIndex::SP loadDiskIndex(const vespalib::string &indexDir) override;
         IDiskIndex::SP reloadDiskIndex(const IDiskIndex &oldIndex) override;
         bool runFusion(const Schema &schema, const vespalib::string &outputDir,
diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp
index 3d1e04196a6..d206388ca04 100644
--- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp
+++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp
@@ -17,13 +17,13 @@ using vespalib::IllegalStateException;
 
 namespace proton {
 
-MemoryIndexWrapper::MemoryIndexWrapper(const search::index::Schema &schema,
-                                       const search::common::FileHeaderContext &fileHeaderContext,
-                                       const TuneFileIndexing &tuneFileIndexing,
-                                       searchcorespi::index::IThreadingService &
-                                       threadingService,
+MemoryIndexWrapper::MemoryIndexWrapper(const search::index::Schema& schema,
+                                       const search::index::IFieldLengthInspector& inspector,
+                                       const search::common::FileHeaderContext& fileHeaderContext,
+                                       const TuneFileIndexing& tuneFileIndexing,
+                                       searchcorespi::index::IThreadingService& threadingService,
                                        search::SerialNum serialNum)
-    : _index(schema, threadingService.indexFieldInverter(),
+    : _index(schema, inspector, threadingService.indexFieldInverter(),
              threadingService.indexFieldWriter()),
       _serialNum(serialNum),
       _fileHeaderContext(fileHeaderContext),
@@ -62,9 +62,7 @@ MemoryIndexWrapper::accept(searchcorespi::IndexSearchableVisitor &visitor) const
 FieldLengthInfo
 MemoryIndexWrapper::get_field_length_info(const vespalib::string& field_name) const
 {
-    // TODO: implement
-    (void) field_name;
-    return FieldLengthInfo();
+    return _index.get_field_length_info(field_name);
 }
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h
index d94a259eb24..dfebaff20f3 100644
--- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h
+++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h
@@ -24,11 +24,11 @@ private:
     const search::TuneFileIndexing _tuneFileIndexing;
 
 public:
-    MemoryIndexWrapper(const search::index::Schema &schema,
-                       const search::common::FileHeaderContext &fileHeaderContext,
-                       const search::TuneFileIndexing &tuneFileIndexing,
-                       searchcorespi::index::IThreadingService &
-                       threadingService,
+    MemoryIndexWrapper(const search::index::Schema& schema,
+                       const search::index::IFieldLengthInspector& inspector,
+                       const search::common::FileHeaderContext& fileHeaderContext,
+                       const search::TuneFileIndexing& tuneFileIndexing,
+                       searchcorespi::index::IThreadingService& threadingService,
                        SerialNum serialNum);
 
     /**