Add unit test for document inverter collection.

10 files changed, 183 insertions, 34 deletions

diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt
index 5022225fbf3..c9c44af3aa3 100644
--- a/searchlib/CMakeLists.txt
+++ b/searchlib/CMakeLists.txt
@@ -176,6 +176,7 @@ vespa_define_module(
     src/tests/memoryindex/compact_words_store
     src/tests/memoryindex/datastore
     src/tests/memoryindex/document_inverter
+    src/tests/memoryindex/document_inverter_collection
     src/tests/memoryindex/field_index
     src/tests/memoryindex/field_index_remover
     src/tests/memoryindex/field_inverter
diff --git a/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp
index d81df4c63fe..bf9bbf1007a 100644
--- a/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp
+++ b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp
@@ -8,25 +8,23 @@
 #include <vespa/searchlib/memoryindex/field_inverter.h>
 #include <vespa/searchlib/memoryindex/i_field_index_collection.h>
 #include <vespa/searchlib/memoryindex/word_store.h>
+#include <vespa/searchlib/test/memoryindex/mock_field_index_collection.h>
 #include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h>
 #include <vespa/vespalib/util/sequencedtaskexecutor.h>
 
 #include <vespa/vespalib/gtest/gtest.h>
 
-namespace search {
+namespace search::memoryindex {
 
 using document::Document;
 using index::DocBuilder;
+using index::FieldLengthCalculator;
 using index::Schema;
 using index::schema::CollectionType;
 using index::schema::DataType;
 using vespalib::SequencedTaskExecutor;
 using vespalib::ISequencedTaskExecutor;
 
-using namespace index;
-
-namespace memoryindex {
-
 namespace {
 
 Document::UP
@@ -91,32 +89,6 @@ makeDoc15(DocBuilder &b)
 
 }
 
-class MockFieldIndexCollection : public IFieldIndexCollection {
-    FieldIndexRemover               &_remover;
-    test::OrderedFieldIndexInserter &_inserter;
-    FieldLengthCalculator           &_calculator;
-
-public:
-    MockFieldIndexCollection(FieldIndexRemover &remover,
-                             test::OrderedFieldIndexInserter &inserter,
-                             FieldLengthCalculator &calculator)
-        : _remover(remover),
-          _inserter(inserter),
-          _calculator(calculator)
-    {
-    }
-
-    FieldIndexRemover &get_remover(uint32_t) override {
-        return _remover;
-    }
-    IOrderedFieldIndexInserter &get_inserter(uint32_t) override {
-        return _inserter;
-    }
-    index::FieldLengthCalculator &get_calculator(uint32_t) override {
-        return _calculator;
-    }
-};
-
 VESPA_THREAD_STACK_TAG(invert_executor)
 VESPA_THREAD_STACK_TAG(push_executor)
 
@@ -129,7 +101,7 @@ struct DocumentInverterTest : public ::testing::Test {
     FieldIndexRemover               _remover;
     test::OrderedFieldIndexInserter _inserter;
     FieldLengthCalculator           _calculator;
-    MockFieldIndexCollection        _fic;
+    test::MockFieldIndexCollection  _fic;
     DocumentInverterContext         _inv_context;
     DocumentInverter                _inv;
 
@@ -306,6 +278,5 @@ TEST_F(DocumentInverterTest, require_that_empty_document_can_be_inverted)
 }
 
 }
-}
 
 GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchlib/src/tests/memoryindex/document_inverter_collection/CMakeLists.txt b/searchlib/src/tests/memoryindex/document_inverter_collection/CMakeLists.txt
new file mode 100644
index 00000000000..2697e9c5626
--- /dev/null
+++ b/searchlib/src/tests/memoryindex/document_inverter_collection/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_document_inverter_collection_test_app TEST
+    SOURCES
+    document_inverter_collection_test.cpp
+    DEPENDS
+    searchlib_test
+    searchlib
+    GTest::GTest
+)
+vespa_add_test(NAME searchlib_document_inverter_collection_test_app COMMAND searchlib_document_inverter_collection_test_app)
diff --git a/searchlib/src/tests/memoryindex/document_inverter_collection/document_inverter_collection_test.cpp b/searchlib/src/tests/memoryindex/document_inverter_collection/document_inverter_collection_test.cpp
new file mode 100644
index 00000000000..a6d675d05dc
--- /dev/null
+++ b/searchlib/src/tests/memoryindex/document_inverter_collection/document_inverter_collection_test.cpp
@@ -0,0 +1,93 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/searchlib/index/field_length_calculator.h>
+#include <vespa/searchlib/memoryindex/document_inverter.h>
+#include <vespa/searchlib/memoryindex/document_inverter_collection.h>
+#include <vespa/searchlib/memoryindex/document_inverter_context.h>
+#include <vespa/searchlib/memoryindex/field_index_remover.h>
+#include <vespa/searchlib/memoryindex/field_inverter.h>
+#include <vespa/searchlib/memoryindex/i_field_index_collection.h>
+#include <vespa/searchlib/memoryindex/word_store.h>
+#include <vespa/searchlib/test/memoryindex/mock_field_index_collection.h>
+#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h>
+#include <vespa/vespalib/util/retain_guard.h>
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
+#include <thread>
+
+#include <vespa/vespalib/gtest/gtest.h>
+
+
+
+namespace search::memoryindex {
+using document::Document;
+using index::FieldLengthCalculator;
+using index::Schema;
+using vespalib::RetainGuard;
+using vespalib::SequencedTaskExecutor;
+using vespalib::ISequencedTaskExecutor;
+
+VESPA_THREAD_STACK_TAG(invert_executor)
+VESPA_THREAD_STACK_TAG(push_executor)
+
+struct DocumentInverterCollectionTest : public ::testing::Test {
+    Schema _schema;
+    std::unique_ptr<ISequencedTaskExecutor> _invertThreads;
+    std::unique_ptr<ISequencedTaskExecutor> _pushThreads;
+    WordStore                       _word_store;
+    FieldIndexRemover               _remover;
+    test::OrderedFieldIndexInserter _inserter;
+    FieldLengthCalculator           _calculator;
+    test::MockFieldIndexCollection  _fic;
+    DocumentInverterContext         _inv_context;
+    DocumentInverterCollection      _inv_collection;
+
+    DocumentInverterCollectionTest()
+        : _schema(),
+          _invertThreads(SequencedTaskExecutor::create(invert_executor, 4)),
+          _pushThreads(SequencedTaskExecutor::create(push_executor, 4)),
+          _word_store(),
+          _remover(_word_store),
+          _inserter(),
+          _calculator(),
+          _fic(_remover, _inserter, _calculator),
+          _inv_context(_schema, *_invertThreads, *_pushThreads, _fic),
+          _inv_collection(_inv_context, 10)
+    {
+    }
+
+};
+
+TEST_F(DocumentInverterCollectionTest, idle_inverter_is_reused)
+{
+    auto& active = _inv_collection.get_active_inverter();
+    for (uint32_t i = 0; i < 4; ++i) {
+        _inv_collection.switch_active_inverter();
+        EXPECT_EQ(&active, &_inv_collection.get_active_inverter());
+    }
+    EXPECT_EQ(1u, _inv_collection.get_num_inverters());
+}
+
+TEST_F(DocumentInverterCollectionTest, busy_inverter_is_not_reused)
+{
+    auto& active = _inv_collection.get_active_inverter();
+    auto retain = std::make_shared<RetainGuard>(active.get_ref_count());
+    _inv_collection.switch_active_inverter();
+    EXPECT_NE(&active, &_inv_collection.get_active_inverter());
+    EXPECT_EQ(2u, _inv_collection.get_num_inverters());
+}
+
+TEST_F(DocumentInverterCollectionTest, number_of_inverters_is_limited_by_max)
+{
+    for (uint32_t i = 0; i < 50; ++i) {
+        auto& active = _inv_collection.get_active_inverter();
+        auto retain = std::make_shared<RetainGuard>(active.get_ref_count());
+        _pushThreads->execute(i, [retain(std::move(retain))] () { std::this_thread::sleep_for(10ms); });
+        _inv_collection.switch_active_inverter();
+    }
+    EXPECT_LE(4u, _inv_collection.get_num_inverters());
+    EXPECT_GE(_inv_collection.get_max_inverters(), _inv_collection.get_num_inverters());
+}
+
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h
index 0a128ec6abd..f69d482f565 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h
@@ -97,6 +97,7 @@ public:
     uint32_t getNumFields() const { return _inverters.size(); }
     void wait_for_zero_ref_count() { _ref_count.waitForZeroRefCount(); }
     bool has_zero_ref_count() { return _ref_count.has_zero_ref_count(); }
+    vespalib::MonitoredRefCount& get_ref_count() noexcept { return _ref_count; }
 };
 
 }
diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter_collection.h b/searchlib/src/vespa/searchlib/memoryindex/document_inverter_collection.h
index deecebc164d..d07cca67e08 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter_collection.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter_collection.h
@@ -26,6 +26,8 @@ public:
     ~DocumentInverterCollection();
     DocumentInverter& get_active_inverter() noexcept { return *_active_inverter; }
     void switch_active_inverter();
+    uint32_t get_num_inverters() const noexcept { return _num_inverters; }
+    uint32_t get_max_inverters() const noexcept { return _max_inverters; }
 };
 
 }
diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/memoryindex/CMakeLists.txt
index 90921cb53b3..c7cc6d77ede 100644
--- a/searchlib/src/vespa/searchlib/test/memoryindex/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/test/memoryindex/CMakeLists.txt
@@ -1,5 +1,6 @@
 # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(searchlib_searchlib_test_memoryindex INTERFACE
+vespa_add_library(searchlib_searchlib_test_memoryindex
     SOURCES
+    mock_field_index_collection.cpp
     DEPENDS
 )
diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/mock_field_index_collection.cpp b/searchlib/src/vespa/searchlib/test/memoryindex/mock_field_index_collection.cpp
new file mode 100644
index 00000000000..1392a92853d
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/test/memoryindex/mock_field_index_collection.cpp
@@ -0,0 +1,37 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "mock_field_index_collection.h"
+#include "ordered_field_index_inserter.h"
+
+namespace search::memoryindex::test {
+
+MockFieldIndexCollection::MockFieldIndexCollection(FieldIndexRemover& remover,
+                                                   OrderedFieldIndexInserter& inserter,
+                                                   index::FieldLengthCalculator& calculator)
+    : _remover(remover),
+      _inserter(inserter),
+      _calculator(calculator)
+{
+}
+
+MockFieldIndexCollection::~MockFieldIndexCollection() = default;
+
+FieldIndexRemover&
+MockFieldIndexCollection::get_remover(uint32_t)
+{
+    return _remover;
+}
+
+IOrderedFieldIndexInserter&
+MockFieldIndexCollection::get_inserter(uint32_t)
+{
+    return _inserter;
+}
+
+index::FieldLengthCalculator&
+MockFieldIndexCollection::get_calculator(uint32_t)
+{
+    return _calculator;
+}
+
+}
diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/mock_field_index_collection.h b/searchlib/src/vespa/searchlib/test/memoryindex/mock_field_index_collection.h
new file mode 100644
index 00000000000..0dd7127f0ae
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/test/memoryindex/mock_field_index_collection.h
@@ -0,0 +1,32 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/memoryindex/i_field_index_collection.h>
+
+namespace search::index { class FieldLengthCalculator; }
+namespace search::memoryindex { class FieldIndexRemover; }
+
+namespace search::memoryindex::test {
+
+class OrderedFieldIndexInserter;
+
+/*
+ * Mockup of field index collection used by unit tests.
+ */
+class MockFieldIndexCollection : public IFieldIndexCollection {
+    FieldIndexRemover&            _remover;
+    OrderedFieldIndexInserter&    _inserter;
+    index::FieldLengthCalculator& _calculator;
+
+public:
+    MockFieldIndexCollection(FieldIndexRemover& remover,
+                             OrderedFieldIndexInserter& inserter,
+                             index::FieldLengthCalculator& calculator);
+    ~MockFieldIndexCollection() override;
+    FieldIndexRemover& get_remover(uint32_t) override;
+    IOrderedFieldIndexInserter& get_inserter(uint32_t) override;
+    index::FieldLengthCalculator& get_calculator(uint32_t) override;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h
index 793a0c31b1c..de7023499a1 100644
--- a/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h
+++ b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h
@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <vespa/searchlib/index/docidandfeatures.h>
 #include <vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h>
 #include <sstream>