diff options
author | Tor Egge <Tor.Egge@online.no> | 2021-11-04 17:19:43 +0100 |
---|---|---|
committer | Tor Egge <Tor.Egge@online.no> | 2021-11-04 17:21:58 +0100 |
commit | 977c9804af8ba92efc2275e9a68f7c939a596309 (patch) | |
tree | 4ac447e7dd3648d7d665ae25149d79968192990f /searchlib | |
parent | a9c05e54b74bf9a6d6d77e484b6738de40ca8274 (diff) |
Move portions of search::memoryindex::test::OrderedFieldIndexInserter to
search::memoryindex::test::OrderedFieldIndexInserterBackend.
Diffstat (limited to 'searchlib')
12 files changed, 301 insertions, 179 deletions
diff --git a/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp index bf9bbf1007a..dec21c64456 100644 --- a/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp +++ b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp @@ -9,7 +9,9 @@ #include <vespa/searchlib/memoryindex/i_field_index_collection.h> #include <vespa/searchlib/memoryindex/word_store.h> #include <vespa/searchlib/test/memoryindex/mock_field_index_collection.h> -#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter_backend.h> +#include <vespa/vespalib/util/gate.h> +#include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/gtest/gtest.h> @@ -99,7 +101,7 @@ struct DocumentInverterTest : public ::testing::Test { std::unique_ptr<ISequencedTaskExecutor> _pushThreads; WordStore _word_store; FieldIndexRemover _remover; - test::OrderedFieldIndexInserter _inserter; + test::OrderedFieldIndexInserterBackend _inserter_backend; FieldLengthCalculator _calculator; test::MockFieldIndexCollection _fic; DocumentInverterContext _inv_context; @@ -117,27 +119,22 @@ struct DocumentInverterTest : public ::testing::Test { DocumentInverterTest() : _schema(makeSchema()), _b(_schema), - _invertThreads(SequencedTaskExecutor::create(invert_executor, 2)), - _pushThreads(SequencedTaskExecutor::create(push_executor, 2)), + _invertThreads(SequencedTaskExecutor::create(invert_executor, 1)), + _pushThreads(SequencedTaskExecutor::create(push_executor, 1)), _word_store(), _remover(_word_store), - _inserter(), + _inserter_backend(), _calculator(), - _fic(_remover, _inserter, _calculator), + _fic(_remover, _inserter_backend, _calculator), _inv_context(_schema, *_invertThreads, *_pushThreads, _fic), _inv(_inv_context) { } void pushDocuments() { - _invertThreads->sync_all(); - uint32_t fieldId = 0; - for (auto &inverter : _inv.getInverters()) { - _inserter.setFieldId(fieldId); - inverter->pushDocuments(); - ++fieldId; - } - _pushThreads->sync_all(); + vespalib::Gate gate; + _inv.pushDocuments(std::make_shared<vespalib::GateCallback>(gate)); + gate.await(); } }; @@ -149,7 +146,7 @@ TEST_F(DocumentInverterTest, require_that_fresh_insert_works) "w=b,a=10," "w=c,a=10," "w=d,a=10", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(DocumentInverterTest, require_that_multiple_docs_work) @@ -164,7 +161,7 @@ TEST_F(DocumentInverterTest, require_that_multiple_docs_work) "w=f,a=11," "f=1,w=a,a=11," "w=g,a=11", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(DocumentInverterTest, require_that_remove_works) @@ -179,7 +176,7 @@ TEST_F(DocumentInverterTest, require_that_remove_works) "w=b,r=10,r=11," "f=1,w=a,r=10," "f=2,w=c,r=12", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(DocumentInverterTest, require_that_reput_works) @@ -193,7 +190,7 @@ TEST_F(DocumentInverterTest, require_that_reput_works) "w=f,a=10," "f=1,w=a,a=10," "w=g,a=10", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(DocumentInverterTest, require_that_abort_pending_doc_works) @@ -214,7 +211,7 @@ TEST_F(DocumentInverterTest, require_that_abort_pending_doc_works) "w=f,a=11," "f=1,w=a,a=11," "w=g,a=11", - _inserter.toStr()); + _inserter_backend.toStr()); _inv.invertDocument(10, *doc10); _inv.invertDocument(11, *doc11); @@ -223,7 +220,7 @@ TEST_F(DocumentInverterTest, require_that_abort_pending_doc_works) _inv.invertDocument(14, *doc14); _inv.removeDocument(11); _inv.removeDocument(13); - _inserter.reset(); + _inserter_backend.reset(); pushDocuments(); EXPECT_EQ("f=0,w=a,a=10," "w=b,a=10," @@ -233,7 +230,7 @@ TEST_F(DocumentInverterTest, require_that_abort_pending_doc_works) "w=doc14,a=14," "w=h,a=12," "w=j,a=14", - _inserter.toStr()); + _inserter_backend.toStr()); _inv.invertDocument(10, *doc10); _inv.invertDocument(11, *doc11); @@ -244,13 +241,13 @@ TEST_F(DocumentInverterTest, require_that_abort_pending_doc_works) _inv.removeDocument(12); _inv.removeDocument(13); _inv.removeDocument(14); - _inserter.reset(); + _inserter_backend.reset(); pushDocuments(); EXPECT_EQ("f=0,w=a,a=10," "w=b,a=10," "w=c,a=10," "w=d,a=10", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(DocumentInverterTest, require_that_mix_of_add_and_remove_works) @@ -266,7 +263,7 @@ TEST_F(DocumentInverterTest, require_that_mix_of_add_and_remove_works) "w=c,r=9,a=10," "w=d,r=10,a=10," "w=z,r=12", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(DocumentInverterTest, require_that_empty_document_can_be_inverted) @@ -274,7 +271,7 @@ TEST_F(DocumentInverterTest, require_that_empty_document_can_be_inverted) _inv.invertDocument(15, *makeDoc15(_b)); pushDocuments(); EXPECT_EQ("", - _inserter.toStr()); + _inserter_backend.toStr()); } } diff --git a/searchlib/src/tests/memoryindex/document_inverter_collection/document_inverter_collection_test.cpp b/searchlib/src/tests/memoryindex/document_inverter_collection/document_inverter_collection_test.cpp index a6d675d05dc..ef08a6fd0e1 100644 --- a/searchlib/src/tests/memoryindex/document_inverter_collection/document_inverter_collection_test.cpp +++ b/searchlib/src/tests/memoryindex/document_inverter_collection/document_inverter_collection_test.cpp @@ -9,7 +9,7 @@ #include <vespa/searchlib/memoryindex/i_field_index_collection.h> #include <vespa/searchlib/memoryindex/word_store.h> #include <vespa/searchlib/test/memoryindex/mock_field_index_collection.h> -#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter_backend.h> #include <vespa/vespalib/util/retain_guard.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <thread> @@ -35,7 +35,7 @@ struct DocumentInverterCollectionTest : public ::testing::Test { std::unique_ptr<ISequencedTaskExecutor> _pushThreads; WordStore _word_store; FieldIndexRemover _remover; - test::OrderedFieldIndexInserter _inserter; + test::OrderedFieldIndexInserterBackend _inserter_backend; FieldLengthCalculator _calculator; test::MockFieldIndexCollection _fic; DocumentInverterContext _inv_context; @@ -47,9 +47,9 @@ struct DocumentInverterCollectionTest : public ::testing::Test { _pushThreads(SequencedTaskExecutor::create(push_executor, 4)), _word_store(), _remover(_word_store), - _inserter(), + _inserter_backend(), _calculator(), - _fic(_remover, _inserter, _calculator), + _fic(_remover, _inserter_backend, _calculator), _inv_context(_schema, *_invertThreads, *_pushThreads, _fic), _inv_collection(_inv_context, 10) { diff --git a/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp b/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp index 9b718052dd0..ed049a82c42 100644 --- a/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp +++ b/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/memoryindex/field_inverter.h> #include <vespa/searchlib/memoryindex/word_store.h> #include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter_backend.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/gtest/gtest.h> @@ -136,8 +137,9 @@ struct FieldInverterTest : public ::testing::Test { DocBuilder _b; WordStore _word_store; FieldIndexRemover _remover; - test::OrderedFieldIndexInserter _inserter; + test::OrderedFieldIndexInserterBackend _inserter_backend; std::vector<std::unique_ptr<FieldLengthCalculator>> _calculators; + std::vector<std::unique_ptr<IOrderedFieldIndexInserter>> _inserters; std::vector<std::unique_ptr<FieldInverter> > _inverters; static Schema makeSchema() { @@ -154,17 +156,19 @@ struct FieldInverterTest : public ::testing::Test { _b(_schema), _word_store(), _remover(_word_store), - _inserter(), + _inserter_backend(), _calculators(), + _inserters(), _inverters() { for (uint32_t fieldId = 0; fieldId < _schema.getNumIndexFields(); ++fieldId) { _calculators.emplace_back(std::make_unique<FieldLengthCalculator>()); + _inserters.emplace_back(std::make_unique<test::OrderedFieldIndexInserter>(_inserter_backend, fieldId)); _inverters.push_back(std::make_unique<FieldInverter>(_schema, fieldId, _remover, - _inserter, + *_inserters.back(), *_calculators.back())); } } @@ -180,11 +184,8 @@ struct FieldInverterTest : public ::testing::Test { } void pushDocuments() { - uint32_t fieldId = 0; for (auto &inverter : _inverters) { - _inserter.setFieldId(fieldId); inverter->pushDocuments(); - ++fieldId; } } @@ -210,7 +211,7 @@ TEST_F(FieldInverterTest, require_that_fresh_insert_works) "w=b,a=10," "w=c,a=10," "w=d,a=10", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(FieldInverterTest, require_that_multiple_docs_work) @@ -225,7 +226,7 @@ TEST_F(FieldInverterTest, require_that_multiple_docs_work) "w=f,a=11," "f=1,w=a,a=11," "w=g,a=11", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(FieldInverterTest, require_that_remove_works) @@ -240,7 +241,7 @@ TEST_F(FieldInverterTest, require_that_remove_works) "w=b,r=10,r=11," "f=1,w=a,r=10," "f=2,w=c,r=12", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(FieldInverterTest, require_that_reput_works) @@ -254,7 +255,7 @@ TEST_F(FieldInverterTest, require_that_reput_works) "w=f,a=10," "f=1,w=a,a=10," "w=g,a=10", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(FieldInverterTest, require_that_abort_pending_doc_works) @@ -275,7 +276,7 @@ TEST_F(FieldInverterTest, require_that_abort_pending_doc_works) "w=f,a=11," "f=1,w=a,a=11," "w=g,a=11", - _inserter.toStr()); + _inserter_backend.toStr()); invertDocument(10, *doc10); invertDocument(11, *doc11); @@ -284,7 +285,7 @@ TEST_F(FieldInverterTest, require_that_abort_pending_doc_works) invertDocument(14, *doc14); removeDocument(11); removeDocument(13); - _inserter.reset(); + _inserter_backend.reset(); pushDocuments(); EXPECT_EQ("f=0,w=a,a=10," "w=b,a=10," @@ -294,7 +295,7 @@ TEST_F(FieldInverterTest, require_that_abort_pending_doc_works) "w=doc14,a=14," "w=h,a=12," "w=j,a=14", - _inserter.toStr()); + _inserter_backend.toStr()); invertDocument(10, *doc10); invertDocument(11, *doc11); @@ -305,13 +306,13 @@ TEST_F(FieldInverterTest, require_that_abort_pending_doc_works) removeDocument(12); removeDocument(13); removeDocument(14); - _inserter.reset(); + _inserter_backend.reset(); pushDocuments(); EXPECT_EQ("f=0,w=a,a=10," "w=b,a=10," "w=c,a=10," "w=d,a=10", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(FieldInverterTest, require_that_mix_of_add_and_remove_works) @@ -327,7 +328,7 @@ TEST_F(FieldInverterTest, require_that_mix_of_add_and_remove_works) "w=c,r=9,a=10," "w=d,r=10,a=10," "w=z,r=12", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(FieldInverterTest, require_that_empty_document_can_be_inverted) @@ -335,13 +336,13 @@ TEST_F(FieldInverterTest, require_that_empty_document_can_be_inverted) invertDocument(15, *makeDoc15(_b)); pushDocuments(); EXPECT_EQ("", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(FieldInverterTest, require_that_multiple_words_at_same_position_works) { invertDocument(16, *makeDoc16(_b)); - _inserter.setVerbose(); + _inserter_backend.setVerbose(); pushDocuments(); EXPECT_EQ("f=0," "w=altbaz,a=16(e=0,w=1,l=5[2])," @@ -351,14 +352,14 @@ TEST_F(FieldInverterTest, require_that_multiple_words_at_same_position_works) "w=foo,a=16(e=0,w=1,l=5[0])," "w=y,a=16(e=0,w=1,l=5[3])," "w=z,a=16(e=0,w=1,l=5[4])", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(FieldInverterTest, require_that_interleaved_features_are_calculated) { invertDocument(17, *makeDoc17(_b)); - _inserter.setVerbose(); - _inserter.set_show_interleaved_features(); + _inserter_backend.setVerbose(); + _inserter_backend.set_show_interleaved_features(); pushDocuments(); EXPECT_EQ("f=1," "w=bar0,a=17(fl=2,occs=1,e=0,w=1,l=2[1])," @@ -369,7 +370,7 @@ TEST_F(FieldInverterTest, require_that_interleaved_features_are_calculated) "f=3," "w=bar2,a=17(fl=3,occs=2,e=0,w=3,l=2[1],e=1,w=4,l=1[0])," "w=foo2,a=17(fl=3,occs=1,e=0,w=3,l=2[0])", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(FieldInverterTest, require_that_average_field_length_is_calculated) @@ -397,7 +398,7 @@ TEST_F(FieldInverterTest, require_that_word_with_NUL_byte_is_truncated) "w=before,a=1," "w=corrupt,a=1," "w=z,a=1", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(FieldInverterTest, require_that_word_with_NUL_byte_is_dropped_when_truncated_to_zero_length) @@ -408,7 +409,7 @@ TEST_F(FieldInverterTest, require_that_word_with_NUL_byte_is_dropped_when_trunca "w=after,a=1," "w=before,a=1," "w=z,a=1", - _inserter.toStr()); + _inserter_backend.toStr()); } } diff --git a/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp b/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp index 5f73b9b8282..969f483eef6 100644 --- a/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp +++ b/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/memoryindex/url_field_inverter.h> #include <vespa/searchlib/memoryindex/word_store.h> #include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter_backend.h> #include <vespa/vespalib/gtest/gtest.h> namespace search { @@ -181,8 +182,9 @@ struct UrlFieldInverterTest : public ::testing::Test { DocBuilder _b; WordStore _word_store; FieldIndexRemover _remover; - test::OrderedFieldIndexInserter _inserter; + test::OrderedFieldIndexInserterBackend _inserter_backend; FieldLengthCalculator _calculator; + std::vector<std::unique_ptr<IOrderedFieldIndexInserter>> _inserters; std::vector<std::unique_ptr<FieldInverter> > _inverters; std::unique_ptr<UrlFieldInverter> _urlInverter; index::SchemaIndexFields _schemaIndexFields; @@ -198,8 +200,9 @@ struct UrlFieldInverterTest : public ::testing::Test { _b(_schema), _word_store(), _remover(_word_store), - _inserter(), + _inserter_backend(), _calculator(), + _inserters(), _inverters(), _urlInverter(), _schemaIndexFields() @@ -207,10 +210,11 @@ struct UrlFieldInverterTest : public ::testing::Test { _schemaIndexFields.setup(_schema); for (uint32_t fieldId = 0; fieldId < _schema.getNumIndexFields(); ++fieldId) { + _inserters.emplace_back(std::make_unique<test::OrderedFieldIndexInserter>(_inserter_backend, fieldId)); _inverters.push_back(std::make_unique<FieldInverter>(_schema, fieldId, _remover, - _inserter, + *_inserters.back(), _calculator)); } index::UriField &urlField = @@ -234,11 +238,8 @@ struct UrlFieldInverterTest : public ::testing::Test { } void pushDocuments() { - uint32_t fieldId = 0; for (auto &inverter : _inverters) { - _inserter.setFieldId(fieldId); inverter->pushDocuments(); - ++fieldId; } } @@ -297,7 +298,7 @@ TEST_F(SingleInverterTest, require_that_single_url_field_works) "w=com,a=10," "w=example,a=10," "w=www,a=10", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(ArrayInverterTest, require_that_array_url_field_works) @@ -340,7 +341,7 @@ TEST_F(ArrayInverterTest, require_that_array_url_field_works) "w=example,a=10," "w=flickr,a=10," "w=www,a=10", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(WeightedSetInverterTest, require_that_weighted_set_field_works) @@ -385,7 +386,7 @@ TEST_F(WeightedSetInverterTest, require_that_weighted_set_field_works) "w=example,a=10," "w=flickr,a=10," "w=www,a=10", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(SingleInverterTest, require_that_annotated_single_url_field_works) @@ -425,7 +426,7 @@ TEST_F(SingleInverterTest, require_that_annotated_single_url_field_works) "w=com,a=10," "w=example,a=10," "w=www,a=10", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(ArrayInverterTest, require_that_annotated_array_url_field_works) @@ -470,13 +471,13 @@ TEST_F(ArrayInverterTest, require_that_annotated_array_url_field_works) "w=example,a=10," "w=flickr,a=10," "w=www,a=10", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(WeightedSetInverterTest, require_that_annotated_weighted_set_field_works) { enableAnnotations(); - _inserter.setVerbose(); + _inserter_backend.setVerbose(); invertDocument(10, *makeDoc10WeightedSet(_b)); pushDocuments(); EXPECT_EQ("f=0," @@ -518,14 +519,14 @@ TEST_F(WeightedSetInverterTest, require_that_annotated_weighted_set_field_works) "w=example,a=10(e=0,w=4,l=5[2])," "w=flickr,a=10(e=1,w=7,l=5[2])," "w=www,a=10(e=0,w=4,l=5[1],e=1,w=7,l=5[1])", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(SingleInverterTest, require_that_empty_single_field_works) { invertDocument(10, *makeDoc10Empty(_b)); pushDocuments(); - EXPECT_EQ("", _inserter.toStr()); + EXPECT_EQ("", _inserter_backend.toStr()); } TEST_F(ArrayInverterTest, require_that_empty_array_field_works) @@ -533,14 +534,14 @@ TEST_F(ArrayInverterTest, require_that_empty_array_field_works) invertDocument(10, *makeDoc10Empty(_b)); pushDocuments(); EXPECT_EQ("", - _inserter.toStr()); + _inserter_backend.toStr()); } TEST_F(WeightedSetInverterTest, require_that_empty_weighted_set_field_works) { invertDocument(10, *makeDoc10Empty(_b)); pushDocuments(); - EXPECT_EQ("", _inserter.toStr()); + EXPECT_EQ("", _inserter_backend.toStr()); } TEST_F(SingleInverterTest, require_that_annotated_empty_single_field_works) @@ -548,7 +549,7 @@ TEST_F(SingleInverterTest, require_that_annotated_empty_single_field_works) enableAnnotations(); invertDocument(10, *makeDoc10Empty(_b)); pushDocuments(); - EXPECT_EQ("", _inserter.toStr()); + EXPECT_EQ("", _inserter_backend.toStr()); } TEST_F(ArrayInverterTest, require_that_annotated_empty_array_field_works) @@ -556,7 +557,7 @@ TEST_F(ArrayInverterTest, require_that_annotated_empty_array_field_works) enableAnnotations(); invertDocument(10, *makeDoc10Empty(_b)); pushDocuments(); - EXPECT_EQ("", _inserter.toStr()); + EXPECT_EQ("", _inserter_backend.toStr()); } TEST_F(WeightedSetInverterTest, require_that_annotated_empty_weighted_set_field_works) @@ -564,7 +565,7 @@ TEST_F(WeightedSetInverterTest, require_that_annotated_empty_weighted_set_field_ enableAnnotations(); invertDocument(10, *makeDoc10Empty(_b)); pushDocuments(); - EXPECT_EQ("", _inserter.toStr()); + EXPECT_EQ("", _inserter_backend.toStr()); } } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h index f69d482f565..dc4fba5d6b0 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h @@ -92,8 +92,6 @@ public: return _inverters[fieldId].get(); } - const std::vector<std::unique_ptr<FieldInverter> > & getInverters() const { return _inverters; } - uint32_t getNumFields() const { return _inverters.size(); } void wait_for_zero_ref_count() { _ref_count.waitForZeroRefCount(); } bool has_zero_ref_count() { return _ref_count.has_zero_ref_count(); } diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/memoryindex/CMakeLists.txt index c7cc6d77ede..ff948166033 100644 --- a/searchlib/src/vespa/searchlib/test/memoryindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/test/memoryindex/CMakeLists.txt @@ -2,5 +2,7 @@ vespa_add_library(searchlib_searchlib_test_memoryindex SOURCES mock_field_index_collection.cpp + ordered_field_index_inserter.cpp + ordered_field_index_inserter_backend.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/mock_field_index_collection.cpp b/searchlib/src/vespa/searchlib/test/memoryindex/mock_field_index_collection.cpp index 1392a92853d..a98e20deba5 100644 --- a/searchlib/src/vespa/searchlib/test/memoryindex/mock_field_index_collection.cpp +++ b/searchlib/src/vespa/searchlib/test/memoryindex/mock_field_index_collection.cpp @@ -6,10 +6,10 @@ namespace search::memoryindex::test { MockFieldIndexCollection::MockFieldIndexCollection(FieldIndexRemover& remover, - OrderedFieldIndexInserter& inserter, + OrderedFieldIndexInserterBackend& inserter_backend, index::FieldLengthCalculator& calculator) : _remover(remover), - _inserter(inserter), + _inserter_backend(inserter_backend), _calculator(calculator) { } @@ -23,9 +23,16 @@ MockFieldIndexCollection::get_remover(uint32_t) } IOrderedFieldIndexInserter& -MockFieldIndexCollection::get_inserter(uint32_t) +MockFieldIndexCollection::get_inserter(uint32_t field_id) { - return _inserter; + if (_inserters.size() <= field_id) { + _inserters.resize(field_id + 1); + } + auto& inserter = _inserters[field_id]; + if (!inserter) { + inserter = std::make_unique<OrderedFieldIndexInserter>(_inserter_backend, field_id); + } + return *inserter; } index::FieldLengthCalculator& diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/mock_field_index_collection.h b/searchlib/src/vespa/searchlib/test/memoryindex/mock_field_index_collection.h index 0dd7127f0ae..c7f2d364c5b 100644 --- a/searchlib/src/vespa/searchlib/test/memoryindex/mock_field_index_collection.h +++ b/searchlib/src/vespa/searchlib/test/memoryindex/mock_field_index_collection.h @@ -3,29 +3,32 @@ #pragma once #include <vespa/searchlib/memoryindex/i_field_index_collection.h> +#include <memory> +#include <vector> namespace search::index { class FieldLengthCalculator; } namespace search::memoryindex { class FieldIndexRemover; } namespace search::memoryindex::test { -class OrderedFieldIndexInserter; +class OrderedFieldIndexInserterBackend; /* * Mockup of field index collection used by unit tests. */ class MockFieldIndexCollection : public IFieldIndexCollection { FieldIndexRemover& _remover; - OrderedFieldIndexInserter& _inserter; + OrderedFieldIndexInserterBackend& _inserter_backend; index::FieldLengthCalculator& _calculator; + std::vector<std::unique_ptr<IOrderedFieldIndexInserter>> _inserters; public: MockFieldIndexCollection(FieldIndexRemover& remover, - OrderedFieldIndexInserter& inserter, + OrderedFieldIndexInserterBackend& inserter_backend, index::FieldLengthCalculator& calculator); ~MockFieldIndexCollection() override; FieldIndexRemover& get_remover(uint32_t) override; - IOrderedFieldIndexInserter& get_inserter(uint32_t) override; + IOrderedFieldIndexInserter& get_inserter(uint32_t field_id) override; index::FieldLengthCalculator& get_calculator(uint32_t) override; }; diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.cpp b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.cpp new file mode 100644 index 00000000000..6dbd434e20a --- /dev/null +++ b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.cpp @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "ordered_field_index_inserter.h" +#include "ordered_field_index_inserter_backend.h" + +namespace search::memoryindex::test { + +OrderedFieldIndexInserter::OrderedFieldIndexInserter(OrderedFieldIndexInserterBackend& backend, uint32_t field_id) + : _backend(backend), + _field_id(field_id) +{ +} + +OrderedFieldIndexInserter::~OrderedFieldIndexInserter() = default; + +void +OrderedFieldIndexInserter::setNextWord(const vespalib::stringref word) +{ + _backend.setNextWord(word); +} + +void +OrderedFieldIndexInserter::add(uint32_t docId, const index::DocIdAndFeatures &features) +{ + _backend.add(docId, features); +} + +vespalib::datastore::EntryRef +OrderedFieldIndexInserter::getWordRef() const +{ + return vespalib::datastore::EntryRef(); +} + +void +OrderedFieldIndexInserter::remove(uint32_t docId) +{ + _backend.remove(docId); +} + +void +OrderedFieldIndexInserter::flush() +{ +} + +void +OrderedFieldIndexInserter::commit() +{ +} + +void +OrderedFieldIndexInserter::rewind() +{ + _backend.rewind(_field_id); +} + +} + diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h index de7023499a1..20b8e58fa60 100644 --- a/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h +++ b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h @@ -2,109 +2,29 @@ #pragma once -#include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h> -#include <sstream> namespace search::memoryindex::test { -class OrderedFieldIndexInserter : public IOrderedFieldIndexInserter { - std::stringstream _ss; - bool _first; - bool _verbose; - bool _show_interleaved_features; - uint32_t _fieldId; - - void addComma() { - if (!_first) { - _ss << ","; - } else { - _first = false; - } - } +class OrderedFieldIndexInserterBackend; +/* + * Test version of ordered field index inserter that uses a backend to create + * a string representation used to validate correct use of ordered field index inserter. + */ +class OrderedFieldIndexInserter : public IOrderedFieldIndexInserter { + OrderedFieldIndexInserterBackend& _backend; + uint32_t _field_id; public: - OrderedFieldIndexInserter() - : _ss(), - _first(true), - _verbose(false), - _show_interleaved_features(false), - _fieldId(0) - { - } - - virtual void setNextWord(const vespalib::stringref word) override { - addComma(); - _ss << "w=" << word; - } - - void setFieldId(uint32_t fieldId) { - _fieldId = fieldId; - } - - virtual void add(uint32_t docId, - const index::DocIdAndFeatures &features) override { - (void) features; - addComma(); - _ss << "a=" << docId; - if (_verbose) { - _ss << "("; - auto wpi = features.word_positions().begin(); - bool firstElement = true; - if (_show_interleaved_features) { - _ss << "fl=" << features.field_length() << - ",occs=" << features.num_occs(); - firstElement = false; - } - for (auto &el : features.elements()) { - if (!firstElement) { - _ss << ","; - } - firstElement = false; - _ss << "e=" << el.getElementId() << ",w=" << - el.getWeight() << ",l=" << - el.getElementLen() << "["; - bool firstWordPos = true; - for (uint32_t i = 0; i < el.getNumOccs(); ++i) { - if (!firstWordPos) { - _ss << ","; - } - firstWordPos = false; - _ss << wpi->getWordPos(); - ++wpi; - } - _ss << "]"; - } - _ss << ")"; - } - } - - virtual vespalib::datastore::EntryRef getWordRef() const override { return vespalib::datastore::EntryRef(); } - - virtual void remove(uint32_t docId) override { - addComma(); - _ss << "r=" << docId; - } - - virtual void flush() override { } - virtual void commit() override { } - virtual void rewind() override { - addComma(); - _ss << "f=" << _fieldId; - } - - std::string toStr() const { - return _ss.str(); - } - - void reset() { - _ss.str(""); - _first = true; - _verbose = false; - } - - void setVerbose() { _verbose = true; } - void set_show_interleaved_features() { _show_interleaved_features = true; } + OrderedFieldIndexInserter(OrderedFieldIndexInserterBackend& backend, uint32_t field_id); + ~OrderedFieldIndexInserter() override; + void setNextWord(const vespalib::stringref word) override; + void add(uint32_t docId, const index::DocIdAndFeatures &features) override; + vespalib::datastore::EntryRef getWordRef() const override; + void remove(uint32_t docId) override; + void flush() override; + void commit() override; + void rewind() override; }; } diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter_backend.cpp b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter_backend.cpp new file mode 100644 index 00000000000..ef35ffa4bdc --- /dev/null +++ b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter_backend.cpp @@ -0,0 +1,101 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "ordered_field_index_inserter_backend.h" +#include <vespa/searchlib/index/docidandfeatures.h> + +namespace search::memoryindex::test { + +OrderedFieldIndexInserterBackend::OrderedFieldIndexInserterBackend() + : _ss(), + _first(true), + _verbose(false), + _show_interleaved_features(false) +{ +} + +OrderedFieldIndexInserterBackend::~OrderedFieldIndexInserterBackend() = default; + +void +OrderedFieldIndexInserterBackend::addComma() +{ + if (!_first) { + _ss << ","; + } else { + _first = false; + } +} + +void +OrderedFieldIndexInserterBackend::setNextWord(const vespalib::stringref word) +{ + addComma(); + _ss << "w=" << word; +} + +void +OrderedFieldIndexInserterBackend::add(uint32_t docId, const index::DocIdAndFeatures &features) +{ + (void) features; + addComma(); + _ss << "a=" << docId; + if (_verbose) { + _ss << "("; + auto wpi = features.word_positions().begin(); + bool firstElement = true; + if (_show_interleaved_features) { + _ss << "fl=" << features.field_length() << + ",occs=" << features.num_occs(); + firstElement = false; + } + for (auto &el : features.elements()) { + if (!firstElement) { + _ss << ","; + } + firstElement = false; + _ss << "e=" << el.getElementId() << ",w=" << + el.getWeight() << ",l=" << + el.getElementLen() << "["; + bool firstWordPos = true; + for (uint32_t i = 0; i < el.getNumOccs(); ++i) { + if (!firstWordPos) { + _ss << ","; + } + firstWordPos = false; + _ss << wpi->getWordPos(); + ++wpi; + } + _ss << "]"; + } + _ss << ")"; + } +} + +void +OrderedFieldIndexInserterBackend::remove(uint32_t docId) +{ + addComma(); + _ss << "r=" << docId; +} + +void +OrderedFieldIndexInserterBackend::rewind(uint32_t field_id) +{ + addComma(); + _ss << "f=" << field_id; +} + +std::string +OrderedFieldIndexInserterBackend::toStr() const +{ + return _ss.str(); +} + +void +OrderedFieldIndexInserterBackend::reset() +{ + _ss.str(""); + _first = true; + _verbose = false; +} + +} diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter_backend.h b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter_backend.h new file mode 100644 index 00000000000..e81b6db5c36 --- /dev/null +++ b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter_backend.h @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <sstream> + +namespace search::index { class DocIdAndFeatures; } + +namespace search::memoryindex::test { + +/* + * Backend for test version of ordered field index inserter that creates + * a string representation used to validate correct use of ordered field index inserter. + */ +class OrderedFieldIndexInserterBackend { + std::stringstream _ss; + bool _first; + bool _verbose; + bool _show_interleaved_features; + void addComma(); +public: + OrderedFieldIndexInserterBackend(); + ~OrderedFieldIndexInserterBackend(); + void setNextWord(const vespalib::stringref word); + void add(uint32_t docId, const index::DocIdAndFeatures &features); + void remove(uint32_t docId); + void rewind(uint32_t field_id); + std::string toStr() const; + void reset(); + void setVerbose() { _verbose = true; } + void set_show_interleaved_features() { _show_interleaved_features = true; } +}; + +} |