diff options
13 files changed, 190 insertions, 87 deletions
diff --git a/searchcore/src/tests/proton/index/CMakeLists.txt b/searchcore/src/tests/proton/index/CMakeLists.txt index a06a3dfddd5..6fffa47f1b9 100644 --- a/searchcore/src/tests/proton/index/CMakeLists.txt +++ b/searchcore/src/tests/proton/index/CMakeLists.txt @@ -29,6 +29,7 @@ vespa_add_executable(searchcore_indexcollection_test_app TEST indexcollection_test.cpp DEPENDS searchcore_index + gtest ) vespa_add_test(NAME searchcore_index_test COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/index_test.sh DEPENDS searchcore_indexmanager_test_app searchcore_fusionrunner_test_app searchcore_diskindexcleaner_test_app searchcore_indexcollection_test_app) diff --git a/searchcore/src/tests/proton/index/indexcollection_test.cpp b/searchcore/src/tests/proton/index/indexcollection_test.cpp index f9e7b22851b..7a42fe574c2 100644 --- a/searchcore/src/tests/proton/index/indexcollection_test.cpp +++ b/searchcore/src/tests/proton/index/indexcollection_test.cpp @@ -2,24 +2,41 @@ #include <vespa/searchcore/proton/matching/fakesearchcontext.h> #include <vespa/searchcorespi/index/warmupindexcollection.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/log/log.h> LOG_SETUP("indexcollection_test"); -using search::queryeval::ISourceSelector; -using search::queryeval::FakeSearchable; -using search::FixedSourceSelector; using namespace proton; using namespace searchcorespi; +using search::FixedSourceSelector; +using search::index::FieldLengthInfo; +using search::queryeval::FakeSearchable; +using search::queryeval::ISourceSelector; using searchcorespi::index::WarmupConfig; -namespace { +class MockIndexSearchable : public FakeIndexSearchable { +private: + FieldLengthInfo _field_length_info; + +public: + MockIndexSearchable() + : _field_length_info() + {} + MockIndexSearchable(const FieldLengthInfo& field_length_info) + : _field_length_info(field_length_info) + {} + FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override { + (void) field_name; + return _field_length_info; + } +}; -class Test : public vespalib::TestApp, - public IWarmupDone +class IndexCollectionTest : public ::testing::Test, + public IWarmupDone { +public: std::shared_ptr<ISourceSelector> _selector; std::shared_ptr<IndexSearchable> _source1; std::shared_ptr<IndexSearchable> _source2; @@ -27,102 +44,120 @@ class Test : public vespalib::TestApp, vespalib::ThreadStackExecutor _executor; std::shared_ptr<IndexSearchable> _warmup; - void requireThatSearchablesCanBeAppended(IndexCollection::UP fsc); - void requireThatSearchablesCanBeReplaced(IndexCollection::UP fsc); - void requireThatReplaceAndRenumberUpdatesCollectionAfterFusion(); - IndexCollection::UP createWarmup(const IndexCollection::SP & prev, const IndexCollection::SP & next); + void expect_searchable_can_be_appended(IndexCollection::UP collection) { + const uint32_t id = 42; + + collection->append(id, _source1); + EXPECT_EQ(1u, collection->getSourceCount()); + EXPECT_EQ(id, collection->getSourceId(0)); + } + + void expect_searchable_can_be_replaced(IndexCollection::UP collection) { + const uint32_t id = 42; + + collection->append(id, _source1); + EXPECT_EQ(1u, collection->getSourceCount()); + EXPECT_EQ(id, collection->getSourceId(0)); + EXPECT_EQ(_source1.get(), &collection->getSearchable(0)); + + collection->replace(id, _source2); + EXPECT_EQ(1u, collection->getSourceCount()); + EXPECT_EQ(id, collection->getSourceId(0)); + EXPECT_EQ(_source2.get(), &collection->getSearchable(0)); + } + + IndexCollection::UP make_unique_collection() const { + return std::make_unique<IndexCollection>(_selector); + } + + IndexCollection::SP make_shared_collection() const { + return std::make_shared<IndexCollection>(_selector); + } + + IndexCollection::UP create_warmup(const IndexCollection::SP& prev, const IndexCollection::SP& next) { + return std::make_unique<WarmupIndexCollection>(WarmupConfig(1.0, false), prev, next, *_warmup, _executor, *this); + } + virtual void warmupDone(ISearchableIndexCollection::SP current) override { (void) current; } -public: - Test() : _selector(new FixedSourceSelector(0, "fs1")), - _source1(new FakeIndexSearchable), - _source2(new FakeIndexSearchable), - _fusion_source(new FakeIndexSearchable), - _executor(1, 128*1024), - _warmup(new FakeIndexSearchable) + IndexCollectionTest() + : _selector(new FixedSourceSelector(0, "fs1")), + _source1(new MockIndexSearchable({3, 5})), + _source2(new MockIndexSearchable({7, 11})), + _fusion_source(new FakeIndexSearchable), + _executor(1, 128*1024), + _warmup(new FakeIndexSearchable) {} - ~Test() {} - - int Main() override; + ~IndexCollectionTest() {} }; -IndexCollection::UP -Test::createWarmup(const IndexCollection::SP & prev, const IndexCollection::SP & next) +TEST_F(IndexCollectionTest, searchable_can_be_appended_to_normal_collection) { - return IndexCollection::UP(new WarmupIndexCollection(WarmupConfig(1.0, false), prev, next, *_warmup, _executor, *this)); + expect_searchable_can_be_appended(make_unique_collection()); } -int -Test::Main() +TEST_F(IndexCollectionTest, searchable_can_be_replaced_in_normal_collection) { - TEST_INIT("indexcollection_test"); - - TEST_DO(requireThatSearchablesCanBeAppended(IndexCollection::UP(new IndexCollection(_selector)))); - TEST_DO(requireThatSearchablesCanBeReplaced(IndexCollection::UP(new IndexCollection(_selector)))); - TEST_DO(requireThatReplaceAndRenumberUpdatesCollectionAfterFusion()); - { - IndexCollection::SP prev(new IndexCollection(_selector)); - IndexCollection::SP next(new IndexCollection(_selector)); - requireThatSearchablesCanBeAppended(createWarmup(prev, next)); - EXPECT_EQUAL(0u, prev->getSourceCount()); - EXPECT_EQUAL(1u, next->getSourceCount()); - } - { - IndexCollection::SP prev(new IndexCollection(_selector)); - IndexCollection::SP next(new IndexCollection(_selector)); - requireThatSearchablesCanBeReplaced(createWarmup(prev, next)); - EXPECT_EQUAL(0u, prev->getSourceCount()); - EXPECT_EQUAL(1u, next->getSourceCount()); - } - - TEST_DONE(); + expect_searchable_can_be_replaced(make_unique_collection()); } -void Test::requireThatSearchablesCanBeAppended(IndexCollection::UP fsc) { - const uint32_t id = 42; - - fsc->append(id, _source1); - EXPECT_EQUAL(1u, fsc->getSourceCount()); - EXPECT_EQUAL(id, fsc->getSourceId(0)); +TEST_F(IndexCollectionTest, searchable_can_be_appended_to_warmup_collection) +{ + auto prev = make_shared_collection(); + auto next = make_shared_collection(); + expect_searchable_can_be_appended(create_warmup(prev, next)); + EXPECT_EQ(0u, prev->getSourceCount()); + EXPECT_EQ(1u, next->getSourceCount()); } -void Test::requireThatSearchablesCanBeReplaced(IndexCollection::UP fsc) { - const uint32_t id = 42; - - fsc->append(id, _source1); - EXPECT_EQUAL(1u, fsc->getSourceCount()); - EXPECT_EQUAL(id, fsc->getSourceId(0)); - EXPECT_EQUAL(_source1.get(), &fsc->getSearchable(0)); - - fsc->replace(id, _source2); - EXPECT_EQUAL(1u, fsc->getSourceCount()); - EXPECT_EQUAL(id, fsc->getSourceId(0)); - EXPECT_EQUAL(_source2.get(), &fsc->getSearchable(0)); +TEST_F(IndexCollectionTest, searchable_can_be_replaced_in_warmup_collection) +{ + auto prev = make_shared_collection(); + auto next = make_shared_collection(); + expect_searchable_can_be_replaced(create_warmup(prev, next)); + EXPECT_EQ(0u, prev->getSourceCount()); + EXPECT_EQ(1u, next->getSourceCount()); } -void Test::requireThatReplaceAndRenumberUpdatesCollectionAfterFusion() { +TEST_F(IndexCollectionTest, replace_and_renumber_updates_collection_after_fusion) +{ IndexCollection fsc(_selector); fsc.append(0, _source1); fsc.append(1, _source1); fsc.append(2, _source1); fsc.append(3, _source2); - EXPECT_EQUAL(4u, fsc.getSourceCount()); + EXPECT_EQ(4u, fsc.getSourceCount()); const uint32_t id_diff = 2; - IndexCollection::UP new_fsc = - IndexCollection::replaceAndRenumber( - _selector, fsc, id_diff, _fusion_source); - EXPECT_EQUAL(2u, new_fsc->getSourceCount()); - EXPECT_EQUAL(0u, new_fsc->getSourceId(0)); - EXPECT_EQUAL(_fusion_source.get(), &new_fsc->getSearchable(0)); - EXPECT_EQUAL(1u, new_fsc->getSourceId(1)); - EXPECT_EQUAL(_source2.get(), &new_fsc->getSearchable(1)); + auto new_fsc = IndexCollection::replaceAndRenumber(_selector, fsc, id_diff, _fusion_source); + EXPECT_EQ(2u, new_fsc->getSourceCount()); + EXPECT_EQ(0u, new_fsc->getSourceId(0)); + EXPECT_EQ(_fusion_source.get(), &new_fsc->getSearchable(0)); + EXPECT_EQ(1u, new_fsc->getSourceId(1)); + EXPECT_EQ(_source2.get(), &new_fsc->getSearchable(1)); } -} // namespace +TEST_F(IndexCollectionTest, returns_field_length_info_for_last_added_searchable) +{ + auto collection = make_unique_collection(); + + collection->append(3, _source1); + collection->append(4, _source2); + + EXPECT_DOUBLE_EQ(7, collection->get_field_length_info("foo").get_average_field_length()); + EXPECT_EQ(11, collection->get_field_length_info("foo").get_num_samples()); +} + +TEST_F(IndexCollectionTest, returns_empty_field_length_info_when_no_searchables_exists) +{ + auto collection = make_unique_collection(); + + EXPECT_DOUBLE_EQ(0, collection->get_field_length_info("foo").get_average_field_length()); + EXPECT_EQ(0, collection->get_field_length_info("foo").get_num_samples()); +} -TEST_APPHOOK(Test); +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp index 96e0ab7b650..d70c321722b 100644 --- a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp @@ -5,6 +5,7 @@ #include <vespa/searchcorespi/index/indexsearchablevisitor.h> using search::TuneFileSearch; +using search::index::FieldLengthInfo; using searchcorespi::index::IndexReadUtilities; namespace proton { @@ -45,5 +46,13 @@ DiskIndexWrapper::accept(searchcorespi::IndexSearchableVisitor &visitor) const visitor.visit(*this); } +FieldLengthInfo +DiskIndexWrapper::get_field_length_info(const vespalib::string& field_name) const +{ + // TODO: implement + (void) field_name; + return FieldLengthInfo(); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.h index c715fc8ba3e..cba29a53b82 100644 --- a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.h @@ -41,6 +41,11 @@ public: void accept(searchcorespi::IndexSearchableVisitor &visitor) const override; /** + * Implements IFieldLengthInspector + */ + search::index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override; + + /** * Implements proton::IDiskIndex */ const vespalib::string &getIndexDir() const override { return _index.getIndexDir(); } diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp index 5bcd2046015..3d1e04196a6 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp @@ -1,17 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "memoryindexwrapper.h" +#include <vespa/searchcorespi/index/indexsearchablevisitor.h> #include <vespa/searchlib/common/serialnumfileheadercontext.h> #include <vespa/searchlib/diskindex/indexbuilder.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/searchcorespi/index/indexsearchablevisitor.h> +using search::SerialNum; using search::TuneFileIndexing; using search::common::FileHeaderContext; using search::common::SerialNumFileHeaderContext; -using search::index::Schema; using search::diskindex::IndexBuilder; -using search::SerialNum; +using search::index::FieldLengthInfo; +using search::index::Schema; using vespalib::IllegalStateException; namespace proton { @@ -58,5 +59,13 @@ MemoryIndexWrapper::accept(searchcorespi::IndexSearchableVisitor &visitor) const visitor.visit(*this); } +FieldLengthInfo +MemoryIndexWrapper::get_field_length_info(const vespalib::string& field_name) const +{ + // TODO: implement + (void) field_name; + return FieldLengthInfo(); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h index 5e3ff79b92e..d94a259eb24 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h @@ -60,6 +60,11 @@ public: void accept(searchcorespi::IndexSearchableVisitor &visitor) const override; /** + * Implements IFieldLengthInspector + */ + search::index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override; + + /** * Implements proton::IMemoryIndex */ bool hasReceivedDocumentInsert() const override { diff --git a/searchcorespi/src/vespa/searchcorespi/index/fakeindexsearchable.h b/searchcorespi/src/vespa/searchcorespi/index/fakeindexsearchable.h index 9eba40e08a4..3a391363e47 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/fakeindexsearchable.h +++ b/searchcorespi/src/vespa/searchcorespi/index/fakeindexsearchable.h @@ -39,6 +39,11 @@ public: (void) visitor; } + search::index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override { + (void) field_name; + return search::index::FieldLengthInfo(); + } + }; } // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp index de4cde956d0..81a43f932ca 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp @@ -13,6 +13,7 @@ LOG_SETUP(".searchcorespi.index.indexcollection"); using namespace search::queryeval; using namespace search::query; using search::attribute::IAttributeContext; +using search::index::FieldLengthInfo; namespace searchcorespi { @@ -238,4 +239,13 @@ IndexCollection::createBlueprint(const IRequestContext & requestContext, return visitor.getResult(); } +FieldLengthInfo +IndexCollection::get_field_length_info(const vespalib::string& field_name) const +{ + if (_sources.empty()) { + return FieldLengthInfo(); + } + return _sources.back().source_wrapper->get_field_length_info(field_name); +} + } // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.h b/searchcorespi/src/vespa/searchcorespi/index/indexcollection.h index eabdddf9dfe..e71f421c3ca 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexcollection.h @@ -57,6 +57,12 @@ public: static ISearchableIndexCollection::UP replaceAndRenumber(const ISourceSelectorSP & selector, const ISearchableIndexCollection &fsc, uint32_t id_diff, const IndexSearchable::SP &new_source); + + // Implements IFieldLengthInspector + /** + * Returns field length info from the newest disk index, or empty info for all fields if no disk index exists. + */ + search::index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override; }; } // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index 668eeba6463..acde26ad554 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -124,6 +124,13 @@ public: } /** + * Implements IFieldLengthInspector + */ + search::index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override { + return _index->get_field_length_info(field_name); + } + + /** * Implements IDiskIndex */ const vespalib::string &getIndexDir() const override { return _index->getIndexDir(); } diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexsearchable.h b/searchcorespi/src/vespa/searchcorespi/index/indexsearchable.h index 03b7f91e47a..96db89b3e61 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexsearchable.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexsearchable.h @@ -4,6 +4,7 @@ #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/common/serialnum.h> +#include <vespa/searchlib/index/i_field_length_inspector.h> #include <vespa/searchlib/query/tree/node.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/field_spec.h> @@ -26,7 +27,8 @@ class IndexSearchableVisitor; * that let the components access a per query attribute context that expose * attribute vectors that can be utilized during query evaluation. **/ -class IndexSearchable : public search::queryeval::Searchable { +class IndexSearchable : public search::queryeval::Searchable, + public search::index::IFieldLengthInspector { protected: using IRequestContext = search::queryeval::IRequestContext; using FieldSpec = search::queryeval::FieldSpec; diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp index 75412dcf8e9..911ff5f9eba 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp @@ -12,16 +12,17 @@ LOG_SETUP(".searchcorespi.index.warmupindexcollection"); namespace searchcorespi { +using fastos::ClockSystem; +using fastos::TimeStamp; +using index::IDiskIndex; +using search::fef::MatchDataLayout; +using search::index::FieldLengthInfo; using search::query::StringBase; using search::queryeval::Blueprint; -using search::fef::MatchDataLayout; -using search::queryeval::SearchIterator; using search::queryeval::ISourceSelector; -using vespalib::makeTask; +using search::queryeval::SearchIterator; using vespalib::makeClosure; -using index::IDiskIndex; -using fastos::TimeStamp; -using fastos::ClockSystem; +using vespalib::makeTask; using TermMap = vespalib::hash_set<vespalib::string>; class FieldTermMap : public vespalib::hash_map<uint32_t, TermMap> @@ -196,6 +197,11 @@ WarmupIndexCollection::accept(IndexSearchableVisitor &visitor) const _next->accept(visitor); } +FieldLengthInfo +WarmupIndexCollection::get_field_length_info(const vespalib::string& field_name) const +{ + return _next->get_field_length_info(field_name); +} void WarmupIndexCollection::append(uint32_t id, const IndexSearchable::SP &source) diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h index f6d6bc89fc4..229d1a7c31d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h +++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h @@ -52,6 +52,9 @@ public: search::SerialNum getSerialNum() const override; void accept(IndexSearchableVisitor &visitor) const override; + // Implements IFieldLengthInspector + search::index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override; + // Implements ISearchableIndexCollection void append(uint32_t id, const IndexSearchable::SP &source) override; void replace(uint32_t id, const IndexSearchable::SP &source) override; |