diff options
12 files changed, 133 insertions, 21 deletions
diff --git a/searchcore/src/tests/proton/index/indexcollection_test.cpp b/searchcore/src/tests/proton/index/indexcollection_test.cpp index 3653ea32bae..7a42fe574c2 100644 --- a/searchcore/src/tests/proton/index/indexcollection_test.cpp +++ b/searchcore/src/tests/proton/index/indexcollection_test.cpp @@ -8,13 +8,31 @@ #include <vespa/log/log.h> LOG_SETUP("indexcollection_test"); -using search::queryeval::ISourceSelector; -using search::queryeval::FakeSearchable; -using search::FixedSourceSelector; using namespace proton; using namespace searchcorespi; +using search::FixedSourceSelector; +using search::index::FieldLengthInfo; +using search::queryeval::FakeSearchable; +using search::queryeval::ISourceSelector; using searchcorespi::index::WarmupConfig; +class MockIndexSearchable : public FakeIndexSearchable { +private: + FieldLengthInfo _field_length_info; + +public: + MockIndexSearchable() + : _field_length_info() + {} + MockIndexSearchable(const FieldLengthInfo& field_length_info) + : _field_length_info(field_length_info) + {} + FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override { + (void) field_name; + return _field_length_info; + } +}; + class IndexCollectionTest : public ::testing::Test, public IWarmupDone { @@ -48,6 +66,14 @@ public: EXPECT_EQ(_source2.get(), &collection->getSearchable(0)); } + IndexCollection::UP make_unique_collection() const { + return std::make_unique<IndexCollection>(_selector); + } + + IndexCollection::SP make_shared_collection() const { + return std::make_shared<IndexCollection>(_selector); + } + IndexCollection::UP create_warmup(const IndexCollection::SP& prev, const IndexCollection::SP& next) { return std::make_unique<WarmupIndexCollection>(WarmupConfig(1.0, false), prev, next, *_warmup, _executor, *this); } @@ -58,8 +84,8 @@ public: IndexCollectionTest() : _selector(new FixedSourceSelector(0, "fs1")), - _source1(new FakeIndexSearchable), - _source2(new FakeIndexSearchable), + _source1(new MockIndexSearchable({3, 5})), + _source2(new MockIndexSearchable({7, 11})), _fusion_source(new FakeIndexSearchable), _executor(1, 128*1024), _warmup(new FakeIndexSearchable) @@ -70,18 +96,18 @@ public: TEST_F(IndexCollectionTest, searchable_can_be_appended_to_normal_collection) { - expect_searchable_can_be_appended(std::make_unique<IndexCollection>(_selector)); + expect_searchable_can_be_appended(make_unique_collection()); } TEST_F(IndexCollectionTest, searchable_can_be_replaced_in_normal_collection) { - expect_searchable_can_be_replaced(std::make_unique<IndexCollection>(_selector)); + expect_searchable_can_be_replaced(make_unique_collection()); } TEST_F(IndexCollectionTest, searchable_can_be_appended_to_warmup_collection) { - auto prev = std::make_shared<IndexCollection>(_selector); - auto next = std::make_shared<IndexCollection>(_selector); + auto prev = make_shared_collection(); + auto next = make_shared_collection(); expect_searchable_can_be_appended(create_warmup(prev, next)); EXPECT_EQ(0u, prev->getSourceCount()); EXPECT_EQ(1u, next->getSourceCount()); @@ -89,8 +115,8 @@ TEST_F(IndexCollectionTest, searchable_can_be_appended_to_warmup_collection) TEST_F(IndexCollectionTest, searchable_can_be_replaced_in_warmup_collection) { - auto prev = std::make_shared<IndexCollection>(_selector); - auto next = std::make_shared<IndexCollection>(_selector); + auto prev = make_shared_collection(); + auto next = make_shared_collection(); expect_searchable_can_be_replaced(create_warmup(prev, next)); EXPECT_EQ(0u, prev->getSourceCount()); EXPECT_EQ(1u, next->getSourceCount()); @@ -115,4 +141,23 @@ TEST_F(IndexCollectionTest, replace_and_renumber_updates_collection_after_fusion EXPECT_EQ(_source2.get(), &new_fsc->getSearchable(1)); } +TEST_F(IndexCollectionTest, returns_field_length_info_for_last_added_searchable) +{ + auto collection = make_unique_collection(); + + collection->append(3, _source1); + collection->append(4, _source2); + + EXPECT_DOUBLE_EQ(7, collection->get_field_length_info("foo").get_average_field_length()); + EXPECT_EQ(11, collection->get_field_length_info("foo").get_num_samples()); +} + +TEST_F(IndexCollectionTest, returns_empty_field_length_info_when_no_searchables_exists) +{ + auto collection = make_unique_collection(); + + EXPECT_DOUBLE_EQ(0, collection->get_field_length_info("foo").get_average_field_length()); + EXPECT_EQ(0, collection->get_field_length_info("foo").get_num_samples()); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp index 96e0ab7b650..d70c321722b 100644 --- a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp @@ -5,6 +5,7 @@ #include <vespa/searchcorespi/index/indexsearchablevisitor.h> using search::TuneFileSearch; +using search::index::FieldLengthInfo; using searchcorespi::index::IndexReadUtilities; namespace proton { @@ -45,5 +46,13 @@ DiskIndexWrapper::accept(searchcorespi::IndexSearchableVisitor &visitor) const visitor.visit(*this); } +FieldLengthInfo +DiskIndexWrapper::get_field_length_info(const vespalib::string& field_name) const +{ + // TODO: implement + (void) field_name; + return FieldLengthInfo(); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.h index c715fc8ba3e..cba29a53b82 100644 --- a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.h @@ -41,6 +41,11 @@ public: void accept(searchcorespi::IndexSearchableVisitor &visitor) const override; /** + * Implements IFieldLengthInspector + */ + search::index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override; + + /** * Implements proton::IDiskIndex */ const vespalib::string &getIndexDir() const override { return _index.getIndexDir(); } diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp index 5bcd2046015..3d1e04196a6 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp @@ -1,17 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "memoryindexwrapper.h" +#include <vespa/searchcorespi/index/indexsearchablevisitor.h> #include <vespa/searchlib/common/serialnumfileheadercontext.h> #include <vespa/searchlib/diskindex/indexbuilder.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/searchcorespi/index/indexsearchablevisitor.h> +using search::SerialNum; using search::TuneFileIndexing; using search::common::FileHeaderContext; using search::common::SerialNumFileHeaderContext; -using search::index::Schema; using search::diskindex::IndexBuilder; -using search::SerialNum; +using search::index::FieldLengthInfo; +using search::index::Schema; using vespalib::IllegalStateException; namespace proton { @@ -58,5 +59,13 @@ MemoryIndexWrapper::accept(searchcorespi::IndexSearchableVisitor &visitor) const visitor.visit(*this); } +FieldLengthInfo +MemoryIndexWrapper::get_field_length_info(const vespalib::string& field_name) const +{ + // TODO: implement + (void) field_name; + return FieldLengthInfo(); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h index 5e3ff79b92e..d94a259eb24 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h @@ -60,6 +60,11 @@ public: void accept(searchcorespi::IndexSearchableVisitor &visitor) const override; /** + * Implements IFieldLengthInspector + */ + search::index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override; + + /** * Implements proton::IMemoryIndex */ bool hasReceivedDocumentInsert() const override { diff --git a/searchcorespi/src/vespa/searchcorespi/index/fakeindexsearchable.h b/searchcorespi/src/vespa/searchcorespi/index/fakeindexsearchable.h index 9eba40e08a4..3a391363e47 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/fakeindexsearchable.h +++ b/searchcorespi/src/vespa/searchcorespi/index/fakeindexsearchable.h @@ -39,6 +39,11 @@ public: (void) visitor; } + search::index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override { + (void) field_name; + return search::index::FieldLengthInfo(); + } + }; } // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp index de4cde956d0..81a43f932ca 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp @@ -13,6 +13,7 @@ LOG_SETUP(".searchcorespi.index.indexcollection"); using namespace search::queryeval; using namespace search::query; using search::attribute::IAttributeContext; +using search::index::FieldLengthInfo; namespace searchcorespi { @@ -238,4 +239,13 @@ IndexCollection::createBlueprint(const IRequestContext & requestContext, return visitor.getResult(); } +FieldLengthInfo +IndexCollection::get_field_length_info(const vespalib::string& field_name) const +{ + if (_sources.empty()) { + return FieldLengthInfo(); + } + return _sources.back().source_wrapper->get_field_length_info(field_name); +} + } // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.h b/searchcorespi/src/vespa/searchcorespi/index/indexcollection.h index eabdddf9dfe..e71f421c3ca 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexcollection.h @@ -57,6 +57,12 @@ public: static ISearchableIndexCollection::UP replaceAndRenumber(const ISourceSelectorSP & selector, const ISearchableIndexCollection &fsc, uint32_t id_diff, const IndexSearchable::SP &new_source); + + // Implements IFieldLengthInspector + /** + * Returns field length info from the newest disk index, or empty info for all fields if no disk index exists. + */ + search::index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override; }; } // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index 668eeba6463..acde26ad554 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -124,6 +124,13 @@ public: } /** + * Implements IFieldLengthInspector + */ + search::index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override { + return _index->get_field_length_info(field_name); + } + + /** * Implements IDiskIndex */ const vespalib::string &getIndexDir() const override { return _index->getIndexDir(); } diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexsearchable.h b/searchcorespi/src/vespa/searchcorespi/index/indexsearchable.h index 03b7f91e47a..96db89b3e61 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexsearchable.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexsearchable.h @@ -4,6 +4,7 @@ #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/common/serialnum.h> +#include <vespa/searchlib/index/i_field_length_inspector.h> #include <vespa/searchlib/query/tree/node.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/field_spec.h> @@ -26,7 +27,8 @@ class IndexSearchableVisitor; * that let the components access a per query attribute context that expose * attribute vectors that can be utilized during query evaluation. **/ -class IndexSearchable : public search::queryeval::Searchable { +class IndexSearchable : public search::queryeval::Searchable, + public search::index::IFieldLengthInspector { protected: using IRequestContext = search::queryeval::IRequestContext; using FieldSpec = search::queryeval::FieldSpec; diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp index 75412dcf8e9..911ff5f9eba 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp @@ -12,16 +12,17 @@ LOG_SETUP(".searchcorespi.index.warmupindexcollection"); namespace searchcorespi { +using fastos::ClockSystem; +using fastos::TimeStamp; +using index::IDiskIndex; +using search::fef::MatchDataLayout; +using search::index::FieldLengthInfo; using search::query::StringBase; using search::queryeval::Blueprint; -using search::fef::MatchDataLayout; -using search::queryeval::SearchIterator; using search::queryeval::ISourceSelector; -using vespalib::makeTask; +using search::queryeval::SearchIterator; using vespalib::makeClosure; -using index::IDiskIndex; -using fastos::TimeStamp; -using fastos::ClockSystem; +using vespalib::makeTask; using TermMap = vespalib::hash_set<vespalib::string>; class FieldTermMap : public vespalib::hash_map<uint32_t, TermMap> @@ -196,6 +197,11 @@ WarmupIndexCollection::accept(IndexSearchableVisitor &visitor) const _next->accept(visitor); } +FieldLengthInfo +WarmupIndexCollection::get_field_length_info(const vespalib::string& field_name) const +{ + return _next->get_field_length_info(field_name); +} void WarmupIndexCollection::append(uint32_t id, const IndexSearchable::SP &source) diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h index f6d6bc89fc4..229d1a7c31d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h +++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h @@ -52,6 +52,9 @@ public: search::SerialNum getSerialNum() const override; void accept(IndexSearchableVisitor &visitor) const override; + // Implements IFieldLengthInspector + search::index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override; + // Implements ISearchableIndexCollection void append(uint32_t id, const IndexSearchable::SP &source) override; void replace(uint32_t id, const IndexSearchable::SP &source) override; |