diff options
author | Geir Storli <geirst@verizonmedia.com> | 2019-06-07 13:23:06 +0000 |
---|---|---|
committer | Geir Storli <geirst@verizonmedia.com> | 2019-06-07 13:23:06 +0000 |
commit | 986ba7289148c5b06deb570316d1a5e4fdaaa78d (patch) | |
tree | 6a85af6ec1c26ca2d0f61e184f8fad8489a112c5 /searchcore/src | |
parent | df0e85d256cd3579ff3d0864bdb5135cfd601edc (diff) |
Test propagation of field length info between indexes in index manager.
Diffstat (limited to 'searchcore/src')
-rw-r--r-- | searchcore/src/tests/proton/index/indexmanager_test.cpp | 116 | ||||
-rw-r--r-- | searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp | 4 |
2 files changed, 92 insertions, 28 deletions
diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index a7ffe1cc45f..d92cc62c5a1 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -41,6 +41,7 @@ using search::TuneFileIndexing; using search::datastore::EntryRef; using search::index::DocBuilder; using search::index::DummyFileHeaderContext; +using search::index::FieldLengthInfo; using search::index::Schema; using search::index::schema::DataType; using search::index::test::MockFieldLengthInspector; @@ -158,6 +159,10 @@ struct IndexManagerTest : public ::testing::Test { uint32_t expNumMemoryIndexes, SerialNum expLastiskIndexSerialNum, SerialNum expLastMemoryIndexSerialNum); + + IIndexCollection::SP get_source_collection() const { + return _index_manager->getMaintainer().getSourceCollection(); + } }; void @@ -216,24 +221,24 @@ IndexManagerTest::assertStats(uint32_t expNumDiskIndexes, uint32_t expNumMemoryI TEST_F(IndexManagerTest, require_that_empty_memory_index_is_not_flushed) { - IIndexCollection::SP sources = _index_manager->getMaintainer().getSourceCollection(); + auto sources = get_source_collection(); EXPECT_EQ(1u, sources->getSourceCount()); flushIndexManager(); - sources = _index_manager->getMaintainer().getSourceCollection(); + sources = get_source_collection(); EXPECT_EQ(1u, sources->getSourceCount()); } TEST_F(IndexManagerTest, require_that_empty_memory_index_is_flushed_if_source_selector_changed) { - IIndexCollection::SP sources = _index_manager->getMaintainer().getSourceCollection(); + auto sources = get_source_collection(); EXPECT_EQ(1u, sources->getSourceCount()); removeDocument(docid, 42); flushIndexManager(); - sources = _index_manager->getMaintainer().getSourceCollection(); + sources = get_source_collection(); EXPECT_EQ(2u, sources->getSourceCount()); } @@ -267,8 +272,7 @@ TEST_F(IndexManagerTest, require_that_memory_index_is_flushed) { addDocument(docid); - IIndexCollection::SP sources = - _index_manager->getMaintainer().getSourceCollection(); + auto sources = get_source_collection(); EXPECT_EQ(1u, sources->getSourceCount()); EXPECT_EQ(1u, sources->getSourceId(0)); @@ -280,7 +284,7 @@ TEST_F(IndexManagerTest, require_that_memory_index_is_flushed) EXPECT_TRUE(FastOS_File::Stat("test_data/index.flush.1", &stat)); EXPECT_EQ(stat._modifiedTime, target.getLastFlushTime().time()); - sources = _index_manager->getMaintainer().getSourceCollection(); + sources = get_source_collection(); EXPECT_EQ(2u, sources->getSourceCount()); EXPECT_EQ(1u, sources->getSourceId(0)); EXPECT_EQ(2u, sources->getSourceId(1)); @@ -343,11 +347,11 @@ uint32_t getSource(const IIndexCollection &sources, uint32_t id) { TEST_F(IndexManagerTest, require_that_put_document_updates_selector) { addDocument(docid); - IIndexCollection::SP sources = _index_manager->getMaintainer().getSourceCollection(); + auto sources = get_source_collection(); EXPECT_EQ(1u, getSource(*sources, docid)); flushIndexManager(); addDocument(docid + 1); - sources = _index_manager->getMaintainer().getSourceCollection(); + sources = get_source_collection(); EXPECT_EQ(1u, getSource(*sources, docid)); EXPECT_EQ(2u, getSource(*sources, docid + 1)); } @@ -355,11 +359,11 @@ TEST_F(IndexManagerTest, require_that_put_document_updates_selector) TEST_F(IndexManagerTest, require_that_remove_document_updates_selector) { Document::UP doc = addDocument(docid); - IIndexCollection::SP sources = _index_manager->getMaintainer().getSourceCollection(); + auto sources = get_source_collection(); EXPECT_EQ(1u, getSource(*sources, docid)); flushIndexManager(); removeDocument(docid, ++_serial_num); - sources = _index_manager->getMaintainer().getSourceCollection(); + sources = get_source_collection(); EXPECT_EQ(2u, getSource(*sources, docid)); } @@ -468,11 +472,10 @@ TEST_F(IndexManagerTest, require_that_fusion_updates_indexes) flushIndexManager(); } uint32_t ids[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; - IIndexCollection::SP - source_list(_index_manager->getMaintainer().getSourceCollection()); - EXPECT_EQ(10u + 1, source_list->getSourceCount()); // disk + mem - EXPECT_EQ(ids[2], getSource(*source_list, docid + 2)); - EXPECT_EQ(ids[6], getSource(*source_list, docid + 6)); + auto sources = get_source_collection(); + EXPECT_EQ(10u + 1, sources->getSourceCount()); // disk + mem + EXPECT_EQ(ids[2], getSource(*sources, docid + 2)); + EXPECT_EQ(ids[6], getSource(*sources, docid + 6)); FusionSpec fusion_spec; fusion_spec.flush_ids.assign(ids, ids + 4); @@ -482,10 +485,10 @@ TEST_F(IndexManagerTest, require_that_fusion_updates_indexes) EXPECT_EQ(1u, fusion_ids.size()); EXPECT_EQ(ids[3], *fusion_ids.begin()); - source_list = _index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQ(10u + 1 - 4 + 1, source_list->getSourceCount()); - EXPECT_EQ(0u, getSource(*source_list, docid + 2)); - EXPECT_EQ(3u, getSource(*source_list, docid + 6)); + sources = get_source_collection(); + EXPECT_EQ(10u + 1 - 4 + 1, sources->getSourceCount()); + EXPECT_EQ(0u, getSource(*sources, docid + 2)); + EXPECT_EQ(3u, getSource(*sources, docid + 6)); } TEST_F(IndexManagerTest, require_that_flush_triggers_fusion) @@ -534,7 +537,7 @@ TEST_F(IndexManagerTest, require_that_fusion_cleans_up_old_indexes) addDocument(docid); flushIndexManager(); // hold reference to index.flush.1 - IIndexCollection::SP fsc = _index_manager->getMaintainer().getSourceCollection(); + auto fsc = get_source_collection(); addDocument(docid + 1); flushIndexManager(); @@ -579,7 +582,7 @@ TEST_F(IndexManagerTest, require_that_disk_indexes_are_loaded_on_startup) ASSERT_TRUE(indexExists("flush", 1)); resetIndexManager(); - IIndexCollection::SP fsc = _index_manager->getMaintainer().getSourceCollection(); + auto fsc = get_source_collection(); EXPECT_EQ(2u, fsc->getSourceCount()); EXPECT_TRUE(contains(*fsc, 1u)); EXPECT_TRUE(contains(*fsc, 2u)); @@ -601,7 +604,7 @@ TEST_F(IndexManagerTest, require_that_disk_indexes_are_loaded_on_startup) ASSERT_TRUE(indexExists("fusion", 2)); resetIndexManager(); - fsc = _index_manager->getMaintainer().getSourceCollection(); + fsc = get_source_collection(); EXPECT_EQ(2u, fsc->getSourceCount()); EXPECT_TRUE(contains(*fsc, 0u)); EXPECT_TRUE(contains(*fsc, 1u)); @@ -620,7 +623,7 @@ TEST_F(IndexManagerTest, require_that_disk_indexes_are_loaded_on_startup) ASSERT_TRUE(indexExists("flush", 3)); resetIndexManager(); - fsc = _index_manager->getMaintainer().getSourceCollection(); + fsc = get_source_collection(); EXPECT_EQ(3u, fsc->getSourceCount()); EXPECT_TRUE(contains(*fsc, 0u)); EXPECT_TRUE(contains(*fsc, 1u)); @@ -756,12 +759,75 @@ TEST_F(IndexManagerTest, require_that_compact_lid_space_works) addDocument(1); addDocument(2); removeDocument(2); - auto fsc = _index_manager->getMaintainer().getSourceCollection(); + auto fsc = get_source_collection(); EXPECT_EQ(3u, fsc->getSourceSelector().getDocIdLimit()); compactLidSpace(2); EXPECT_EQ(2u, fsc->getSourceSelector().getDocIdLimit()); } +template <typename IndexType> +IndexType* +as_index_type(const IIndexCollection& col, uint32_t source_id) +{ + auto& searchable = col.getSearchable(source_id); + auto* result = dynamic_cast<IndexType *>(&searchable); + assert(result != nullptr); + return result; +} + +IMemoryIndex* +as_memory_index(const IIndexCollection& col, uint32_t source_id) +{ + return as_index_type<IMemoryIndex>(col, source_id); +} + +IDiskIndex* +as_disk_index(const IIndexCollection& col, uint32_t source_id) +{ + return as_index_type<IDiskIndex>(col, source_id); +} + +void +expect_field_length_info(double exp_average, uint32_t exp_samples, const IndexSearchable& searchable) +{ + auto info = searchable.get_field_length_info(field_name); + EXPECT_DOUBLE_EQ(exp_average, info.get_average_field_length()); + EXPECT_EQ(exp_samples, info.get_num_samples()); +} + +TEST_F(IndexManagerTest, field_length_info_is_propagated_to_disk_index_and_next_memory_index_during_flush) +{ + addDocument(1); + addDocument(2); + + auto sources = get_source_collection(); + ASSERT_EQ(1, sources->getSourceCount()); + auto *first_index = as_memory_index(*sources, 0); + expect_field_length_info(1, 2, *first_index); + + flushIndexManager(); + + sources = get_source_collection(); + ASSERT_EQ(2, sources->getSourceCount()); + expect_field_length_info(1, 2, *as_disk_index(*sources, 0)); + auto *second_index = as_memory_index(*sources, 1); + EXPECT_NE(first_index, second_index); + expect_field_length_info(1, 2, *second_index); +} + +TEST_F(IndexManagerTest, field_length_info_is_loaded_from_disk_index_during_startup) +{ + addDocument(1); + addDocument(2); + flushIndexManager(); + resetIndexManager(); + + auto sources = get_source_collection(); + ASSERT_EQ(2, sources->getSourceCount()); + expect_field_length_info(1, 2, *as_disk_index(*sources, 0)); + expect_field_length_info(1, 2, *as_memory_index(*sources, 1)); +} + } // namespace int diff --git a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp index d70c321722b..4cfaac9aac0 100644 --- a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp @@ -49,9 +49,7 @@ DiskIndexWrapper::accept(searchcorespi::IndexSearchableVisitor &visitor) const FieldLengthInfo DiskIndexWrapper::get_field_length_info(const vespalib::string& field_name) const { - // TODO: implement - (void) field_name; - return FieldLengthInfo(); + return _index.get_field_length_info(field_name); } } // namespace proton |