diff options
Diffstat (limited to 'searchlib/src/tests/docstore')
16 files changed, 105 insertions, 42 deletions
diff --git a/searchlib/src/tests/docstore/chunk/CMakeLists.txt b/searchlib/src/tests/docstore/chunk/CMakeLists.txt index 9d5213f51e3..1068863e846 100644 --- a/searchlib/src/tests/docstore/chunk/CMakeLists.txt +++ b/searchlib/src/tests/docstore/chunk/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchlib_chunk_test_app TEST SOURCES chunk_test.cpp diff --git a/searchlib/src/tests/docstore/chunk/chunk_test.cpp b/searchlib/src/tests/docstore/chunk/chunk_test.cpp index 28bd5208b73..5863297be51 100644 --- a/searchlib/src/tests/docstore/chunk/chunk_test.cpp +++ b/searchlib/src/tests/docstore/chunk/chunk_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> @@ -19,10 +19,10 @@ TEST("require that Chunk obey limits") { Chunk c(0, Chunk::Config(256)); EXPECT_TRUE(c.hasRoom(1000)); // At least 1 is allowed no matter what the size is. - c.append(1, "abc", 3); + c.append(1, {"abc", 3}); EXPECT_TRUE(c.hasRoom(229)); EXPECT_FALSE(c.hasRoom(230)); - c.append(2, "abc", 3); + c.append(2, {"abc", 3}); EXPECT_TRUE(c.hasRoom(20)); } @@ -30,11 +30,11 @@ TEST("require that Chunk can produce unique list") { const char *d = "ABCDEF"; Chunk c(0, Chunk::Config(100)); - c.append(1, d, 1); - c.append(2, d, 2); - c.append(3, d, 3); - c.append(2, d, 4); - c.append(1, d, 5); + c.append(1, {d, 1}); + c.append(2, {d, 2}); + c.append(3, {d, 3}); + c.append(2, {d, 4}); + c.append(1, {d, 5}); EXPECT_EQUAL(5u, c.count()); const Chunk::LidList & all = c.getLids(); EXPECT_EQUAL(5u, all.size()); diff --git a/searchlib/src/tests/docstore/document_store/CMakeLists.txt b/searchlib/src/tests/docstore/document_store/CMakeLists.txt index f09c0953da9..9386ab8fa33 100644 --- a/searchlib/src/tests/docstore/document_store/CMakeLists.txt +++ b/searchlib/src/tests/docstore/document_store/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchlib_document_store_test_app TEST SOURCES document_store_test.cpp diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp index 99a9cdec17e..a35aeb5e851 100644 --- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp +++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/docstore/logdocumentstore.h> #include <vespa/searchlib/docstore/value.h> diff --git a/searchlib/src/tests/docstore/document_store/visitcache_test.cpp b/searchlib/src/tests/docstore/document_store/visitcache_test.cpp index 3f80bb6004f..6bed51732a8 100644 --- a/searchlib/src/tests/docstore/document_store/visitcache_test.cpp +++ b/searchlib/src/tests/docstore/document_store/visitcache_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/docstore/visitcache.h> diff --git a/searchlib/src/tests/docstore/document_store_visitor/CMakeLists.txt b/searchlib/src/tests/docstore/document_store_visitor/CMakeLists.txt index 6fbf55fdb04..3f51a6b1fd0 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/CMakeLists.txt +++ b/searchlib/src/tests/docstore/document_store_visitor/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchlib_document_store_visitor_test_app TEST SOURCES document_store_visitor_test.cpp diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp index 9bc649149d0..8c7522bbd0a 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp +++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/docstore/documentstore.h> diff --git a/searchlib/src/tests/docstore/file_chunk/CMakeLists.txt b/searchlib/src/tests/docstore/file_chunk/CMakeLists.txt index 7ccefdb7702..4c3e5f2358e 100644 --- a/searchlib/src/tests/docstore/file_chunk/CMakeLists.txt +++ b/searchlib/src/tests/docstore/file_chunk/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchlib_file_chunk_test_app TEST SOURCES file_chunk_test.cpp diff --git a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp index 64ae4dfd2f2..8f506b7ca2d 100644 --- a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp +++ b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/docstore/filechunk.h> @@ -43,10 +43,10 @@ struct BucketizerObserver : public IBucketizer { document::BucketId getBucketOf(const vespalib::GenerationHandler::Guard &guard, uint32_t lid) const override { (void) guard; lids.push_back(lid); - return document::BucketId(); + return {}; } vespalib::GenerationHandler::Guard getGuard() const override { - return vespalib::GenerationHandler::Guard(); + return {}; } }; @@ -129,7 +129,7 @@ struct WriteFixture : public FixtureBase { } WriteFixture &append(uint32_t lid) { vespalib::string data = getData(lid); - chunk.append(nextSerialNum(), lid, data.c_str(), data.size(), CpuUsage::Category::WRITE); + chunk.append(nextSerialNum(), lid, {data.c_str(), data.size()}, CpuUsage::Category::WRITE); return *this; } void updateLidMap(uint32_t docIdLimit) { @@ -202,7 +202,7 @@ assertUpdateLidMap(FixtureType &f) f.assertBucketizer(expLids); size_t entrySize = 10 + 8; EXPECT_EQUAL(9 * entrySize, f.chunk.getAddedBytes()); - EXPECT_EQUAL(3u, f.chunk.getBloatCount()); + EXPECT_EQUAL(3u, f.chunk.getErasedCount()); EXPECT_EQUAL(3 * entrySize, f.chunk.getErasedBytes()); } diff --git a/searchlib/src/tests/docstore/lid_info/CMakeLists.txt b/searchlib/src/tests/docstore/lid_info/CMakeLists.txt index ae35ab9c032..419a79b4714 100644 --- a/searchlib/src/tests/docstore/lid_info/CMakeLists.txt +++ b/searchlib/src/tests/docstore/lid_info/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchlib_lid_info_test_app TEST SOURCES lid_info_test.cpp diff --git a/searchlib/src/tests/docstore/lid_info/lid_info_test.cpp b/searchlib/src/tests/docstore/lid_info/lid_info_test.cpp index 0f0cabfc443..045f4a474df 100644 --- a/searchlib/src/tests/docstore/lid_info/lid_info_test.cpp +++ b/searchlib/src/tests/docstore/lid_info/lid_info_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/docstore/lid_info.h> diff --git a/searchlib/src/tests/docstore/logdatastore/CMakeLists.txt b/searchlib/src/tests/docstore/logdatastore/CMakeLists.txt index 5c8b1c33d00..479d34bad9d 100644 --- a/searchlib/src/tests/docstore/logdatastore/CMakeLists.txt +++ b/searchlib/src/tests/docstore/logdatastore/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchlib_logdatastore_test_app TEST SOURCES logdatastore_test.cpp diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index c76f73e7477..72bdd533719 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/repo/configbuilder.h> diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.sh b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.sh index c31a2ba51c6..6b2d0ac5255 100755 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.sh +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -e if [ -z "$SOURCE_DIRECTORY" ]; then diff --git a/searchlib/src/tests/docstore/store_by_bucket/CMakeLists.txt b/searchlib/src/tests/docstore/store_by_bucket/CMakeLists.txt index c30d99cc882..71abe3c6564 100644 --- a/searchlib/src/tests/docstore/store_by_bucket/CMakeLists.txt +++ b/searchlib/src/tests/docstore/store_by_bucket/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchlib_store_by_bucket_test_app TEST SOURCES store_by_bucket_test.cpp diff --git a/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp b/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp index 13aa1880e8c..d6dde433365 100644 --- a/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp +++ b/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp @@ -1,13 +1,12 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/document/base/documentid.h> -#include <vespa/searchlib/docstore/storebybucket.h> +#include <vespa/searchlib/docstore/compacter.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_set.h> -#include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/log/log.h> @@ -26,8 +25,8 @@ createPayload(BucketId b) { } uint32_t userId(size_t i) { return i%100; } -void -add(StoreByBucket & sbb, size_t i) { +BucketId +createBucketId(size_t i) { constexpr size_t USED_BITS=5; vespalib::asciistream os; os << "id:a:b:n=" << userId(i) << ":" << i; @@ -35,14 +34,20 @@ add(StoreByBucket & sbb, size_t i) { BucketId b = docId.getGlobalId().convertToBucketId(); EXPECT_EQUAL(userId(i), docId.getGlobalId().getLocationSpecificBits()); b.setUsedBits(USED_BITS); + return b; +} +void +add(StoreByBucket & sbb, size_t i) { + BucketId b = createBucketId(i); vespalib::string s = createPayload(b); - sbb.add(b, i%10, i, s.c_str(), s.size()); + sbb.add(b, i%10, i, {s.c_str(), s.size()}); } class VerifyBucketOrder : public StoreByBucket::IWrite { public: - VerifyBucketOrder() : _lastLid(0), _lastBucketId(0), _uniqueUser(), _uniqueBucket() { } - void write(BucketId bucketId, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz) override { + VerifyBucketOrder() : _lastLid(0), _lastBucketId(0), _uniqueUser(), _uniqueBucket(){ } + ~VerifyBucketOrder() override; + void write(BucketId bucketId, uint32_t chunkId, uint32_t lid, vespalib::ConstBufferRef data) override { (void) chunkId; EXPECT_LESS_EQUAL(_lastBucketId.toKey(), bucketId.toKey()); if (_lastBucketId != bucketId) { @@ -55,34 +60,92 @@ public: } _lastLid = lid; _lastBucketId = bucketId; - EXPECT_EQUAL(0, memcmp(buffer, createPayload(bucketId).c_str(), sz)); + EXPECT_EQUAL(0, memcmp(data.data(), createPayload(bucketId).c_str(), data.size())); } - ~VerifyBucketOrder() override; + private: uint32_t _lastLid; BucketId _lastBucketId; vespalib::hash_set<uint32_t> _uniqueUser; vespalib::hash_set<uint64_t> _uniqueBucket; + }; VerifyBucketOrder::~VerifyBucketOrder() = default; +struct StoreIndex : public StoreByBucket::StoreIndex { + ~StoreIndex() override; + void store(const StoreByBucket::Index &index) override { + _where.push_back(index); + } + std::vector<StoreByBucket::Index> _where; +}; +StoreIndex::~StoreIndex() = default; + +struct Iterator : public StoreByBucket::IndexIterator { + explicit Iterator(const std::vector<StoreByBucket::Index> & where) : _where(where), _current(0) {} + + bool has_next() noexcept override { + return _current < _where.size(); + } + + StoreByBucket::Index next() noexcept override { + return _where[_current++]; + } + + const std::vector<StoreByBucket::Index> & _where; + uint32_t _current; +}; + TEST("require that StoreByBucket gives bucket by bucket and ordered within") { std::mutex backing_lock; vespalib::MemoryDataStore backing(vespalib::alloc::Alloc::alloc(256), &backing_lock); vespalib::ThreadStackExecutor executor(8); - StoreByBucket sbb(backing, executor, CompressionConfig::LZ4); - for (size_t i(1); i <=500; i++) { + StoreIndex storeIndex; + StoreByBucket sbb(storeIndex, backing, executor, CompressionConfig::LZ4); + for (size_t i(1); i <= 500u; i++) { add(sbb, i); } - for (size_t i(1000); i > 500; i--) { + for (size_t i(1000); i > 500u; i--) { add(sbb, i); } - EXPECT_EQUAL(32u, sbb.getBucketCount()); - EXPECT_EQUAL(1000u, sbb.getLidCount()); + sbb.close(); + std::sort(storeIndex._where.begin(), storeIndex._where.end()); + EXPECT_EQUAL(1000u, storeIndex._where.size()); VerifyBucketOrder vbo; - sbb.drain(vbo); + Iterator all(storeIndex._where); + sbb.drain(vbo, all); +} + +constexpr uint32_t NUM_PARTS = 3; + +void +verifyIter(BucketIndexStore &store, uint32_t partId, uint32_t expected_count) { + auto iter = store.createIterator(partId); + uint32_t count(0); + while (iter->has_next()) { + StoreByBucket::Index idx = iter->next(); + EXPECT_EQUAL(store.toPartitionId(idx._bucketId), partId); + count++; + } + EXPECT_EQUAL(expected_count, count); +} + +TEST("test that iterators cover the whole corpus and maps to correct partid") { + + BucketIndexStore bucketIndexStore(32, NUM_PARTS); + for (size_t i(1); i <= 500u; i++) { + bucketIndexStore.store(StoreByBucket::Index(createBucketId(i), 1, 2, i)); + } + bucketIndexStore.prepareForIterate(); + EXPECT_EQUAL(500u, bucketIndexStore.getLidCount()); + EXPECT_EQUAL(32u, bucketIndexStore.getBucketCount()); + constexpr uint32_t COUNT_0 = 175, COUNT_1 = 155, COUNT_2 = 170; + verifyIter(bucketIndexStore, 0, COUNT_0); + verifyIter(bucketIndexStore, 1, COUNT_1); + verifyIter(bucketIndexStore, 2, COUNT_2); + EXPECT_EQUAL(500u, COUNT_0 + COUNT_1 + COUNT_2); } TEST_MAIN() { TEST_RUN_ALL(); } |