diff options
Diffstat (limited to 'searchlib/src/tests/memoryindex/datastore')
7 files changed, 814 insertions, 0 deletions
diff --git a/searchlib/src/tests/memoryindex/datastore/.gitignore b/searchlib/src/tests/memoryindex/datastore/.gitignore new file mode 100644 index 00000000000..98f4acc70a8 --- /dev/null +++ b/searchlib/src/tests/memoryindex/datastore/.gitignore @@ -0,0 +1,8 @@ +.depend +Makefile +datastore_test +featurestore_test +wordstore_test +searchlib_datastore_test_app +searchlib_featurestore_test_app +searchlib_wordstore_test_app diff --git a/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt b/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt new file mode 100644 index 00000000000..da45288fe5e --- /dev/null +++ b/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_datastore_test_app + SOURCES + datastore_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_datastore_test_app COMMAND searchlib_datastore_test_app) +vespa_add_executable(searchlib_featurestore_test_app + SOURCES + featurestore_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_featurestore_test_app COMMAND searchlib_featurestore_test_app) +vespa_add_executable(searchlib_wordstore_test_app + SOURCES + wordstore_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_wordstore_test_app COMMAND searchlib_wordstore_test_app) diff --git a/searchlib/src/tests/memoryindex/datastore/DESC b/searchlib/src/tests/memoryindex/datastore/DESC new file mode 100644 index 00000000000..56725396b65 --- /dev/null +++ b/searchlib/src/tests/memoryindex/datastore/DESC @@ -0,0 +1 @@ +datastore test. Take a look at datastore_test.cpp and wordstore_test.cpp for details. diff --git a/searchlib/src/tests/memoryindex/datastore/FILES b/searchlib/src/tests/memoryindex/datastore/FILES new file mode 100644 index 00000000000..6cbbaf6a328 --- /dev/null +++ b/searchlib/src/tests/memoryindex/datastore/FILES @@ -0,0 +1,2 @@ +datastore_test.cpp +wordstore_test.cpp diff --git a/searchlib/src/tests/memoryindex/datastore/datastore_test.cpp b/searchlib/src/tests/memoryindex/datastore/datastore_test.cpp new file mode 100644 index 00000000000..be55dd7ee1e --- /dev/null +++ b/searchlib/src/tests/memoryindex/datastore/datastore_test.cpp @@ -0,0 +1,432 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("datastore_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchlib/btree/datastore.h> +#include <vespa/searchlib/btree/datastore.hpp> + +namespace search { +namespace btree { + +class MyStore : public DataStore<int, EntryRefT<3, 2> > { +private: + typedef DataStore<int, EntryRefT<3, 2> > ParentType; + using ParentType::_buffers; + using ParentType::_states; + using ParentType::_activeBufferIds; +public: + MyStore() {} + + void + holdBuffer(uint32_t bufferId) + { + ParentType::holdBuffer(bufferId); + } + + void + holdElem(EntryRef ref, uint64_t len) + { + ParentType::holdElem(ref, len); + } + + void + transferHoldLists(generation_t generation) + { + ParentType::transferHoldLists(generation); + } + + void trimElemHoldList(generation_t usedGen) { + ParentType::trimElemHoldList(usedGen); + } + void incDead(EntryRef ref, uint64_t dead) { + ParentType::incDead(ref, dead); + } + void ensureBufferCapacity(size_t sizeNeeded) { + ParentType::ensureBufferCapacity(0, sizeNeeded); + } + void enableFreeLists() { + ParentType::enableFreeLists(); + } + + void + switchActiveBuffer(void) + { + ParentType::switchActiveBuffer(0, 0u); + } + std::vector<void *> & buffers() { return _buffers; } + std::vector<BufferState> &statesVec() { return _states; } + size_t activeBufferId() const { return _activeBufferIds[0]; } +}; + +typedef MyStore::RefType MyRef; + +class Test : public vespalib::TestApp { +private: + bool assertMemStats(const DataStoreBase::MemStats & exp, + const DataStoreBase::MemStats & act); + void requireThatEntryRefIsWorking(); + void requireThatAlignedEntryRefIsWorking(); + void requireThatEntriesCanBeAddedAndRetrieved(); + void requireThatAddEntryTriggersChangeOfBuffer(); + void requireThatWeCanHoldAndTrimBuffers(); + void requireThatWeCanHoldAndTrimElements(); + void requireThatWeCanUseFreeLists(); + void requireThatMemoryStatsAreCalculated(); + void requireThatMemoryUsageIsCalculated(); + + void + requireThatWecanDisableElemHoldList(void); +public: + int Main(); +}; + +bool +Test::assertMemStats(const DataStoreBase::MemStats & exp, + const DataStoreBase::MemStats & act) +{ + if (!EXPECT_EQUAL(exp._allocElems, act._allocElems)) return false; + if (!EXPECT_EQUAL(exp._usedElems, act._usedElems)) return false; + if (!EXPECT_EQUAL(exp._deadElems, act._deadElems)) return false; + if (!EXPECT_EQUAL(exp._holdElems, act._holdElems)) return false; + if (!EXPECT_EQUAL(exp._freeBuffers, act._freeBuffers)) return false; + if (!EXPECT_EQUAL(exp._activeBuffers, act._activeBuffers)) return false; + if (!EXPECT_EQUAL(exp._holdBuffers, act._holdBuffers)) return false; + return true; +} + +void +Test::requireThatEntryRefIsWorking() +{ + typedef EntryRefT<22> MyRefType; + EXPECT_EQUAL(4194304u, MyRefType::offsetSize()); + EXPECT_EQUAL(1024u, MyRefType::numBuffers()); + { + MyRefType r(0, 0); + EXPECT_EQUAL(0u, r.offset()); + EXPECT_EQUAL(0u, r.bufferId()); + } + { + MyRefType r(237, 13); + EXPECT_EQUAL(237u, r.offset()); + EXPECT_EQUAL(13u, r.bufferId()); + } + { + MyRefType r(4194303, 1023); + EXPECT_EQUAL(4194303u, r.offset()); + EXPECT_EQUAL(1023u, r.bufferId()); + } + { + MyRefType r1(6498, 76); + MyRefType r2(r1); + EXPECT_EQUAL(r1.offset(), r2.offset()); + EXPECT_EQUAL(r1.bufferId(), r2.bufferId()); + } +} + +void +Test::requireThatAlignedEntryRefIsWorking() +{ + typedef AlignedEntryRefT<22, 2> MyRefType; // 4 byte alignement + EXPECT_EQUAL(4 * 4194304u, MyRefType::offsetSize()); + EXPECT_EQUAL(1024u, MyRefType::numBuffers()); + EXPECT_EQUAL(0u, MyRefType::align(0)); + EXPECT_EQUAL(4u, MyRefType::align(1)); + EXPECT_EQUAL(4u, MyRefType::align(2)); + EXPECT_EQUAL(4u, MyRefType::align(3)); + EXPECT_EQUAL(4u, MyRefType::align(4)); + EXPECT_EQUAL(8u, MyRefType::align(5)); + { + MyRefType r(0, 0); + EXPECT_EQUAL(0u, r.offset()); + EXPECT_EQUAL(0u, r.bufferId()); + } + { + MyRefType r(237, 13); + EXPECT_EQUAL(MyRefType::align(237), r.offset()); + EXPECT_EQUAL(13u, r.bufferId()); + } + { + MyRefType r(MyRefType::offsetSize() - 4, 1023); + EXPECT_EQUAL(MyRefType::align(MyRefType::offsetSize() - 4), r.offset()); + EXPECT_EQUAL(1023u, r.bufferId()); + } +} + +void +Test::requireThatEntriesCanBeAddedAndRetrieved() +{ + typedef DataStore<int> IntStore; + IntStore ds; + EntryRef r1 = ds.addEntry(10); + EntryRef r2 = ds.addEntry(20); + EntryRef r3 = ds.addEntry(30); + EXPECT_EQUAL(1u, IntStore::RefType(r1).offset()); + EXPECT_EQUAL(2u, IntStore::RefType(r2).offset()); + EXPECT_EQUAL(3u, IntStore::RefType(r3).offset()); + EXPECT_EQUAL(0u, IntStore::RefType(r1).bufferId()); + EXPECT_EQUAL(0u, IntStore::RefType(r2).bufferId()); + EXPECT_EQUAL(0u, IntStore::RefType(r3).bufferId()); + EXPECT_EQUAL(10, ds.getEntry(r1)); + EXPECT_EQUAL(20, ds.getEntry(r2)); + EXPECT_EQUAL(30, ds.getEntry(r3)); +} + +void +Test::requireThatAddEntryTriggersChangeOfBuffer() +{ + typedef DataStore<uint64_t, EntryRefT<10, 10> > Store; + Store s; + uint64_t num = 0; + uint32_t lastId = 0; + uint64_t lastNum = 0; + for (;;++num) { + EntryRef r = s.addEntry(num); + EXPECT_EQUAL(num, s.getEntry(r)); + uint32_t bufferId = Store::RefType(r).bufferId(); + if (bufferId > lastId) { + LOG(info, "Changed to bufferId %u after %" PRIu64 " nums", bufferId, num); + EXPECT_EQUAL(Store::RefType::offsetSize() - (lastId == 0), + num - lastNum); + lastId = bufferId; + lastNum = num; + } + if (bufferId == 2) { + break; + } + } + EXPECT_EQUAL(Store::RefType::offsetSize() * 2 - 1, num); + LOG(info, "Added %" PRIu64 " nums in 2 buffers", num); +} + +void +Test::requireThatWeCanHoldAndTrimBuffers() +{ + MyStore s; + EXPECT_EQUAL(0u, MyRef(s.addEntry(1)).bufferId()); + s.switchActiveBuffer(); + EXPECT_EQUAL(1u, s.activeBufferId()); + s.holdBuffer(0); // hold last buffer + s.transferHoldLists(10); + + EXPECT_EQUAL(1u, MyRef(s.addEntry(2)).bufferId()); + s.switchActiveBuffer(); + EXPECT_EQUAL(2u, s.activeBufferId()); + s.holdBuffer(1); // hold last buffer + s.transferHoldLists(20); + + EXPECT_EQUAL(2u, MyRef(s.addEntry(3)).bufferId()); + s.switchActiveBuffer(); + EXPECT_EQUAL(3u, s.activeBufferId()); + s.holdBuffer(2); // hold last buffer + s.transferHoldLists(30); + + EXPECT_EQUAL(3u, MyRef(s.addEntry(4)).bufferId()); + s.holdBuffer(3); // hold current buffer + s.transferHoldLists(40); + + EXPECT_TRUE(s.statesVec()[0].size() != 0); + EXPECT_TRUE(s.statesVec()[1].size() != 0); + EXPECT_TRUE(s.statesVec()[2].size() != 0); + EXPECT_TRUE(s.statesVec()[3].size() != 0); + s.trimHoldLists(11); + EXPECT_TRUE(s.statesVec()[0].size() == 0); + EXPECT_TRUE(s.statesVec()[1].size() != 0); + EXPECT_TRUE(s.statesVec()[2].size() != 0); + EXPECT_TRUE(s.statesVec()[3].size() != 0); + + s.switchActiveBuffer(); + EXPECT_EQUAL(0u, s.activeBufferId()); + EXPECT_EQUAL(0u, MyRef(s.addEntry(5)).bufferId()); + s.trimHoldLists(41); + EXPECT_TRUE(s.statesVec()[0].size() != 0); + EXPECT_TRUE(s.statesVec()[1].size() == 0); + EXPECT_TRUE(s.statesVec()[2].size() == 0); + EXPECT_TRUE(s.statesVec()[3].size() == 0); +} + +void +Test::requireThatWeCanHoldAndTrimElements() +{ + MyStore s; + MyRef r1 = s.addEntry(1); + s.holdElem(r1, 1); + s.transferHoldLists(10); + MyRef r2 = s.addEntry(2); + s.holdElem(r2, 1); + s.transferHoldLists(20); + MyRef r3 = s.addEntry(3); + s.holdElem(r3, 1); + s.transferHoldLists(30); + EXPECT_EQUAL(1, s.getEntry(r1)); + EXPECT_EQUAL(2, s.getEntry(r2)); + EXPECT_EQUAL(3, s.getEntry(r3)); + s.trimElemHoldList(11); + EXPECT_EQUAL(0, s.getEntry(r1)); + EXPECT_EQUAL(2, s.getEntry(r2)); + EXPECT_EQUAL(3, s.getEntry(r3)); + s.trimElemHoldList(31); + EXPECT_EQUAL(0, s.getEntry(r1)); + EXPECT_EQUAL(0, s.getEntry(r2)); + EXPECT_EQUAL(0, s.getEntry(r3)); +} + +void +Test::requireThatWeCanUseFreeLists() +{ + MyStore s; + s.enableFreeLists(); + MyRef r1 = s.addEntry2(1); + s.holdElem(r1, 1); + s.transferHoldLists(10); + MyRef r2 = s.addEntry2(2); + s.holdElem(r2, 1); + s.transferHoldLists(20); + s.trimElemHoldList(11); + MyRef r3 = s.addEntry2(3); // reuse r1 + EXPECT_EQUAL(r1.offset(), r3.offset()); + EXPECT_EQUAL(r1.bufferId(), r3.bufferId()); + MyRef r4 = s.addEntry2(4); + EXPECT_EQUAL(r2.offset() + 1, r4.offset()); + s.trimElemHoldList(21); + MyRef r5 = s.addEntry2(5); // reuse r2 + EXPECT_EQUAL(r2.offset(), r5.offset()); + EXPECT_EQUAL(r2.bufferId(), r5.bufferId()); + MyRef r6 = s.addEntry2(6); + EXPECT_EQUAL(r4.offset() + 1, r6.offset()); + EXPECT_EQUAL(3, s.getEntry(r1)); + EXPECT_EQUAL(5, s.getEntry(r2)); + EXPECT_EQUAL(3, s.getEntry(r3)); + EXPECT_EQUAL(4, s.getEntry(r4)); + EXPECT_EQUAL(5, s.getEntry(r5)); + EXPECT_EQUAL(6, s.getEntry(r6)); +} + +void +Test::requireThatMemoryStatsAreCalculated() +{ + MyStore s; + DataStoreBase::MemStats m; + m._allocElems = MyRef::offsetSize(); + m._usedElems = 1; // ref = 0 is reserved + m._deadElems = 1; // ref = 0 is reserved + m._holdElems = 0; + m._activeBuffers = 1; + m._freeBuffers = MyRef::numBuffers() - 1; + m._holdBuffers = 0; + EXPECT_TRUE(assertMemStats(m, s.getMemStats())); + + // add entry + MyRef r = s.addEntry(10); + m._usedElems++; + EXPECT_TRUE(assertMemStats(m, s.getMemStats())); + + // inc dead + s.incDead(r, 1); + m._deadElems++; + EXPECT_TRUE(assertMemStats(m, s.getMemStats())); + + // hold buffer + s.addEntry(20); + s.addEntry(30); + s.holdBuffer(r.bufferId()); + s.transferHoldLists(100); + m._usedElems += 2; + m._holdElems += 2; // used - dead + m._activeBuffers--; + m._holdBuffers++; + EXPECT_TRUE(assertMemStats(m, s.getMemStats())); + + // new active buffer + s.switchActiveBuffer(); + s.addEntry(40); + m._allocElems += MyRef::offsetSize(); + m._usedElems++; + m._activeBuffers++; + m._freeBuffers--; + + // trim hold buffer + s.trimHoldLists(101); + m._allocElems -= MyRef::offsetSize(); + m._usedElems = 1; + m._deadElems = 0; + m._holdElems = 0; + m._freeBuffers = MyRef::numBuffers() - 1; + m._holdBuffers = 0; + EXPECT_TRUE(assertMemStats(m, s.getMemStats())); +} + +void +Test::requireThatMemoryUsageIsCalculated() +{ + MyStore s; + MyRef r = s.addEntry(10); + s.addEntry(20); + s.addEntry(30); + s.addEntry(40); + s.incDead(r, 1); + s.holdBuffer(r.bufferId()); + s.transferHoldLists(100); + MemoryUsage m = s.getMemoryUsage(); + EXPECT_EQUAL(MyRef::offsetSize() * sizeof(int), m.allocatedBytes()); + EXPECT_EQUAL(5 * sizeof(int), m.usedBytes()); + EXPECT_EQUAL(2 * sizeof(int), m.deadBytes()); + EXPECT_EQUAL(3 * sizeof(int), m.allocatedBytesOnHold()); + s.trimHoldLists(101); +} + + +void +Test::requireThatWecanDisableElemHoldList(void) +{ + MyStore s; + MyRef r1 = s.addEntry(10); + MyRef r2 = s.addEntry(20); + MyRef r3 = s.addEntry(30); + (void) r3; + MemoryUsage m = s.getMemoryUsage(); + EXPECT_EQUAL(MyRef::offsetSize() * sizeof(int), m.allocatedBytes()); + EXPECT_EQUAL(4 * sizeof(int), m.usedBytes()); + EXPECT_EQUAL(1 * sizeof(int), m.deadBytes()); + EXPECT_EQUAL(0 * sizeof(int), m.allocatedBytesOnHold()); + s.holdElem(r1, 1); + m = s.getMemoryUsage(); + EXPECT_EQUAL(MyRef::offsetSize() * sizeof(int), m.allocatedBytes()); + EXPECT_EQUAL(4 * sizeof(int), m.usedBytes()); + EXPECT_EQUAL(1 * sizeof(int), m.deadBytes()); + EXPECT_EQUAL(1 * sizeof(int), m.allocatedBytesOnHold()); + s.disableElemHoldList(); + s.holdElem(r2, 1); + m = s.getMemoryUsage(); + EXPECT_EQUAL(MyRef::offsetSize() * sizeof(int), m.allocatedBytes()); + EXPECT_EQUAL(4 * sizeof(int), m.usedBytes()); + EXPECT_EQUAL(2 * sizeof(int), m.deadBytes()); + EXPECT_EQUAL(1 * sizeof(int), m.allocatedBytesOnHold()); + s.transferHoldLists(100); + s.trimHoldLists(101); +} + +int +Test::Main() +{ + TEST_INIT("datastore_test"); + + requireThatEntryRefIsWorking(); + requireThatAlignedEntryRefIsWorking(); + requireThatEntriesCanBeAddedAndRetrieved(); + requireThatAddEntryTriggersChangeOfBuffer(); + requireThatWeCanHoldAndTrimBuffers(); + requireThatWeCanHoldAndTrimElements(); + requireThatWeCanUseFreeLists(); + requireThatMemoryStatsAreCalculated(); + requireThatMemoryUsageIsCalculated(); + requireThatWecanDisableElemHoldList(); + + TEST_DONE(); +} + +} +} + +TEST_APPHOOK(search::btree::Test); + diff --git a/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp b/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp new file mode 100644 index 00000000000..87d32c90b78 --- /dev/null +++ b/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp @@ -0,0 +1,245 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("featurestore_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchlib/memoryindex/featurestore.h> + +using namespace search::btree; +using namespace search::index; + +namespace search +{ + + +namespace memoryindex +{ + + +class Test : public vespalib::TestApp +{ +private: + Schema _schema; + + const Schema & + getSchema(void) const + { + return _schema; + } + + bool + assertFeatures(const DocIdAndFeatures &exp, + const DocIdAndFeatures &act); + + void + requireThatFeaturesCanBeAddedAndRetrieved(void); + + void + requireThatNextWordsAreWorking(void); + void + requireThatAddFeaturesTriggersChangeOfBuffer(void); + +public: + Test(void); + + int + Main(void); +}; + + +bool +Test::assertFeatures(const DocIdAndFeatures &exp, + const DocIdAndFeatures &act) +{ + // docid is not encoded as part of features + if (!EXPECT_EQUAL(exp._elements.size(), + act._elements.size())) + return false; + for (size_t i = 0; i < exp._elements.size(); ++i) { + if (!EXPECT_EQUAL(exp._elements[i]._elementId, + act._elements[i]._elementId)) + return false; + if (!EXPECT_EQUAL(exp._elements[i]._numOccs, + act._elements[i]._numOccs)) + return false; + if (!EXPECT_EQUAL(exp._elements[i]._weight, act._elements[i]._weight)) + return false; + if (!EXPECT_EQUAL(exp._elements[i]._elementLen, + act._elements[i]._elementLen)) + return false; + } + if (!EXPECT_EQUAL(exp._wordPositions.size(), act._wordPositions.size())) + return false; + for (size_t i = 0; i < exp._wordPositions.size(); ++i) { + if (!EXPECT_EQUAL(exp._wordPositions[i]._wordPos, + act._wordPositions[i]._wordPos)) return false; + } + return true; +} + + +DocIdAndFeatures +getFeatures(uint32_t numOccs, + int32_t weight, + uint32_t elemLen) +{ + DocIdAndFeatures f; + f._docId = 0; + f._elements.push_back(WordDocElementFeatures(0)); + f._elements.back().setNumOccs(numOccs); + f._elements.back().setWeight(weight); + f._elements.back().setElementLen(elemLen); + for (uint32_t i = 0; i < numOccs; ++i) { + f._wordPositions.push_back(WordDocElementWordPosFeatures(i)); + } + return f; +} + + +void +Test::requireThatFeaturesCanBeAddedAndRetrieved(void) +{ + FeatureStore fs(getSchema()); + DocIdAndFeatures act; + EntryRef r1; + EntryRef r2; + std::pair<EntryRef, uint64_t> r; + { + DocIdAndFeatures f = getFeatures(2, 4, 8); + r = fs.addFeatures(0, f); + r1 = r.first; + EXPECT_TRUE(r.second > 0); + EXPECT_EQUAL(FeatureStore::RefType::align(1u), + FeatureStore::RefType(r1).offset()); + EXPECT_EQUAL(0u, FeatureStore::RefType(r1).bufferId()); + LOG(info, + "bits(%" PRIu64 "), ref.offset(%" PRIu64 "), ref.bufferId(%u)", + r.second, + FeatureStore::RefType(r1).offset(), + FeatureStore::RefType(r1).bufferId()); + fs.getFeatures(0, r1, act); + // weight not encoded for single value + EXPECT_TRUE(assertFeatures(getFeatures(2, 1, 8), act)); + } + { + DocIdAndFeatures f = getFeatures(4, 8, 16); + r = fs.addFeatures(1, f); + r2 = r.first; + EXPECT_TRUE(r.second > 0); + EXPECT_TRUE(FeatureStore::RefType(r2).offset() > + FeatureStore::RefType(r1).offset()); + EXPECT_EQUAL(0u, FeatureStore::RefType(r1).bufferId()); + LOG(info, + "bits(%" PRIu64 "), ref.offset(%" PRIu64 "), ref.bufferId(%u)", + r.second, + FeatureStore::RefType(r2).offset(), + FeatureStore::RefType(r2).bufferId()); + fs.getFeatures(1, r2, act); + EXPECT_TRUE(assertFeatures(f, act)); + } +} + + +void +Test::requireThatNextWordsAreWorking(void) +{ + FeatureStore fs(getSchema()); + DocIdAndFeatures act; + EntryRef r1; + EntryRef r2; + std::pair<EntryRef, uint64_t> r; + { + DocIdAndFeatures f = getFeatures(2, 4, 8); + r = fs.addFeatures(0, f); + r1 = r.first; + EXPECT_TRUE(r.second > 0); + EXPECT_EQUAL(FeatureStore::RefType::align(1u), + FeatureStore::RefType(r1).offset()); + EXPECT_EQUAL(0u, FeatureStore::RefType(r1).bufferId()); + LOG(info, + "bits(%" PRIu64 "), ref.offset(%" PRIu64 "), ref.bufferId(%u)", + r.second, + FeatureStore::RefType(r1).offset(), + FeatureStore::RefType(r1).bufferId()); + fs.getFeatures(0, r1, act); + // weight not encoded for single value + EXPECT_TRUE(assertFeatures(getFeatures(2, 1, 8), act)); + } + { + DocIdAndFeatures f = getFeatures(4, 8, 16); + r = fs.addFeatures(1, f); + r2 = r.first; + EXPECT_TRUE(r.second > 0); + EXPECT_TRUE(FeatureStore::RefType(r2).offset() > + FeatureStore::RefType(r1).offset()); + EXPECT_EQUAL(0u, FeatureStore::RefType(r1).bufferId()); + LOG(info, + "bits(%" PRIu64 "), ref.offset(%" PRIu64 "), ref.bufferId(%u)", + r.second, + FeatureStore::RefType(r2).offset(), + FeatureStore::RefType(r2).bufferId()); + fs.getFeatures(1, r2, act); + EXPECT_TRUE(assertFeatures(f, act)); + } +} + + +void +Test::requireThatAddFeaturesTriggersChangeOfBuffer(void) +{ + FeatureStore fs(getSchema()); + size_t cnt = 1; + DocIdAndFeatures act; + uint32_t lastId = 0; + for (;;++cnt) { + uint32_t numOccs = (cnt % 100) + 1; + DocIdAndFeatures f = getFeatures(numOccs, 1, numOccs + 1); + std::pair<EntryRef, uint64_t> r = fs.addFeatures(0, f); + fs.getFeatures(0, r.first, act); + EXPECT_TRUE(assertFeatures(f, act)); + uint32_t bufferId = FeatureStore::RefType(r.first).bufferId(); + if (bufferId > lastId) { + LOG(info, + "Changed to bufferId %u after %zu feature sets", + bufferId, cnt); + lastId = bufferId; + } + if (bufferId == 1) { + break; + } + } + EXPECT_EQUAL(1u, lastId); + LOG(info, "Added %zu feature sets in 1 buffer", cnt); +} + + +Test::Test() + : _schema() +{ + _schema.addIndexField(Schema::IndexField("f0", Schema::STRING)); + _schema.addIndexField(Schema::IndexField("f1", + Schema::STRING, + Schema::WEIGHTEDSET)); +} + + +int +Test::Main() +{ + TEST_INIT("featurestore_test"); + + requireThatFeaturesCanBeAddedAndRetrieved(); + requireThatNextWordsAreWorking(); + requireThatAddFeaturesTriggersChangeOfBuffer(); + + TEST_DONE(); +} + + +} + + +} + + +TEST_APPHOOK(search::memoryindex::Test); diff --git a/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp b/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp new file mode 100644 index 00000000000..825992b3b4f --- /dev/null +++ b/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("wordstore_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchlib/memoryindex/wordstore.h> + +using namespace search::btree; + +namespace search { +namespace memoryindex { + +class Test : public vespalib::TestApp { +private: + void requireThatWordsCanBeAddedAndRetrieved(); + void requireThatAddWordTriggersChangeOfBuffer(); +public: + int Main(); +}; + +void +Test::requireThatWordsCanBeAddedAndRetrieved() +{ + std::string w1 = "require"; + std::string w2 = "that"; + std::string w3 = "words"; + WordStore ws; + EntryRef r1 = ws.addWord(w1); + EntryRef r2 = ws.addWord(w2); + EntryRef r3 = ws.addWord(w3); + uint32_t invp = WordStore::RefType::align(1); // Reserved as invalid + uint32_t w1s = w1.size() + 1; + uint32_t w1p = WordStore::RefType::pad(w1s); + uint32_t w2s = w2.size() + 1; + uint32_t w2p = WordStore::RefType::pad(w2s); + EXPECT_EQUAL(invp, WordStore::RefType(r1).offset()); + EXPECT_EQUAL(invp + w1s + w1p, WordStore::RefType(r2).offset()); + EXPECT_EQUAL(invp + w1s + w1p + w2s + w2p, WordStore::RefType(r3).offset()); + EXPECT_EQUAL(0u, WordStore::RefType(r1).bufferId()); + EXPECT_EQUAL(0u, WordStore::RefType(r2).bufferId()); + EXPECT_EQUAL(0u, WordStore::RefType(r3).bufferId()); + EXPECT_EQUAL(std::string("require"), ws.getWord(r1)); + EXPECT_EQUAL(std::string("that"), ws.getWord(r2)); + EXPECT_EQUAL(std::string("words"), ws.getWord(r3)); +} + +void +Test::requireThatAddWordTriggersChangeOfBuffer() +{ + WordStore ws; + size_t word = 0; + uint32_t lastId = 0; + size_t lastWord = 0; + char wordStr[10]; + size_t entrySize = WordStore::RefType::align(6 + 1); + size_t initBufferSpace = 1024u * WordStore::RefType::align(1); + size_t bufferSpace = initBufferSpace; + size_t bufferWords = (bufferSpace - WordStore::RefType::align(1)) / + entrySize; + size_t usedSpace = 0; + size_t sumBufferWords = 0; + for (;;++word) { + sprintf(wordStr, "%6zu", word); + // all words uses 12 bytes (include padding) + EntryRef r = ws.addWord(std::string(wordStr)); + EXPECT_EQUAL(std::string(wordStr), ws.getWord(r)); + uint32_t bufferId = WordStore::RefType(r).bufferId(); + if (bufferId > lastId) { + LOG(info, + "Changed to bufferId %u after %zu words", + bufferId, word); + EXPECT_EQUAL(bufferWords, word - lastWord); + lastId = bufferId; + lastWord = word; + usedSpace += bufferWords * entrySize; + sumBufferWords += bufferWords; + bufferSpace = usedSpace + initBufferSpace; + bufferWords = bufferSpace / entrySize; + } + if (bufferId == 4) { + break; + } + } + // each buffer can have offsetSize / 12 words + EXPECT_EQUAL(sumBufferWords, word); + LOG(info, "Added %zu words in 4 buffers", word); +} + +int +Test::Main() +{ + TEST_INIT("wordstore_test"); + + requireThatWordsCanBeAddedAndRetrieved(); + requireThatAddWordTriggersChangeOfBuffer(); + + TEST_DONE(); +} + +} +} + +TEST_APPHOOK(search::memoryindex::Test); + |