diff options
Diffstat (limited to 'searchcore/src/tests')
10 files changed, 428 insertions, 50 deletions
diff --git a/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp b/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp index 37cf201e354..f95ea478ce1 100644 --- a/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp @@ -137,6 +137,7 @@ struct Fixture : public DirectoryHandler EXPECT_TRUE(hasAttributeDir(dir)); auto writer = dir->getWriter(); writer->createInvalidSnapshot(serialNum); + vespalib::mkdir(writer->getSnapshotDir(serialNum), false); writer->markValidSnapshot(serialNum); TEST_DO(assertAttributeDiskDir("foo")); } @@ -162,6 +163,7 @@ struct Fixture : public DirectoryHandler auto dir = createFooAttrDir(); auto writer = dir->getWriter(); writer->createInvalidSnapshot(serialNum); + vespalib::mkdir(writer->getSnapshotDir(serialNum), false); writer->markValidSnapshot(serialNum); } @@ -208,8 +210,10 @@ TEST_F("Test that we can prune attribute snapshots", Fixture) TEST_DO(f.assertNotAttributeDiskDir("foo")); auto writer = dir->getWriter(); writer->createInvalidSnapshot(2); + vespalib::mkdir(writer->getSnapshotDir(2), false); writer->markValidSnapshot(2); writer->createInvalidSnapshot(4); + vespalib::mkdir(writer->getSnapshotDir(4), false); writer->markValidSnapshot(4); writer.reset(); TEST_DO(f.assertAttributeDiskDir("foo")); diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 6a098667be8..d88199e6ae8 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -662,7 +662,7 @@ TEST_F("requireThatSerialNumberIsReadOnLoad", Fixture) { void crippleFusion(uint32_t fusionId) { vespalib::asciistream ost; ost << index_dir << "/index.flush." << fusionId << "/serial.dat"; - FastOS_File(ost.str().c_str()).Delete(); + FastOS_File(ost.str().data()).Delete(); } TEST_F("requireThatFailedFusionIsRetried", Fixture) { diff --git a/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp b/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp index 2bfd907b4a3..b64b2526f06 100644 --- a/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp +++ b/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp @@ -8,20 +8,21 @@ using namespace proton::matching; using vespalib::Box; using vespalib::make_box; -typedef MatchLoopCommunicator::Range Range; -typedef MatchLoopCommunicator::RangePair RangePair; -typedef MatchLoopCommunicator::feature_t feature_t; -typedef MatchLoopCommunicator::Matches Matches; +using Range = MatchLoopCommunicator::Range; +using RangePair = MatchLoopCommunicator::RangePair; +using Matches = MatchLoopCommunicator::Matches; +using Hit = MatchLoopCommunicator::Hit; +using Hits = MatchLoopCommunicator::Hits; -std::vector<feature_t> makeScores(size_t id) { +Hits makeScores(size_t id) { switch (id) { - case 0: return make_box<feature_t>(5.4, 4.4, 3.4, 2.4, 1.4); - case 1: return make_box<feature_t>(5.3, 4.3, 3.3, 2.3, 1.3); - case 2: return make_box<feature_t>(5.2, 4.2, 3.2, 2.2, 1.2); - case 3: return make_box<feature_t>(5.1, 4.1, 3.1, 2.1, 1.1); - case 4: return make_box<feature_t>(5.0, 4.0, 3.0, 2.0, 1.0); + case 0: return make_box<Hit>({1, 5.4}, {2, 4.4}, {3, 3.4}, {4, 2.4}, {5, 1.4}); + case 1: return make_box<Hit>({11, 5.3}, {12, 4.3}, {13, 3.3}, {14, 2.3}, {15, 1.3}); + case 2: return make_box<Hit>({21, 5.2}, {22, 4.2}, {23, 3.2}, {24, 2.2}, {25, 1.2}); + case 3: return make_box<Hit>({31, 5.1}, {32, 4.1}, {33, 3.1}, {34, 2.1}, {35, 1.1}); + case 4: return make_box<Hit>({41, 5.0}, {42, 4.0}, {43, 3.0}, {44, 2.0}, {45, 1.0}); } - return Box<feature_t>(); + return Box<Hit>(); } RangePair makeRanges(size_t id) { @@ -35,43 +36,65 @@ RangePair makeRanges(size_t id) { return std::make_pair(Range(-50, -60), Range(60, 50)); } +void equal(size_t count, const Hits & a, const Hits & b) { + EXPECT_EQUAL(count, b.size()); + for (size_t i(0); i < count; i++) { + EXPECT_EQUAL(a[i].first, b[i].first); + EXPECT_EQUAL(a[i].second , b[i].second); + } +} + +struct EveryOdd : public search::queryeval::IDiversifier { + bool accepted(uint32_t docId) override { + return docId & 0x01; + } +}; + TEST_F("require that selectBest gives appropriate results for single thread", MatchLoopCommunicator(num_threads, 3)) { - EXPECT_EQUAL(2u, f1.selectBest(make_box<feature_t>(5, 4))); - EXPECT_EQUAL(3u, f1.selectBest(make_box<feature_t>(5, 4, 3))); - EXPECT_EQUAL(3u, f1.selectBest(make_box<feature_t>(5, 4, 3, 2))); + TEST_DO(equal(2u, make_box<Hit>({1, 5}, {2, 4}), f1.selectBest(make_box<Hit>({1, 5}, {2, 4})))); + TEST_DO(equal(3u, make_box<Hit>({1, 5}, {2, 4}, {3, 3}), f1.selectBest(make_box<Hit>({1, 5}, {2, 4}, {3, 3})))); + TEST_DO(equal(3u, make_box<Hit>({1, 5}, {2, 4}, {3, 3}), f1.selectBest(make_box<Hit>({1, 5}, {2, 4}, {3, 3}, {4, 2})))); +} + +TEST_F("require that selectBest gives appropriate results for single thread with filter", + MatchLoopCommunicator(num_threads, 3, std::make_unique<EveryOdd>())) +{ + TEST_DO(equal(1u, make_box<Hit>({1, 5}), f1.selectBest(make_box<Hit>({1, 5}, {2, 4})))); + TEST_DO(equal(2u, make_box<Hit>({1, 5}, {3, 3}), f1.selectBest(make_box<Hit>({1, 5}, {2, 4}, {3, 3})))); + TEST_DO(equal(3u, make_box<Hit>({1, 5}, {3, 3}, {5, 1}), f1.selectBest(make_box<Hit>({1, 5}, {2, 4}, {3, 3}, {4, 2}, {5, 1}, {6, 0})))); } TEST_MT_F("require that selectBest works with no hits", 10, MatchLoopCommunicator(num_threads, 10)) { - EXPECT_EQUAL(0u, f1.selectBest(Box<feature_t>())); + EXPECT_TRUE(f1.selectBest(Box<Hit>()).empty()); } TEST_MT_F("require that selectBest works with too many hits from all threads", 5, MatchLoopCommunicator(num_threads, 13)) { if (thread_id < 3) { - EXPECT_EQUAL(3u, f1.selectBest(makeScores(thread_id))); + TEST_DO(equal(3u, makeScores(thread_id), f1.selectBest(makeScores(thread_id)))); } else { - EXPECT_EQUAL(2u, f1.selectBest(makeScores(thread_id))); + TEST_DO(equal(2u, makeScores(thread_id), f1.selectBest(makeScores(thread_id)))); } } TEST_MT_F("require that selectBest works with some exhausted threads", 5, MatchLoopCommunicator(num_threads, 22)) { if (thread_id < 2) { - EXPECT_EQUAL(5u, f1.selectBest(makeScores(thread_id))); + TEST_DO(equal(5u, makeScores(thread_id), f1.selectBest(makeScores(thread_id)))); } else { - EXPECT_EQUAL(4u, f1.selectBest(makeScores(thread_id))); + TEST_DO(equal(4u, makeScores(thread_id), f1.selectBest(makeScores(thread_id)))); } } TEST_MT_F("require that selectBest can select all hits from all threads", 5, MatchLoopCommunicator(num_threads, 100)) { - EXPECT_EQUAL(5u, f1.selectBest(makeScores(thread_id))); + EXPECT_EQUAL(5u, f1.selectBest(makeScores(thread_id)).size()); } TEST_MT_F("require that selectBest works with some empty threads", 10, MatchLoopCommunicator(num_threads, 7)) { if (thread_id < 2) { - EXPECT_EQUAL(2u, f1.selectBest(makeScores(thread_id))); + TEST_DO(equal(2u, makeScores(thread_id), f1.selectBest(makeScores(thread_id)))); } else if (thread_id < 5) { - EXPECT_EQUAL(1u, f1.selectBest(makeScores(thread_id))); + TEST_DO(equal(1u, makeScores(thread_id), f1.selectBest(makeScores(thread_id)))); } else { - EXPECT_EQUAL(0u, f1.selectBest(makeScores(thread_id))); + EXPECT_TRUE(f1.selectBest(makeScores(thread_id)).empty()); } } diff --git a/searchcore/src/tests/proton/matching/matching_stats_test.cpp b/searchcore/src/tests/proton/matching/matching_stats_test.cpp index 8b58a9e271c..85960de2021 100644 --- a/searchcore/src/tests/proton/matching/matching_stats_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_stats_test.cpp @@ -240,6 +240,65 @@ TEST("requireThatPartitionsAreAddedCorrectly") { EXPECT_EQUAL(1.0, all1.getPartition(1).wait_time_max()); } +TEST("requireThatSoftDoomIsSetAndAdded") { + MatchingStats stats; + MatchingStats stats2; + EXPECT_EQUAL(0ul, stats.softDoomed()); + EXPECT_EQUAL(0.5, stats.softDoomFactor()); + stats.softDoomFactor(0.7); + stats.softDoomed(3); + EXPECT_EQUAL(3ul, stats.softDoomed()); + EXPECT_EQUAL(0.7, stats.softDoomFactor()); + stats2.add(stats); + EXPECT_EQUAL(3ul, stats2.softDoomed()); + EXPECT_EQUAL(0.5, stats2.softDoomFactor()); // Not affected by add +} + +TEST("requireThatSoftDoomFacorIsComputedCorrectlyForDownAdjustment") { + MatchingStats stats; + EXPECT_EQUAL(0ul, stats.softDoomed()); + EXPECT_EQUAL(0.5, stats.softDoomFactor()); + stats.softDoomed(1); + stats.updatesoftDoomFactor(1.0, 0.5, 2.0); + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.47, stats.softDoomFactor()); + stats.updatesoftDoomFactor(1.0, 0.5, 2.0); + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.44, stats.softDoomFactor()); + stats.updatesoftDoomFactor(0.0009, 0.5, 2.0); // hard limits less than 1ms should be ignored + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.44, stats.softDoomFactor()); + stats.updatesoftDoomFactor(1.0, 0.0009, 2.0); // soft limits less than 1ms should be ignored + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.44, stats.softDoomFactor()); + stats.updatesoftDoomFactor(1.0, 0.5, 10.0); // Prevent changes above 10% + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.396, stats.softDoomFactor()); +} + +TEST("requireThatSoftDoomFacorIsComputedCorrectlyForUpAdjustment") { + MatchingStats stats; + EXPECT_EQUAL(0ul, stats.softDoomed()); + EXPECT_EQUAL(0.5, stats.softDoomFactor()); + stats.softDoomed(1); + stats.updatesoftDoomFactor(1.0, 0.9, 0.1); + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.508, stats.softDoomFactor()); + stats.updatesoftDoomFactor(1.0, 0.9, 0.1); + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.516, stats.softDoomFactor()); + stats.updatesoftDoomFactor(0.0009, 0.9, 0.1); // hard limits less than 1ms should be ignored + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.516, stats.softDoomFactor()); + stats.updatesoftDoomFactor(1.0, 0.0009, 0.1); // soft limits less than 1ms should be ignored + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.516, stats.softDoomFactor()); + stats.softDoomFactor(0.1); + stats.updatesoftDoomFactor(1.0, 0.9, 0.001); // Prevent changes above 5% + EXPECT_EQUAL(1ul, stats.softDoomed()); + EXPECT_EQUAL(0.105, stats.softDoomFactor()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 61823a17f09..82aab72068d 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -63,6 +63,12 @@ using search::queryeval::SearchIterator; using search::queryeval::SimpleBlueprint; using search::queryeval::SimpleResult; using search::queryeval::ParallelWeakAndBlueprint; +using search::queryeval::RankBlueprint; +using search::queryeval::AndBlueprint; +using search::queryeval::IntermediateBlueprint; +using search::queryeval::AndNotBlueprint; +using search::queryeval::SourceBlenderBlueprint; + using std::string; using std::vector; namespace fef_test = search::fef::test; @@ -106,6 +112,8 @@ class Test : public vespalib::TestApp { void requireThatWeakAndBlueprintsAreCreatedCorrectly(); void requireThatParallelWandBlueprintsAreCreatedCorrectly(); void requireThatWhiteListBlueprintCanBeUsed(); + void requireThatRankBlueprintStaysOnTopAfterWhiteListing(); + void requireThatAndNotBlueprintStaysOnTopAfterWhiteListing(); void requireThatSameElementTermsAreProperlyPrefixed(); void requireThatSameElementDoesNotAllocateMatchData(); void requireThatSameElementIteratorsCanBeBuilt(); @@ -879,6 +887,54 @@ Test::requireThatWhiteListBlueprintCanBeUsed() EXPECT_EQUAL(exp, act); } +template<typename T1, typename T2> +void verifyThatRankBlueprintAndAndNotStaysOnTopAfterWhiteListing(QueryBuilder<ProtonNodeTypes> & builder) { + builder.addStringTerm("foo", field, field_id, string_weight); + builder.addStringTerm("bar", field, field_id, string_weight); + builder.addStringTerm("baz", field, field_id, string_weight); + std::string stackDump = StackDumpCreator::create(*builder.build()); + Query query; + query.buildTree(stackDump, "", ViewResolver(), plain_index_env); + FakeSearchContext context(42); + context.addIdx(0).idx(0).getFake() + .addResult(field, "foo", FakeResult().doc(1)); + context.setLimit(42); + + query.setWhiteListBlueprint(std::make_unique<SimpleBlueprint>(SimpleResult())); + + FakeRequestContext requestContext; + MatchDataLayout mdl; + query.reserveHandles(requestContext, context, mdl); + const IntermediateBlueprint * root = dynamic_cast<const T1 *>(query.peekRoot()); + ASSERT_TRUE(root != nullptr); + EXPECT_EQUAL(2u, root->childCnt()); + const IntermediateBlueprint * second = dynamic_cast<const T2 *>(&root->getChild(0)); + ASSERT_TRUE(second != nullptr); + EXPECT_EQUAL(2u, second->childCnt()); + const AndBlueprint * first = dynamic_cast<const AndBlueprint *>(&second->getChild(0)); + ASSERT_TRUE(first != nullptr); + EXPECT_EQUAL(2u, first->childCnt()); + EXPECT_TRUE(dynamic_cast<const SourceBlenderBlueprint *>(&first->getChild(0))); + EXPECT_TRUE(dynamic_cast<const SimpleBlueprint *>(&first->getChild(1))); + EXPECT_TRUE(dynamic_cast<const SourceBlenderBlueprint *>(&second->getChild(1))); + EXPECT_TRUE(dynamic_cast<const SourceBlenderBlueprint *>(&root->getChild(1))); +} + +void Test::requireThatRankBlueprintStaysOnTopAfterWhiteListing() { + QueryBuilder<ProtonNodeTypes> builder; + builder.addRank(2); + builder.addAndNot(2); + verifyThatRankBlueprintAndAndNotStaysOnTopAfterWhiteListing<RankBlueprint, AndNotBlueprint>(builder); +} + +void Test::requireThatAndNotBlueprintStaysOnTopAfterWhiteListing() { + QueryBuilder<ProtonNodeTypes> builder; + builder.addAndNot(2); + builder.addRank(2); + verifyThatRankBlueprintAndAndNotStaysOnTopAfterWhiteListing<AndNotBlueprint, RankBlueprint>(builder); +} + + search::query::Node::UP make_same_element_stack_dump(const vespalib::string &prefix, const vespalib::string &term_prefix) { @@ -984,6 +1040,8 @@ Test::Main() TEST_CALL(requireThatWeakAndBlueprintsAreCreatedCorrectly); TEST_CALL(requireThatParallelWandBlueprintsAreCreatedCorrectly); TEST_CALL(requireThatWhiteListBlueprintCanBeUsed); + TEST_CALL(requireThatRankBlueprintStaysOnTopAfterWhiteListing); + TEST_CALL(requireThatAndNotBlueprintStaysOnTopAfterWhiteListing); TEST_CALL(requireThatSameElementTermsAreProperlyPrefixed); TEST_CALL(requireThatSameElementDoesNotAllocateMatchData); TEST_CALL(requireThatSameElementIteratorsCanBeBuilt); diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index 045c8de9384..dfb1268aaa6 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -12,10 +12,11 @@ #include <vespa/searchcore/proton/server/bootstrapconfig.h> #include <vespa/searchcore/proton/server/bootstrapconfigmanager.h> #include <vespa/searchcore/proton/server/documentdbconfigmanager.h> -#include <vespa/searchcore/proton/server/i_document_db_config_owner.h> +#include <vespa/searchcore/proton/server/document_db_config_owner.h> #include <vespa/searchcore/proton/server/proton_config_snapshot.h> #include <vespa/searchcore/proton/server/proton_configurer.h> #include <vespa/searchcore/proton/server/i_proton_configurer_owner.h> +#include <vespa/searchcore/proton/server/i_proton_disk_layout.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/vespalib/testkit/testapp.h> @@ -208,13 +209,13 @@ struct ConfigFixture { struct MyProtonConfigurerOwner; -struct MyDocumentDBConfigOwner : public IDocumentDBConfigOwner +struct MyDocumentDBConfigOwner : public DocumentDBConfigOwner { vespalib::string _name; MyProtonConfigurerOwner &_owner; MyDocumentDBConfigOwner(const vespalib::string &name, MyProtonConfigurerOwner &owner) - : IDocumentDBConfigOwner(), + : DocumentDBConfigOwner(), _name(name), _owner(owner) { @@ -224,28 +225,43 @@ struct MyDocumentDBConfigOwner : public IDocumentDBConfigOwner void reconfigure(const DocumentDBConfig::SP & config) override; }; -struct MyProtonConfigurerOwner : public IProtonConfigurerOwner +struct MyLog +{ + std::vector<vespalib::string> _log; + + MyLog() + : _log() + { + } + + void appendLog(vespalib::string logEntry) + { + _log.emplace_back(logEntry); + } +}; + +struct MyProtonConfigurerOwner : public IProtonConfigurerOwner, + public MyLog { using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>; vespalib::ThreadStackExecutor _executor; std::map<DocTypeName, std::shared_ptr<MyDocumentDBConfigOwner>> _dbs; - std::vector<vespalib::string> _log; MyProtonConfigurerOwner() : IProtonConfigurerOwner(), + MyLog(), _executor(1, 128 * 1024), - _dbs(), - _log() + _dbs() { } virtual ~MyProtonConfigurerOwner() { } - virtual IDocumentDBConfigOwner *addDocumentDB(const DocTypeName &docTypeName, - document::BucketSpace bucketSpace, - const vespalib::string &configId, - const std::shared_ptr<BootstrapConfig> &bootstrapConfig, - const std::shared_ptr<DocumentDBConfig> &documentDBConfig, - InitializeThreads initializeThreads) override + virtual std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName &docTypeName, + document::BucketSpace bucketSpace, + const vespalib::string &configId, + const std::shared_ptr<BootstrapConfig> &bootstrapConfig, + const std::shared_ptr<DocumentDBConfig> &documentDBConfig, + InitializeThreads initializeThreads) override { (void) bucketSpace; (void) configId; @@ -257,7 +273,7 @@ struct MyProtonConfigurerOwner : public IProtonConfigurerOwner std::ostringstream os; os << "add db " << docTypeName.getName() << " " << documentDBConfig->getGeneration(); _log.push_back(os.str()); - return db.get(); + return db; } virtual void removeDocumentDB(const DocTypeName &docTypeName) override { ASSERT_FALSE(_dbs.find(docTypeName) == _dbs.end()); @@ -286,17 +302,48 @@ MyDocumentDBConfigOwner::reconfigure(const DocumentDBConfig::SP & config) _owner.reconfigureDocumentDB(_name, config); } +struct MyProtonDiskLayout : public IProtonDiskLayout +{ + MyLog &_log; + + MyProtonDiskLayout(MyLog &myLog) + : _log(myLog) + { + } + void remove(const DocTypeName &docTypeName) override { + std::ostringstream os; + os << "remove dbdir " << docTypeName.getName(); + _log.appendLog(os.str()); + } + void initAndPruneUnused(const std::set<DocTypeName> &docTypeNames) override { + std::ostringstream os; + os << "initial dbs "; + bool first = true; + for (const auto &docTypeName : docTypeNames) { + if (!first) { + os << ","; + } + first = false; + os << docTypeName.getName(); + } + _log.appendLog(os.str()); + } +}; + struct Fixture { MyProtonConfigurerOwner _owner; ConfigFixture _config; + std::unique_ptr<IProtonDiskLayout> _diskLayout; ProtonConfigurer _configurer; Fixture() : _owner(), _config("test"), - _configurer(_owner._executor, _owner) + _diskLayout(), + _configurer(_owner._executor, _owner, _diskLayout) { + _diskLayout = std::make_unique<MyProtonDiskLayout>(_owner); } ~Fixture() { } @@ -338,14 +385,14 @@ TEST_F("require that nothing is applied before initial config", Fixture()) TEST_F("require that initial config is applied", Fixture()) { f.applyInitialConfig(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"})); } TEST_F("require that new config is blocked", Fixture()) { f.applyInitialConfig(); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"})); } TEST_F("require that new config can be unblocked", Fixture()) @@ -353,14 +400,14 @@ TEST_F("require that new config can be unblocked", Fixture()) f.applyInitialConfig(); f.reconfigure(); f.allowReconfig(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"})); } TEST_F("require that initial config is not reapplied due to config unblock", Fixture()) { f.applyInitialConfig(); f.allowReconfig(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"})); } TEST_F("require that we can add document db", Fixture()) @@ -369,7 +416,7 @@ TEST_F("require that we can add document db", Fixture()) f.allowReconfig(); f.addDocType("foobar"); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3","reconf db _alwaysthere_ 3", "add db foobar 3"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3","reconf db _alwaysthere_ 3", "add db foobar 3"})); } TEST_F("require that we can remove document db", Fixture()) @@ -379,7 +426,7 @@ TEST_F("require that we can remove document db", Fixture()) f.allowReconfig(); f.removeDocType("foobar"); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "remove db foobar"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "remove db foobar", "remove dbdir foobar"})); } TEST_F("require that document db adds and reconfigs are intermingled", Fixture()) @@ -392,7 +439,7 @@ TEST_F("require that document db adds and reconfigs are intermingled", Fixture() f.addDocType("foobar"); f.addDocType("zbar"); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "add db abar 3", "reconf db foobar 3", "add db zbar 3"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "add db abar 3", "reconf db foobar 3", "add db zbar 3"})); } TEST_F("require that document db removes are applied at end", Fixture()) @@ -403,7 +450,7 @@ TEST_F("require that document db removes are applied at end", Fixture()) f.allowReconfig(); f.removeDocType("abar"); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db abar 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "reconf db foobar 3", "remove db abar"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,abar,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db abar 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "reconf db foobar 3", "remove db abar", "remove dbdir abar"})); } TEST_F("require that new configs can be blocked again", Fixture()) @@ -413,7 +460,7 @@ TEST_F("require that new configs can be blocked again", Fixture()) f.allowReconfig(); f.disableReconfig(); f.reconfigure(); - TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"})); + TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"})); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt b/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt new file mode 100644 index 00000000000..f63fa21a954 --- /dev/null +++ b/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_proton_disk_layout_test_app TEST + SOURCES + proton_disk_layout_test.cpp + DEPENDS + searchcore_server + searchcore_fconfig +) +vespa_add_test(NAME searchcore_proton_disk_layout_test_app COMMAND searchcore_proton_disk_layout_test_app) diff --git a/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp new file mode 100644 index 00000000000..edb4250ce76 --- /dev/null +++ b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp @@ -0,0 +1,178 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcore/proton/server/proton_disk_layout.h> +#include <vespa/searchcore/proton/common/doctypename.h> +#include <vespa/searchlib/index/dummyfileheadercontext.h> +#include <vespa/searchlib/transactionlog/translogserver.h> +#include <vespa/searchlib/transactionlog/translogclient.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/util/stringfmt.h> + +using search::index::DummyFileHeaderContext; +using search::transactionlog::TransLogClient; +using search::transactionlog::TransLogServer; +using proton::DocTypeName; +using proton::ProtonDiskLayout; + +static constexpr unsigned int tlsPort = 9018; + +static const vespalib::string baseDir("testdb"); +static const vespalib::string documentsDir(baseDir + "/documents"); + +struct FixtureBase +{ + FixtureBase() { vespalib::rmdir(baseDir, true); } + ~FixtureBase() { vespalib::rmdir(baseDir, true); } +}; + +struct DiskLayoutFixture { + DummyFileHeaderContext _fileHeaderContext; + TransLogServer _tls; + vespalib::string _tlsSpec; + ProtonDiskLayout _diskLayout; + + DiskLayoutFixture(); + ~DiskLayoutFixture(); + + void createDirs(const std::set<vespalib::string> &dirs) { + for (const auto &dir : dirs) { + vespalib::mkdir(documentsDir + "/" + dir, false); + } + } + void createDomains(const std::set<vespalib::string> &domains) { + TransLogClient tlc(_tlsSpec); + for (const auto &domain : domains) { + ASSERT_TRUE(tlc.create(domain)); + } + } + + std::set<vespalib::string> listDomains() { + std::vector<vespalib::string> domainVector; + TransLogClient tlc(_tlsSpec); + ASSERT_TRUE(tlc.listDomains(domainVector)); + std::set<vespalib::string> domains; + for (const auto &domain : domainVector) { + domains.emplace(domain); + } + return domains; + } + + std::set<vespalib::string> listDirs() { + std::set<vespalib::string> dirs; + auto names = vespalib::listDirectory(documentsDir); + for (const auto &name : names) { + if (vespalib::isDirectory(documentsDir + "/" + name)) { + dirs.emplace(name); + } + } + return dirs; + } + + void initAndPruneUnused(const std::set<vespalib::string> names) + { + std::set<DocTypeName> docTypeNames; + for (const auto &name: names) { + docTypeNames.emplace(name); + } + _diskLayout.initAndPruneUnused(docTypeNames); + } + + void assertDirs(const std::set<vespalib::string> &expDirs) { + EXPECT_EQUAL(expDirs, listDirs()); + } + + void assertDomains(const std::set<vespalib::string> &expDomains) + { + EXPECT_EQUAL(expDomains, listDomains()); + } +}; + +DiskLayoutFixture::DiskLayoutFixture() + : _fileHeaderContext(), + _tls("tls", tlsPort, baseDir, _fileHeaderContext), + _tlsSpec(vespalib::make_string("tcp/localhost:%u", tlsPort)), + _diskLayout(baseDir, _tlsSpec) +{ +} + +DiskLayoutFixture::~DiskLayoutFixture() = default; + +struct Fixture : public FixtureBase, public DiskLayoutFixture +{ + Fixture() + : FixtureBase(), + DiskLayoutFixture() + { + } +}; + +TEST_F("require that empty config is ok", Fixture) { + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_F("require that disk layout is preserved", FixtureBase) +{ + { + DiskLayoutFixture diskLayout; + diskLayout.createDirs({"foo", "bar"}); + diskLayout.createDomains({"bar", "baz"}); + } + { + DiskLayoutFixture diskLayout; + TEST_DO(diskLayout.assertDirs({"foo", "bar"})); + TEST_DO(diskLayout.assertDomains({"bar", "baz"})); + } +} + +TEST_F("require that used dir is preserved", Fixture) +{ + f.createDirs({"foo"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"foo"}); + TEST_DO(f.assertDirs({"foo"})); + TEST_DO(f.assertDomains({"foo"})); +} + +TEST_F("require that unused dir is removed", Fixture) +{ + f.createDirs({"foo"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"bar"}); + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_F("require that interrupted remove is completed", Fixture) +{ + f.createDirs({"foo.removed"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"foo"}); + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_F("require that early interrupted remove is completed", Fixture) +{ + f.createDirs({"foo", "foo.removed"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"foo"}); + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_F("require that live document db dir remove works", Fixture) +{ + f.createDirs({"foo"}); + f.createDomains({"foo"}); + f.initAndPruneUnused({"foo"}); + TEST_DO(f.assertDirs({"foo"})); + TEST_DO(f.assertDomains({"foo"})); + f._diskLayout.remove(DocTypeName("foo")); + TEST_DO(f.assertDirs({})); + TEST_DO(f.assertDomains({})); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp index 7be774f7291..72e558fd25f 100644 --- a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp @@ -294,9 +294,9 @@ TEST("require that added attribute aspect with flushed attribute after interrupt auto dir = diskLayout->createAttributeDir("a"); auto writer = dir->getWriter(); writer->createInvalidSnapshot(INIT_SERIAL_NUM); - writer->markValidSnapshot(INIT_SERIAL_NUM); auto snapshotdir = writer->getSnapshotDir(INIT_SERIAL_NUM); vespalib::mkdir(snapshotdir); + writer->markValidSnapshot(INIT_SERIAL_NUM); auto av = AttributeFactory::createAttribute(snapshotdir + "/a", Config(BasicType::STRING)); av->save(); diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp index 8206eba6350..0a520044985 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp @@ -68,7 +68,7 @@ public: DocsumReply::Docsum docsum; docsum.docid = 10 + i; docsum.gid = h.gid; - docsum.setData(_reply.c_str(), _reply.size()); + docsum.setData(_reply.data(), _reply.size()); retval->docsums.push_back(docsum); } return retval; |