diff options
Diffstat (limited to 'storage/src/tests')
150 files changed, 40231 insertions, 0 deletions
diff --git a/storage/src/tests/.gitignore b/storage/src/tests/.gitignore new file mode 100644 index 00000000000..9023e5da3b4 --- /dev/null +++ b/storage/src/tests/.gitignore @@ -0,0 +1,22 @@ +*.o +*.lo +.depend.NEW +.depend +.deps +.libs +.config.log +Makefile +testrunner +vdsroot +*.core +state +*.So +test.vlog +dirconfig.tmp +.*.swp +metricsreport.html +piefile.html +piefile-customcols.html +palette.html +use_new_storage_core +storage_testrunner_app diff --git a/storage/src/tests/CMakeLists.txt b/storage/src/tests/CMakeLists.txt new file mode 100644 index 00000000000..894ea7b4d25 --- /dev/null +++ b/storage/src/tests/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(storage_testrunner_app + SOURCES + testrunner.cpp + DEPENDS + storage_teststorageserver + storage_testbucketmover + storage_teststorageutil + storage_testvisiting + storage_testbucketdb + storage_testcommon + storage_testhostreporter + storage_testdistributor + storage_testpersistence + storage_testfilestorage + storage_testmemory + storage_teststatus + storage + AFTER + storage_storageconfig +) +vespa_add_test(NAME storage_testrunner_app COMMAND storage_testrunner_app) diff --git a/storage/src/tests/bucketdb/.gitignore b/storage/src/tests/bucketdb/.gitignore new file mode 100644 index 00000000000..4e71c44a596 --- /dev/null +++ b/storage/src/tests/bucketdb/.gitignore @@ -0,0 +1,12 @@ +*.So +*.core +*.lo +*.o +.*.swp +.config.log +.depend +.depend.NEW +.deps +.libs +Makefile +testrunner diff --git a/storage/src/tests/bucketdb/CMakeLists.txt b/storage/src/tests/bucketdb/CMakeLists.txt new file mode 100644 index 00000000000..95228966589 --- /dev/null +++ b/storage/src/tests/bucketdb/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storage_testbucketdb + SOURCES + initializertest.cpp + bucketmanagertest.cpp + judyarraytest.cpp + judymultimaptest.cpp + lockablemaptest.cpp + bucketinfotest.cpp + distribution_hash_normalizer_test.cpp + DEPENDS + AFTER + storage_storageconfig +) diff --git a/storage/src/tests/bucketdb/bucketinfotest.cpp b/storage/src/tests/bucketdb/bucketinfotest.cpp new file mode 100644 index 00000000000..eef4c6d7739 --- /dev/null +++ b/storage/src/tests/bucketdb/bucketinfotest.cpp @@ -0,0 +1,201 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <boost/assign.hpp> +#include <boost/random.hpp> +#include <cppunit/extensions/HelperMacros.h> +#include <map> +#include <vector> +#include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/storage/distributor/bucketdb/bucketinfo.h> + +namespace storage { + +namespace distributor { + +struct BucketInfoTest : public CppUnit::TestFixture { + void testBucketInfoEntriesWithNewestTimestampsAreKept(); + void testOrder(); + void testHasInvalidCopy(); + void testAddNodeSetsTrustedWhenConsistent(); + void testTrustedResetWhenCopiesBecomeInconsistent(); + void testTrustedResetWhenTrustedCopiesGoOutOfSync(); + void testTrustedNotResetWhenNonTrustedCopiesStillOutOfSync(); + + CPPUNIT_TEST_SUITE(BucketInfoTest); + CPPUNIT_TEST(testBucketInfoEntriesWithNewestTimestampsAreKept); + CPPUNIT_TEST(testOrder); + CPPUNIT_TEST(testHasInvalidCopy); + CPPUNIT_TEST(testAddNodeSetsTrustedWhenConsistent); + CPPUNIT_TEST_IGNORED(testTrustedResetWhenCopiesBecomeInconsistent); + CPPUNIT_TEST(testTrustedResetWhenTrustedCopiesGoOutOfSync); + CPPUNIT_TEST(testTrustedNotResetWhenNonTrustedCopiesStillOutOfSync); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(BucketInfoTest); + +BucketInfo +getBucketInfo(std::string nodeList, std::string order) { + BucketInfo info; + + std::vector<uint16_t> ordering; + { + vespalib::StringTokenizer tokenizer(order, ","); + for (uint32_t i = 0; i < tokenizer.size(); i++) { + ordering.push_back(atoi(tokenizer[i].c_str())); + } + } + + vespalib::StringTokenizer tokenizer(nodeList, ","); + for (uint32_t i = 0; i < tokenizer.size(); i++) { + info.addNode(BucketCopy(0, + atoi(tokenizer[i].c_str()), + api::BucketInfo(1,1,1)), + ordering); + } + + return info; +} + +std::string +nodeList(const BucketInfo& info) { + std::ostringstream ost; + for (uint32_t i = 0; i < info.getNodeCount(); i++) { + if (i != 0) { + ost << ","; + } + ost << (int)info.getNodeRef(i).getNode(); + } + return ost.str(); +} + +// Since we keep bucket info in memory for a period of time before applying +// to bucket db, we maintain timestamps to prevent external load happening +// in the meantime from having their updates lost when we perform a batch +// insert. This also applies for when we postpone db updates in persistence +// message tracker until we've received a reply from all copies. +void +BucketInfoTest::testBucketInfoEntriesWithNewestTimestampsAreKept() +{ + BucketInfo bi; + std::vector<uint16_t> idealState; + idealState.push_back(0); + + bi.addNode(BucketCopy(5, 0, api::BucketInfo(1,1,1)), idealState); + CPPUNIT_ASSERT_EQUAL(api::BucketInfo(1,1,1), + bi.getNode(0)->getBucketInfo()); + + bi.addNode(BucketCopy(5, 0, api::BucketInfo(2,2,2)), idealState); + CPPUNIT_ASSERT_EQUAL(api::BucketInfo(1,1,1), + bi.getNode(0)->getBucketInfo()); + + bi.addNode(BucketCopy(4, 0, api::BucketInfo(3,3,3)), idealState); + CPPUNIT_ASSERT_EQUAL(api::BucketInfo(1,1,1), + bi.getNode(0)->getBucketInfo()); + + bi.addNode(BucketCopy(7, 0, api::BucketInfo(4,4,4)), idealState); + CPPUNIT_ASSERT_EQUAL(api::BucketInfo(4,4,4), + bi.getNode(0)->getBucketInfo()); + + bi.addNode(BucketCopy(2, 1, api::BucketInfo(4,4,4)), idealState); + CPPUNIT_ASSERT_EQUAL(api::BucketInfo(4,4,4), + bi.getNode(1)->getBucketInfo()); +} + +void +BucketInfoTest::testOrder() { + + CPPUNIT_ASSERT_EQUAL(std::string("2,0,1"), nodeList(getBucketInfo("0,1,2", "2,0,1"))); + CPPUNIT_ASSERT_EQUAL(std::string("2,0,1"), nodeList(getBucketInfo("1,0,2", "2,0,1"))); + CPPUNIT_ASSERT_EQUAL(std::string("1,0,2"), nodeList(getBucketInfo("1,2,0", "1"))); + CPPUNIT_ASSERT_EQUAL(std::string("2,1,0,3,4"), nodeList(getBucketInfo("0,1,2,3,4", "2,1"))); +} + +void +BucketInfoTest::testHasInvalidCopy() +{ + std::vector<uint16_t> order; + + BucketInfo info; + info.addNode(BucketCopy(0, 0, api::BucketInfo(10, 100, 1000)), order); + info.addNode(BucketCopy(0, 1, api::BucketInfo(10, 100, 1000)), order); + CPPUNIT_ASSERT(!info.hasInvalidCopy()); + + info.addNode(BucketCopy(0, 2, api::BucketInfo()), order); + CPPUNIT_ASSERT(info.hasInvalidCopy()); + +} + +void +BucketInfoTest::testAddNodeSetsTrustedWhenConsistent() +{ + std::vector<uint16_t> order; + + { + BucketInfo info; + info.addNode(BucketCopy(0, 0, api::BucketInfo(0x1, 2, 144)).setTrusted(), order); + info.addNode(BucketCopy(0, 1, api::BucketInfo(0x1, 2, 144)), order); + CPPUNIT_ASSERT(info.getNode(1)->trusted()); + } + + { + BucketInfo info; + info.addNode(BucketCopy(0, 0, api::BucketInfo(0x1, 1, 2)).setTrusted(), order); + info.addNode(BucketCopy(0, 1, api::BucketInfo(0x2, 2, 3)), order); + info.addNode(BucketCopy(0, 2, api::BucketInfo(0x3, 3, 4)), order); + + BucketCopy copy(1, 1, api::BucketInfo(0x1, 1, 2)); + info.updateNode(copy); + CPPUNIT_ASSERT(info.getNode(1)->trusted()); + CPPUNIT_ASSERT(!info.getNode(2)->trusted()); + } +} + +void +BucketInfoTest::testTrustedResetWhenCopiesBecomeInconsistent() +{ + CPPUNIT_FAIL("TODO: test this!"); +} + +void +BucketInfoTest::testTrustedResetWhenTrustedCopiesGoOutOfSync() +{ + std::vector<uint16_t> order; + + BucketInfo info; + info.addNode(BucketCopy(0, 0, api::BucketInfo(10, 100, 1000)).setTrusted(), order); + info.addNode(BucketCopy(0, 1, api::BucketInfo(10, 100, 1000)), order); + CPPUNIT_ASSERT(info.getNode(0)->trusted()); + CPPUNIT_ASSERT(info.getNode(1)->trusted()); + + info.updateNode(BucketCopy(0, 1, api::BucketInfo(20, 200, 2000)).setTrusted()); + CPPUNIT_ASSERT(!info.getNode(0)->trusted()); + CPPUNIT_ASSERT(!info.getNode(1)->trusted()); +} + +void +BucketInfoTest::testTrustedNotResetWhenNonTrustedCopiesStillOutOfSync() +{ + std::vector<uint16_t> order; + + BucketInfo info; + info.addNode(BucketCopy(0, 0, api::BucketInfo(10, 100, 1000)).setTrusted(), order); + info.addNode(BucketCopy(0, 1, api::BucketInfo(20, 200, 2000)), order); + info.addNode(BucketCopy(0, 2, api::BucketInfo(30, 300, 3000)), order); + CPPUNIT_ASSERT(info.getNode(0)->trusted()); + CPPUNIT_ASSERT(!info.getNode(1)->trusted()); + CPPUNIT_ASSERT(!info.getNode(2)->trusted()); + + info.updateNode(BucketCopy(0, 1, api::BucketInfo(21, 201, 2001))); + + CPPUNIT_ASSERT(info.getNode(0)->trusted()); + CPPUNIT_ASSERT(!info.getNode(1)->trusted()); + CPPUNIT_ASSERT(!info.getNode(2)->trusted()); +} + +} + +} // storage + diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp new file mode 100644 index 00000000000..ee2e3f6ef7f --- /dev/null +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -0,0 +1,1323 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".test.bucketdb.bucketmanager"); + +#include <vespa/config/helper/configgetter.h> +#include <cppunit/extensions/HelperMacros.h> +#include <vespa/document/config/config-documenttypes.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/storage/bucketdb/bucketmanager.h> +#include <vespa/storage/common/bucketmessages.h> +#include <vespa/storageframework/defaultimplementation/clock/realclock.h> +#include <vespa/storage/persistence/filestorage/filestormanager.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/dummystoragelink.h> +#include <tests/common/testhelper.h> +#include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vdslib/state/random.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/storageapi/message/bucket.h> +#include <thread> +#include <future> + +using config::ConfigGetter; +using document::DocumenttypesConfig; +using config::FileSpec; +using document::DocumentType; +using document::DocumentTypeRepo; + +namespace storage { + +struct TestBucketInfo { + uint32_t crc; + uint32_t size; + uint32_t count; + uint32_t partition; + + api::BucketInfo getInfo() const + { return api::BucketInfo(crc, count, size); } +}; + +std::ostream& operator<<(std::ostream& out, const TestBucketInfo& info) { + out << "TestBucketInfo(" << info.crc << ", " << info.size + << ", " << info.count << ", " << info.partition << ")"; + return out; +} + +class ConcurrentOperationFixture; +struct TestParams; + +struct BucketManagerTest : public CppUnit::TestFixture { +public: + CPPUNIT_TEST_SUITE(BucketManagerTest); + CPPUNIT_TEST(testRequestBucketInfoWithList); + CPPUNIT_TEST(testDistributionBitGenerationEmpty); + CPPUNIT_TEST(testDistributionBitChangeOnCreateBucket); + CPPUNIT_TEST(testMinUsedBitsFromComponentIsHonored); + CPPUNIT_TEST(testRemoveLastModifiedOK); + CPPUNIT_TEST(testRemoveLastModifiedFailed); + CPPUNIT_TEST(testSwallowNotifyBucketChangeReply); + CPPUNIT_TEST(testMetricsGeneration); + CPPUNIT_TEST(testSplitReplyOrderedAfterBucketReply); + CPPUNIT_TEST(testJoinReplyOrderedAfterBucketReply); + CPPUNIT_TEST(testDeleteReplyOrderedAfterBucketReply); + CPPUNIT_TEST(testOnlyEnqueueWhenProcessingRequest); + CPPUNIT_TEST(testOrderRepliesAfterBucketSpecificRequest); + CPPUNIT_TEST(testQueuedRepliesOnlyDispatchedWhenAllProcessingDone); + CPPUNIT_TEST(testMutationRepliesForSplitBucketAreEnqueued); + CPPUNIT_TEST(testMutationRepliesForDeletedBucketAreEnqueued); + CPPUNIT_TEST(testMutationRepliesForJoinedBucketAreEnqueued); + CPPUNIT_TEST(testConflictingPutRepliesAreEnqueued); + CPPUNIT_TEST(testConflictingUpdateRepliesAreEnqueued); + CPPUNIT_TEST(testRemappedMutationIsCheckedAgainstOriginalBucket); + CPPUNIT_TEST(testBucketConflictSetIsClearedBetweenBlockingRequests); + CPPUNIT_TEST(testConflictSetOnlyClearedAfterAllBucketRequestsDone); + CPPUNIT_TEST(testRejectRequestWithMismatchingDistributionHash); + CPPUNIT_TEST(testDbNotIteratedWhenAllRequestsRejected); + CPPUNIT_TEST(testReceivedDistributionHashIsNormalized); + + // FIXME(vekterli): test is not deterministic and enjoys failing + // sporadically when running under Valgrind. See bug 5932891. + CPPUNIT_TEST_IGNORED(testRequestBucketInfoWithState); + CPPUNIT_TEST_SUITE_END(); + + std::unique_ptr<TestServiceLayerApp> _node; + std::unique_ptr<DummyStorageLink> _top; + BucketManager *_manager; + DummyStorageLink* _bottom; + FileStorManager* _filestorManager; + std::map<document::BucketId, TestBucketInfo> _bucketInfo; + uint32_t _emptyBuckets; + document::Document::SP _document; + + void setupTestEnvironment(bool fakePersistenceLayer = true, + bool noDelete = false); + void addBucketsToDB(uint32_t count); + bool wasBlockedDueToLastModified(api::StorageMessage* msg, + uint64_t lastModified); + bool wasBlockedDueToLastModified(api::StorageMessage::SP msg); + void insertSingleBucket(const document::BucketId& bucket, + const api::BucketInfo& info); + void waitUntilRequestsAreProcessing(size_t nRequests = 1); + void doTestMutationOrdering( + ConcurrentOperationFixture& fixture, + const TestParams& params); + void doTestConflictingReplyIsEnqueued( + const document::BucketId& bucket, + const api::StorageCommand::SP& treeMutationCmd, + const api::MessageType& treeMutationReplyType); + + void scheduleBucketInfoRequestWithConcurrentOps( + ConcurrentOperationFixture& fixture, + const document::BucketId& bucketForRemove, + const document::BucketId& bucketForSplit, + api::Timestamp mutationTimestamp); + void sendSingleBucketInfoRequest(const document::BucketId& id); + void assertRequestWithBadHashIsRejected( + ConcurrentOperationFixture& fixture); + + + void testRequestBucketInfoWithState(); + void testRequestBucketInfoWithList(); + void testDistributionBitGenerationEmpty(); + void testDistributionBitChangeOnCreateBucket(); + void testMinUsedBitsFromComponentIsHonored(); + + void testRemoveLastModifiedOK(); + void testRemoveLastModifiedFailed(); + + void testSwallowNotifyBucketChangeReply(); + void testMetricsGeneration(); + void testSplitReplyOrderedAfterBucketReply(); + void testJoinReplyOrderedAfterBucketReply(); + void testDeleteReplyOrderedAfterBucketReply(); + void testOnlyEnqueueWhenProcessingRequest(); + void testOrderRepliesAfterBucketSpecificRequest(); + void testQueuedRepliesOnlyDispatchedWhenAllProcessingDone(); + void testMutationRepliesForSplitBucketAreEnqueued(); + void testMutationRepliesForDeletedBucketAreEnqueued(); + void testMutationRepliesForJoinedBucketAreEnqueued(); + void testConflictingPutRepliesAreEnqueued(); + void testConflictingUpdateRepliesAreEnqueued(); + void testRemappedMutationIsCheckedAgainstOriginalBucket(); + void testBucketConflictSetIsClearedBetweenBlockingRequests(); + void testConflictSetOnlyClearedAfterAllBucketRequestsDone(); + void testRejectRequestWithMismatchingDistributionHash(); + void testDbNotIteratedWhenAllRequestsRejected(); + void testReceivedDistributionHashIsNormalized(); + +public: + static constexpr uint32_t DIR_SPREAD = 3; + static constexpr uint32_t MESSAGE_WAIT_TIME = 60*2; + + void setUp() { + _emptyBuckets = 0; + } + + void tearDown() { + } + + friend class ConcurrentOperationFixture; +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(BucketManagerTest); + +#define ASSERT_DUMMYLINK_REPLY_COUNT(link, count) \ + if (link->getNumReplies() != count) { \ + std::ostringstream ost; \ + ost << "Expected there to be " << count << " replies in link, but " \ + << "found " << link->getNumReplies() << ":\n"; \ + for (uint32_t i=0; i<link->getNumReplies(); ++i) { \ + ost << link->getReply(i)->getType() << "\n"; \ + } \ + CPPUNIT_FAIL(ost.str()); \ + } + +void BucketManagerTest::setupTestEnvironment(bool fakePersistenceLayer, + bool noDelete) +{ + if (!noDelete) { + assert(system("rm -rf vdsroot") == 0); + } + assert(system("mkdir -p vdsroot/disks/d0") == 0); + assert(system("mkdir -p vdsroot/disks/d1") == 0); + vdstestlib::DirConfig config(getStandardConfig(true)); + + DocumentTypeRepo::SP repo(new DocumentTypeRepo( + *ConfigGetter<DocumenttypesConfig>::getConfig("config-doctypes", + FileSpec("config-doctypes.cfg")))); + _top.reset(new DummyStorageLink); + _node.reset(new TestServiceLayerApp( + DiskCount(2), NodeIndex(0), config.getConfigId())); + _node->setTypeRepo(repo); + _node->setupDummyPersistence(); + // Set up the 3 links + StorageLink::UP manager(new BucketManager("", _node->getComponentRegister())); + _manager = (BucketManager*) manager.get(); + _top->push_back(std::move(manager)); + if (fakePersistenceLayer) { + StorageLink::UP bottom(new DummyStorageLink); + _bottom = (DummyStorageLink*) bottom.get(); + _top->push_back(std::move(bottom)); + } else { + StorageLink::UP bottom(new FileStorManager( + config.getConfigId(), _node->getPartitions(), + _node->getPersistenceProvider(), _node->getComponentRegister())); + _filestorManager = (FileStorManager*) bottom.get(); + _top->push_back(std::move(bottom)); + } + // Generate a doc to use for testing.. + const DocumentType &type(*_node->getTypeRepo() + ->getDocumentType("text/html")); + _document.reset(new document::Document(type, document::DocumentId( + document::DocIdString("test", "ntnu")))); +} + +void BucketManagerTest::addBucketsToDB(uint32_t count) +{ + _bucketInfo.clear(); + _emptyBuckets = 0; + lib::RandomGen randomizer(25423); + while (_bucketInfo.size() < count) { + document::BucketId id(16, randomizer.nextUint32()); + id = id.stripUnused(); + if (_bucketInfo.size() == 0) { + id = _node->getBucketIdFactory().getBucketId( + _document->getId()).stripUnused(); + } + TestBucketInfo info; + info.crc = randomizer.nextUint32(); + info.size = randomizer.nextUint32(); + info.count = randomizer.nextUint32(1, 0xFFFF); + + info.partition = _node->getPartition(id); + _bucketInfo[id] = info; + } + + // Make sure we have at least one empty bucket + TestBucketInfo& info = (++_bucketInfo.begin())->second; + CPPUNIT_ASSERT(info.size != 0); + info.size = 0; + info.count = 0; + info.crc = 0; + ++_emptyBuckets; + for (std::map<document::BucketId, TestBucketInfo>::iterator it + = _bucketInfo.begin(); it != _bucketInfo.end(); ++it) + { + bucketdb::StorageBucketInfo entry; + entry.disk = it->second.partition; + entry.setBucketInfo(api::BucketInfo(it->second.crc, + it->second.count, + it->second.size)); + _node->getStorageBucketDatabase().insert(it->first, entry, "foo"); + } +} + +bool +BucketManagerTest::wasBlockedDueToLastModified(api::StorageMessage* msg, + uint64_t lastModified) +{ + setupTestEnvironment(); + document::BucketId id(16, 1); + api::BucketInfo info(1, 2, 3); + info.setLastModified(api::Timestamp(1234)); + + { + bucketdb::StorageBucketInfo entry; + entry.setBucketInfo(info); + entry.disk = 0; + _node->getStorageBucketDatabase().insert(id, entry, "foo"); + } + + _top->open(); + + _top->sendDown(api::StorageMessage::SP(msg)); + if (_top->getNumReplies() == 1) { + CPPUNIT_ASSERT_EQUAL(0, (int)_bottom->getNumCommands()); + CPPUNIT_ASSERT(!static_cast<api::StorageReply&>( + *_top->getReply(0)).getResult().success()); + return true; + } else { + CPPUNIT_ASSERT_EQUAL(0, (int)_top->getNumReplies()); + + // Check that bucket database now has the operation's timestamp as last modified. + { + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get(id, "foo")); + CPPUNIT_ASSERT_EQUAL(lastModified, entry->info.getLastModified()); + } + + return false; + } +} + +void BucketManagerTest::testRemoveLastModifiedOK() +{ + CPPUNIT_ASSERT(!wasBlockedDueToLastModified( + new api::RemoveCommand(document::BucketId(16, 1), + document::DocumentId("userdoc:m:1:foo"), + api::Timestamp(1235)), + 1235)); +} + + +void BucketManagerTest::testRemoveLastModifiedFailed() +{ + CPPUNIT_ASSERT(wasBlockedDueToLastModified( + new api::RemoveCommand(document::BucketId(16, 1), + document::DocumentId("userdoc:m:1:foo"), + api::Timestamp(1233)), + 1233)); +} + +void BucketManagerTest::testDistributionBitGenerationEmpty() +{ + TestName("BucketManagerTest::testDistributionBitGenerationEmpty()"); + setupTestEnvironment(); + _manager->doneInit(); + vespalib::Monitor l; + _manager->updateMetrics(BucketManager::MetricLockGuard(l)); + CPPUNIT_ASSERT_EQUAL(58u, _node->getStateUpdater().getReportedNodeState()->getMinUsedBits()); +} + +void BucketManagerTest::testDistributionBitChangeOnCreateBucket() +{ + TestName("BucketManagerTest::testDistributionBitChangeOnCreateBucket()"); + setupTestEnvironment(); + addBucketsToDB(30); + _top->open(); + _node->getDoneInitializeHandler().notifyDoneInitializing(); + _manager->doneInit(); + _manager->updateMinUsedBits(); + CPPUNIT_ASSERT_EQUAL(16u, _node->getStateUpdater().getReportedNodeState()->getMinUsedBits()); + + std::shared_ptr<api::CreateBucketCommand> cmd( + new api::CreateBucketCommand(document::BucketId(4, 5678))); + _top->sendDown(cmd); + CPPUNIT_ASSERT_EQUAL(4u, _node->getStateUpdater().getReportedNodeState()->getMinUsedBits()); +} + +void BucketManagerTest::testMinUsedBitsFromComponentIsHonored() +{ + TestName("BucketManagerTest::testMinUsedBitsFromComponentIsHonored()"); + setupTestEnvironment(); + // Let these differ in order to test state update behavior. + _node->getComponentRegister().getMinUsedBitsTracker().setMinUsedBits(10); + lib::NodeState ns( + *_node->getStateUpdater().getReportedNodeState()); + ns.setMinUsedBits(13); + _node->getStateUpdater().setReportedNodeState(ns); + addBucketsToDB(30); + _top->open(); + // Don't update metrics, as these will always overwrite the min used bits + // if it differs from the db. + + // 12 >= 10, so no update of reported state (left at 13; this should of + // course not happen in practice, but used for faking in the test) + std::shared_ptr<api::CreateBucketCommand> cmd( + new api::CreateBucketCommand(document::BucketId(12, 5678))); + _top->sendDown(cmd); + CPPUNIT_ASSERT_EQUAL(13u, _node->getStateUpdater().getReportedNodeState()->getMinUsedBits()); +} + +void BucketManagerTest::testRequestBucketInfoWithState() +{ + TestName("BucketManagerTest::testRequestBucketInfoWithState()"); + // Test prior to building bucket cache + setupTestEnvironment(); + addBucketsToDB(30); + /* Currently this is just queued up + { + std::shared_ptr<api::RequestBucketInfoCommand> cmd( + new api::RequestBucketInfoCommand( + 0, lib::ClusterState("distributor:3 .2.s:d storage:1"))); + _top->sendDown(cmd); + _top->waitForMessages(1, 5); + CPPUNIT_ASSERT_EQUAL((size_t) 1, _top->getNumReplies()); + std::shared_ptr<api::RequestBucketInfoReply> reply( + std::dynamic_pointer_cast<api::RequestBucketInfoReply>( + _top->getReply(0))); + _top->reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::NOT_READY), + reply->getResult()); + } */ + std::vector<lib::ClusterState> states; + states.push_back(lib::ClusterState("version:0")); + states.push_back(lib::ClusterState("version:1 distributor:1 storage:1")); + states.push_back(lib::ClusterState( + "version:2 distributor:3 .1.s:i .2.s:d storage:4")); + states.push_back(lib::ClusterState( + "version:3 distributor:3 .1.s:i .2.s:d storage:4 .3.s:d")); + states.push_back(lib::ClusterState( + "version:4 distributor:3 .1.s:i .2.s:d storage:4")); + + _node->setClusterState(states.back()); + for (uint32_t i=0; i<states.size(); ++i) { + api::SetSystemStateCommand::SP cmd( + new api::SetSystemStateCommand(states[i])); + _manager->onDown(cmd); + } + + // Send a request bucket info command that will be outdated and failed. + std::shared_ptr<api::RequestBucketInfoCommand> cmd1( + new api::RequestBucketInfoCommand(0, states[1])); + // Send two request bucket info commands that will be processed together + // when the bucket manager is idle, as states are equivalent + std::shared_ptr<api::RequestBucketInfoCommand> cmd2( + new api::RequestBucketInfoCommand(0, states[2])); + std::shared_ptr<api::RequestBucketInfoCommand> cmd3( + new api::RequestBucketInfoCommand(0, states[3])); + + // Tag server initialized before starting + _top->open(); + _manager->startWorkerThread(); + _node->getDoneInitializeHandler().notifyDoneInitializing(); + _manager->doneInit(); + + LOG(info, "Sending 3 different request bucket info messages"); + _top->sendDown(cmd1); + _top->sendDown(cmd2); + _top->sendDown(cmd3); + + { + LOG(info, "Waiting for response from 3 request bucket info messages"); + _top->waitForMessages(3, 5); + ASSERT_DUMMYLINK_REPLY_COUNT(_top, 3); + std::map<uint64_t, api::RequestBucketInfoReply::SP> replies; + for (uint32_t i=0; i<3; ++i) { + replies[_top->getReply(i)->getMsgId()] + = std::dynamic_pointer_cast<api::RequestBucketInfoReply>( + _top->getReply(i)); + } + std::shared_ptr<api::RequestBucketInfoReply> reply1( + replies[cmd1->getMsgId()]); + std::shared_ptr<api::RequestBucketInfoReply> reply2( + replies[cmd2->getMsgId()]); + std::shared_ptr<api::RequestBucketInfoReply> reply3( + replies[cmd3->getMsgId()]); + _top->reset(); + CPPUNIT_ASSERT(reply1.get()); + CPPUNIT_ASSERT(reply2.get()); + CPPUNIT_ASSERT(reply3.get()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::REJECTED, + "Ignoring bucket info request for cluster state version 1 as " + "versions from version 2 differs from this state."), + reply1->getResult()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::REJECTED, + "There is already a newer bucket info request for " + "this node from distributor 0"), + reply2->getResult()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), + reply3->getResult()); + api::RequestBucketInfoReply::Entry entry; + + CPPUNIT_ASSERT_EQUAL((size_t) 18, reply3->getBucketInfo().size()); + entry = api::RequestBucketInfoReply::Entry( + document::BucketId(16, 0xe8c8), api::BucketInfo(0x79d04f78, 11153, 1851385240u)); + CPPUNIT_ASSERT_EQUAL(entry, reply3->getBucketInfo()[0]); + } +} + +namespace { + struct PopenWrapper { + FILE* _file; + std::vector<char> _buffer; + uint32_t _index; + uint32_t _size; + bool _eof; + + PopenWrapper(const std::string& cmd) + : _buffer(65536, '\0'), _index(0), _size(0), _eof(false) + { + _file = popen(cmd.c_str(), "r"); + if (_file == 0) { + throw vespalib::Exception("Failed to run '" + cmd + + "' in popen: " + strerror(errno), VESPA_STRLOC); + } + } + + const char* getNextLine() { + if (_eof && _size == 0) return 0; + // Check if we have a newline waiting + char* newline = strchr(&_buffer[_index], '\n'); + // If not try to get one + if (_eof) { + newline = &_buffer[_index + _size]; + } else if (newline == 0) { + // If we index is passed half the buffer, reposition + if (_index > _buffer.size() / 2) { + memcpy(&_buffer[0], &_buffer[_index], _size); + _index = 0; + } + // Verify we have space to write to + if (_index + _size >= _buffer.size()) { + throw vespalib::Exception("No newline could be find in " + "half the buffer size. Wrapper not designed to " + "handle that long lines (1)", VESPA_STRLOC); + } + // Fill up buffer + size_t bytesRead = fread(&_buffer[_index + _size], + 1, _buffer.size() - _index - _size - 1, + _file); + if (bytesRead == 0) { + if (!feof(_file)) { + throw vespalib::Exception("Failed to run fgets: " + + std::string(strerror(errno)), VESPA_STRLOC); + } else { + _eof = true; + } + } else { + _size += bytesRead; + } + newline = strchr(&_buffer[_index], '\n'); + if (newline == 0) { + if (_eof) { + if (_size == 0) return 0; + } else { + throw vespalib::Exception("No newline could be find in " + "half the buffer size. Wrapper not designed to " + "handle that long lines (2)", VESPA_STRLOC); + } + } + } + *newline = '\0'; + ++newline; + const char* line = &_buffer[_index]; + uint32_t strlen = (newline - line); + _index += strlen; + _size -= strlen; + return line; + } + }; +} + +void BucketManagerTest::testRequestBucketInfoWithList() +{ + TestName("BucketManagerTest::testRequestBucketInfoWithList()"); + setupTestEnvironment(); + addBucketsToDB(30); + _top->open(); + _node->getDoneInitializeHandler().notifyDoneInitializing(); + _top->doneInit(); + { + std::vector<document::BucketId> bids; + bids.push_back(document::BucketId(16, 0xe8c8)); + + std::shared_ptr<api::RequestBucketInfoCommand> cmd( + new api::RequestBucketInfoCommand(bids)); + + _top->sendDown(cmd); + _top->waitForMessages(1, 5); + ASSERT_DUMMYLINK_REPLY_COUNT(_top, 1); + std::shared_ptr<api::RequestBucketInfoReply> reply( + std::dynamic_pointer_cast<api::RequestBucketInfoReply>( + _top->getReply(0))); + _top->reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), + reply->getResult()); + if (reply->getBucketInfo().size() > 1) { + std::cerr << "Too many replies found\n"; + for (uint32_t i=0; i<reply->getBucketInfo().size(); ++i) { + std::cerr << reply->getBucketInfo()[i] << "\n"; + } + } + CPPUNIT_ASSERT_EQUAL((size_t) 1, reply->getBucketInfo().size()); + api::RequestBucketInfoReply::Entry entry( + document::BucketId(16, 0xe8c8), + api::BucketInfo(0x79d04f78, 11153, 1851385240u)); + CPPUNIT_ASSERT_EQUAL(entry, reply->getBucketInfo()[0]); + } +} + +void +BucketManagerTest::testSwallowNotifyBucketChangeReply() +{ + TestName("BucketManagerTest::testSwallowNotifyBucketChangeReply()"); + setupTestEnvironment(); + addBucketsToDB(30); + _top->open(); + _node->getDoneInitializeHandler().notifyDoneInitializing(); + _top->doneInit(); + + api::NotifyBucketChangeCommand cmd(document::BucketId(1, 16), + api::BucketInfo()); + std::shared_ptr<api::NotifyBucketChangeReply> reply( + new api::NotifyBucketChangeReply(cmd)); + + _top->sendDown(reply); + // Should not leave the bucket manager. + CPPUNIT_ASSERT_EQUAL(0, (int)_bottom->getNumCommands()); +} + +void +BucketManagerTest::testMetricsGeneration() +{ + setupTestEnvironment(); + _top->open(); + // Add 3 buckets; 2 ready, 1 active. 300 docs total, 600 bytes total. + for (int i = 0; i < 3; ++i) { + bucketdb::StorageBucketInfo entry; + entry.disk = 0; + api::BucketInfo info(50, 100, 200); + if (i > 0) { + info.setReady(); + if (i == 2) { + info.setActive(); + } + } + entry.setBucketInfo(info); + _node->getStorageBucketDatabase().insert(document::BucketId(16, i), + entry, "foo"); + } + _node->getDoneInitializeHandler().notifyDoneInitializing(); + _top->doneInit(); + vespalib::Monitor l; + _manager->updateMetrics(BucketManager::MetricLockGuard(l)); + + CPPUNIT_ASSERT_EQUAL(size_t(2), _manager->_metrics->disks.size()); + const DataStoredMetrics& m(*_manager->_metrics->disks[0]); + CPPUNIT_ASSERT_EQUAL(int64_t(3), m.buckets.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(300), m.docs.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(600), m.bytes.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(1), m.active.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(2), m.ready.getLast()); +} + +void +BucketManagerTest::insertSingleBucket(const document::BucketId& bucket, + const api::BucketInfo& info) +{ + bucketdb::StorageBucketInfo entry; + entry.disk = 0; + entry.setBucketInfo(info); + _node->getStorageBucketDatabase().insert(bucket, entry, "foo"); +} + +void +BucketManagerTest::waitUntilRequestsAreProcessing(size_t nRequests) +{ + while (_manager->bucketInfoRequestsCurrentlyProcessing() != nRequests) { + std::this_thread::yield(); + } +} + +namespace { + +struct WithBuckets { + std::map<document::BucketId, api::BucketInfo> _bucketsAndInfo; + + WithBuckets& add(const document::BucketId& id, + const api::BucketInfo& info) + { + _bucketsAndInfo[id] = info; + return *this; + } +}; + +} // anon ns + +class ConcurrentOperationFixture { +public: + ConcurrentOperationFixture(BucketManagerTest& self) + : _self(self), + _state("distributor:1 storage:1") + { + _self.setupTestEnvironment(); + _self._top->open(); + _self._node->getDoneInitializeHandler().notifyDoneInitializing(); + _self._manager->startWorkerThread(); + _self._top->doneInit(); + + // Need a cluster state to work with initially, so that processing + // bucket requests can calculate a target distributor. + _self._node->setClusterState(_state); + _self._manager->onDown( + std::make_shared<api::SetSystemStateCommand>(_state)); + } + + void setUp(const WithBuckets& buckets) { + for (auto& b : buckets._bucketsAndInfo) { + _self.insertSingleBucket(b.first, b.second); + } + } + + auto acquireBucketLock(const document::BucketId& bucket) { + return _self._node->getStorageBucketDatabase().get(bucket, "foo"); + } + + auto createRemoveCommand(const document::BucketId& bucket, + api::Timestamp timestamp = 123456) const + { + // Note: this is a dummy message; its contained document ID will not + // map to the provided bucket ID (at least it's extremely unlikely..) + return std::make_shared<api::RemoveCommand>( + bucket, + document::DocumentId("id:foo:testdoctype1::bar"), + timestamp); + } + + auto createPutCommand(const document::BucketId& bucket) const { + auto doc = _self._node->getTestDocMan().createDocument( + "a foo walks into a bar", "id:foo:testdoctype1::bar1"); + return std::make_shared<api::PutCommand>( + bucket, std::move(doc), api::Timestamp(123456)); + } + + auto createUpdateCommand(const document::BucketId& bucket) const { + auto update = std::make_shared<document::DocumentUpdate>( + *_self._node->getTestDocMan().getTypeRepo() + .getDocumentType("testdoctype1"), + document::DocumentId("id:foo:testdoctype1::bar2")); + return std::make_shared<api::UpdateCommand>( + bucket, update, api::Timestamp(123456)); + } + + auto createFullFetchCommand() const { + return std::make_shared<api::RequestBucketInfoCommand>(0, _state); + } + + auto createFullFetchCommandWithHash(vespalib::stringref hash) const { + return std::make_shared<api::RequestBucketInfoCommand>(0, _state, hash); + } + + auto acquireBucketLockAndSendInfoRequest(const document::BucketId& bucket) { + auto guard = acquireBucketLock(bucket); + // Send down processing command which will block. + _self._top->sendDown(createFullFetchCommand()); + // Have to wait until worker thread has started chewing on request + // before we can continue, or we can end up in a race where processing + // does not start until _after_ we've sent up our bucket-deleting + // message. Since we hold a bucket lock, the below function can never + // transition false->true->false under our feet, only false->true. + _self.waitUntilRequestsAreProcessing(1); + return guard; + } + + // Currently assumes there is only 1 command of cmd's message type in + // the bottom storage link. + void bounceWithReply(api::StorageCommand& cmd, + api::ReturnCode::Result code = api::ReturnCode::OK, + const document::BucketId& remapTo = document::BucketId()) + { + _self._bottom->waitForMessages(1, BucketManagerTest::MESSAGE_WAIT_TIME); + // Bounce it back up with an implicitly OK status. This should cause the + // bucket manager to avoid reporting deleted buckets in its result set + // since these have been "tainted" by a concurrent removal. + std::unique_ptr<api::StorageReply> reply(cmd.makeReply()); + if (remapTo.getRawId() != 0) { + dynamic_cast<api::BucketReply&>(*reply).remapBucketId(remapTo); + } + reply->setResult(code); + _self._bottom->getAndRemoveMessage(cmd.getType()); + _self._bottom->sendUp(std::move(reply)); + } + + auto awaitAndGetReplies(size_t nReplies) { + _self._top->waitForMessages( + nReplies, BucketManagerTest::MESSAGE_WAIT_TIME); + return _self._top->getReplies(); + } + + void assertOrderedAfterBucketReply(size_t nBucketReplies, + const api::MessageType& msgType) + { + const size_t nTotal = nBucketReplies + 1; + auto replies = awaitAndGetReplies(nTotal); + CPPUNIT_ASSERT_EQUAL(nTotal, replies.size()); + for (size_t i = 0; i < nBucketReplies; ++i) { + CPPUNIT_ASSERT_EQUAL(api::MessageType::REQUESTBUCKETINFO_REPLY, + replies[i]->getType()); + } + CPPUNIT_ASSERT_EQUAL(msgType, replies[nBucketReplies]->getType()); + } + + void assertReplyOrdering( + const std::vector<const api::MessageType*>& replyTypes) + { + auto replies = awaitAndGetReplies(replyTypes.size()); + CPPUNIT_ASSERT_EQUAL(replyTypes.size(), replies.size()); + for (size_t i = 0; i < replyTypes.size(); ++i) { + CPPUNIT_ASSERT_EQUAL(*replyTypes[i], replies[i]->getType()); + } + } + + void clearReceivedReplies() { + _self._top->getRepliesOnce(); + } + +private: + BucketManagerTest& _self; + lib::ClusterState _state; +}; + +void +BucketManagerTest::testSplitReplyOrderedAfterBucketReply() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId bucketA(17, 0); + document::BucketId bucketB(17, 1); + fixture.setUp(WithBuckets() + .add(bucketA, api::BucketInfo(50, 100, 200)) + .add(bucketB, api::BucketInfo(100, 200, 400))); + auto guard = fixture.acquireBucketLockAndSendInfoRequest(bucketB); + + // Split bucket A to model a concurrent modification to an already fetched + // bucket. + auto splitCmd = std::make_shared<api::SplitBucketCommand>(bucketA); + _top->sendDown(splitCmd); + fixture.bounceWithReply(*splitCmd); + // Let bucket manager breathe again. + guard.unlock(); + + fixture.assertOrderedAfterBucketReply( + 1, api::MessageType::SPLITBUCKET_REPLY); +} + +void +BucketManagerTest::testJoinReplyOrderedAfterBucketReply() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId bucketA(17, 0); + document::BucketId bucketB(17, 1 << 16); + document::BucketId parent(16, 0); + fixture.setUp(WithBuckets() + .add(bucketA, api::BucketInfo(50, 100, 200)) + .add(bucketB, api::BucketInfo(100, 200, 400))); + auto guard = fixture.acquireBucketLockAndSendInfoRequest(bucketB); + + auto joinCmd = std::make_shared<api::JoinBucketsCommand>(parent); + joinCmd->getSourceBuckets().assign({bucketA, bucketB}); + _top->sendDown(joinCmd); + fixture.bounceWithReply(*joinCmd); + + guard.unlock(); + fixture.assertOrderedAfterBucketReply( + 1, api::MessageType::JOINBUCKETS_REPLY); +} + +// Technically, deletes being ordered after bucket info replies won't help +// correctness since buckets are removed from the distributor DB upon _sending_ +// the delete and not receiving it. +void +BucketManagerTest::testDeleteReplyOrderedAfterBucketReply() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId bucketA(17, 0); + document::BucketId bucketB(17, 1); + fixture.setUp(WithBuckets() + .add(bucketA, api::BucketInfo(50, 100, 200)) + .add(bucketB, api::BucketInfo(100, 200, 400))); + auto guard = fixture.acquireBucketLockAndSendInfoRequest(bucketB); + + auto deleteCmd = std::make_shared<api::DeleteBucketCommand>(bucketA); + _top->sendDown(deleteCmd); + fixture.bounceWithReply(*deleteCmd); + + guard.unlock(); + + fixture.assertOrderedAfterBucketReply( + 1, api::MessageType::DELETEBUCKET_REPLY); +} + +void +BucketManagerTest::testOnlyEnqueueWhenProcessingRequest() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId bucketA(17, 0); + fixture.setUp(WithBuckets() + .add(bucketA, api::BucketInfo(50, 100, 200))); + + // Process delete command _before_ processing bucket requests. + auto deleteCmd = std::make_shared<api::DeleteBucketCommand>(bucketA); + _top->sendDown(deleteCmd); + fixture.bounceWithReply(*deleteCmd); + // Should arrive happily on the top. + _top->waitForMessages(1, MESSAGE_WAIT_TIME); +} + +// Bucket info requests that contain a specific set of buckets are handled +// differently than full bucket info fetches and are not delegated to the +// worker thread. We still require that any split/joins etc are ordered after +// this reply if their reply is sent up concurrently. +void +BucketManagerTest::testOrderRepliesAfterBucketSpecificRequest() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId bucketA(17, 0); + fixture.setUp(WithBuckets() + .add(bucketA, api::BucketInfo(50, 100, 200))); + + auto guard = fixture.acquireBucketLock(bucketA); + + auto infoRoundtrip = std::async(std::launch::async, [&]() { + std::vector<document::BucketId> buckets{bucketA}; + auto infoCmd = std::make_shared<api::RequestBucketInfoCommand>(buckets); + // Can't complete until `guard` has been unlocked. + _top->sendDown(infoCmd); + // Barrier: bucket reply and subsequent split reply + _top->waitForMessages(2, MESSAGE_WAIT_TIME); + }); + waitUntilRequestsAreProcessing(); + // Barrier: roundtrip thread now blocked. Send a split whose reply shall + // be enqueued since there's a RequestBucketInfo currently doing its thing. + auto splitCmd = std::make_shared<api::SplitBucketCommand>(bucketA); + _top->sendDown(splitCmd); + // Enqueuing happens synchronously in this thread, so no need for further + // synchronization. + fixture.bounceWithReply(*splitCmd); + + guard.unlock(); + infoRoundtrip.get(); + // At this point, we know 2 messages are in the top queue since the + // async future guarantees this for completion. + fixture.assertOrderedAfterBucketReply( + 1, api::MessageType::SPLITBUCKET_REPLY); +} + +// Test is similar to testOrderRepliesAfterBucketSpecificRequest, but has +// two concurrent bucket info request processing instances going on; one in +// the worker thread and one in the message chain itself. Since we only have +// one queue, we must wait with dispatching replies until _all_ processing +// has ceased. +void +BucketManagerTest::testQueuedRepliesOnlyDispatchedWhenAllProcessingDone() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId bucketA(17, 0); + fixture.setUp(WithBuckets() + .add(bucketA, api::BucketInfo(50, 100, 200))); + + auto guard = fixture.acquireBucketLock(bucketA); + + auto singleBucketInfo = std::async(std::launch::async, [&]() { + std::vector<document::BucketId> buckets{bucketA}; + auto infoCmd = std::make_shared<api::RequestBucketInfoCommand>(buckets); + _top->sendDown(infoCmd); + _top->waitForMessages(3, MESSAGE_WAIT_TIME); + }); + waitUntilRequestsAreProcessing(1); + auto fullFetch = std::async(std::launch::async, [&]() { + _top->sendDown(fixture.createFullFetchCommand()); + _top->waitForMessages(3, MESSAGE_WAIT_TIME); + }); + waitUntilRequestsAreProcessing(2); + auto splitCmd = std::make_shared<api::SplitBucketCommand>(bucketA); + _top->sendDown(splitCmd); + fixture.bounceWithReply(*splitCmd); + + guard.unlock(); + singleBucketInfo.get(); + fullFetch.get(); + + fixture.assertOrderedAfterBucketReply( + 2, api::MessageType::SPLITBUCKET_REPLY); +} + +// Hide boring, repetetive code to allow for chaining of setters (and auto- +// generation of getters and member vars) behind a macro. +#ifdef BUILDER_PARAM +# error "Redefinition of existing macro `BUILDER_PARAM`" +#endif +#define BUILDER_PARAM(type, name) \ + type _ ## name; \ + auto& name(const type& name ## _) { _ ## name = name ## _; return *this; } \ + const type & name() const { return _ ## name; } + +struct TestParams { + BUILDER_PARAM(document::BucketId, bucket); + BUILDER_PARAM(document::BucketId, remappedTo); + BUILDER_PARAM(api::StorageCommand::SP, documentMutation); + BUILDER_PARAM(api::StorageCommand::SP, treeMutation); + BUILDER_PARAM(std::vector<const api::MessageType*>, expectedOrdering); +}; + +void +BucketManagerTest::doTestMutationOrdering( + ConcurrentOperationFixture& fixture, + const TestParams& params) +{ + fixture.setUp(WithBuckets() + .add(params.bucket(), api::BucketInfo(50, 100, 200))); + // Have to send down mutating command _before_ we take bucket lock, as the + // bucket manager acquires a lock for bucket on the way down in order to + // check the timestamp of the message vs the last modified timestamp of + // the bucket itself (offers some time travelling clock protection). + _top->sendDown(params.documentMutation()); + auto guard = fixture.acquireBucketLockAndSendInfoRequest(params.bucket()); + + _top->sendDown(params.treeMutation()); + // Unless "conflicting" mutation replies are enqueued after splits et al, + // they will bypass the lock and arrive in an inverse order of execution + // at the distributor. Note that we send replies in the opposite order their + // commands were sent down, but this is an artifact of ordering commands + // to avoid test deadlocks, and priorities may alter the execution order + // anyway. The important thing is that reply orders are not altered. + fixture.bounceWithReply(*params.treeMutation()); + fixture.bounceWithReply(*params.documentMutation(), + api::ReturnCode::OK, + params.remappedTo()); + guard.unlock(); + + fixture.assertReplyOrdering(params.expectedOrdering()); +} + +void +BucketManagerTest::doTestConflictingReplyIsEnqueued( + const document::BucketId& bucket, + const api::StorageCommand::SP& treeMutationCmd, + const api::MessageType& treeMutationReplyType) +{ + ConcurrentOperationFixture fixture(*this); + + // We don't check all combinations of document operation replies vs + // bucket operation replies, just RemoveReply vs all bucket ops. + auto params = TestParams() + .bucket(bucket) + .documentMutation(fixture.createRemoveCommand(bucket)) + .treeMutation(treeMutationCmd) + .expectedOrdering({&api::MessageType::REQUESTBUCKETINFO_REPLY, + &treeMutationReplyType, + &api::MessageType::REMOVE_REPLY}); + + doTestMutationOrdering(fixture, params); +} + +void +BucketManagerTest::testMutationRepliesForSplitBucketAreEnqueued() +{ + document::BucketId bucket(17, 0); + doTestConflictingReplyIsEnqueued( + bucket, + std::make_shared<api::SplitBucketCommand>(bucket), + api::MessageType::SPLITBUCKET_REPLY); +} + +void +BucketManagerTest::testMutationRepliesForDeletedBucketAreEnqueued() +{ + document::BucketId bucket(17, 0); + doTestConflictingReplyIsEnqueued( + bucket, + std::make_shared<api::DeleteBucketCommand>(bucket), + api::MessageType::DELETEBUCKET_REPLY); +} + +void +BucketManagerTest::testMutationRepliesForJoinedBucketAreEnqueued() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId bucketA(17, 0); + document::BucketId bucketB(17, 1 << 16); + document::BucketId parent(16, 0); + // We only test for the parent bucket, since that's what queued operations + // will be remapped to after a successful join. + auto joinCmd = std::make_shared<api::JoinBucketsCommand>(parent); + joinCmd->getSourceBuckets().assign({bucketA, bucketB}); + + auto params = TestParams() + .bucket(parent) + .documentMutation(fixture.createRemoveCommand(parent)) + .treeMutation(joinCmd) + .expectedOrdering({&api::MessageType::REQUESTBUCKETINFO_REPLY, + &api::MessageType::JOINBUCKETS_REPLY, + &api::MessageType::REMOVE_REPLY}); + + doTestMutationOrdering(fixture, params); +} + +void +BucketManagerTest::testConflictingPutRepliesAreEnqueued() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId bucket(17, 0); + + auto params = TestParams() + .bucket(bucket) + .documentMutation(fixture.createPutCommand(bucket)) + .treeMutation(std::make_shared<api::SplitBucketCommand>(bucket)) + .expectedOrdering({&api::MessageType::REQUESTBUCKETINFO_REPLY, + &api::MessageType::SPLITBUCKET_REPLY, + &api::MessageType::PUT_REPLY}); + + doTestMutationOrdering(fixture, params); +} + +void +BucketManagerTest::testConflictingUpdateRepliesAreEnqueued() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId bucket(17, 0); + + auto params = TestParams() + .bucket(bucket) + .documentMutation(fixture.createUpdateCommand(bucket)) + .treeMutation(std::make_shared<api::SplitBucketCommand>(bucket)) + .expectedOrdering({&api::MessageType::REQUESTBUCKETINFO_REPLY, + &api::MessageType::SPLITBUCKET_REPLY, + &api::MessageType::UPDATE_REPLY}); + + doTestMutationOrdering(fixture, params); +} + +/** + * After a split or join, any messages bound for the original bucket(s) that + * are currently in the persistence queues will be remapped to the bucket + * resulting from the operation. We have to make sure remapped operations are + * enqueued as well. + */ +void +BucketManagerTest::testRemappedMutationIsCheckedAgainstOriginalBucket() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId bucket(17, 0); + document::BucketId remappedToBucket(18, 0); + + auto params = TestParams() + .bucket(bucket) + .documentMutation(fixture.createRemoveCommand(bucket)) + .remappedTo(remappedToBucket) + .treeMutation(std::make_shared<api::SplitBucketCommand>(bucket)) + .expectedOrdering({&api::MessageType::REQUESTBUCKETINFO_REPLY, + &api::MessageType::SPLITBUCKET_REPLY, + &api::MessageType::REMOVE_REPLY}); + + doTestMutationOrdering(fixture, params); +} + +void +BucketManagerTest::scheduleBucketInfoRequestWithConcurrentOps( + ConcurrentOperationFixture& fixture, + const document::BucketId& bucketForRemove, + const document::BucketId& bucketForSplit, + api::Timestamp mutationTimestamp) +{ + auto mutation( + fixture.createRemoveCommand(bucketForRemove, mutationTimestamp)); + _top->sendDown(mutation); + auto guard = fixture.acquireBucketLockAndSendInfoRequest( + bucketForRemove); + + auto conflictingOp( + std::make_shared<api::SplitBucketCommand>(bucketForSplit)); + _top->sendDown(conflictingOp); + fixture.bounceWithReply(*conflictingOp); + fixture.bounceWithReply(*mutation); + guard.unlock(); +} + +void +BucketManagerTest::testBucketConflictSetIsClearedBetweenBlockingRequests() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId firstConflictBucket(17, 0); + document::BucketId secondConflictBucket(18, 0); + + fixture.setUp(WithBuckets() + .add(firstConflictBucket, api::BucketInfo(50, 100, 200)) + .add(secondConflictBucket, api::BucketInfo(60, 200, 300))); + + // Do a single round of starting and completing a request bucket info + // command with queueing and adding of `firstConflictBucket` to the set + // of conflicting buckets. + scheduleBucketInfoRequestWithConcurrentOps( + fixture, firstConflictBucket, + firstConflictBucket, api::Timestamp(1000)); + + // Barrier for completion of first round of replies. Subsequently remove + // all replies to get a clean slate. + fixture.awaitAndGetReplies(3); + fixture.clearReceivedReplies(); + + // Do a second round with a different bucket as the conflict. The + // mutation towards the first conflict bucket should now _not_ be queued + // as it was for an entirely different request bucket round. + scheduleBucketInfoRequestWithConcurrentOps( + fixture, firstConflictBucket, + secondConflictBucket, api::Timestamp(1001)); + + // Remove is not ordered after the split here since it should not be + // queued. + fixture.assertReplyOrdering({&api::MessageType::REMOVE_REPLY, + &api::MessageType::REQUESTBUCKETINFO_REPLY, + &api::MessageType::SPLITBUCKET_REPLY}); +} + +void +BucketManagerTest::sendSingleBucketInfoRequest(const document::BucketId& id) +{ + std::vector<document::BucketId> buckets{id}; + auto infoCmd = std::make_shared<api::RequestBucketInfoCommand>(buckets); + _top->sendDown(infoCmd); +} + +void +BucketManagerTest::testConflictSetOnlyClearedAfterAllBucketRequestsDone() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId bucketA(16, 0); + document::BucketId bucketB(16, 1); + + fixture.setUp(WithBuckets() + .add(bucketA, api::BucketInfo(50, 100, 200)) + .add(bucketB, api::BucketInfo(60, 200, 300))); + + auto mutation = fixture.createRemoveCommand(bucketA); + _top->sendDown(mutation); + + auto guardA = fixture.acquireBucketLock(bucketA); + auto guardB = fixture.acquireBucketLock(bucketB); + + auto singleBucketInfoA = std::async(std::launch::async, [&]() { + sendSingleBucketInfoRequest(bucketA); + _top->waitForMessages(4, MESSAGE_WAIT_TIME); + }); + waitUntilRequestsAreProcessing(1); + auto singleBucketInfoB = std::async(std::launch::async, [&]() { + sendSingleBucketInfoRequest(bucketB); + _top->waitForMessages(4, MESSAGE_WAIT_TIME); + }); + // Barrier: after this point, both tasks are in the protected section. + // Neither async bucket info request can proceed as long as there are + // guards holding their desired bucket locks. + waitUntilRequestsAreProcessing(2); + + auto conflictingOp = std::make_shared<api::SplitBucketCommand>(bucketA); + _top->sendDown(conflictingOp); + fixture.bounceWithReply(*conflictingOp); + // Releasing guard A (and allowing the request for A to go through) should + // _not_ clear the conflict set. I.e. if we send a mutation reply for a + // conflicted bucket up at this point, it should be enqueued after the + // split reply. + guardA.unlock(); + _top->waitForMessages(1, MESSAGE_WAIT_TIME); // Completion barrier for A. + fixture.bounceWithReply(*mutation); + // Allow B to go through. This _should_ clear the conflict set and dequeue + // any conflicted mutations after their conflicting ops. + guardB.unlock(); + singleBucketInfoA.get(); + singleBucketInfoB.get(); + // Note: request bucket info reply is dispatched up _before_ protected + // section guard goes out of scope, so reply is ordered before conflicts. + fixture.assertReplyOrdering({&api::MessageType::REQUESTBUCKETINFO_REPLY, + &api::MessageType::REQUESTBUCKETINFO_REPLY, + &api::MessageType::SPLITBUCKET_REPLY, + &api::MessageType::REMOVE_REPLY}); +} + +void +BucketManagerTest::assertRequestWithBadHashIsRejected( + ConcurrentOperationFixture& fixture) +{ + // Test by default sets up 10 nodes in config. Pretend we only know of 3. + auto infoCmd = fixture.createFullFetchCommandWithHash("(0;0;1;2)"); + _top->sendDown(infoCmd); + auto replies = fixture.awaitAndGetReplies(1); + auto& reply = dynamic_cast<api::RequestBucketInfoReply&>(*replies[0]); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::REJECTED, + reply.getResult().getResult()); +} + +void +BucketManagerTest::testRejectRequestWithMismatchingDistributionHash() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId bucket(17, 0); + fixture.setUp(WithBuckets().add(bucket, api::BucketInfo(50, 100, 200))); + assertRequestWithBadHashIsRejected(fixture); +} + +void +BucketManagerTest::testDbNotIteratedWhenAllRequestsRejected() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId bucket(17, 0); + fixture.setUp(WithBuckets().add(bucket, api::BucketInfo(50, 100, 200))); + auto guard = fixture.acquireBucketLock(bucket); + // We've got a bucket locked, so iff the manager actually starts processing + // buckets even though it has no requests active, it will stall while + // waiting for the lock to be released. When we then send down an additional + // bucket info request, this request will either be rejected immediately (if + // the db is NOT processed) or time out and fail the test. + assertRequestWithBadHashIsRejected(fixture); + fixture.clearReceivedReplies(); + + auto infoCmd = fixture.createFullFetchCommandWithHash("(0;0;1;2)"); + _top->sendDown(infoCmd); + auto replies = fixture.awaitAndGetReplies(1); +} + +/** + * Accept bucket info requests if their distribution hash is a valid permutation + * of our own config (i.e. they are set-wise identical even though the + * ordering of nodes may differ). See VESPA-1980 for context. + */ +void +BucketManagerTest::testReceivedDistributionHashIsNormalized() +{ + ConcurrentOperationFixture fixture(*this); + document::BucketId bucket(17, 0); + fixture.setUp(WithBuckets().add(bucket, api::BucketInfo(50, 100, 200))); + + // Test is configured with 10 nodes in increasing order. Jumble the order + // around. + auto infoCmd = fixture.createFullFetchCommandWithHash( + "(0;2;1;3;9;6;4;5;8;7;0)"); + _top->sendDown(infoCmd); + auto replies = fixture.awaitAndGetReplies(1); + auto& reply = dynamic_cast<api::RequestBucketInfoReply&>(*replies[0]); + // Should NOT have been rejected despite hash not matching config order + // verbatim. + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::OK, reply.getResult().getResult()); +} + +} // storage diff --git a/storage/src/tests/bucketdb/distribution_hash_normalizer_test.cpp b/storage/src/tests/bucketdb/distribution_hash_normalizer_test.cpp new file mode 100644 index 00000000000..7734e1054ff --- /dev/null +++ b/storage/src/tests/bucketdb/distribution_hash_normalizer_test.cpp @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/bucketdb/distribution_hash_normalizer.h> +#include <string> + +namespace storage { + +using Normalizer = DistributionHashNormalizer; + +class DistributionHashNormalizerTest : public CppUnit::TestFixture { +public: + CPPUNIT_TEST_SUITE(DistributionHashNormalizerTest); + CPPUNIT_TEST(orderNonHierarchicRootGroupNodesByDistributionKey); + CPPUNIT_TEST(mayHaveSameGroupIndexAsNodeIndex); + CPPUNIT_TEST(emitOptionalCapacityForRootGroup); + CPPUNIT_TEST(emitOptionalCapacityForSubGroups); + CPPUNIT_TEST(hierarchicGroupsAreOrderedByGroupIndex); + CPPUNIT_TEST(subgroupsOrderedOnEachNestingLevel); + CPPUNIT_TEST(distributionSpecIsCopiedVerbatim); + CPPUNIT_TEST(emptyInputYieldsEmptyOutput); + CPPUNIT_TEST(parseFailureReturnsInputVerbatim); + CPPUNIT_TEST_SUITE_END(); + + void orderNonHierarchicRootGroupNodesByDistributionKey(); + void mayHaveSameGroupIndexAsNodeIndex(); + void emitOptionalCapacityForRootGroup(); + void emitOptionalCapacityForSubGroups(); + void hierarchicGroupsAreOrderedByGroupIndex(); + void subgroupsOrderedOnEachNestingLevel(); + void distributionSpecIsCopiedVerbatim(); + void emptyInputYieldsEmptyOutput(); + void parseFailureReturnsInputVerbatim(); + +private: + DistributionHashNormalizer _normalizer; +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(DistributionHashNormalizerTest); + +void +DistributionHashNormalizerTest::orderNonHierarchicRootGroupNodesByDistributionKey() +{ + // Group index is first in list. + CPPUNIT_ASSERT_EQUAL(vespalib::string("(1;0;2;3;4;7)"), + _normalizer.normalize("(1;4;7;2;0;3)")); +} + +void +DistributionHashNormalizerTest::mayHaveSameGroupIndexAsNodeIndex() +{ + CPPUNIT_ASSERT_EQUAL(vespalib::string("(0;0;2;3;4;7)"), + _normalizer.normalize("(0;4;7;2;0;3)")); +} + +void +DistributionHashNormalizerTest::emitOptionalCapacityForRootGroup() +{ + CPPUNIT_ASSERT_EQUAL(vespalib::string("(0c12.5;1;2;3;4;7)"), + _normalizer.normalize("(0c12.5;1;4;7;2;3)")); +} + +void +DistributionHashNormalizerTest::emitOptionalCapacityForSubGroups() +{ + CPPUNIT_ASSERT_EQUAL(vespalib::string("(0d1|*(1c5.5;1)(2;2)(3c7;3))"), + _normalizer.normalize("(0d1|*(2;2)(1c5.5;1)(3c7;3))")); +} + +void +DistributionHashNormalizerTest::hierarchicGroupsAreOrderedByGroupIndex() +{ + CPPUNIT_ASSERT_EQUAL(vespalib::string("(0d1|*(0;0)(1;1)(3;3))"), + _normalizer.normalize("(0d1|*(3;3)(1;1)(0;0))")); +} + +void +DistributionHashNormalizerTest::subgroupsOrderedOnEachNestingLevel() +{ + CPPUNIT_ASSERT_EQUAL(vespalib::string("(0d1|*(1d3|*(2;2)(3;3))" + "(4;1)(7d2|*(5;5)(6;6)))"), + _normalizer.normalize("(0d1|*(7d2|*(6;6)(5;5))" + "(1d3|*(2;2)(3;3))(4;1))")); +} + +void +DistributionHashNormalizerTest::distributionSpecIsCopiedVerbatim() +{ + // Definitely don't want to do any ordering of the distribution spec. + CPPUNIT_ASSERT_EQUAL(vespalib::string("(0d3|2|1|*(0;0)(1;1)(3;3))"), + _normalizer.normalize("(0d3|2|1|*(3;3)(1;1)(0;0))")); +} + +void +DistributionHashNormalizerTest::emptyInputYieldsEmptyOutput() +{ + // Technically a parse failure (only 4.2 has this behavior), but it's + // explicitly checked for in BucketManager, so let's test it explicitly + // here as well. + CPPUNIT_ASSERT_EQUAL(vespalib::string(""), _normalizer.normalize("")); +} + +// In the (unlikely) case that the parser somehow fails to capture all possible +// valid values of the distribution hash, fall back to returning the non- +// normalized string. A log warning will also be emitted (though that's not +// testable). +void +DistributionHashNormalizerTest::parseFailureReturnsInputVerbatim() +{ + CPPUNIT_ASSERT_EQUAL(vespalib::string("onkel skrue"), + _normalizer.normalize("onkel skrue")); +} + +} // storage + diff --git a/storage/src/tests/bucketdb/initializertest.cpp b/storage/src/tests/bucketdb/initializertest.cpp new file mode 100644 index 00000000000..169150a7ff9 --- /dev/null +++ b/storage/src/tests/bucketdb/initializertest.cpp @@ -0,0 +1,924 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Tests storage initialization without depending on persistence layer. + */ +#include <vespa/fastos/fastos.h> +#include <vespa/storage/bucketdb/storagebucketdbinitializer.h> + +#include <vespa/document/base/testdocman.h> +#include <vespa/log/log.h> +#include <vespa/storage/persistence/filestorage/filestormanager.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/state.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/dummystoragelink.h> +#include <tests/common/testhelper.h> +#include <vespa/vdstestlib/cppunit/dirconfig.h> +#include <vespa/vdstestlib/cppunit/macros.h> + +LOG_SETUP(".test.bucketdb.initializing"); + +namespace storage { + +typedef uint16_t PartitionId; + +struct InitializerTest : public CppUnit::TestFixture { + + class InitParams { + vdstestlib::DirConfig config; + bool configFinalized; + + public: + uint32_t bucketBitsUsed; + NodeIndex nodeIndex; + NodeCount nodeCount; + Redundancy redundancy; + uint32_t docsPerDisk; + DiskCount diskCount; + std::set<uint32_t> disksDown; + bool bucketWrongDisk; + bool bucketMultipleDisks; + bool failingListRequest; + bool failingInfoRequest; + + InitParams() + : config(getStandardConfig(true)), + configFinalized(false), + bucketBitsUsed(4), + nodeIndex(0), + nodeCount(10), + redundancy(2), + docsPerDisk(10), + diskCount(5), + bucketWrongDisk(false), + bucketMultipleDisks(false), + failingListRequest(false), + failingInfoRequest(false) {} + + void setAllFailures() { + bucketWrongDisk = true; + bucketMultipleDisks = true; + failingListRequest = true; + failingInfoRequest = true; + } + + vdstestlib::DirConfig& getConfig() { + if (!configFinalized) { + config.getConfig("stor-server") + .setValue("node_index", nodeIndex); + config.getConfig("stor-distribution") + .setValue("redundancy", redundancy); + configFinalized = true; + } + return config; + } + + }; + + document::TestDocMan _docMan; + + void testInitialization(InitParams& params); + + /** + * Test that the status page can be shown during init without a deadlock + * or crash or anything. Don't validate much output, it might change. + */ + void testStatusPage(); + + /** Test initializing with an empty node. */ + void testInitEmptyNode() { + InitParams params; + params.docsPerDisk = 0; + testInitialization(params); + } + /** Test initializing with some data on single disk. */ + void testInitSingleDisk() { + InitParams params; + params.diskCount = DiskCount(1); + testInitialization(params); + } + /** Test initializing with multiple disks. */ + void testInitMultiDisk() { + InitParams params; + testInitialization(params); + } + /** Test initializing with one of the disks being bad. */ + void testInitFailingMiddleDisk() { + InitParams params; + params.disksDown.insert(1); + testInitialization(params); + } + /** Test initializing with last disk being bad. */ + void testInitFailingLastDisk() { + InitParams params; + params.disksDown.insert(params.diskCount - 1); + testInitialization(params); + } + /** Test initializing with bucket on wrong disk. */ + void testInitBucketOnWrongDisk() { + InitParams params; + params.bucketWrongDisk = true; + params.bucketBitsUsed = 58; + testInitialization(params); + } + /** Test initializing with bucket on multiple disks. */ + void testInitBucketOnMultipleDisks() { + InitParams params; + params.bucketMultipleDisks = true; + params.bucketBitsUsed = 58; + testInitialization(params); + } + /** Test initializing with failing list request. */ + void testInitFailingListRequest() { + InitParams params; + params.failingListRequest = true; + testInitialization(params); + } + void testInitFailingInfoRequest() { + InitParams params; + params.failingInfoRequest = true; + testInitialization(params); + } + /** Test initializing with everything being wrong at once. */ + void testAllFailures() { + InitParams params; + params.docsPerDisk = 100; + params.diskCount = DiskCount(10); + params.disksDown.insert(0); + params.disksDown.insert(2); + params.disksDown.insert(3); + params.disksDown.insert(9); + params.setAllFailures(); + testInitialization(params); + } + void testCommandBlockingDuringInit(); + + void testBucketProgressCalculator(); + + void testBucketsInitializedByLoad(); + + CPPUNIT_TEST_SUITE(InitializerTest); + CPPUNIT_TEST(testInitEmptyNode); + CPPUNIT_TEST(testInitSingleDisk); + CPPUNIT_TEST(testInitMultiDisk); + CPPUNIT_TEST(testInitFailingMiddleDisk); + CPPUNIT_TEST(testInitFailingLastDisk); + CPPUNIT_TEST(testInitBucketOnWrongDisk); + //CPPUNIT_TEST(testInitBucketOnMultipleDisks); + //CPPUNIT_TEST(testStatusPage); + //CPPUNIT_TEST(testCommandBlockingDuringInit); + //CPPUNIT_TEST(testAllFailures); + CPPUNIT_TEST(testBucketProgressCalculator); + CPPUNIT_TEST(testBucketsInitializedByLoad); + CPPUNIT_TEST_SUITE_END(); + +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(InitializerTest); + +namespace { +// Data kept on buckets we're using in test. +struct BucketData { + api::BucketInfo info; + + BucketData() : info(0, 0, 0, 0, 0) { + } + + BucketData operator+(const BucketData& other) const { + BucketData copy; + copy.info.setDocumentCount( + info.getDocumentCount() + other.info.getDocumentCount()); + copy.info.setTotalDocumentSize( + info.getTotalDocumentSize() + + other.info.getTotalDocumentSize()); + copy.info.setChecksum( + info.getChecksum() * other.info.getChecksum()); + return copy; + } +}; +// Data reciding on one disk +typedef std::map<document::BucketId, BucketData> DiskData; +struct BucketInfoLogger { + std::map<PartitionId, DiskData>& map; + + BucketInfoLogger(std::map<PartitionId, DiskData>& m) + : map(m) {} + + StorBucketDatabase::Decision operator()( + uint64_t revBucket, StorBucketDatabase::Entry& entry) + { + document::BucketId bucket( + document::BucketId::keyToBucketId(revBucket)); + CPPUNIT_ASSERT(bucket.getRawId() != 0); + CPPUNIT_ASSERT_MSG( + "Found invalid bucket in database: " + bucket.toString() + + " " + entry.getBucketInfo().toString(), + entry.getBucketInfo().valid()); + DiskData& ddata(map[entry.disk]); + BucketData& bdata(ddata[bucket]); + bdata.info = entry.getBucketInfo(); + return StorBucketDatabase::CONTINUE; + } +}; +std::map<PartitionId, DiskData> +createMapFromBucketDatabase(StorBucketDatabase& db) { + std::map<PartitionId, DiskData> result; + BucketInfoLogger infoLogger(result); + db.all(infoLogger, "createmap"); + return result; +} +// Create data we want to have in this test +std::map<PartitionId, DiskData> +buildBucketInfo(const document::TestDocMan& docMan, + InitializerTest::InitParams& params) +{ + std::map<PartitionId, DiskData> result; + for (uint32_t i=0; i<params.diskCount; ++i) { + if (params.disksDown.find(i) == params.disksDown.end()) { + result[i]; + } + } + lib::Distribution distribution( + lib::Distribution::getDefaultDistributionConfig( + params.redundancy, params.nodeCount)); + document::BucketIdFactory bucketIdFactory; + lib::NodeState nodeState; + nodeState.setDiskCount(params.diskCount); + + uint64_t totalDocs = params.docsPerDisk * params.diskCount; + for (uint32_t i=0, n=totalDocs; i<n; ++i) { + bool useWrongDisk = false; + if (i == 1 && params.bucketWrongDisk) { + useWrongDisk = true; + } + document::Document::SP doc(docMan.createRandomDocument(i)); + if (i == 3 && params.bucketMultipleDisks) { + doc = docMan.createRandomDocument(i - 1); + useWrongDisk = true; + } + document::BucketId bid(bucketIdFactory.getBucketId(doc->getId())); + bid.setUsedBits(params.bucketBitsUsed); + bid = bid.stripUnused(); + uint32_t partition(distribution.getIdealDisk( + nodeState, params.nodeIndex, bid, + lib::Distribution::IDEAL_DISK_EVEN_IF_DOWN)); + if (params.disksDown.find(partition) != params.disksDown.end()) { + continue; + } + if (useWrongDisk) { + int correctPart = partition; + partition = (partition + 1) % params.diskCount;; + while (params.disksDown.find(partition) != params.disksDown.end()) { + partition = (partition + 1) % params.diskCount;; + } + LOG(info, "Putting bucket %s on wrong disk %u instead of %u", + bid.toString().c_str(), partition, correctPart); + } + LOG(info, "Putting bucket %s on disk %u", + bid.toString().c_str(), partition); + BucketData& data(result[partition][bid]); + data.info.setDocumentCount(data.info.getDocumentCount() + 1); + data.info.setTotalDocumentSize( + data.info.getTotalDocumentSize() + 100); + data.info.setChecksum(data.info.getChecksum() * 3); + } + return result; +} +void verifyEqual(std::map<PartitionId, DiskData>& org, + std::map<PartitionId, DiskData>& existing) +{ + uint32_t equalCount = 0; + std::map<PartitionId, DiskData>::const_iterator part1(org.begin()); + std::map<PartitionId, DiskData>::const_iterator part2(existing.begin()); + while (part1 != org.end() && part2 != existing.end()) { + if (part1->first < part2->first) { + if (!part1->second.empty()) { + std::ostringstream ost; + ost << "No data in partition " << part1->first << " found."; + CPPUNIT_FAIL(ost.str()); + } + ++part1; + } else if (part1->first > part2->first) { + if (!part2->second.empty()) { + std::ostringstream ost; + ost << "Found data in partition " << part2->first + << " which should not exist."; + CPPUNIT_FAIL(ost.str()); + } + ++part2; + } else { + DiskData::const_iterator bucket1(part1->second.begin()); + DiskData::const_iterator bucket2(part2->second.begin()); + while (bucket1 != part1->second.end() + && bucket2 != part2->second.end()) + { + if (bucket1->first < bucket2->first) { + std::ostringstream ost; + ost << "No data in partition " << part1->first + << " for bucket " << bucket1->first << " found."; + CPPUNIT_FAIL(ost.str()); + } else if (bucket1->first.getId() > bucket2->first.getId()) + { + std::ostringstream ost; + ost << "Found data in partition " << part2->first + << " for bucket " << bucket2->first + << " which should not exist."; + CPPUNIT_FAIL(ost.str()); + } else if (!(bucket1->second.info == bucket2->second.info)) { + std::ostringstream ost; + ost << "Bucket " << bucket1->first << " on partition " + << part1->first << " has bucket info " + << bucket2->second.info << " and not " + << bucket1->second.info << " as expected."; + CPPUNIT_FAIL(ost.str()); + } + ++bucket1; + ++bucket2; + ++equalCount; + } + if (bucket1 != part1->second.end()) { + std::ostringstream ost; + ost << "No data in partition " << part1->first + << " for bucket " << bucket1->first << " found."; + CPPUNIT_FAIL(ost.str()); + } + if (bucket2 != part2->second.end()) { + std::ostringstream ost; + ost << "Found data in partition " << part2->first + << " for bucket " << bucket2->first + << " which should not exist."; + CPPUNIT_FAIL(ost.str()); + } + ++part1; + ++part2; + } + } + if (part1 != org.end() && !part1->second.empty()) { + std::ostringstream ost; + ost << "No data in partition " << part1->first << " found."; + CPPUNIT_FAIL(ost.str()); + } + if (part2 != existing.end() && !part2->second.empty()) { + std::ostringstream ost; + ost << "Found data in partition " << part2->first + << " which should not exist."; + CPPUNIT_FAIL(ost.str()); + } + //std::cerr << "\n " << equalCount << " buckets were matched. "; +} + +struct MessageCallback +{ +public: + virtual ~MessageCallback() {} + virtual void onMessage(const api::StorageMessage&) = 0; +}; + +struct FakePersistenceLayer : public StorageLink { + StorBucketDatabase& bucketDatabase; + std::map<PartitionId, DiskData>& data; + std::string firstFatal; + std::string fatalError; + MessageCallback* messageCallback; + + FakePersistenceLayer(std::map<PartitionId, DiskData>& d, + StorBucketDatabase& db) + : StorageLink("fakepersistencelayer"), + bucketDatabase(db), + data(d), + messageCallback(0) + { + } + + void fatal(vespalib::stringref error) { + fatalError = error; + if (firstFatal.empty()) firstFatal = fatalError; + } + const BucketData* getBucketData(PartitionId partition, + const document::BucketId& bucket, + vespalib::stringref opname) + { + std::map<PartitionId, DiskData>::const_iterator it( + data.find(partition)); + if (it == data.end()) { + std::ostringstream ost; + ost << bucket << " is stated to be on partition " << partition + << " in operation " << opname << ", but we have no data for " + << "it there."; + fatal(ost.str()); + } else { + DiskData::const_iterator it2(it->second.find(bucket)); + if (it2 == it->second.end()) { + std::ostringstream ost; + ost << "Have no data for " << bucket << " on disk " << partition + << " in operation " << opname; + fatal(ost.str()); + } else { + const BucketData& bucketData(it2->second); + return &bucketData; + } + } + return 0; + } + virtual bool onDown(const api::StorageMessage::SP& msg) { + fatalError = ""; + if (messageCallback) { + messageCallback->onMessage(*msg); + } + if (msg->getType() == api::MessageType::INTERNAL) { + api::InternalCommand& cmd( + dynamic_cast<api::InternalCommand&>(*msg)); + if (cmd.getType() == ReadBucketList::ID) { + ReadBucketList& rbl(dynamic_cast<ReadBucketList&>(cmd)); + ReadBucketListReply::SP reply(new ReadBucketListReply(rbl)); + std::map<PartitionId, DiskData>::const_iterator it( + data.find(rbl.getPartition())); + if (it == data.end()) { + std::ostringstream ost; + ost << "Got list request to partition " + << rbl.getPartition() + << " for which we should not get a request"; + fatal(ost.str()); + } else { + for (DiskData::const_iterator it2 = it->second.begin(); + it2 != it->second.end(); ++it2) + { + reply->getBuckets().push_back(it2->first); + } + } + if (!fatalError.empty()) { + reply->setResult(api::ReturnCode( + api::ReturnCode::INTERNAL_FAILURE, fatalError)); + } + sendUp(reply); + } else if (cmd.getType() == ReadBucketInfo::ID) { + ReadBucketInfo& rbi(dynamic_cast<ReadBucketInfo&>(cmd)); + ReadBucketInfoReply::SP reply(new ReadBucketInfoReply(rbi)); + StorBucketDatabase::WrappedEntry entry( + bucketDatabase.get(rbi.getBucketId(), "fakelayer")); + if (!entry.exist()) { + fatal("Bucket " + rbi.getBucketId().toString() + + " did not exist in bucket database but we got " + + "read bucket info request for it."); + } else { + const BucketData* bucketData(getBucketData( + entry->disk, rbi.getBucketId(), "readbucketinfo")); + if (bucketData != 0) { + entry->setBucketInfo(bucketData->info); + entry.write(); + } + } + if (!fatalError.empty()) { + reply->setResult(api::ReturnCode( + api::ReturnCode::INTERNAL_FAILURE, fatalError)); + } + sendUp(reply); + } else if (cmd.getType() == InternalBucketJoinCommand::ID) { + InternalBucketJoinCommand& ibj( + dynamic_cast<InternalBucketJoinCommand&>(cmd)); + InternalBucketJoinReply::SP reply( + new InternalBucketJoinReply(ibj)); + StorBucketDatabase::WrappedEntry entry( + bucketDatabase.get(ibj.getBucketId(), "fakelayer")); + if (!entry.exist()) { + fatal("Bucket " + ibj.getBucketId().toString() + + " did not exist in bucket database but we got " + + "read bucket info request for it."); + } else { + const BucketData* source(getBucketData( + ibj.getDiskOfInstanceToJoin(), ibj.getBucketId(), + "internaljoinsource")); + const BucketData* target(getBucketData( + ibj.getDiskOfInstanceToKeep(), ibj.getBucketId(), + "internaljointarget")); + if (source != 0 && target != 0) { + entry->setBucketInfo((*source + *target).info); + entry.write(); + } + } + if (!fatalError.empty()) { + reply->setResult(api::ReturnCode( + api::ReturnCode::INTERNAL_FAILURE, fatalError)); + } + sendUp(reply); + } else { + return false; + } + return true; + } + return false; + } +}; + +} // end of anonymous namespace + +#define CPPUNIT_ASSERT_METRIC_SET(x) \ + CPPUNIT_ASSERT(initializer->getMetrics().x.getValue() > 0); + +void +InitializerTest::testInitialization(InitParams& params) +{ + std::map<PartitionId, DiskData> data(buildBucketInfo(_docMan, params)); + + spi::PartitionStateList partitions(params.diskCount); + for (std::set<uint32_t>::const_iterator it = params.disksDown.begin(); + it != params.disksDown.end(); ++it) + { + partitions[*it] = spi::PartitionState( + spi::PartitionState::DOWN, "Set down in test"); + } + TestServiceLayerApp node(params.diskCount, params.nodeIndex, + params.getConfig().getConfigId()); + DummyStorageLink top; + StorageBucketDBInitializer* initializer; + FakePersistenceLayer* bottom; + top.push_back(StorageLink::UP(initializer = new StorageBucketDBInitializer( + params.getConfig().getConfigId(), + partitions, + node.getDoneInitializeHandler(), + node.getComponentRegister()))); + top.push_back(StorageLink::UP(bottom = new FakePersistenceLayer( + data, node.getStorageBucketDatabase()))); + + LOG(info, "STARTING INITIALIZATION"); + top.open(); + + /* + FileChanger updater(config, nodeIndex, params, orgBucketDatabase); + if (params.bucketWrongDisk) updater.moveBucketWrongDisk(); + if (params.bucketMultipleDisks) updater.copyBucketWrongDisk(); + if (params.failingListRequest) { + updater.removeDirPermission(6, 'r'); + updater.removeBucketsFromDBAtPath(6); + } + if (params.failingInfoRequest) { + updater.removeFilePermission(); + orgBucketDatabase.erase(updater.getBucket(8)); + } + */ + + node.waitUntilInitialized(initializer); + + std::map<PartitionId, DiskData> initedBucketDatabase( + createMapFromBucketDatabase(node.getStorageBucketDatabase())); + verifyEqual(data, initedBucketDatabase); + /* + if (params.bucketWrongDisk) { + CPPUNIT_ASSERT_METRIC_SET(_wrongDisk); + } + if (params.bucketMultipleDisks) { + CPPUNIT_ASSERT_METRIC_SET(_joinedCount); + } + */ +} + +/* +namespace { + enum State { LISTING, INFO, DONE }; + void verifyStatusContent(StorageBucketDBInitializer& initializer, + State state) + { + std::ostringstream ost; + initializer.reportStatus(ost, framework::HttpUrlPath("")); + std::string status = ost.str(); + + if (state == LISTING) { + CPPUNIT_ASSERT_CONTAIN("List phase completed: false", status); + CPPUNIT_ASSERT_CONTAIN("Initialization completed: false", status); + } else if (state == INFO) { + CPPUNIT_ASSERT_CONTAIN("List phase completed: true", status); + CPPUNIT_ASSERT_CONTAIN("Initialization completed: false", status); + } else if (state == DONE) { + CPPUNIT_ASSERT_CONTAIN("List phase completed: true", status); + CPPUNIT_ASSERT_CONTAIN("Initialization completed: true", status); + } + } +} + +void +InitializerTest::testStatusPage() +{ + // Set up surrounding system to create a single bucket for us to + // do init on. + vdstestlib::DirConfig config(getStandardConfig(true)); + uint16_t nodeIndex( + config.getConfig("stor-server").getValue("node_index", 0)); + InitParams params; + params.docsPerDisk = 1; + params.diskCount = 1; + std::map<document::BucketId, api::BucketInfo> orgBucketDatabase( + buildBucketInfo(_docMan, config, nodeIndex, 1, 1, params.disksDown)); + FileChanger updater(config, nodeIndex, params, orgBucketDatabase); + + // Set up the initializer. + DummyStorageServer server(config.getConfigId()); + DummyStorageLink top; + DummyStorageLink *bottom; + StorageBucketDBInitializer* initializer; + top.push_back(StorageLink::UP(initializer = new StorageBucketDBInitializer( + config.getConfigId(), server))); + top.push_back(StorageLink::UP(bottom = new DummyStorageLink)); + + // Grab bucket database lock for bucket to init to lock the initializer + // in the init stage + StorBucketDatabase::WrappedEntry entry( + server.getStorageBucketDatabase().get( + updater.getBucket(0), "testCommandBlocking", + StorBucketDatabase::LOCK_IF_NONEXISTING_AND_NOT_CREATING)); + // Start the initializer + top.open(); + bottom->waitForMessages(1, 30); + verifyStatusContent(*initializer, LISTING); + // Attempt to send put. Should be blocked + // Attempt to send request bucket info. Should be blocked. + // Attempt to send getNodeState. Should not be blocked. + + // Unlock bucket in bucket database so listing step can complete. + // Await read info request being sent down. + entry.unlock(); + bottom->waitForMessages(1, 30); + verifyStatusContent(*initializer, INFO); + + ReadBucketInfo& cmd(dynamic_cast<ReadBucketInfo&>(*bottom->getCommand(0))); + ReadBucketInfoReply::SP reply(new ReadBucketInfoReply(cmd)); + bottom->sendUp(reply); + + node.waitUntilInitialized(initializer); + verifyStatusContent(*initializer, DONE); + +} + +#define ASSERT_BLOCKED(top, bottom, blocks) \ + if (blocks) { \ + top.waitForMessages(1, 30); \ + CPPUNIT_ASSERT_EQUAL(size_t(1), top.getReplies().size()); \ + CPPUNIT_ASSERT_EQUAL(size_t(0), bottom.getCommands().size()); \ + api::StorageReply& reply(dynamic_cast<api::StorageReply&>( \ + *top.getReply(0))); \ + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ABORTED, \ + reply.getResult().getResult()); \ + top.reset(); \ + } else { \ + bottom.waitForMessages(1, 30); \ + CPPUNIT_ASSERT_EQUAL(size_t(0), top.getReplies().size()); \ + CPPUNIT_ASSERT_EQUAL(size_t(1), bottom.getCommands().size()); \ + api::StorageCommand& command(dynamic_cast<api::StorageCommand&>( \ + *bottom.getCommand(0))); \ + (void) command; \ + bottom.reset(); \ + } + +namespace { + void verifyBlockingOn(DummyStorageLink& top, + DummyStorageLink& bottom, + bool blockEnabled) + { + // Attempt to send get. Should be blocked if block enabled + { + api::GetCommand::SP cmd(new api::GetCommand( + document::BucketId(16, 4), + document::DocumentId("userdoc:ns:4:test"), true)); + top.sendDown(cmd); + ASSERT_BLOCKED(top, bottom, blockEnabled); + } + // Attempt to send request bucket info. Should be blocked if enabled. + { + api::RequestBucketInfoCommand::SP cmd( + new api::RequestBucketInfoCommand( + 0, lib::ClusterState(""))); + top.sendDown(cmd); + ASSERT_BLOCKED(top, bottom, blockEnabled); + } + // Attempt to send getNodeState. Should not be blocked. + { + api::GetNodeStateCommand::SP cmd(new api::GetNodeStateCommand( + lib::NodeState::UP(0))); + top.sendDown(cmd); + ASSERT_BLOCKED(top, bottom, false); + } + } +} + +void +InitializerTest::testCommandBlockingDuringInit() +{ + // Set up surrounding system to create a single bucket for us to + // do init on. + vdstestlib::DirConfig config(getStandardConfig(true)); + uint16_t nodeIndex( + config.getConfig("stor-server").getValue("node_index", 0)); + InitParams params; + params.docsPerDisk = 1; + params.diskCount = 1; + std::map<document::BucketId, api::BucketInfo> orgBucketDatabase( + buildBucketInfo(_docMan, config, nodeIndex, 1, 1, params.disksDown)); + FileChanger updater(config, nodeIndex, params, orgBucketDatabase); + + // Set up the initializer. + DummyStorageServer server(config.getConfigId()); + DummyStorageLink top; + DummyStorageLink *bottom; + StorageBucketDBInitializer* initializer; + top.push_back(StorageLink::UP(initializer = new StorageBucketDBInitializer( + config.getConfigId(), server))); + top.push_back(StorageLink::UP(bottom = new DummyStorageLink)); + + // Grab bucket database lock for bucket to init to lock the initializer + // in the init stage + StorBucketDatabase::WrappedEntry entry( + server.getStorageBucketDatabase().get( + updater.getBucket(0), "testCommandBlocking", + StorBucketDatabase::LOCK_IF_NONEXISTING_AND_NOT_CREATING)); + // Start the initializer + top.open(); + verifyBlockingOn(top, *bottom, true); + // Attempt to send put. Should be blocked + // Attempt to send request bucket info. Should be blocked. + // Attempt to send getNodeState. Should not be blocked. + + // Unlock bucket in bucket database so listing step can complete. + // Await read info request being sent down. + entry.unlock(); + bottom->waitForMessages(1, 30); + dynamic_cast<ReadBucketInfo&>(*bottom->getCommand(0)); + CPPUNIT_ASSERT(!server.isInitialized()); + bottom->reset(); + + // Retry - Should now not block + verifyBlockingOn(top, *bottom, false); +} +*/ + +void +InitializerTest::testBucketProgressCalculator() +{ + using document::BucketId; + StorageBucketDBInitializer::BucketProgressCalculator calc; + // We consider the given bucket as not being completed, so progress + // will be _up to_, not _including_ the bucket. This means we can never + // reach 1.0, so progress completion must be handled by other logic! + CPPUNIT_ASSERT_EQUAL(0.0, calc.calculateProgress(BucketId(1, 0))); + CPPUNIT_ASSERT_EQUAL(0.0, calc.calculateProgress(BucketId(32, 0))); + + CPPUNIT_ASSERT_EQUAL(0.5, calc.calculateProgress(BucketId(1, 1))); + + CPPUNIT_ASSERT_EQUAL(0.25, calc.calculateProgress(BucketId(2, 2))); + CPPUNIT_ASSERT_EQUAL(0.5, calc.calculateProgress(BucketId(2, 1))); + CPPUNIT_ASSERT_EQUAL(0.75, calc.calculateProgress(BucketId(2, 3))); + + CPPUNIT_ASSERT_EQUAL(0.875, calc.calculateProgress(BucketId(3, 7))); +} + +struct DatabaseInsertCallback : MessageCallback +{ + DiskData& _data; + StorBucketDatabase& _database; + TestServiceLayerApp& _app; + const InitializerTest::InitParams& _params; + bool _invoked; + double _lastSeenProgress; + uint8_t _expectedReadBucketPriority; + std::ostringstream _errors; + DatabaseInsertCallback(DiskData& data, + StorBucketDatabase& db, + TestServiceLayerApp& app, + const InitializerTest::InitParams& params) + : _data(data), + _database(db), + _app(app), + _params(params), + _invoked(false), + _lastSeenProgress(0), + _expectedReadBucketPriority(255) + {} + + void onMessage(const api::StorageMessage& msg) + { + // Always make sure we're not set as initialized while we're still + // processing messages! Also ensure progress never goes down. + lib::NodeState::CSP reportedState( + _app.getStateUpdater().getReportedNodeState()); + double progress(reportedState->getInitProgress().getValue()); + LOG(debug, "reported progress is now %g", progress); + // CppUnit exceptions are swallowed... + if (progress >= 1.0) { + _errors << "progress exceeded 1.0: " << progress << "\n"; + } + if (progress < _lastSeenProgress) { + _errors << "progress went down! " + << _lastSeenProgress << " -> " << progress + << "\n"; + } + // 16 bits is allowed before we have listed any buckets at all + // since we at that point have no idea and have not reported anything + // back to the fleetcontroller. + if (_params.bucketBitsUsed != reportedState->getMinUsedBits() + && !(reportedState->getMinUsedBits() == 16 && !_invoked)) + { + _errors << "reported state contains wrong min used bits. " + << "expected " << _params.bucketBitsUsed + << ", but got " << reportedState->getMinUsedBits() + << "\n"; + } + _lastSeenProgress = progress; + if (_invoked) { + return; + } + + if (msg.getType() == api::MessageType::INTERNAL) { + const api::InternalCommand& cmd( + dynamic_cast<const api::InternalCommand&>(msg)); + if (cmd.getType() == ReadBucketInfo::ID) { + if (cmd.getPriority() != _expectedReadBucketPriority) { + _errors << "expected ReadBucketInfo priority of " + << static_cast<int>(_expectedReadBucketPriority) + << ", was " << static_cast<int>(cmd.getPriority()); + } + // As soon as we get the first ReadBucketInfo, we insert new buckets + // into the the bucket database in order to simulate external + // load init. Kinda hacky, but should work as long as initializer + // always does at least 1 extra iteration pass (which we use + // config overrides to ensure happens). + _invoked = true; + for (int i = 0; i < 4; ++i) { + document::BucketId bid(16 + i, 8); // not the first, nor the last bucket + BucketData d; + StorBucketDatabase::WrappedEntry entry( + _database.get(bid, "DatabaseInsertCallback::onMessage", + StorBucketDatabase::LOCK_IF_NONEXISTING_AND_NOT_CREATING)); + if (entry.exist()) { + _errors << "db entry for " << bid << " already existed"; + } + if (i < 5) { + d.info = api::BucketInfo(3+i, 4+i, 5+i, 6+i, 7+i); + } + _data[bid] = d; + entry->disk = 0; + entry->setBucketInfo(d.info); + entry.write(); + } + } + } + } +}; + +void +InitializerTest::testBucketsInitializedByLoad() +{ + InitParams params; + params.docsPerDisk = 100; + params.diskCount = DiskCount(1); + params.getConfig().getConfig("stor-bucket-init").setValue("max_pending_info_reads_per_disk", 1); + params.getConfig().getConfig("stor-bucket-init").setValue("min_pending_info_reads_per_disk", 1); + params.getConfig().getConfig("stor-bucket-init") + .setValue("info_read_priority", 231); + + std::map<PartitionId, DiskData> data(buildBucketInfo(_docMan, params)); + + spi::PartitionStateList partitions(params.diskCount); + TestServiceLayerApp node(params.diskCount, params.nodeIndex, + params.getConfig().getConfigId()); + DummyStorageLink top; + StorageBucketDBInitializer* initializer; + FakePersistenceLayer* bottom; + top.push_back(StorageLink::UP(initializer = new StorageBucketDBInitializer( + params.getConfig().getConfigId(), + partitions, + node.getDoneInitializeHandler(), + node.getComponentRegister()))); + top.push_back(StorageLink::UP(bottom = new FakePersistenceLayer( + data, node.getStorageBucketDatabase()))); + + DatabaseInsertCallback callback(data[0], node.getStorageBucketDatabase(), + node, params); + callback._expectedReadBucketPriority = 231; + + bottom->messageCallback = &callback; + + top.open(); + + node.waitUntilInitialized(initializer); + // Must explicitly wait until initializer has closed to ensure node state + // has been set. + top.close(); + + CPPUNIT_ASSERT(callback._invoked); + CPPUNIT_ASSERT_EQUAL(std::string(), callback._errors.str()); + + std::map<PartitionId, DiskData> initedBucketDatabase( + createMapFromBucketDatabase(node.getStorageBucketDatabase())); + verifyEqual(data, initedBucketDatabase); + + lib::NodeState::CSP reportedState( + node.getStateUpdater().getReportedNodeState()); + + double progress(reportedState->getInitProgress().getValue()); + CPPUNIT_ASSERT(progress >= 1.0); + CPPUNIT_ASSERT(progress < 1.0001); + + CPPUNIT_ASSERT_EQUAL(params.bucketBitsUsed, + reportedState->getMinUsedBits()); +} + +} // storage diff --git a/storage/src/tests/bucketdb/judyarraytest.cpp b/storage/src/tests/bucketdb/judyarraytest.cpp new file mode 100644 index 00000000000..235c0c9eb5c --- /dev/null +++ b/storage/src/tests/bucketdb/judyarraytest.cpp @@ -0,0 +1,287 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storage/bucketdb/judyarray.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <boost/assign.hpp> +#include <boost/random.hpp> +#include <cppunit/extensions/HelperMacros.h> +#include <map> +#include <vector> + +namespace storage { + +struct JudyArrayTest : public CppUnit::TestFixture { + void testIterating(); + void testDualArrayFunctions(); + void testComparing(); + void testSize(); + void testStress(); + + CPPUNIT_TEST_SUITE(JudyArrayTest); + CPPUNIT_TEST(testIterating); + CPPUNIT_TEST(testDualArrayFunctions); + CPPUNIT_TEST(testSize); + CPPUNIT_TEST(testStress); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(JudyArrayTest); + +namespace { + std::vector<std::pair<JudyArray::key_type, JudyArray::data_type> > + getJudyArrayContents(const JudyArray& array) { + std::vector<std::pair<JudyArray::key_type, JudyArray::data_type> > vals; + for (JudyArray::const_iterator it = array.begin(); + it != array.end(); ++it) + { + vals.push_back(std::make_pair(it.key(), it.value())); + } + return vals; + } +} + +void +JudyArrayTest::testIterating() +{ + JudyArray array; + // Test that things are sane for empty document + CPPUNIT_ASSERT_EQUAL(array.begin(), array.end()); + // Add some values + using namespace boost::assign; + std::vector<std::pair<JudyArray::key_type, JudyArray::data_type> > values + = map_list_of(3,2)(5,12)(15,8)(13,10)(7,6)(9,4); + for (uint32_t i=0; i<values.size(); ++i) { + array.insert(values[i].first, values[i].second); + } + // Create expected result + std::sort(values.begin(), values.end()); + // Test that we can iterate through const iterator + std::vector<std::pair<JudyArray::key_type, JudyArray::data_type> > + foundVals = getJudyArrayContents(array); + CPPUNIT_ASSERT_EQUAL(values, foundVals); + + { // Test that both postfix operator work + JudyArray::iterator it = array.begin(); + JudyArray::iterator it2 = it++; + CPPUNIT_ASSERT_EQUAL(JudyArray::value_type(values[0]), *it2); + CPPUNIT_ASSERT_EQUAL(JudyArray::value_type(values[1]), *it); + + // And that iterator comparisons work + CPPUNIT_ASSERT_EQUAL(it2, array.begin()); + CPPUNIT_ASSERT_EQUAL(it, ++array.begin()); + CPPUNIT_ASSERT(!(it == it2)); + CPPUNIT_ASSERT(it != it2); + } + { // Test that we can alter through non-const iterator + JudyArray::iterator it = array.begin(); + ++it; + ++it; + it.setValue(20); + CPPUNIT_ASSERT_EQUAL((JudyArray::key_type) 7, it.key()); + CPPUNIT_ASSERT_EQUAL((JudyArray::data_type) 20, array[7]); + it.remove(); + CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 5, + getJudyArrayContents(array).size()); + CPPUNIT_ASSERT_EQUAL(array.end(), array.find(7)); + values.erase(values.begin() + 2); + CPPUNIT_ASSERT_EQUAL(values, getJudyArrayContents(array)); + // And that we can continue iterating after removing. + ++it; + CPPUNIT_ASSERT_EQUAL((JudyArray::key_type) 9, it.key()); + CPPUNIT_ASSERT_EQUAL((JudyArray::data_type) 4, array[9]); + } + { // Test printing of iterators + JudyArray::ConstIterator cit = array.begin(); + CPPUNIT_ASSERT_MATCH_REGEX( + "^ConstIterator\\(Key: 3, Valp: 0x[0-9a-f]{1,16}, Val: 2\\)$", + cit.toString()); + JudyArray::Iterator it = array.end(); + CPPUNIT_ASSERT_MATCH_REGEX( + "^Iterator\\(Key: 0, Valp: 0\\)$", + it.toString()); + } +} + +void +JudyArrayTest::testDualArrayFunctions() +{ + JudyArray array1; + JudyArray array2; + // Add values to array1 + using namespace boost::assign; + std::vector<std::pair<JudyArray::key_type, JudyArray::data_type> > values1 + = map_list_of(3,2)(5,12)(15,8)(13,10)(7,6)(9,4); + for (uint32_t i=0; i<values1.size(); ++i) { + array1.insert(values1[i].first, values1[i].second); + } + // Add values to array2 + std::vector<std::pair<JudyArray::key_type, JudyArray::data_type> > values2 + = map_list_of(4,5)(9,40); + for (uint32_t i=0; i<values2.size(); ++i) { + array2.insert(values2[i].first, values2[i].second); + } + // Create expected result + std::sort(values1.begin(), values1.end()); + std::sort(values2.begin(), values2.end()); + + CPPUNIT_ASSERT_EQUAL(values1, getJudyArrayContents(array1)); + CPPUNIT_ASSERT_EQUAL(values2, getJudyArrayContents(array2)); + CPPUNIT_ASSERT(array1 > array2); + CPPUNIT_ASSERT(array1 != array2); + array1.swap(array2); + CPPUNIT_ASSERT_EQUAL(values1, getJudyArrayContents(array2)); + CPPUNIT_ASSERT_EQUAL(values2, getJudyArrayContents(array1)); + CPPUNIT_ASSERT(array1 < array2); + CPPUNIT_ASSERT(array1 != array2); + + // Test some operators + JudyArray array3; + for (uint32_t i=0; i<values1.size(); ++i) { + array3.insert(values1[i].first, values1[i].second); + } + CPPUNIT_ASSERT(array1 != array3); + CPPUNIT_ASSERT_EQUAL(array2, array3); + CPPUNIT_ASSERT(array2 >= array3); + CPPUNIT_ASSERT(array2 <= array3); + CPPUNIT_ASSERT(!(array2 < array3)); + CPPUNIT_ASSERT(!(array2 > array3)); +} + +void +JudyArrayTest::testSize() +{ + JudyArray array; + CPPUNIT_ASSERT_EQUAL(array.begin(), array.end()); + CPPUNIT_ASSERT(array.empty()); + CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 0, array.size()); + CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 0, array.getMemoryUsage()); + + // Test each method one can insert stuff into array + array.insert(4, 3); + CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); + array.insert(4, 7); + CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); + if (sizeof(JudyArray::size_type) == 4) { + CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 12, array.getMemoryUsage()); + } else if (sizeof(JudyArray::size_type) == 8) { + CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 24, array.getMemoryUsage()); + } else CPPUNIT_FAIL("Unknown size of type"); + + array[6] = 8; + CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); + array[6] = 10; + CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); + if (sizeof(JudyArray::size_type) == 4) { + CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 20, array.getMemoryUsage()); + } else if (sizeof(JudyArray::size_type) == 8) { + CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 40, array.getMemoryUsage()); + } else CPPUNIT_FAIL("Unknown size of type"); + + bool preExisted; + array.find(8, true, preExisted); + CPPUNIT_ASSERT_EQUAL(false, preExisted); + CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); + array.find(8, true, preExisted); + CPPUNIT_ASSERT_EQUAL(true, preExisted); + CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); + CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 3, array.size()); + if (sizeof(JudyArray::size_type) == 4) { + CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 28, array.getMemoryUsage()); + } else if (sizeof(JudyArray::size_type) == 8) { + CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 56, array.getMemoryUsage()); + } else CPPUNIT_FAIL("Unknown size of type"); + + // Test each method one can remove stuff in array with + array.erase(8); + CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); + array.erase(8); + CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); + CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 2, array.size()); + if (sizeof(JudyArray::size_type) == 4) { + CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 20, array.getMemoryUsage()); + } else if (sizeof(JudyArray::size_type) == 8) { + CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 40, array.getMemoryUsage()); + } else CPPUNIT_FAIL("Unknown size of type"); +} + +namespace { + template<typename T> + std::string toString(const T& m) { + std::cerr << "#"; + std::ostringstream ost; + ost << m; + return ost.str(); + } +} + +void +JudyArrayTest::testStress() +{ + // Do a lot of random stuff to both judy array and std::map. Ensure equal + // behaviour + + JudyArray judyArray; + typedef std::map<JudyArray::key_type, JudyArray::data_type> StdMap; + StdMap stdMap; + + boost::rand48 rnd(55); + + for (uint32_t checkpoint=0; checkpoint<50; ++checkpoint) { + for (uint32_t opnr=0; opnr<500; ++opnr) { + int optype = rnd() % 100; + if (optype < 30) { // Insert + JudyArray::key_type key(rnd() % 500); + JudyArray::key_type value(rnd()); + judyArray.insert(key, value); + stdMap[key] = value; + //std::pair<StdMap::iterator, bool> result + // = stdMap.insert(std::make_pair(key, value)); + //if (!result.second) result.first->second = value; + } else if (optype < 50) { // operator[] + JudyArray::key_type key(rnd() % 500); + JudyArray::key_type value(rnd()); + judyArray[key] = value; + stdMap[key] = value; + } else if (optype < 70) { // erase() + JudyArray::key_type key(rnd() % 500); + CPPUNIT_ASSERT_EQUAL_MSG( + toString(judyArray) + toString(stdMap), + stdMap.erase(key), judyArray.erase(key)); + } else if (optype < 75) { // size() + CPPUNIT_ASSERT_EQUAL_MSG( + toString(judyArray) + toString(stdMap), + stdMap.size(), judyArray.size()); + } else if (optype < 78) { // empty() + CPPUNIT_ASSERT_EQUAL_MSG( + toString(judyArray) + toString(stdMap), + stdMap.empty(), judyArray.empty()); + } else { // find() + JudyArray::key_type key(rnd() % 500); + JudyArray::iterator it = judyArray.find(key); + StdMap::iterator it2 = stdMap.find(key); + CPPUNIT_ASSERT_EQUAL_MSG( + toString(judyArray) + toString(stdMap), + it2 == stdMap.end(), it == judyArray.end()); + if (it != judyArray.end()) { + CPPUNIT_ASSERT_EQUAL_MSG( + toString(judyArray) + toString(stdMap), + it.key(), it2->first); + CPPUNIT_ASSERT_EQUAL_MSG( + toString(judyArray) + toString(stdMap), + it.value(), it2->second); + } + } + } + // Ensure judy array contents is equal to std::map's at this point + StdMap tmpMap; + for (JudyArray::const_iterator it = judyArray.begin(); + it != judyArray.end(); ++it) + { + tmpMap[it.key()] = it.value(); + } + CPPUNIT_ASSERT_EQUAL(stdMap, tmpMap); + } +} + +} // storage diff --git a/storage/src/tests/bucketdb/judymultimaptest.cpp b/storage/src/tests/bucketdb/judymultimaptest.cpp new file mode 100644 index 00000000000..f63fad9aa06 --- /dev/null +++ b/storage/src/tests/bucketdb/judymultimaptest.cpp @@ -0,0 +1,172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storage/bucketdb/judymultimap.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <boost/assign.hpp> +#include <boost/random.hpp> +#include <cppunit/extensions/HelperMacros.h> +#include <map> +#include <vector> + +namespace storage { + +struct JudyMultiMapTest : public CppUnit::TestFixture { + void testSimpleUsage(); + void testIterator(); + + CPPUNIT_TEST_SUITE(JudyMultiMapTest); + CPPUNIT_TEST(testSimpleUsage); + CPPUNIT_TEST(testIterator); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(JudyMultiMapTest); + +namespace { + struct B; + struct C; + + struct A { + int _val1; + int _val2; + int _val3; + + A() {} + A(const B& b); + A(const C& c); + A(int val1, int val2, int val3) + : _val1(val1), _val2(val2), _val3(val3) {} + + static bool mayContain(const A&) { return true; } + + bool operator==(const A& a) const { + return (_val1 == a._val1 && _val2 == a._val2 && _val3 == a._val3); + } + }; + + struct B { + int _val1; + int _val2; + + B() {} + B(const A& a) : _val1(a._val1), _val2(a._val2) {} + B(int val1, int val2) : _val1(val1), _val2(val2) {} + + static bool mayContain(const A& a) { return (a._val3 == 0); } + }; + + struct C { + int _val1; + + C() {} + C(const A& a) : _val1(a._val1) {} + C(int val1) : _val1(val1) {} + + static bool mayContain(const A& a) + { return (a._val2 == 0 && a._val3 == 0); } + }; + + A::A(const B& b) : _val1(b._val1), _val2(b._val2), _val3(0) {} + A::A(const C& c) : _val1(c._val1), _val2(0), _val3(0) {} + + std::ostream& operator<<(std::ostream& out, const A& a) { + return out << "A(" << a._val1 << ", " << a._val2 << ", " + << a._val3 << ")"; + } + std::ostream& operator<<(std::ostream& out, const B& b) { + return out << "B(" << b._val1 << ", " << b._val2 << ")"; + } + std::ostream& operator<<(std::ostream& out, const C& c) { + return out << "C(" << c._val1 << ")"; + } +} + +void +JudyMultiMapTest::testSimpleUsage() { + typedef JudyMultiMap<C, B, A> MultiMap; + MultiMap multiMap; + // Do some insertions + bool preExisted; + CPPUNIT_ASSERT(multiMap.empty()); + multiMap.insert(16, A(1, 2, 3), preExisted); + CPPUNIT_ASSERT_EQUAL(false, preExisted); + multiMap.insert(11, A(4, 6, 0), preExisted); + CPPUNIT_ASSERT_EQUAL(false, preExisted); + multiMap.insert(14, A(42, 0, 0), preExisted); + CPPUNIT_ASSERT_EQUAL(false, preExisted); + CPPUNIT_ASSERT_EQUAL_MSG(multiMap.toString(), + (MultiMap::size_type) 3, multiMap.size()); + + multiMap.insert(11, A(4, 7, 0), preExisted); + CPPUNIT_ASSERT_EQUAL(true, preExisted); + CPPUNIT_ASSERT_EQUAL((MultiMap::size_type) 3, multiMap.size()); + CPPUNIT_ASSERT(!multiMap.empty()); + + // Access some elements + CPPUNIT_ASSERT_EQUAL(A(4, 7, 0), multiMap[11]); + CPPUNIT_ASSERT_EQUAL(A(1, 2, 3), multiMap[16]); + CPPUNIT_ASSERT_EQUAL(A(42,0, 0), multiMap[14]); + + // Do removes + CPPUNIT_ASSERT(multiMap.erase(12) == 0); + CPPUNIT_ASSERT_EQUAL((MultiMap::size_type) 3, multiMap.size()); + + CPPUNIT_ASSERT(multiMap.erase(14) == 1); + CPPUNIT_ASSERT_EQUAL((MultiMap::size_type) 2, multiMap.size()); + + CPPUNIT_ASSERT(multiMap.erase(11) == 1); + CPPUNIT_ASSERT(multiMap.erase(16) == 1); + CPPUNIT_ASSERT_EQUAL((MultiMap::size_type) 0, multiMap.size()); + CPPUNIT_ASSERT(multiMap.empty()); +} + +void +JudyMultiMapTest::testIterator() +{ + typedef JudyMultiMap<C, B, A> MultiMap; + MultiMap multiMap; + bool preExisted; + // Do some insertions + multiMap.insert(16, A(1, 2, 3), preExisted); + multiMap.insert(11, A(4, 6, 0), preExisted); + multiMap.insert(14, A(42, 0, 0), preExisted); + + MultiMap::Iterator iter = multiMap.begin(); + CPPUNIT_ASSERT_EQUAL((uint64_t)11, (uint64_t)iter.key()); + CPPUNIT_ASSERT_EQUAL(A(4, 6, 0), iter.value()); + iter++; + CPPUNIT_ASSERT_EQUAL((uint64_t)14, (uint64_t)iter.key()); + CPPUNIT_ASSERT_EQUAL(A(42, 0, 0), iter.value()); + iter++; + CPPUNIT_ASSERT_EQUAL((uint64_t)16, (uint64_t)iter.key()); + CPPUNIT_ASSERT_EQUAL(A(1, 2, 3), iter.value()); + iter--; + CPPUNIT_ASSERT_EQUAL((uint64_t)14, (uint64_t)iter.key()); + CPPUNIT_ASSERT_EQUAL(A(42, 0, 0), iter.value()); + iter++; + CPPUNIT_ASSERT_EQUAL((uint64_t)16, (uint64_t)iter.key()); + CPPUNIT_ASSERT_EQUAL(A(1, 2, 3), iter.value()); + iter--; + iter--; + CPPUNIT_ASSERT_EQUAL((uint64_t)11,(uint64_t) iter.key()); + CPPUNIT_ASSERT_EQUAL(A(4, 6, 0), iter.value()); + iter++; + iter++; + iter++; + CPPUNIT_ASSERT_EQUAL(multiMap.end(), iter); + iter--; + CPPUNIT_ASSERT_EQUAL((uint64_t)16, (uint64_t)iter.key()); + CPPUNIT_ASSERT_EQUAL(A(1, 2, 3), iter.value()); + iter--; + CPPUNIT_ASSERT_EQUAL((uint64_t)14, (uint64_t)iter.key()); + CPPUNIT_ASSERT_EQUAL(A(42, 0, 0), iter.value()); + iter--; + CPPUNIT_ASSERT_EQUAL((uint64_t)11,(uint64_t) iter.key()); + CPPUNIT_ASSERT_EQUAL(A(4, 6, 0), iter.value()); + + +} + +} // storage + diff --git a/storage/src/tests/bucketdb/lockablemaptest.cpp b/storage/src/tests/bucketdb/lockablemaptest.cpp new file mode 100644 index 00000000000..0f35f51afbd --- /dev/null +++ b/storage/src/tests/bucketdb/lockablemaptest.cpp @@ -0,0 +1,1262 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/document_runnable.h> +#include <vespa/storage/bucketdb/judymultimap.h> +#include <vespa/storage/bucketdb/lockablemap.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <cppunit/extensions/HelperMacros.h> + +namespace storage { + +struct LockableMapTest : public CppUnit::TestFixture { + void testSimpleUsage(); + void testComparison(); + void testIterating(); + void testChunkedIterationIsTransparentAcrossChunkSizes(); + void testCanAbortDuringChunkedIteration(); + void testThreadSafetyStress(); + void testFindBuckets(); + void testFindBuckets2(); + void testFindBuckets3(); + void testFindBuckets4(); + void testFindBuckets5(); + void testFindBucketsSimple(); + void testFindNoBuckets(); + void testFindAll(); + void testFindAll2(); + void testFindAllUnusedBitIsSet(); + void testFindAllInconsistentlySplit(); + void testFindAllInconsistentlySplit2(); + void testFindAllInconsistentlySplit3(); + void testFindAllInconsistentlySplit4(); + void testFindAllInconsistentlySplit5(); + void testFindAllInconsistentlySplit6(); + void testFindAllInconsistentBelow16Bits(); + void testCreate(); + void testCreate2(); + void testCreate3(); + void testCreate4(); + void testCreate5(); + void testCreate6(); + void testCreateEmpty(); + void testIsConsistent(); + + CPPUNIT_TEST_SUITE(LockableMapTest); + CPPUNIT_TEST(testSimpleUsage); + CPPUNIT_TEST(testComparison); + CPPUNIT_TEST(testIterating); + CPPUNIT_TEST(testChunkedIterationIsTransparentAcrossChunkSizes); + CPPUNIT_TEST(testCanAbortDuringChunkedIteration); + CPPUNIT_TEST(testThreadSafetyStress); + CPPUNIT_TEST(testFindBuckets); + CPPUNIT_TEST(testFindBuckets2); + CPPUNIT_TEST(testFindBuckets3); + CPPUNIT_TEST(testFindBuckets4); + CPPUNIT_TEST(testFindBuckets5); + CPPUNIT_TEST(testFindBucketsSimple); + CPPUNIT_TEST(testFindNoBuckets); + CPPUNIT_TEST(testFindAll); + CPPUNIT_TEST(testFindAll2); + CPPUNIT_TEST(testFindAllUnusedBitIsSet); + CPPUNIT_TEST(testFindAllInconsistentlySplit); + CPPUNIT_TEST(testFindAllInconsistentlySplit2); + CPPUNIT_TEST(testFindAllInconsistentlySplit3); + CPPUNIT_TEST(testFindAllInconsistentlySplit4); + CPPUNIT_TEST(testFindAllInconsistentlySplit5); + CPPUNIT_TEST(testFindAllInconsistentlySplit6); + CPPUNIT_TEST(testFindAllInconsistentBelow16Bits); + CPPUNIT_TEST(testCreate); + CPPUNIT_TEST(testCreate2); + CPPUNIT_TEST(testCreate3); + CPPUNIT_TEST(testCreate4); + CPPUNIT_TEST(testCreate5); + CPPUNIT_TEST(testCreate6); + CPPUNIT_TEST(testCreateEmpty); + CPPUNIT_TEST(testIsConsistent); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(LockableMapTest); + +namespace { + struct A : public boost::operators<A> { + int _val1; + int _val2; + int _val3; + + A() : _val1(0), _val2(0), _val3(0) {} + A(int val1, int val2, int val3) + : _val1(val1), _val2(val2), _val3(val3) {} + + static bool mayContain(const A&) { return true; } + + bool operator==(const A& a) const { + return (_val1 == a._val1 && _val2 == a._val2 && _val3 == a._val3); + } + bool operator<(const A& a) const { + if (_val1 != a._val1) return (_val1 < a._val1); + if (_val2 != a._val2) return (_val2 < a._val2); + return (_val3 < a._val3); + } + }; + + std::ostream& operator<<(std::ostream& out, const A& a) { + return out << "A(" << a._val1 << ", " << a._val2 << ", " + << a._val3 << ")"; + } + + typedef LockableMap<JudyMultiMap<A> > Map; +} + +void +LockableMapTest::testSimpleUsage() { + // Tests insert, erase, size, empty, operator[] + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + // Do some insertions + CPPUNIT_ASSERT(map.empty()); + bool preExisted; + map.insert(16, A(1, 2, 3), "foo", preExisted); + CPPUNIT_ASSERT_EQUAL(false, preExisted); + map.insert(11, A(4, 6, 0), "foo", preExisted); + CPPUNIT_ASSERT_EQUAL(false, preExisted); + map.insert(14, A(42, 0, 0), "foo", preExisted); + CPPUNIT_ASSERT_EQUAL(false, preExisted); + CPPUNIT_ASSERT_EQUAL_MSG(map.toString(), + (Map::size_type) 3, map.size()); + + map.insert(11, A(4, 7, 0), "foo", preExisted); + CPPUNIT_ASSERT_EQUAL(true, preExisted); + CPPUNIT_ASSERT_EQUAL((Map::size_type) 3, map.size()); + CPPUNIT_ASSERT(!map.empty()); + + // Access some elements + CPPUNIT_ASSERT_EQUAL(A(4, 7, 0), *map.get(11, "foo")); + CPPUNIT_ASSERT_EQUAL(A(1, 2, 3), *map.get(16, "foo")); + CPPUNIT_ASSERT_EQUAL(A(42,0, 0), *map.get(14, "foo")); + + // Do removes + CPPUNIT_ASSERT(map.erase(12, "foo") == 0); + CPPUNIT_ASSERT_EQUAL((Map::size_type) 3, map.size()); + + CPPUNIT_ASSERT(map.erase(14, "foo") == 1); + CPPUNIT_ASSERT_EQUAL((Map::size_type) 2, map.size()); + + CPPUNIT_ASSERT(map.erase(11, "foo") == 1); + CPPUNIT_ASSERT(map.erase(16, "foo") == 1); + CPPUNIT_ASSERT_EQUAL((Map::size_type) 0, map.size()); + CPPUNIT_ASSERT(map.empty()); +} + +void +LockableMapTest::testComparison() { + typedef LockableMap<JudyMultiMap<A> > Map; + Map map1; + Map map2; + bool preExisted; + + // Check empty state is correct + CPPUNIT_ASSERT_EQUAL(map1, map2); + CPPUNIT_ASSERT(map1 <= map2); + CPPUNIT_ASSERT(map1 >= map2); + CPPUNIT_ASSERT(!(map1 < map2)); + CPPUNIT_ASSERT(!(map1 > map2)); + CPPUNIT_ASSERT(!(map1 != map2)); + + // Check that different lengths are oki + map1.insert(4, A(1, 2, 3), "foo", preExisted); + CPPUNIT_ASSERT(!(map1 == map2)); + CPPUNIT_ASSERT(!(map1 <= map2)); + CPPUNIT_ASSERT(!(map1 < map2)); + CPPUNIT_ASSERT(map1 >= map2); + CPPUNIT_ASSERT(map1 > map2); + CPPUNIT_ASSERT(map1 != map2); + + // Check that equal elements are oki + map2.insert(4, A(1, 2, 3), "foo", preExisted); + CPPUNIT_ASSERT_EQUAL(map1, map2); + CPPUNIT_ASSERT(map1 <= map2); + CPPUNIT_ASSERT(map1 >= map2); + CPPUNIT_ASSERT(!(map1 < map2)); + CPPUNIT_ASSERT(!(map1 > map2)); + CPPUNIT_ASSERT(!(map1 != map2)); + + // Check that non-equal values are oki + map1.insert(6, A(1, 2, 6), "foo", preExisted); + map2.insert(6, A(1, 2, 3), "foo", preExisted); + CPPUNIT_ASSERT(!(map1 == map2)); + CPPUNIT_ASSERT(!(map1 <= map2)); + CPPUNIT_ASSERT(!(map1 < map2)); + CPPUNIT_ASSERT(map1 >= map2); + CPPUNIT_ASSERT(map1 > map2); + CPPUNIT_ASSERT(map1 != map2); + + // Check that non-equal keys are oki + map1.erase(6, "foo"); + map1.insert(7, A(1, 2, 3), "foo", preExisted); + CPPUNIT_ASSERT(!(map1 == map2)); + CPPUNIT_ASSERT(!(map1 <= map2)); + CPPUNIT_ASSERT(!(map1 < map2)); + CPPUNIT_ASSERT(map1 >= map2); + CPPUNIT_ASSERT(map1 > map2); + CPPUNIT_ASSERT(map1 != map2); +} + +namespace { + struct NonConstProcessor { + Map::Decision operator()(int key, A& a) { + (void) key; + ++a._val2; + return Map::UPDATE; + } + }; + struct EntryProcessor { + mutable uint32_t count; + mutable std::vector<std::string> log; + mutable std::vector<Map::Decision> behaviour; + + EntryProcessor() : count(0), log(), behaviour() {} + EntryProcessor(const std::vector<Map::Decision>& decisions) + : count(0), log(), behaviour(decisions) {} + + Map::Decision operator()(uint64_t key, A& a) const { + std::ostringstream ost; + ost << key << " - " << a; + log.push_back(ost.str()); + Map::Decision d = Map::CONTINUE; + if (behaviour.size() > count) { + d = behaviour[count++]; + } + if (d == Map::UPDATE) { + ++a._val3; + } + return d; + } + + std::string toString() { + std::ostringstream ost; + for (uint32_t i=0; i<log.size(); ++i) ost << log[i] << "\n"; + return ost.str(); + } + }; +} + +void +LockableMapTest::testIterating() { + Map map; + bool preExisted; + map.insert(16, A(1, 2, 3), "foo", preExisted); + map.insert(11, A(4, 6, 0), "foo", preExisted); + map.insert(14, A(42, 0, 0), "foo", preExisted); + // Test that we can use functor with non-const function + { + NonConstProcessor ncproc; + map.each(ncproc, "foo"); // Locking both for each element + CPPUNIT_ASSERT_EQUAL(A(4, 7, 0), *map.get(11, "foo")); + CPPUNIT_ASSERT_EQUAL(A(42,1, 0), *map.get(14, "foo")); + CPPUNIT_ASSERT_EQUAL(A(1, 3, 3), *map.get(16, "foo")); + map.all(ncproc, "foo"); // And for all + CPPUNIT_ASSERT_EQUAL(A(4, 8, 0), *map.get(11, "foo")); + CPPUNIT_ASSERT_EQUAL(A(42,2, 0), *map.get(14, "foo")); + CPPUNIT_ASSERT_EQUAL(A(1, 4, 3), *map.get(16, "foo")); + } + // Test that we can use const functors directly.. + map.each(EntryProcessor(), "foo"); + + // Test iterator bounds + { + EntryProcessor proc; + map.each(proc, "foo", 11, 16); + std::string expected("11 - A(4, 8, 0)\n" + "14 - A(42, 2, 0)\n" + "16 - A(1, 4, 3)\n"); + CPPUNIT_ASSERT_EQUAL(expected, proc.toString()); + + EntryProcessor proc2; + map.each(proc2, "foo", 12, 15); + expected = "14 - A(42, 2, 0)\n"; + CPPUNIT_ASSERT_EQUAL(expected, proc2.toString()); + } + // Test that we can abort iterating + { + std::vector<Map::Decision> decisions; + decisions.push_back(Map::CONTINUE); + decisions.push_back(Map::ABORT); + EntryProcessor proc(decisions); + map.each(proc, "foo"); + std::string expected("11 - A(4, 8, 0)\n" + "14 - A(42, 2, 0)\n"); + CPPUNIT_ASSERT_EQUAL(expected, proc.toString()); + } + // Test that we can remove during iteration + { + std::vector<Map::Decision> decisions; + decisions.push_back(Map::CONTINUE); + decisions.push_back(Map::REMOVE); + EntryProcessor proc(decisions); + map.each(proc, "foo"); + std::string expected("11 - A(4, 8, 0)\n" + "14 - A(42, 2, 0)\n" + "16 - A(1, 4, 3)\n"); + CPPUNIT_ASSERT_EQUAL(expected, proc.toString()); + CPPUNIT_ASSERT_EQUAL_MSG(map.toString(), + (Map::size_type) 2, map.size()); + CPPUNIT_ASSERT_EQUAL(A(4, 8, 0), *map.get(11, "foo")); + CPPUNIT_ASSERT_EQUAL(A(1, 4, 3), *map.get(16, "foo")); + Map::WrappedEntry entry = map.get(14, "foo"); + CPPUNIT_ASSERT(!entry.exist()); + } +} + +void +LockableMapTest::testChunkedIterationIsTransparentAcrossChunkSizes() +{ + Map map; + bool preExisted; + map.insert(16, A(1, 2, 3), "foo", preExisted); + map.insert(11, A(4, 6, 0), "foo", preExisted); + map.insert(14, A(42, 0, 0), "foo", preExisted); + NonConstProcessor ncproc; // Increments 2nd value in all entries. + // chunkedAll with chunk size of 1 + map.chunkedAll(ncproc, "foo", 1); + CPPUNIT_ASSERT_EQUAL(A(4, 7, 0), *map.get(11, "foo")); + CPPUNIT_ASSERT_EQUAL(A(42, 1, 0), *map.get(14, "foo")); + CPPUNIT_ASSERT_EQUAL(A(1, 3, 3), *map.get(16, "foo")); + // chunkedAll with chunk size larger than db size + map.chunkedAll(ncproc, "foo", 100); + CPPUNIT_ASSERT_EQUAL(A(4, 8, 0), *map.get(11, "foo")); + CPPUNIT_ASSERT_EQUAL(A(42, 2, 0), *map.get(14, "foo")); + CPPUNIT_ASSERT_EQUAL(A(1, 4, 3), *map.get(16, "foo")); +} + +void +LockableMapTest::testCanAbortDuringChunkedIteration() +{ + Map map; + bool preExisted; + map.insert(16, A(1, 2, 3), "foo", preExisted); + map.insert(11, A(4, 6, 0), "foo", preExisted); + map.insert(14, A(42, 0, 0), "foo", preExisted); + + std::vector<Map::Decision> decisions; + decisions.push_back(Map::CONTINUE); + decisions.push_back(Map::ABORT); + EntryProcessor proc(decisions); + map.chunkedAll(proc, "foo", 100); + std::string expected("11 - A(4, 6, 0)\n" + "14 - A(42, 0, 0)\n"); + CPPUNIT_ASSERT_EQUAL(expected, proc.toString()); +} + +namespace { + struct LoadGiver : public document::Runnable { + typedef std::shared_ptr<LoadGiver> SP; + Map& _map; + uint32_t _counter; + + LoadGiver(Map& map) : _map(map), _counter(0) {} + }; + + struct InsertEraseLoadGiver : public LoadGiver { + InsertEraseLoadGiver(Map& map) : LoadGiver(map) {} + + void run() { + // Screws up order of buckets by xor'ing with 12345. + // Only operate on last 32k super buckets. + while (running()) { + uint32_t bucket = ((_counter ^ 12345) % 0x8000) + 0x8000; + if (bucket % 7 < 3) { + bool preExisted; + _map.insert(bucket, A(bucket, 0, _counter), "foo", + preExisted); + } + if (bucket % 5 < 2) { + _map.erase(bucket, "foo"); + } + ++_counter; + } + } + }; + + struct GetLoadGiver : public LoadGiver { + GetLoadGiver(Map& map) : LoadGiver(map) {} + + void run() { + // It's legal to keep entries as long as you only request higher + // buckets. So, to test this, keep entries until you request one + // that is smaller than those stored. + std::vector<std::pair<uint32_t, Map::WrappedEntry> > stored; + while (running()) { + uint32_t bucket = (_counter ^ 52721) % 0x10000; + if (!stored.empty() && stored.back().first > bucket) { + stored.clear(); + } + stored.push_back(std::pair<uint32_t, Map::WrappedEntry>( + bucket, _map.get(bucket, "foo", _counter % 3 == 0))); + ++_counter; + } + } + }; + + struct AllLoadGiver : public LoadGiver { + AllLoadGiver(Map& map) : LoadGiver(map) {} + + void run() { + while (running()) { + _map.all(*this, "foo"); + ++_counter; + } + } + + Map::Decision operator()(int key, A& a) { + //std::cerr << (void*) this << " - " << key << "\n"; + (void) key; + ++a._val2; + return Map::CONTINUE; + } + }; + + struct EachLoadGiver : public LoadGiver { + EachLoadGiver(Map& map) : LoadGiver(map) {} + + void run() { + while (running()) { + _map.each(*this, "foo"); + ++_counter; + } + } + + Map::Decision operator()(int key, A& a) { + //std::cerr << (void*) this << " - " << key << "\n"; + (void) key; + ++a._val2; + return Map::CONTINUE; + } + }; + + struct RandomRangeLoadGiver : public LoadGiver { + RandomRangeLoadGiver(Map& map) : LoadGiver(map) {} + + void run() { + while (running()) { + uint32_t min = (_counter ^ 23426) % 0x10000; + uint32_t max = (_counter ^ 40612) % 0x10000; + if (min > max) { + uint32_t tmp = min; + min = max; + max = tmp; + } + if (_counter % 7 < 5) { + _map.each(*this, "foo", min, max); + } else { + _map.all(*this, "foo", min, max); + } + ++_counter; + } + } + + Map::Decision operator()(int key, A& a) { + //std::cerr << "."; + (void) key; + ++a._val2; + return Map::CONTINUE; + } + }; + + struct GetNextLoadGiver : public LoadGiver { + GetNextLoadGiver(Map& map) : LoadGiver(map) {} + + void run() { + while (running()) { + uint32_t bucket = (_counter ^ 60417) % 0xffff; + if (_counter % 7 < 5) { + _map.each(*this, "foo", bucket + 1, 0xffff); + } else { + _map.all(*this, "foo", bucket + 1, 0xffff); + } + ++_counter; + } + } + + Map::Decision operator()(int key, A& a) { + //std::cerr << "."; + (void) key; + ++a._val2; + return Map::ABORT; + } + }; +} + +void +LockableMapTest::testThreadSafetyStress() { + uint32_t duration = 2 * 1000; + std::cerr << "\nRunning LockableMap threadsafety test for " + << (duration / 1000) << " seconds.\n"; + // Set up multiple threads going through the bucket database at the same + // time. Ensuring all works and there are no deadlocks. + + // Initial database of 32k elements which should always be present. + // Next 32k elements may exist (loadgivers may erase and create them, "foo") + Map map; + for (uint32_t i=0; i<65536; ++i) { + bool preExisted; + map.insert(i, A(i, 0, i ^ 12345), "foo", preExisted); + } + std::vector<LoadGiver::SP> loadgivers; + for (uint32_t i=0; i<8; ++i) { + loadgivers.push_back(LoadGiver::SP(new InsertEraseLoadGiver(map))); + } + for (uint32_t i=0; i<2; ++i) { + loadgivers.push_back(LoadGiver::SP(new GetLoadGiver(map))); + } + for (uint32_t i=0; i<2; ++i) { + loadgivers.push_back(LoadGiver::SP(new AllLoadGiver(map))); + } + for (uint32_t i=0; i<2; ++i) { + loadgivers.push_back(LoadGiver::SP(new EachLoadGiver(map))); + } + for (uint32_t i=0; i<2; ++i) { + loadgivers.push_back(LoadGiver::SP(new RandomRangeLoadGiver(map))); + } + for (uint32_t i=0; i<2; ++i) { + loadgivers.push_back(LoadGiver::SP(new GetNextLoadGiver(map))); + } + + FastOS_ThreadPool pool(128 * 1024); + for (uint32_t i=0; i<loadgivers.size(); ++i) { + CPPUNIT_ASSERT(loadgivers[i]->start(pool)); + } + FastOS_Thread::Sleep(duration); + std::cerr << "Closing down test\n"; + for (uint32_t i=0; i<loadgivers.size(); ++i) { + CPPUNIT_ASSERT(loadgivers[i]->stop()); + } +// FastOS_Thread::Sleep(duration); +// std::cerr << "Didn't manage to shut down\n"; +// map._lockedKeys.print(std::cerr, true, ""); + + for (uint32_t i=0; i<loadgivers.size(); ++i) { + CPPUNIT_ASSERT(loadgivers[i]->join()); + } + std::cerr << "Loadgiver counts:"; + for (uint32_t i=0; i<loadgivers.size(); ++i) { + std::cerr << " " << loadgivers[i]->_counter; + } + std::cerr << "\nTest completed\n"; +} + +#if 0 +namespace { +struct Hex { + document::BucketId::Type val; + + Hex(document::BucketId::Type v) : val(v) {} + bool operator==(const Hex& h) const { return val == h.val; } +}; + +std::ostream& operator<<(std::ostream& out, const Hex& h) { + out << std::hex << h.val << std::dec; + return out; +} + +void +printBucket(const std::string s, const document::BucketId& b) { + std::cerr << s << "bucket=" << b << ", reversed=" << b.stripUnused().toKey() << ", hex=" << Hex(b.stripUnused().toKey()) << "\n"; +} + +void +printBuckets(const std::map<document::BucketId, Map::WrappedEntry>& results) { + for (std::map<document::BucketId, Map::WrappedEntry>::const_iterator iter = results.begin(); + iter != results.end(); + iter++) { + printBucket("Returned ", iter->first); + } +} + +} +#endif + +void +LockableMapTest::testFindBucketsSimple() { +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(17, 0x0ffff); + id1 = id1.stripUnused(); + + document::BucketId id2(18, 0x1ffff); + id2 = id2.stripUnused(); + + document::BucketId id3(18, 0x3ffff); + id3 = id3.stripUnused(); + + bool preExisted; + map.insert(id1.toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.toKey(), A(2,3,4), "foo", preExisted); + map.insert(id3.toKey(), A(3,4,5), "foo", preExisted); + + document::BucketId id(22, 0xfffff); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getContained(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)1, results.size()); + CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3]); +#endif +} + +void +LockableMapTest::testFindBuckets() { +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(16, 0x0ffff); + document::BucketId id2(17, 0x0ffff); + document::BucketId id3(17, 0x1ffff); + document::BucketId id4(19, 0xfffff); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); + map.insert(id4.stripUnused().toKey(), A(4,5,6), "foo", preExisted); + + document::BucketId id(22, 0xfffff); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getContained(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)3, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); + CPPUNIT_ASSERT_EQUAL(A(4,5,6), *results[id4.stripUnused()]); + CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); +#endif +} + +void +LockableMapTest::testFindBuckets2() { // ticket 3121525 +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(16, 0x0ffff); + document::BucketId id2(17, 0x0ffff); + document::BucketId id3(17, 0x1ffff); + document::BucketId id4(18, 0x1ffff); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); + map.insert(id4.stripUnused().toKey(), A(4,5,6), "foo", preExisted); + + document::BucketId id(22, 0x1ffff); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getContained(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)3, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); + CPPUNIT_ASSERT_EQUAL(A(4,5,6), *results[id4.stripUnused()]); + CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); +#endif +} + +void +LockableMapTest::testFindBuckets3() { // ticket 3121525 +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(16, 0x0ffff); + document::BucketId id2(17, 0x0ffff); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + + document::BucketId id(22, 0x1ffff); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getContained(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)1, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); +#endif +} + +void +LockableMapTest::testFindBuckets4() { // ticket 3121525 +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(16, 0x0ffff); + document::BucketId id2(17, 0x0ffff); + document::BucketId id3(19, 0x1ffff); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); + + document::BucketId id(18, 0x1ffff); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getContained(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)1, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); +#endif +} + +void +LockableMapTest::testFindBuckets5() { // ticket 3121525 +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(16, 0x0ffff); + document::BucketId id2(17, 0x0ffff); + document::BucketId id3(19, 0x5ffff); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); + + document::BucketId id(18, 0x1ffff); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getContained(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)1, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); +#endif +} + +void +LockableMapTest::testFindNoBuckets() { +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id(16, 0x0ffff); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getAll(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)0, results.size()); +#endif +} + +void +LockableMapTest::testFindAll() { +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(16, 0x0aaaa); // contains id2-id7 + document::BucketId id2(17, 0x0aaaa); // contains id3-id4 + document::BucketId id3(20, 0xcaaaa); + document::BucketId id4(20, 0xeaaaa); + document::BucketId id5(17, 0x1aaaa); // contains id6-id7 + document::BucketId id6(20, 0xdaaaa); + document::BucketId id7(20, 0xfaaaa); + document::BucketId id8(20, 0xceaaa); + document::BucketId id9(17, 0x1ffff); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); + map.insert(id4.stripUnused().toKey(), A(4,5,6), "foo", preExisted); + map.insert(id5.stripUnused().toKey(), A(5,6,7), "foo", preExisted); + map.insert(id6.stripUnused().toKey(), A(6,7,8), "foo", preExisted); + map.insert(id7.stripUnused().toKey(), A(7,8,9), "foo", preExisted); + map.insert(id8.stripUnused().toKey(), A(8,9,10), "foo", preExisted); + map.insert(id9.stripUnused().toKey(), A(9,10,11), "foo", preExisted); + //printBucket("Inserted ", id1); + //printBucket("Inserted ", id2); + //printBucket("Inserted ", id3); + //printBucket("Inserted ", id4); + //printBucket("Inserted ", id5); + //printBucket("Inserted ", id6); + //printBucket("Inserted ", id7); + //printBucket("Inserted ", id8); + //printBucket("Inserted ", id9); + + document::BucketId id(17, 0x1aaaa); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getAll(id, "foo"); + + //std::cerr << "Done: getAll() for bucket " << id << "\n"; + //printBuckets(results); + + CPPUNIT_ASSERT_EQUAL((size_t)4, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // super bucket + CPPUNIT_ASSERT_EQUAL(A(5,6,7), *results[id5.stripUnused()]); // most specific match (exact match) + CPPUNIT_ASSERT_EQUAL(A(6,7,8), *results[id6.stripUnused()]); // sub bucket + CPPUNIT_ASSERT_EQUAL(A(7,8,9), *results[id7.stripUnused()]); // sub bucket + + id = document::BucketId(16, 0xffff); + results = map.getAll(id, "foo"); + + //std::cerr << "Done: getAll() for bucket " << id << "\n"; + //printBuckets(results); + + CPPUNIT_ASSERT_EQUAL((size_t)1, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(9,10,11), *results[id9.stripUnused()]); // sub bucket +#endif +} + +void +LockableMapTest::testFindAll2() { // Ticket 3121525 +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(17, 0x00001); + document::BucketId id2(17, 0x10001); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + + document::BucketId id(16, 0x00001); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getAll(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)2, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // sub bucket + CPPUNIT_ASSERT_EQUAL(A(2,3,4), *results[id2.stripUnused()]); // sub bucket +#endif +} + +void +LockableMapTest::testFindAllUnusedBitIsSet() { // ticket 2938896 +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(24, 0x000dc7089); + document::BucketId id2(33, 0x0053c7089); + document::BucketId id3(33, 0x1053c7089); + document::BucketId id4(24, 0x000bc7089); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); + map.insert(id4.stripUnused().toKey(), A(4,5,6), "foo", preExisted); + + document::BucketId id(33, 0x1053c7089); + id.setUsedBits(32); // Bit 33 is set, but unused + std::map<document::BucketId, Map::WrappedEntry> results = + map.getAll(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)2, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(2,3,4), *results[id2.stripUnused()]); // sub bucket + CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); // sub bucket +#endif +} + +void +LockableMapTest::testFindAllInconsistentlySplit() { // Ticket 2938896 +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(16, 0x00001); // contains id2-id3 + document::BucketId id2(17, 0x00001); + document::BucketId id3(17, 0x10001); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); + + document::BucketId id(16, 0x00001); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getAll(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)3, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // most specific match (exact match) + CPPUNIT_ASSERT_EQUAL(A(2,3,4), *results[id2.stripUnused()]); // sub bucket + CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); // sub bucket +#endif +} + +void +LockableMapTest::testFindAllInconsistentlySplit2() { // ticket 3121525 +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(17, 0x10000); + document::BucketId id2(27, 0x007228034); // contains id3 + document::BucketId id3(29, 0x007228034); + document::BucketId id4(17, 0x1ffff); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); + map.insert(id4.stripUnused().toKey(), A(4,5,6), "foo", preExisted); + + document::BucketId id(32, 0x027228034); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getAll(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)2, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(2,3,4), *results[id2.stripUnused()]); // super bucket + CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); // most specific match (super bucket) +#endif +} + +void +LockableMapTest::testFindAllInconsistentlySplit3() { // ticket 3121525 +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(16, 0x0ffff); // contains id2 + document::BucketId id2(17, 0x0ffff); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + + document::BucketId id(22, 0x1ffff); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getAll(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)1, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // super bucket +#endif +} + +void +LockableMapTest::testFindAllInconsistentlySplit4() { // ticket 3121525 +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(16, 0x0ffff); // contains id2-id3 + document::BucketId id2(17, 0x0ffff); + document::BucketId id3(19, 0x1ffff); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); + + document::BucketId id(18, 0x1ffff); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getAll(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)2, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // super bucket + CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); // sub bucket +#endif +} + +void +LockableMapTest::testFindAllInconsistentlySplit5() { // ticket 3121525 +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(16, 0x0ffff); // contains id2-id3 + document::BucketId id2(17, 0x0ffff); + document::BucketId id3(19, 0x5ffff); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); + + document::BucketId id(18, 0x1ffff); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getAll(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)2, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // super bucket + CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); // sub bucket +#endif +} + +void +LockableMapTest::testFindAllInconsistentlySplit6() { + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(16, 0x0ffff); // contains id2-id3 + document::BucketId id2(18, 0x1ffff); + document::BucketId id3(19, 0x7ffff); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); + + document::BucketId id(18, 0x3ffff); + std::map<document::BucketId, Map::WrappedEntry> results = + map.getAll(id, "foo"); + + CPPUNIT_ASSERT_EQUAL((size_t)2, results.size()); + + CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // super bucket + CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); // sub bucket +} + +void +LockableMapTest::testFindAllInconsistentBelow16Bits() +{ + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + + document::BucketId id1(1, 0x1); // contains id2-id3 + document::BucketId id2(3, 0x1); + document::BucketId id3(4, 0xD); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); + map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); + + document::BucketId id(3, 0x5); + + std::map<document::BucketId, Map::WrappedEntry> results = + map.getAll(id, "foo"); + + CPPUNIT_ASSERT_EQUAL(size_t(2), results.size()); + + CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // super bucket + CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); // sub bucket +} + +void +LockableMapTest::testCreate() { +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + { + document::BucketId id1(58, 0x43d6c878000004d2ull); + + std::map<document::BucketId, Map::WrappedEntry> entries( + map.getContained(id1, "foo")); + + CPPUNIT_ASSERT_EQUAL((size_t)0, entries.size()); + + Map::WrappedEntry entry = map.createAppropriateBucket(36, "", id1); + CPPUNIT_ASSERT_EQUAL(document::BucketId(36,0x8000004d2ull), + entry.getBucketId()); + } + { + document::BucketId id1(58, 0x423bf1e0000004d2ull); + + std::map<document::BucketId, Map::WrappedEntry> entries( + map.getContained(id1, "foo")); + CPPUNIT_ASSERT_EQUAL((size_t)0, entries.size()); + + Map::WrappedEntry entry = map.createAppropriateBucket(36, "", id1); + CPPUNIT_ASSERT_EQUAL(document::BucketId(36,0x0000004d2ull), + entry.getBucketId()); + } + + CPPUNIT_ASSERT_EQUAL((size_t)2, map.size()); +#endif +} + +void +LockableMapTest::testCreate2() { +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + { + document::BucketId id1(58, 0xeaf77782000004d2); + Map::WrappedEntry entry( + map.get(id1.stripUnused().toKey(), "foo", true)); + } + { + document::BucketId id1(58, 0x00000000000004d2); + std::map<document::BucketId, Map::WrappedEntry> entries( + map.getContained(id1, "foo")); + + CPPUNIT_ASSERT_EQUAL((size_t)0, entries.size()); + + Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); + + CPPUNIT_ASSERT_EQUAL(document::BucketId(34, 0x0000004d2ull), + entry.getBucketId()); + } + + CPPUNIT_ASSERT_EQUAL((size_t)2, map.size()); +#endif +} + +void +LockableMapTest::testCreate3() { +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + { + document::BucketId id1(58, 0xeaf77780000004d2); + Map::WrappedEntry entry( + map.get(id1.stripUnused().toKey(), "foo", true)); + } + { + document::BucketId id1(58, 0xeaf77782000004d2); + Map::WrappedEntry entry( + map.get(id1.stripUnused().toKey(), "foo", true)); + } + { + document::BucketId id1(58, 0x00000000000004d2); + std::map<document::BucketId, Map::WrappedEntry> entries( + map.getContained(id1, "foo")); + + CPPUNIT_ASSERT_EQUAL((size_t)0, entries.size()); + + Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); + CPPUNIT_ASSERT_EQUAL(document::BucketId(40, 0x0000004d2ull), + entry.getBucketId()); + } +#endif +} + +void +LockableMapTest::testCreate4() { +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + { + document::BucketId id1(16, 0x00000000000004d1); + Map::WrappedEntry entry( + map.get(id1.stripUnused().toKey(), "foo", true)); + } + { + document::BucketId id1(40, 0x00000000000004d2); + Map::WrappedEntry entry( + map.get(id1.stripUnused().toKey(), "foo", true)); + } + { + document::BucketId id1(58, 0x00000000010004d2); + Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); + + CPPUNIT_ASSERT_EQUAL(document::BucketId(25, 0x0010004d2ull), + entry.getBucketId()); + } +#endif +} + +void +LockableMapTest::testCreate6() { +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + { + document::BucketId id1(0x8c000000000004d2); + Map::WrappedEntry entry( + map.get(id1.stripUnused().toKey(), "foo", true)); + } + + { + document::BucketId id1(0xeb54b3ac000004d2); + Map::WrappedEntry entry( + map.get(id1.stripUnused().toKey(), "foo", true)); + } + + { + document::BucketId id1(0x88000002000004d2); + Map::WrappedEntry entry( + map.get(id1.stripUnused().toKey(), "foo", true)); + } + { + document::BucketId id1(0x84000001000004d2); + Map::WrappedEntry entry( + map.get(id1.stripUnused().toKey(), "foo", true)); + } + { + document::BucketId id1(0xe9944a44000004d2); + Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); + CPPUNIT_ASSERT_EQUAL(document::BucketId(0x90000004000004d2), + entry.getBucketId()); + } +#endif +} + + +void +LockableMapTest::testCreate5() { +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + { + document::BucketId id1(58, 0xeaf77780000004d2); + Map::WrappedEntry entry( + map.get(id1.stripUnused().toKey(), "foo", true)); + } + { + document::BucketId id1(40, 0x00000000000004d1); + + Map::WrappedEntry entry( + map.get(id1.stripUnused().toKey(), "foo", true)); + } + { + document::BucketId id1(58, 0x00000000010004d2); + Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); + CPPUNIT_ASSERT_EQUAL(document::BucketId(25, 0x0010004d2ull), + entry.getBucketId()); + } +#endif +} + +void +LockableMapTest::testCreateEmpty() { +#if __WORDSIZE == 64 + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + { + document::BucketId id1(58, 0x00000000010004d2); + Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x0000004d2ull), + entry.getBucketId()); + } +#endif +} + +void +LockableMapTest::testIsConsistent() +{ + typedef LockableMap<JudyMultiMap<A> > Map; + Map map; + document::BucketId id1(16, 0x00001); // contains id2-id3 + document::BucketId id2(17, 0x00001); + + bool preExisted; + map.insert(id1.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + { + Map::WrappedEntry entry( + map.get(id1.stripUnused().toKey(), "foo", true)); + CPPUNIT_ASSERT(map.isConsistent(entry)); + } + map.insert(id2.stripUnused().toKey(), A(1,2,3), "foo", preExisted); + { + Map::WrappedEntry entry( + map.get(id1.stripUnused().toKey(), "foo", true)); + CPPUNIT_ASSERT(!map.isConsistent(entry)); + } +} + +} // storage diff --git a/storage/src/tests/bucketmover/CMakeLists.txt b/storage/src/tests/bucketmover/CMakeLists.txt new file mode 100644 index 00000000000..2d02cdc4942 --- /dev/null +++ b/storage/src/tests/bucketmover/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storage_testbucketmover + SOURCES + bucketmovertest.cpp + htmltabletest.cpp + DEPENDS + AFTER + storage_storageconfig +) diff --git a/storage/src/tests/bucketmover/bucketmovertest.cpp b/storage/src/tests/bucketmover/bucketmovertest.cpp new file mode 100644 index 00000000000..2720e6bac2a --- /dev/null +++ b/storage/src/tests/bucketmover/bucketmovertest.cpp @@ -0,0 +1,190 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storage/bucketdb/storbucketdb.h> +#include <vespa/storage/common/bucketmessages.h> +#include <vespa/storage/bucketmover/bucketmover.h> +#include <tests/common/dummystoragelink.h> +#include <tests/common/testhelper.h> +#include <tests/common/teststorageapp.h> +#include <vespa/storage/bucketdb/storbucketdb.h> + +bool debug = false; + +namespace storage { +namespace bucketmover { + +struct BucketMoverTest : public CppUnit::TestFixture { +public: + void setUp(); + void tearDown(); + + void testNormalUsage(); + void testMaxPending(); + void testErrorHandling(); + + CPPUNIT_TEST_SUITE(BucketMoverTest); + CPPUNIT_TEST(testNormalUsage); + CPPUNIT_TEST(testMaxPending); + CPPUNIT_TEST(testErrorHandling); + CPPUNIT_TEST_SUITE_END(); + + std::unique_ptr<TestServiceLayerApp> _node; + std::unique_ptr<ServiceLayerComponent> _component; + std::unique_ptr<BucketMover> _bucketMover; + DummyStorageLink* after; + +private: + void addBucket(const document::BucketId& id, uint16_t idealDiff); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(BucketMoverTest); + +void +BucketMoverTest::tearDown() +{ + _node.reset(0); +} + +void +BucketMoverTest::setUp() +{ + try { + _node.reset(new TestServiceLayerApp(DiskCount(4))); + _node->setupDummyPersistence(); + } catch (config::InvalidConfigException& e) { + fprintf(stderr, "%s\n", e.what()); + } + + _component.reset(new ServiceLayerComponent(_node->getComponentRegister(), "foo")); + _bucketMover.reset(new BucketMover("raw:", _node->getComponentRegister())); + after = new DummyStorageLink(); + _bucketMover->push_back(StorageLink::UP(after)); +} + +void +BucketMoverTest::addBucket(const document::BucketId& id, + uint16_t idealDiff) +{ + StorBucketDatabase::WrappedEntry entry( + _component->getBucketDatabase().get( + id, + "", + StorBucketDatabase::CREATE_IF_NONEXISTING)); + + entry->setBucketInfo(api::BucketInfo(1,1,1)); + + uint16_t idealDisk = _component->getIdealPartition(id); + entry->disk = (idealDisk + idealDiff) % _component->getDiskCount(); + entry.write(); +} + +void +BucketMoverTest::testNormalUsage() +{ + for (uint32_t i = 1; i < 4; ++i) { + addBucket(document::BucketId(16, i), 1); + } + for (uint32_t i = 4; i < 6; ++i) { + addBucket(document::BucketId(16, i), 0); + } + + _bucketMover->open(); + _bucketMover->tick(); + + std::vector<api::StorageMessage::SP> msgs = after->getCommandsOnce(); + CPPUNIT_ASSERT_EQUAL( + std::string("BucketDiskMoveCommand(" + "BucketId(0x4000000000000002), source 3, target 2)"), + msgs[0]->toString()); + CPPUNIT_ASSERT_EQUAL( + std::string("BucketDiskMoveCommand(" + "BucketId(0x4000000000000001), source 2, target 1)"), + msgs[1]->toString()); + CPPUNIT_ASSERT_EQUAL( + std::string("BucketDiskMoveCommand(" + "BucketId(0x4000000000000003), source 1, target 0)"), + msgs[2]->toString()); + + for (uint32_t i = 0; i < 2; ++i) { + after->sendUp(std::shared_ptr<api::StorageMessage>( + ((api::StorageCommand*)msgs[i].get())-> + makeReply().release())); + } + + _bucketMover->tick(); + CPPUNIT_ASSERT_EQUAL(0, (int)after->getNumCommands()); + + _bucketMover->finishCurrentRun(); +} + +void +BucketMoverTest::testMaxPending() +{ + for (uint32_t i = 1; i < 100; ++i) { + addBucket(document::BucketId(16, i), 1); + } + for (uint32_t i = 101; i < 200; ++i) { + addBucket(document::BucketId(16, i), 0); + } + + _bucketMover->open(); + _bucketMover->tick(); + + std::vector<api::StorageMessage::SP> msgs = after->getCommandsOnce(); + // 5 is the max pending default config. + CPPUNIT_ASSERT_EQUAL(5, (int)msgs.size()); + + after->sendUp(std::shared_ptr<api::StorageMessage>( + ((api::StorageCommand*)msgs[3].get())-> + makeReply().release())); + + _bucketMover->tick(); + + std::vector<api::StorageMessage::SP> msgs2 = after->getCommandsOnce(); + CPPUNIT_ASSERT_EQUAL(1, (int)msgs2.size()); +} + +void +BucketMoverTest::testErrorHandling() +{ + for (uint32_t i = 1; i < 100; ++i) { + addBucket(document::BucketId(16, i), 1); + } + for (uint32_t i = 101; i < 200; ++i) { + addBucket(document::BucketId(16, i), 0); + } + + _bucketMover->open(); + _bucketMover->tick(); + + std::vector<api::StorageMessage::SP> msgs = after->getCommandsOnce(); + // 5 is the max pending default config. + CPPUNIT_ASSERT_EQUAL(5, (int)msgs.size()); + + BucketDiskMoveCommand& cmd = static_cast<BucketDiskMoveCommand&>(*msgs[0]); + uint32_t targetDisk = cmd.getDstDisk(); + + std::unique_ptr<api::StorageReply> reply(cmd.makeReply().release()); + reply->setResult(api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE, "foobar")); + after->sendUp(std::shared_ptr<api::StorageMessage>(reply.release())); + + for (uint32_t i = 1; i < msgs.size(); ++i) { + after->sendUp(std::shared_ptr<api::StorageMessage>( + ((api::StorageCommand*)msgs[i].get())-> + makeReply().release())); + } + + _bucketMover->tick(); + + std::vector<api::StorageMessage::SP> msgs2 = after->getCommandsOnce(); + CPPUNIT_ASSERT_EQUAL(5, (int)msgs2.size()); + + for (uint32_t i = 0; i < msgs2.size(); ++i) { + BucketDiskMoveCommand& bdm = static_cast<BucketDiskMoveCommand&>(*msgs2[i]); + CPPUNIT_ASSERT(bdm.getDstDisk() != targetDisk); + } +} + +} // bucketmover +} // storage diff --git a/storage/src/tests/bucketmover/htmltabletest.cpp b/storage/src/tests/bucketmover/htmltabletest.cpp new file mode 100644 index 00000000000..98cf68d489a --- /dev/null +++ b/storage/src/tests/bucketmover/htmltabletest.cpp @@ -0,0 +1,100 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storage/bucketmover/htmltable.h> +#include <tests/common/testhelper.h> + +namespace storage { + +struct HtmlTableTest : public CppUnit::TestFixture { + + void testPercentageColumn(); + void testByteSizeColumn(); + + CPPUNIT_TEST_SUITE(HtmlTableTest); + CPPUNIT_TEST(testPercentageColumn); + CPPUNIT_TEST(testByteSizeColumn); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(HtmlTableTest); + +void HtmlTableTest::testPercentageColumn() +{ + // With total hardcoded to 100 + { + HtmlTable table("disk"); + PercentageColumn perc("fillrate", 100); + perc.addColorLimit(70, Column::LIGHT_GREEN); + perc.addColorLimit(85, Column::LIGHT_YELLOW); + perc.addColorLimit(100, Column::LIGHT_RED); + table.addColumn(perc); + table.addRow(0); + table.addRow(1); + table.addRow(2); + perc[0] = 30; + perc[1] = 80; + perc[2] = 100; + std::ostringstream ost; + table.print(ost); + std::string expected( +"<table border=\"1\" cellpadding=\"2\" cellspacing=\"0\">\n" +"<tr><th>disk</th><th>fillrate</th></tr>\n" +"<tr><td>0</td><td bgcolor=\"#a0ffa0\" align=\"right\">30.00 %</td></tr>\n" +"<tr><td>1</td><td bgcolor=\"#ffffa0\" align=\"right\">80.00 %</td></tr>\n" +"<tr><td>2</td><td bgcolor=\"#ffa0a0\" align=\"right\">100.00 %</td></tr>\n" +"</table>\n"); + CPPUNIT_ASSERT_EQUAL(expected, ost.str()); + } + // With automatically gathered total + { + HtmlTable table("disk"); + PercentageColumn perc("fillrate"); + table.addColumn(perc); + table.addRow(0); + table.addRow(1); + table.addRow(2); + perc[0] = 30; + perc[1] = 80; + perc[2] = 100; + std::ostringstream ost; + table.print(ost); + std::string expected( + "<table border=\"1\" cellpadding=\"2\" cellspacing=\"0\">\n" + "<tr><th>disk</th><th>fillrate</th></tr>\n" + "<tr><td>0</td><td align=\"right\">14.29 %</td></tr>\n" + "<tr><td>1</td><td align=\"right\">38.10 %</td></tr>\n" + "<tr><td>2</td><td align=\"right\">47.62 %</td></tr>\n" + "</table>\n"); + CPPUNIT_ASSERT_EQUAL(expected, ost.str()); + } +} + +void HtmlTableTest::testByteSizeColumn() +{ + { + HtmlTable table("disk"); + ByteSizeColumn size("size"); + table.addColumn(size); + table.addRow(0); + table.addRow(1); + table.addRow(2); + // Biggest value enforce the denomination + size[0] = 42123; + size[1] = 124123151; + size[2] = 6131231; + std::ostringstream ost; + table.print(ost); + std::string expected( + "<table border=\"1\" cellpadding=\"2\" cellspacing=\"0\">\n" + "<tr><th>disk</th><th>size</th></tr>\n" + "<tr><td>0</td><td align=\"right\">0 MB</td></tr>\n" + "<tr><td>1</td><td align=\"right\">118 MB</td></tr>\n" + "<tr><td>2</td><td align=\"right\">5 MB</td></tr>\n" + "</table>\n"); + CPPUNIT_ASSERT_EQUAL(expected, ost.str()); + } + +} + +} // storage diff --git a/storage/src/tests/common/.gitignore b/storage/src/tests/common/.gitignore new file mode 100644 index 00000000000..333f254ba10 --- /dev/null +++ b/storage/src/tests/common/.gitignore @@ -0,0 +1,8 @@ +*.So +*.lo +.*.swp +.depend +.depend.NEW +.deps +.libs +Makefile diff --git a/storage/src/tests/common/CMakeLists.txt b/storage/src/tests/common/CMakeLists.txt new file mode 100644 index 00000000000..309308473e1 --- /dev/null +++ b/storage/src/tests/common/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storage_testcommon + SOURCES + dummystoragelink.cpp + testhelper.cpp + metricstest.cpp + storagelinktest.cpp + teststorageapp.cpp + DEPENDS + AFTER + storage_storageconfig +) diff --git a/storage/src/tests/common/dummystoragelink.cpp b/storage/src/tests/common/dummystoragelink.cpp new file mode 100644 index 00000000000..d05241cb5b5 --- /dev/null +++ b/storage/src/tests/common/dummystoragelink.cpp @@ -0,0 +1,191 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/storageframework/defaultimplementation/clock/realclock.h> +#include <sys/time.h> +#include "dummystoragelink.h" + +namespace storage { + +DummyStorageLink* DummyStorageLink::_last(0); + +DummyStorageLink::DummyStorageLink() + : StorageLink("Dummy storage link"), + _commands(), + _replies(), + _injected(), + _autoReply(false), + _useDispatch(false), + _ignore(false), + _waitMonitor() +{ + _last = this; +} + +DummyStorageLink::~DummyStorageLink() +{ + // Often a chain with dummy link on top is deleted in unit tests. + // If they haven't been closed already, close them for a cleaner + // shutdown + if (getState() == OPENED) { + close(); + flush(); + } + closeNextLink(); + reset(); +} + +bool +DummyStorageLink::handleInjectedReply() +{ + vespalib::LockGuard guard(_lock); + if (!_injected.empty()) { + sendUp(*_injected.begin()); + _injected.pop_front(); + return true; + } + return false; +} + +bool DummyStorageLink::onDown(const api::StorageMessage::SP& cmd) +{ + if (_ignore) { + return false; + } + bool injected = handleInjectedReply(); + if (!injected && _autoReply) { + if (!cmd->getType().isReply()) { + std::shared_ptr<api::StorageReply> reply( + std::dynamic_pointer_cast<api::StorageCommand>(cmd) + ->makeReply().release()); + reply->setResult(api::ReturnCode( + api::ReturnCode::OK, "Automatically generated reply")); + sendUp(reply); + } + } + if (isBottom()) { + vespalib::MonitorGuard lock(_waitMonitor); + { + vespalib::LockGuard guard(_lock); + _commands.push_back(cmd); + } + lock.broadcast(); + return true; + } + return StorageLink::onDown(cmd); +} + +bool DummyStorageLink::onUp(const api::StorageMessage::SP& reply) { + if (isTop()) { + vespalib::MonitorGuard lock(_waitMonitor); + { + vespalib::LockGuard guard(_lock); + _replies.push_back(reply); + } + lock.broadcast(); + return true; + } + return StorageLink::onUp(reply); + +} + +void DummyStorageLink::injectReply(api::StorageReply* reply) +{ + assert(reply); + vespalib::LockGuard guard(_lock); + _injected.push_back(std::shared_ptr<api::StorageReply>(reply)); +} + +void DummyStorageLink::reset() { + vespalib::MonitorGuard lock(_waitMonitor); + vespalib::LockGuard guard(_lock); + _commands.clear(); + _replies.clear(); + _injected.clear(); +} + +void DummyStorageLink::waitForMessages(unsigned int msgCount, int timeout) +{ + framework::defaultimplementation::RealClock clock; + framework::MilliSecTime endTime( + clock.getTimeInMillis() + framework::MilliSecTime(timeout * 1000)); + vespalib::MonitorGuard lock(_waitMonitor); + while (_commands.size() + _replies.size() < msgCount) { + if (timeout != 0 && clock.getTimeInMillis() > endTime) { + std::ostringstream ost; + ost << "Timed out waiting for " << msgCount << " messages to " + << "arrive in dummy storage link. Only " + << (_commands.size() + _replies.size()) << " messages seen " + << "after timout of " << timeout << " seconds was reached."; + throw vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); + } + if (timeout >= 0) { + lock.wait((endTime - clock.getTimeInMillis()).getTime()); + } else { + lock.wait(); + } + } +} + +void DummyStorageLink::waitForMessage(const api::MessageType& type, int timeout) +{ + framework::defaultimplementation::RealClock clock; + framework::MilliSecTime endTime( + clock.getTimeInMillis() + framework::MilliSecTime(timeout * 1000)); + vespalib::MonitorGuard lock(_waitMonitor); + while (true) { + for (uint32_t i=0; i<_commands.size(); ++i) { + if (_commands[i]->getType() == type) return; + } + for (uint32_t i=0; i<_replies.size(); ++i) { + if (_replies[i]->getType() == type) return; + } + if (timeout != 0 && clock.getTimeInMillis() > endTime) { + std::ostringstream ost; + ost << "Timed out waiting for " << type << " message to " + << "arrive in dummy storage link. Only " + << (_commands.size() + _replies.size()) << " messages seen " + << "after timout of " << timeout << " seconds was reached."; + if (_commands.size() == 1) { + ost << " Found command of type " << _commands[0]->getType(); + } + if (_replies.size() == 1) { + ost << " Found command of type " << _replies[0]->getType(); + } + throw vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); + } + if (timeout >= 0) { + lock.wait((endTime - clock.getTimeInMillis()).getTime()); + } else { + lock.wait(); + } + } +} + +api::StorageMessage::SP +DummyStorageLink::getAndRemoveMessage(const api::MessageType& type) +{ + vespalib::MonitorGuard lock(_waitMonitor); + for (std::vector<api::StorageMessage::SP>::iterator it = _commands.begin(); + it != _commands.end(); ++it) + { + if ((*it)->getType() == type) { + api::StorageMessage::SP result(*it); + _commands.erase(it); + return result; + } + } + for (std::vector<api::StorageMessage::SP>::iterator it = _replies.begin(); + it != _replies.end(); ++it) + { + if ((*it)->getType() == type) { + api::StorageMessage::SP result(*it); + _replies.erase(it); + return result; + } + } + std::ostringstream ost; + ost << "No message of type " << type << " found."; + throw vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); +} + +} // storage diff --git a/storage/src/tests/common/dummystoragelink.h b/storage/src/tests/common/dummystoragelink.h new file mode 100644 index 00000000000..072d961cbc0 --- /dev/null +++ b/storage/src/tests/common/dummystoragelink.h @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/sync.h> +#include <list> +#include <sstream> +#include <vespa/storageapi/messageapi/storagecommand.h> +#include <string> +#include <vector> +#include <vespa/storage/common/storagelink.h> +#include <vespa/storage/common/bucketmessages.h> +#include <vespa/storageapi/message/internal.h> + +class FastOS_ThreadPool; + +namespace storage { + +class DummyStorageLink : public StorageLink { + + mutable vespalib::Lock _lock; // to protect below containers: + std::vector<api::StorageMessage::SP> _commands; + std::vector<api::StorageMessage::SP> _replies; + std::list<api::StorageMessage::SP> _injected; + + bool _autoReply; + bool _useDispatch; + bool _ignore; + static DummyStorageLink* _last; + vespalib::Monitor _waitMonitor; + +public: + DummyStorageLink(); + ~DummyStorageLink(); + + bool onDown(const api::StorageMessage::SP&); + bool onUp(const api::StorageMessage::SP&); + + void addOnTopOfChain(StorageLink& link) { + link.addTestLinkOnTop(this); + } + + void print(std::ostream& ost, bool verbose, const std::string& indent) const + { + (void) verbose; + ost << indent << "DummyStorageLink(" + << "autoreply = " << (_autoReply ? "on" : "off") + << ", dispatch = " << (_useDispatch ? "on" : "off") + << ", " << _commands.size() << " commands" + << ", " << _replies.size() << " replies"; + if (_injected.size() > 0) + ost << ", " << _injected.size() << " injected"; + ost << ")"; + } + + void injectReply(api::StorageReply* reply); + void reset(); + void setAutoreply(bool autoReply) { _autoReply = autoReply; } + void setIgnore(bool ignore) { _ignore = ignore; } + // Timeout is given in seconds + void waitForMessages(unsigned int msgCount = 1, int timeout = -1); + // Wait for a single message of a given type + void waitForMessage(const api::MessageType&, int timeout = -1); + + api::StorageMessage::SP getCommand(size_t i) const { + vespalib::LockGuard guard(_lock); + api::StorageMessage::SP ret = _commands[i]; + return ret; + } + api::StorageMessage::SP getReply(size_t i) const { + vespalib::LockGuard guard(_lock); + api::StorageMessage::SP ret = _replies[i]; + return ret; + } + size_t getNumCommands() const { + vespalib::LockGuard guard(_lock); + return _commands.size(); + } + size_t getNumReplies() const { + vespalib::LockGuard guard(_lock); + return _replies.size(); + } + + const std::vector<api::StorageMessage::SP>& getCommands() const + { return _commands; } + const std::vector<api::StorageMessage::SP>& getReplies() const + { return _replies; } + + std::vector<api::StorageMessage::SP> getCommandsOnce() { + vespalib::MonitorGuard lock(_waitMonitor); + std::vector<api::StorageMessage::SP> retval; + { + vespalib::LockGuard guard(_lock); + retval.swap(_commands); + } + return retval; + } + + std::vector<api::StorageMessage::SP> getRepliesOnce() { + vespalib::MonitorGuard lock(_waitMonitor); + std::vector<api::StorageMessage::SP> retval; + { + vespalib::LockGuard guard(_lock); + retval.swap(_replies); + } + return retval; + } + + api::StorageMessage::SP getAndRemoveMessage(const api::MessageType&); + + static DummyStorageLink* getLast() { return _last; } +private: + /** + * Auto-reply with an injected message if one is available and return + * whether such an injection took place. + */ + bool handleInjectedReply(); +}; + +} + diff --git a/storage/src/tests/common/hostreporter/CMakeLists.txt b/storage/src/tests/common/hostreporter/CMakeLists.txt new file mode 100644 index 00000000000..f0cb197c5e2 --- /dev/null +++ b/storage/src/tests/common/hostreporter/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storage_testhostreporter + SOURCES + cpureportertest.cpp + memreportertest.cpp + networkreportertest.cpp + versionreportertest.cpp + diskreportertest.cpp + util.cpp + hostinfotest.cpp + DEPENDS + AFTER + storage_storageconfig +) diff --git a/storage/src/tests/common/hostreporter/cpureportertest.cpp b/storage/src/tests/common/hostreporter/cpureportertest.cpp new file mode 100644 index 00000000000..56a929c3aff --- /dev/null +++ b/storage/src/tests/common/hostreporter/cpureportertest.cpp @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/storage/common/hostreporter/cpureporter.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/util/jsonstream.h> +#include "util.h" + +LOG_SETUP(".test.cpureporter"); + +namespace storage { +namespace { +using Object = vespalib::JsonStream::Object; +using End = vespalib::JsonStream::End; +} + +struct CpuReporterTest : public CppUnit::TestFixture +{ + void testCpuReporter(); + + CPPUNIT_TEST_SUITE(CpuReporterTest); + CPPUNIT_TEST(testCpuReporter); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(CpuReporterTest); + +void +CpuReporterTest::testCpuReporter() +{ + CpuReporter cpuReporter; + vespalib::Slime slime; + util::reporterToSlime(cpuReporter, slime); + CPPUNIT_ASSERT(1.0 <= slime.get()["cpu"]["context switches"].asDouble()); + CPPUNIT_ASSERT(1.0 <= slime.get()["cpu"]["cputotal"]["user"].asDouble()); + CPPUNIT_ASSERT(1.0 <= slime.get()["cpu"]["cputotal"]["user"].asDouble()); + CPPUNIT_ASSERT(1.0 <= slime.get()["cpu"]["cputotal"]["user"].asDouble()); +} +} // storage diff --git a/storage/src/tests/common/hostreporter/diskreportertest.cpp b/storage/src/tests/common/hostreporter/diskreportertest.cpp new file mode 100644 index 00000000000..158a77c2e7e --- /dev/null +++ b/storage/src/tests/common/hostreporter/diskreportertest.cpp @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/storage/common/hostreporter/diskreporter.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/util/jsonstream.h> +#include "util.h" + +LOG_SETUP(".test.diskreporter"); + +namespace storage { + +struct DiskReporterTest : public CppUnit::TestFixture +{ + void testDiskReporter(); + + CPPUNIT_TEST_SUITE(DiskReporterTest); + CPPUNIT_TEST(testDiskReporter); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(DiskReporterTest); + +void +DiskReporterTest::testDiskReporter() +{ + DiskReporter diskReporter; + vespalib::Slime slime; + util::reporterToSlime(diskReporter, slime); + CPPUNIT_ASSERT(0 < slime.get()["disk"].toString().size()); +} +} // storage diff --git a/storage/src/tests/common/hostreporter/hostinfotest.cpp b/storage/src/tests/common/hostreporter/hostinfotest.cpp new file mode 100644 index 00000000000..99954c19840 --- /dev/null +++ b/storage/src/tests/common/hostreporter/hostinfotest.cpp @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/storage/common/hostreporter/hostinfo.h> +#include <vespa/storage/common/hostreporter/hostreporter.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/util/jsonstream.h> +#include "util.h" + +LOG_SETUP(".test.hostinforeporter"); + +namespace storage { +namespace { +using Object = vespalib::JsonStream::Object; +using End = vespalib::JsonStream::End; +using JsonFormat = vespalib::slime::JsonFormat; +using Memory = vespalib::slime::Memory; + +class DummyReporter: public HostReporter { +public: + void report(vespalib::JsonStream& jsonreport) override { + jsonreport << "dummy" << Object() << "foo" << "bar" << End(); + } +}; +} + +struct HostInfoReporterTest : public CppUnit::TestFixture +{ + void testHostInfoReporter(); + + CPPUNIT_TEST_SUITE(HostInfoReporterTest); + CPPUNIT_TEST(testHostInfoReporter); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(HostInfoReporterTest); + +void +HostInfoReporterTest::testHostInfoReporter() +{ + HostInfo hostinfo; + DummyReporter dummyReporter; + hostinfo.registerReporter(&dummyReporter); + vespalib::asciistream json; + vespalib::JsonStream stream(json, true); + + stream << Object(); + hostinfo.printReport(stream); + stream << End(); + + std::string jsonData = json.str(); + vespalib::Slime slime; + JsonFormat::decode(Memory(jsonData), slime); + CPPUNIT_ASSERT(slime.get()["dummy"]["foo"].asString() == "bar"); + CPPUNIT_ASSERT(0 < slime.get()["network"]["lo"]["input"]["packets"].asLong()); + CPPUNIT_ASSERT(1.0 <= slime.get()["cpu"]["context switches"].asDouble()); +} +} // storage + diff --git a/storage/src/tests/common/hostreporter/memreportertest.cpp b/storage/src/tests/common/hostreporter/memreportertest.cpp new file mode 100644 index 00000000000..3eedfd48a3c --- /dev/null +++ b/storage/src/tests/common/hostreporter/memreportertest.cpp @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/storage/common/hostreporter/memreporter.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/util/jsonstream.h> +#include "util.h" + +LOG_SETUP(".test.memreporter"); + +namespace storage { +namespace { +using Object = vespalib::JsonStream::Object; +using End = vespalib::JsonStream::End; +} + +struct MemReporterTest : public CppUnit::TestFixture +{ + void testMemReporter(); + + CPPUNIT_TEST_SUITE(MemReporterTest); + CPPUNIT_TEST(testMemReporter); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(MemReporterTest); + +void +MemReporterTest::testMemReporter() +{ + MemReporter memReporter; + vespalib::Slime slime; + util::reporterToSlime(memReporter, slime); + CPPUNIT_ASSERT(0 < slime.get()["memory"]["total memory"].asLong()); + CPPUNIT_ASSERT(0 < slime.get()["memory"]["free memory"].asLong()); + CPPUNIT_ASSERT(0 < slime.get()["memory"]["disk cache"].asLong()); + CPPUNIT_ASSERT(0 < slime.get()["memory"]["active memory"].asLong()); + CPPUNIT_ASSERT(0 < slime.get()["memory"]["inactive memory"].asLong()); + CPPUNIT_ASSERT(0 <= slime.get()["memory"]["swap total"].asLong()); + CPPUNIT_ASSERT(0 <= slime.get()["memory"]["swap free"].asLong()); + CPPUNIT_ASSERT(0 < slime.get()["memory"]["dirty"].asLong()); +} +} // storage diff --git a/storage/src/tests/common/hostreporter/networkreportertest.cpp b/storage/src/tests/common/hostreporter/networkreportertest.cpp new file mode 100644 index 00000000000..cba5717adce --- /dev/null +++ b/storage/src/tests/common/hostreporter/networkreportertest.cpp @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/storage/common/hostreporter/networkreporter.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/util/jsonstream.h> +#include "util.h" + +LOG_SETUP(".test.networkreporter"); + +namespace storage { +namespace { +using Object = vespalib::JsonStream::Object; +using End = vespalib::JsonStream::End; +} + +struct NetworkReporterTest : public CppUnit::TestFixture +{ + void testNetworkReporter(); + + CPPUNIT_TEST_SUITE(NetworkReporterTest); + CPPUNIT_TEST(testNetworkReporter); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(NetworkReporterTest); + +void +NetworkReporterTest::testNetworkReporter() +{ + NetworkReporter networkReporter; + vespalib::Slime slime; + util::reporterToSlime(networkReporter, slime); + CPPUNIT_ASSERT(0 < slime.get()["network"]["lo"]["input"]["bytes"].asLong()); + CPPUNIT_ASSERT(0 < slime.get()["network"]["lo"]["input"]["packets"].asLong()); + CPPUNIT_ASSERT(0 < slime.get()["network"]["lo"]["output"]["bytes"].asLong()); + CPPUNIT_ASSERT(0 < slime.get()["network"]["lo"]["output"]["packets"].asLong()); +} +} // storage diff --git a/storage/src/tests/common/hostreporter/util.cpp b/storage/src/tests/common/hostreporter/util.cpp new file mode 100644 index 00000000000..37d5803070d --- /dev/null +++ b/storage/src/tests/common/hostreporter/util.cpp @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "util.h" +#include <vespa/storage/common/hostreporter/hostreporter.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/util/jsonstream.h> + +namespace storage { +namespace util { +namespace { +using Object = vespalib::JsonStream::Object; +using End = vespalib::JsonStream::End; +using JsonFormat = vespalib::slime::JsonFormat; +using Memory = vespalib::slime::Memory; +} + +void +reporterToSlime(HostReporter &hostReporter, vespalib::Slime &slime) { + vespalib::asciistream json; + vespalib::JsonStream stream(json, true); + + stream << Object(); + hostReporter.report(stream); + stream << End(); + std::string jsonData = json.str(); + size_t parsedSize = JsonFormat::decode(Memory(jsonData), slime); + + if (jsonData.size() != parsedSize) { + CPPUNIT_FAIL("Sizes of jsonData mismatched, probably not json:\n" + jsonData); + } +} +} +} diff --git a/storage/src/tests/common/hostreporter/util.h b/storage/src/tests/common/hostreporter/util.h new file mode 100644 index 00000000000..e7fcf418bd3 --- /dev/null +++ b/storage/src/tests/common/hostreporter/util.h @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#ifndef VESPA_STORAGE_COMMON_UTIL +#define VESPA_STORAGE_COMMON_UTIL + +#include <vespa/storage/common/hostreporter/hostreporter.h> +#include <vespa/vespalib/data/slime/slime.h> + +namespace storage { +namespace util { + +void +reporterToSlime(HostReporter &hostReporter, vespalib::Slime &slime); +} +} + +#endif // VESPA_STORAGE_COMMON_UTIL diff --git a/storage/src/tests/common/hostreporter/versionreportertest.cpp b/storage/src/tests/common/hostreporter/versionreportertest.cpp new file mode 100644 index 00000000000..43c6e64b0de --- /dev/null +++ b/storage/src/tests/common/hostreporter/versionreportertest.cpp @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/storage/common/hostreporter/versionreporter.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/util/jsonstream.h> +#include "util.h" + +LOG_SETUP(".test.versionreporter"); + +namespace storage { +namespace { +using Object = vespalib::JsonStream::Object; +using End = vespalib::JsonStream::End; +} + +struct VersionReporterTest : public CppUnit::TestFixture +{ + void testVersionReporter(); + + CPPUNIT_TEST_SUITE(VersionReporterTest); + CPPUNIT_TEST(testVersionReporter); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(VersionReporterTest); + +void +VersionReporterTest::testVersionReporter() +{ + VersionReporter versionReporter; + vespalib::Slime slime; + util::reporterToSlime(versionReporter, slime); + std::string version = slime.get()["vtag"]["version"].asString().make_string().c_str(); + CPPUNIT_ASSERT(version.length() > 2); + CPPUNIT_ASSERT(version.find(".") > 0); +} +} // storage diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp new file mode 100644 index 00000000000..e06b2183380 --- /dev/null +++ b/storage/src/tests/common/metricstest.cpp @@ -0,0 +1,393 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/bucket/bucketid.h> +#include <vespa/log/log.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> +#include <vespa/storage/bucketdb/bucketmanager.h> +#include <vespa/storageframework/storageframework.h> +#include <vespa/storage/common/statusmetricconsumer.h> +#include <vespa/storage/persistence/filestorage/filestormanager.h> +#include <vespa/storage/visiting/visitormetrics.h> +#include <vespa/documentapi/loadtypes/loadtype.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/testhelper.h> +#include <tests/common/dummystoragelink.h> +#include <iostream> +#include <string> +#include <chrono> + +LOG_SETUP(".test.metrics"); + +namespace storage { + +struct MetricsTest : public CppUnit::TestFixture { + FastOS_ThreadPool _threadPool; + framework::defaultimplementation::FakeClock* _clock; + std::unique_ptr<TestServiceLayerApp> _node; + std::unique_ptr<DummyStorageLink> _top; + std::unique_ptr<StatusMetricConsumer> _metricsConsumer; + std::unique_ptr<vdstestlib::DirConfig> _config; + std::unique_ptr<metrics::MetricSet> _topSet; + std::unique_ptr<metrics::MetricManager> _metricManager; + std::shared_ptr<FileStorMetrics> _filestorMetrics; + std::shared_ptr<BucketManagerMetrics> _bucketManagerMetrics; + std::shared_ptr<VisitorMetrics> _visitorMetrics; + + void createSnapshotForPeriod(std::chrono::seconds secs); + void assertMetricLastValue(const std::string& name, + int interval, + uint64_t expected); + + MetricsTest(); + + void setUp(); + void tearDown(); + void runLoad(uint32_t count = 1); + void createFakeLoad(); + + void testFileStorMetrics(); + void testSnapshotPresenting(); + void testHtmlMetricsReport(); + void testCurrentGaugeValuesOverrideSnapshotValues(); + void testVerboseReportIncludesNonSetMetricsEvenAfterSnapshot(); + + CPPUNIT_TEST_SUITE(MetricsTest); + CPPUNIT_TEST(testFileStorMetrics); + CPPUNIT_TEST(testSnapshotPresenting); + CPPUNIT_TEST(testHtmlMetricsReport); + CPPUNIT_TEST(testCurrentGaugeValuesOverrideSnapshotValues); + CPPUNIT_TEST(testVerboseReportIncludesNonSetMetricsEvenAfterSnapshot); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(MetricsTest); + +namespace { + struct MetricClock : public metrics::MetricManager::Timer + { + framework::Clock& _clock; + MetricClock(framework::Clock& c) : _clock(c) {} + virtual time_t getTime() const + { return _clock.getTimeInSeconds().getTime(); } + virtual time_t getTimeInMilliSecs() const + { return _clock.getTimeInMillis().getTime(); } + }; +} + +MetricsTest::MetricsTest() + : _threadPool(256*1024), + _clock(0), + _top(), + _metricsConsumer() +{ +} + +void MetricsTest::setUp() { + assert(system("rm -rf vdsroot") == 0); + _config.reset(new vdstestlib::DirConfig(getStandardConfig(true))); + try { + _node.reset(new TestServiceLayerApp(DiskCount(4), NodeIndex(0), + _config->getConfigId())); + _node->setupDummyPersistence(); + _clock = &_node->getClock(); + _clock->setAbsoluteTimeInSeconds(1000000); + _top.reset(new DummyStorageLink); + } catch (config::InvalidConfigException& e) { + fprintf(stderr, "%s\n", e.what()); + } + _metricManager.reset(new metrics::MetricManager( + std::unique_ptr<metrics::MetricManager::Timer>( + new MetricClock(*_clock)))); + _topSet.reset(new metrics::MetricSet("vds", "", "")); + { + metrics::MetricLockGuard guard(_metricManager->getMetricLock()); + _metricManager->registerMetric(guard, *_topSet); + } + + _metricsConsumer.reset(new StatusMetricConsumer( + _node->getComponentRegister(), + *_metricManager, + "status")); + + uint16_t diskCount = _node->getPartitions().size(); + documentapi::LoadTypeSet::SP loadTypes(_node->getLoadTypes()); + + _filestorMetrics.reset(new FileStorMetrics( + _node->getLoadTypes()->getMetricLoadTypes())); + _filestorMetrics->initDiskMetrics( + diskCount, loadTypes->getMetricLoadTypes(), 1); + _topSet->registerMetric(*_filestorMetrics); + + _bucketManagerMetrics.reset(new BucketManagerMetrics); + _bucketManagerMetrics->setDisks(diskCount); + _topSet->registerMetric(*_bucketManagerMetrics); + + _visitorMetrics.reset(new VisitorMetrics); + _visitorMetrics->initThreads(4, loadTypes->getMetricLoadTypes()); + _topSet->registerMetric(*_visitorMetrics); + _metricManager->init(_config->getConfigId(), _node->getThreadPool()); +} + +void MetricsTest::tearDown() { + _metricManager->stop(); + _metricsConsumer.reset(0); + _topSet.reset(0); + _metricManager.reset(0); + _top.reset(0); + _node.reset(0); + _config.reset(0); + _filestorMetrics.reset(); + _bucketManagerMetrics.reset(); + _visitorMetrics.reset(); +} + +void MetricsTest::createFakeLoad() +{ + _clock->addSecondsToTime(1); + _metricManager->timeChangedNotification(); + uint32_t n = 5; + for (uint32_t i=0; i<_bucketManagerMetrics->disks.size(); ++i) { + DataStoredMetrics& metrics(*_bucketManagerMetrics->disks[i]); + metrics.docs.inc(10 * n); + metrics.bytes.inc(10240 * n); + } + _filestorMetrics->directoryEvents.inc(5); + _filestorMetrics->partitionEvents.inc(4); + _filestorMetrics->diskEvents.inc(3); + for (uint32_t i=0; i<_filestorMetrics->disks.size(); ++i) { + FileStorDiskMetrics& disk(*_filestorMetrics->disks[i]); + disk.queueSize.addValue(4 * n); + disk.averageQueueWaitingTime[documentapi::LoadType::DEFAULT].addValue(10 * n); + disk.pendingMerges.addValue(4 * n); + for (uint32_t j=0; j<disk.threads.size(); ++j) { + FileStorThreadMetrics& thread(*disk.threads[j]); + thread.operations.inc(120 * n); + thread.failedOperations.inc(2 * n); + + using documentapi::LoadType; + + thread.put[LoadType::DEFAULT].count.inc(10 * n); + thread.put[LoadType::DEFAULT].latency.addValue(5 * n); + thread.get[LoadType::DEFAULT].count.inc(12 * n); + thread.get[LoadType::DEFAULT].notFound.inc(2 * n); + thread.get[LoadType::DEFAULT].latency.addValue(3 * n); + thread.remove[LoadType::DEFAULT].count.inc(6 * n); + thread.remove[LoadType::DEFAULT].notFound.inc(1 * n); + thread.remove[LoadType::DEFAULT].latency.addValue(2 * n); + thread.update[LoadType::DEFAULT].count.inc(2 * n); + thread.update[LoadType::DEFAULT].notFound.inc(1 * n); + thread.update[LoadType::DEFAULT].latencyRead.addValue(2 * n); + thread.update[LoadType::DEFAULT].latency.addValue(7 * n); + thread.revert[LoadType::DEFAULT].count.inc(2 * n); + thread.revert[LoadType::DEFAULT].notFound.inc(n / 2); + thread.revert[LoadType::DEFAULT].latency.addValue(2 * n); + thread.visit[LoadType::DEFAULT].count.inc(6 * n); + + thread.deleteBuckets.count.inc(1 * n); + thread.repairs.count.inc(3 * n); + thread.repairFixed.inc(1 * n); + thread.splitBuckets.count.inc(20 * n); + thread.movedBuckets.count.inc(1 * n); + thread.readBucketInfo.count.inc(2 * n); + thread.internalJoin.count.inc(3 * n); + + thread.mergeBuckets.count.inc(2 * n); + thread.bytesMerged.inc(1000 * n); + thread.getBucketDiff.count.inc(4 * n); + thread.getBucketDiffReply.inc(4 * n); + thread.applyBucketDiff.count.inc(4 * n); + thread.applyBucketDiffReply.inc(4 * n); + thread.mergeLatencyTotal.addValue(300 * n); + thread.mergeMetadataReadLatency.addValue(20 * n); + thread.mergeDataReadLatency.addValue(40 * n); + thread.mergeDataWriteLatency.addValue(50 * n); + thread.mergeAverageDataReceivedNeeded.addValue(0.8); + } + } + for (uint32_t i=0; i<_visitorMetrics->threads.size(); ++i) { + VisitorThreadMetrics& thread(*_visitorMetrics->threads[i]); + thread.queueSize.addValue(2); + thread.averageQueueWaitingTime[documentapi::LoadType::DEFAULT].addValue(10); + thread.averageVisitorLifeTime[documentapi::LoadType::DEFAULT].addValue(1000); + thread.createdVisitors[documentapi::LoadType::DEFAULT].inc(5 * n); + thread.abortedVisitors[documentapi::LoadType::DEFAULT].inc(1 * n); + thread.completedVisitors[documentapi::LoadType::DEFAULT].inc(4 * n); + thread.failedVisitors[documentapi::LoadType::DEFAULT].inc(2 * n); + } + _clock->addSecondsToTime(60); + _metricManager->timeChangedNotification(); + while (uint64_t(_metricManager->getLastProcessedTime()) + < _clock->getTimeInSeconds().getTime()) + { + FastOS_Thread::Sleep(5); + _metricManager->timeChangedNotification(); + } +} + +void MetricsTest::testFileStorMetrics() { + createFakeLoad(); + std::ostringstream ost; + framework::HttpUrlPath path("metrics?interval=-1&format=text"); + bool retVal = _metricsConsumer->reportStatus(ost, path); + CPPUNIT_ASSERT_MESSAGE("_metricsConsumer->reportStatus failed", retVal); + std::string s = ost.str(); + CPPUNIT_ASSERT_MESSAGE("No get statistics in:\n" + s, + s.find("vds.filestor.alldisks.allthreads.get.sum.count count=240") != std::string::npos); + CPPUNIT_ASSERT_MESSAGE("No put statistics in:\n" + s, + s.find("vds.filestor.alldisks.allthreads.put.sum.count count=200") != std::string::npos); + CPPUNIT_ASSERT_MESSAGE("No remove statistics in:\n" + s, + s.find("vds.filestor.alldisks.allthreads.remove.sum.count count=120") != std::string::npos); + CPPUNIT_ASSERT_MESSAGE("No removenotfound stats in:\n" + s, + s.find("vds.filestor.alldisks.allthreads.remove.sum.not_found count=20") != std::string::npos); +} + +#define ASSERT_METRIC(interval, metric, count) \ +{ \ + std::ostringstream pathost; \ + pathost << "metrics?interval=" << interval << "&format=text"; \ + std::ostringstream ost;\ + framework::HttpUrlPath path(pathost.str()); \ + bool retVal = _metricsConsumer->reportStatus(ost, path); \ + CPPUNIT_ASSERT_MESSAGE("_metricsConsumer->reportStatus failed", retVal); \ + std::string s = ost.str(); \ + if (count == -1) { \ + CPPUNIT_ASSERT_MESSAGE(std::string("Metric ") + metric + " was set", \ + s.find(metric) == std::string::npos); \ + } else { \ + std::ostringstream valueost; \ + valueost << metric << " count=" << count; \ + CPPUNIT_ASSERT_MESSAGE("Did not find value " + valueost.str() \ + + " in metric dump " + s, \ + s.find(valueost.str()) != std::string::npos); \ + } \ +} + +void MetricsTest::testSnapshotPresenting() { + FileStorDiskMetrics& disk0(*_filestorMetrics->disks[0]); + FileStorThreadMetrics& thread0(*disk0.threads[0]); + + LOG(info, "Adding to get metric"); + + using documentapi::LoadType; + thread0.get[LoadType::DEFAULT].count.inc(1); + + LOG(info, "Waiting for 5 minute snapshot to be taken"); + // Wait until active metrics have been added to 5 min snapshot and reset + for (uint32_t i=0; i<6; ++i) { + _clock->addSecondsToTime(60); + _metricManager->timeChangedNotification(); + while ( + uint64_t(_metricManager->getLastProcessedTime()) + < _clock->getTimeInSeconds().getTime()) + { + FastOS_Thread::Sleep(1); + } + } + LOG(info, "5 minute snapshot should have been taken. Adding put count"); + + thread0.put[LoadType::DEFAULT].count.inc(1); + + // Verify that active metrics have set put count but not get count + ASSERT_METRIC(-2, "vds.filestor.alldisks.allthreads.put.sum.count", 1); + ASSERT_METRIC(-2, "vds.filestor.alldisks.allthreads.get.sum.count", -1); + + // Verify that 5 min metrics have set get count but not put count + ASSERT_METRIC(300, "vds.filestor.alldisks.allthreads.put.sum.count", -1); + ASSERT_METRIC(300, "vds.filestor.alldisks.allthreads.get.sum.count", 1); + + // Verify that the total metrics is equal to 5 minute + ASSERT_METRIC(0, "vds.filestor.alldisks.allthreads.put.sum.count", -1); + ASSERT_METRIC(0, "vds.filestor.alldisks.allthreads.get.sum.count", 1); + + // Verify that total + active have set both + ASSERT_METRIC(-1, "vds.filestor.alldisks.allthreads.put.sum.count", 1); + ASSERT_METRIC(-1, "vds.filestor.alldisks.allthreads.get.sum.count", 1); +} + +void MetricsTest::testHtmlMetricsReport() { + createFakeLoad(); + _clock->addSecondsToTime(6 * 60); + _metricManager->timeChangedNotification(); + _metricsConsumer->waitUntilTimeProcessed(_clock->getTimeInSeconds()); + createFakeLoad(); + std::ostringstream ost; + framework::HttpUrlPath path("metrics?interval=300&format=html"); + bool retVal = _metricsConsumer->reportStatus(ost, path); + CPPUNIT_ASSERT_MESSAGE("_metricsConsumer->reportStatus failed", retVal); + std::string s = ost.str(); + // Not actually testing against content. Better to manually verify that + // HTML look sane after changes. + //std::cerr << s << "\n"; + { + std::ofstream out("metricsreport.html"); + out << s; + out.close(); + } +} + +void +MetricsTest::assertMetricLastValue(const std::string& name, + int interval, + uint64_t expected) +{ + std::ostringstream path; + path << "metrics?interval=" << interval + << "&format=text&pattern=" << name + << "&verbosity=2"; + std::ostringstream report; + framework::HttpUrlPath uri(path.str()); + CPPUNIT_ASSERT(_metricsConsumer->reportStatus(report, uri)); + std::ostringstream expectedSubstr; + expectedSubstr << " last=" << expected; + auto str = report.str(); + CPPUNIT_ASSERT_MESSAGE("Did not find value " + expectedSubstr.str() + + " in metric dump " + str, + str.find(expectedSubstr.str()) != std::string::npos); +} + +using namespace std::chrono_literals; + +void +MetricsTest::createSnapshotForPeriod(std::chrono::seconds secs) +{ + _clock->addSecondsToTime(secs.count()); + _metricManager->timeChangedNotification(); + while (uint64_t(_metricManager->getLastProcessedTime()) + < _clock->getTimeInSeconds().getTime()) + { + std::this_thread::sleep_for(100ms); + } +} + +void +MetricsTest::testCurrentGaugeValuesOverrideSnapshotValues() +{ + auto& metrics(*_bucketManagerMetrics->disks[0]); + metrics.docs.set(1000); + // Take a 5 minute snapshot of active metrics (1000 docs). + createSnapshotForPeriod(5min); + metrics.docs.set(2000); + // Active metrics are now 2000 docs. Asking for metric snapshots with + // an interval of -1 implies that the _active_ metric values should + // be added to the total snapshot, which in the case of gauge metrics + // only makes sense if the _active_ gauge value gets reported back. + // In this case it means we should observe 2000 docs, not 1000. + assertMetricLastValue("vds.datastored.alldisks.docs", -1, 2000); +} + +void +MetricsTest::testVerboseReportIncludesNonSetMetricsEvenAfterSnapshot() +{ + createSnapshotForPeriod(5min); + // When using verbosity=2 (which is what the system test framework invokes), + // all metrics should be included regardless of whether they've been set or + // not. In this case, the bytes gauge metric has not been set explicitly + // but should be reported as zero. + assertMetricLastValue("vds.datastored.alldisks.bytes", -1, 0); +} + +} // storage diff --git a/storage/src/tests/common/storagelinktest.cpp b/storage/src/tests/common/storagelinktest.cpp new file mode 100644 index 00000000000..34b774ac424 --- /dev/null +++ b/storage/src/tests/common/storagelinktest.cpp @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <tests/common/storagelinktest.h> +#include <iostream> +#include <string> +#include <vespa/storageapi/message/stat.h> + +namespace storage { + +CPPUNIT_TEST_SUITE_REGISTRATION(StorageLinkTest); + +StorageLinkTest::StorageLinkTest() + : _threadPool(1024), + _feeder(), + _middle(0), + _replier(0) {} + +void StorageLinkTest::setUp() { + _feeder.reset(new DummyStorageLink()); + _middle = new DummyStorageLink(); + _replier = new DummyStorageLink(); + _feeder->push_back(StorageLink::UP(_middle)); + _feeder->push_back(StorageLink::UP(_replier)); + _replier->setAutoreply(true); +} + +void StorageLinkTest::testPrinting() { + std::ostringstream actual; + actual << *_feeder; + std::string expected = +"StorageChain(3)\n" +" DummyStorageLink(autoreply = off, dispatch = off, 0 commands, 0 replies)\n" +" DummyStorageLink(autoreply = off, dispatch = off, 0 commands, 0 replies)\n" +" DummyStorageLink(autoreply = on, dispatch = off, 0 commands, 0 replies)"; + + CPPUNIT_ASSERT_EQUAL(expected, actual.str()); +} + +void StorageLinkTest::testNotImplemented() { + _feeder->open(); + // Test that a message that nobody handles fails with NOT_IMPLEMENTED + _replier->setIgnore(true); + _feeder->sendDown(api::StorageCommand::SP( + new api::StatBucketCommand(document::BucketId(0), ""))); + _feeder->close(); + _feeder->flush(); + CPPUNIT_ASSERT_EQUAL((size_t) 1, _feeder->getNumReplies()); + CPPUNIT_ASSERT_EQUAL( + dynamic_cast<api::StatBucketReply&>( + *_feeder->getReply(0)).getResult(), + api::ReturnCode(api::ReturnCode::NOT_IMPLEMENTED, "Statbucket")); + _feeder->reset(); + _replier->setIgnore(false); +} + +} // storage diff --git a/storage/src/tests/common/storagelinktest.h b/storage/src/tests/common/storagelinktest.h new file mode 100644 index 00000000000..efeebb1146e --- /dev/null +++ b/storage/src/tests/common/storagelinktest.h @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cppunit/extensions/HelperMacros.h> +#include <vespa/fastos/fastos.h> +#include <tests/common/dummystoragelink.h> + +namespace storage { + +struct StorageLinkTest : public CppUnit::TestFixture { + FastOS_ThreadPool _threadPool; + std::unique_ptr<DummyStorageLink> _feeder; + DummyStorageLink* _middle; + DummyStorageLink* _replier; + + StorageLinkTest(); + + void setUp(); + + void testPrinting(); + void testNotImplemented(); + + static bool callOnUp(StorageLink& link, + const api::StorageMessage::SP& msg) + { + return link.onUp(msg); + } + static bool callOnDown(StorageLink& link, + const api::StorageMessage::SP& msg) + { + return link.onDown(msg); + } + static void callOnFlush(StorageLink& link, bool downwards) + { + link.onFlush(downwards); + } + + CPPUNIT_TEST_SUITE(StorageLinkTest); + CPPUNIT_TEST(testPrinting); + CPPUNIT_TEST(testNotImplemented); + CPPUNIT_TEST_SUITE_END(); +}; + +} + diff --git a/storage/src/tests/common/testhelper.cpp b/storage/src/tests/common/testhelper.cpp new file mode 100644 index 00000000000..b8b42124d39 --- /dev/null +++ b/storage/src/tests/common/testhelper.cpp @@ -0,0 +1,209 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <tests/common/testhelper.h> + +#include <vespa/log/log.h> +#include <vespa/vespalib/io/fileutil.h> + +LOG_SETUP(".testhelper"); + +namespace storage { + +namespace { + bool useNewStorageCore() { + if ( // Unit test directory + vespalib::fileExists("use_new_storage_core") || + // src/cpp directory + vespalib::fileExists("../use_new_storage_core") || + // Top build directory where storage-HEAD remains + vespalib::fileExists("../../../../use_new_storage_core")) + { + std::cerr << "Using new storage core for unit tests\n"; + return true; + } + return false; + } + bool newStorageCore(useNewStorageCore()); +} + +void addStorageDistributionConfig(vdstestlib::DirConfig& dc) +{ + vdstestlib::DirConfig::Config* config; + config = &dc.getConfig("stor-distribution", true); + config->clear(); + config->set("group[1]"); + config->set("group[0].name", "invalid"); + config->set("group[0].index", "invalid"); + config->set("group[0].nodes[50]"); + config->set("redundancy", "2"); + + for (uint32_t i = 0; i < 50; i++) { + std::ostringstream key; key << "group[0].nodes[" << i << "].index"; + std::ostringstream val; val << i; + config->set(key.str(), val.str()); + } +} + +vdstestlib::DirConfig getStandardConfig(bool storagenode) { + std::string clusterName("storage"); + vdstestlib::DirConfig dc; + vdstestlib::DirConfig::Config* config; + config = &dc.addConfig("fleetcontroller"); + config->set("cluster_name", clusterName); + config->set("index", "0"); + config->set("zookeeper_server", "\"\""); + config->set("total_distributor_count", "10"); + config->set("total_storage_count", "10"); + config = &dc.addConfig("upgrading"); + config = &dc.addConfig("load-type"); + config->set("type[10]"); + config->set("type[0].id", "1"); + config->set("type[0].name", "\"maintenance.inconsistent.join\""); + config->set("type[0].priority", "\"high_3\""); + config->set("type[1].id", "2"); + config->set("type[1].name", "\"maintenance.inconsistent.split\""); + config->set("type[1].priority", "\"normal_1\""); + config->set("type[2].id", "3"); + config->set("type[2].name", "\"maintenance.active.incorrectamount\""); + config->set("type[2].priority", "\"normal_2\""); + config->set("type[3].id", "4"); + config->set("type[3].name", "\"maintenance.active.wrongcopy\""); + config->set("type[3].priority", "\"normal_3\""); + config->set("type[4].id", "5"); + config->set("type[4].name", "\"maintenance.size.split\""); + config->set("type[4].priority", "\"normal_4\""); + config->set("type[5].id", "6"); + config->set("type[5].name", "\"maintenance.size.join\""); + config->set("type[5].priority", "\"normal_5\""); + config->set("type[6].id", "7"); + config->set("type[6].name", "\"maintenance.merge.toofewcopies\""); + config->set("type[6].priority", "\"normal_6\""); + config->set("type[7].id", "8"); + config->set("type[7].name", "\"maintenance.merge.toomanycopies\""); + config->set("type[7].priority", "\"low_1\""); + config->set("type[8].id", "9"); + config->set("type[8].name", "\"maintenance.merge.outofsync\""); + config->set("type[8].priority", "\"low_2\""); + config->set("type[9].id", "10"); + config->set("type[9].name", "\"maintenance.move\""); + config->set("type[9].priority", "\"low_3\""); + config = &dc.addConfig("bucket"); + config = &dc.addConfig("messagebus"); + config = &dc.addConfig("stor-prioritymapping"); + config = &dc.addConfig("stor-bucketdbupdater"); + config = &dc.addConfig("stor-bucket-init"); + config = &dc.addConfig("metricsmanager"); + config->set("consumer[2]"); + config->set("consumer[0].name", "\"status\""); + config->set("consumer[0].addedmetrics[1]"); + config->set("consumer[0].addedmetrics[0]", "\"*\""); + config->set("consumer[1].name", "\"statereporter\""); + config->set("consumer[1].addedmetrics[1]"); + config->set("consumer[1].addedmetrics[0]", "\"*\""); + config = &dc.addConfig("stor-communicationmanager"); + config->set("rpcport", "0"); + config->set("mbusport", "0"); + config = &dc.addConfig("stor-bucketdb"); + config->set("chunklevel", "0"); + config = &dc.addConfig("stor-distributormanager"); + config->set("splitcount", "1000"); + config->set("splitsize", "10000000"); + config->set("joincount", "500"); + config->set("joinsize", "5000000"); + config = &dc.addConfig("stor-opslogger"); + config = &dc.addConfig("persistence"); + config->set("abort_operations_with_changed_bucket_ownership", "true"); + config = &dc.addConfig("stor-filestor"); + // Easier to see what goes wrong with only 1 thread per disk. + config->set("minimum_file_meta_slots", "2"); + config->set("minimum_file_header_block_size", "368"); + config->set("minimum_file_size", "4096"); + config->set("threads[1]"); + config->set("threads[0].lowestpri 255"); + config->set("dir_spread", "4"); + config->set("dir_levels", "0"); + config->set("use_new_core", newStorageCore ? "true" : "false"); + config->set("maximum_versions_of_single_document_stored", "0"); + //config->set("enable_slotfile_cache", "false"); + // Unit tests typically use fake low time values, so don't complain + // about them or compact/delete them by default. Override in tests testing that + // behavior + config->set("time_future_limit", "5"); + config->set("time_past_limit", "2000000000"); + config->set("keep_remove_time_period", "2000000000"); + config->set("revert_time_period", "2000000000"); + // Don't want test to call exit() + config->set("fail_disk_after_error_count", "0"); + config = &dc.addConfig("stor-bouncer"); + config = &dc.addConfig("stor-integritychecker"); + config = &dc.addConfig("stor-bucketmover"); + config = &dc.addConfig("stor-messageforwarder"); + config = &dc.addConfig("stor-server"); + config->set("cluster_name", clusterName); + config->set("enable_dead_lock_detector", "false"); + config->set("enable_dead_lock_detector_warnings", "false"); + config->set("max_merges_per_node", "25"); + config->set("max_merge_queue_size", "20"); + config->set("root_folder", + (storagenode ? "vdsroot" : "vdsroot.distributor")); + config->set("is_distributor", + (storagenode ? "false" : "true")); + config = &dc.addConfig("stor-devices"); + config->set("root_folder", + (storagenode ? "vdsroot" : "vdsroot.distributor")); + config = &dc.addConfig("stor-status"); + config->set("httpport", "0"); + config = &dc.addConfig("stor-visitor"); + config->set("defaultdocblocksize", "8192"); + // By default, need "old" behaviour of maxconcurrent + config->set("maxconcurrentvisitors_fixed", "4"); + config->set("maxconcurrentvisitors_variable", "0"); + config = &dc.addConfig("stor-visitordispatcher"); + addFileConfig(dc, "documenttypes", "config-doctypes.cfg"); + addStorageDistributionConfig(dc); + return dc; +} + +void addSlobrokConfig(vdstestlib::DirConfig& dc, + const mbus::Slobrok& slobrok) +{ + std::ostringstream ost; + ost << "tcp/localhost:" << slobrok.port(); + vdstestlib::DirConfig::Config* config; + config = &dc.getConfig("slobroks", true); + config->clear(); + config->set("slobrok[1]"); + config->set("slobrok[0].connectionspec", ost.str()); +} + +void addFileConfig(vdstestlib::DirConfig& dc, + const std::string& configDefName, + const std::string& fileName) +{ + vdstestlib::DirConfig::Config* config; + config = &dc.getConfig(configDefName, true); + config->clear(); + std::ifstream in(fileName.c_str()); + std::string line; + while (std::getline(in, line, '\n')) { + std::string::size_type pos = line.find(' '); + if (pos == std::string::npos) { + config->set(line); + } else { + config->set(line.substr(0, pos), line.substr(pos + 1)); + } + } + in.close(); +} + +TestName::TestName(const std::string& n) + : name(n) +{ + LOG(debug, "Starting test %s", name.c_str()); +} + +TestName::~TestName() { + LOG(debug, "Done with test %s", name.c_str()); +} + +} // storage diff --git a/storage/src/tests/common/testhelper.h b/storage/src/tests/common/testhelper.h new file mode 100644 index 00000000000..be2c3e7ec66 --- /dev/null +++ b/storage/src/tests/common/testhelper.h @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include <vespa/vdstestlib/cppunit/dirconfig.h> +#include <vespa/vdstestlib/cppunit/macros.h> + + +#include <fstream> +#include <vespa/fastos/fastos.h> +#include <vespa/messagebus/testlib/slobrok.h> +#include <sstream> + +#define ASSERT_REPLY_COUNT(count, dummylink) \ + { \ + std::ostringstream msgost; \ + if ((dummylink).getNumReplies() != count) { \ + for (uint32_t ijx=0; ijx<(dummylink).getNumReplies(); ++ijx) { \ + msgost << (dummylink).getReply(ijx)->toString(true) << "\n"; \ + } \ + } \ + CPPUNIT_ASSERT_EQUAL_MSG(msgost.str(), size_t(count), \ + (dummylink).getNumReplies()); \ + } +#define ASSERT_COMMAND_COUNT(count, dummylink) \ + { \ + std::ostringstream msgost; \ + if ((dummylink).getNumCommands() != count) { \ + for (uint32_t ijx=0; ijx<(dummylink).getNumCommands(); ++ijx) { \ + msgost << (dummylink).getCommand(ijx)->toString(true) << "\n"; \ + } \ + } \ + CPPUNIT_ASSERT_EQUAL_MSG(msgost.str(), size_t(count), \ + (dummylink).getNumCommands()); \ + } + +namespace storage { + +void addFileConfig(vdstestlib::DirConfig& dc, + const std::string& configDefName, + const std::string& fileName); + + +void addStorageDistributionConfig(vdstestlib::DirConfig& dc); + +vdstestlib::DirConfig getStandardConfig(bool storagenode); + +void addSlobrokConfig(vdstestlib::DirConfig& dc, + const mbus::Slobrok& slobrok); + +// Class used to print start and end of test. Enable debug when you want to see +// which test creates what output or where we get stuck +struct TestName { + std::string name; + TestName(const std::string& n); + ~TestName(); +}; + +} // storage + diff --git a/storage/src/tests/common/testnodestateupdater.h b/storage/src/tests/common/testnodestateupdater.h new file mode 100644 index 00000000000..9f5b2d8ba51 --- /dev/null +++ b/storage/src/tests/common/testnodestateupdater.h @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class storage::TestNodeStateUpdater + * \ingroup common + * + * \brief Test implementation of the node state updater. + */ + +#pragma once + +#include <vespa/storage/common/nodestateupdater.h> + +namespace storage { + +struct TestNodeStateUpdater : public NodeStateUpdater +{ + lib::NodeState::CSP _reported; + lib::NodeState::CSP _current; + lib::ClusterState::CSP _cluster; + std::vector<StateListener*> _listeners; + +public: + TestNodeStateUpdater(const lib::NodeType& type) { + _reported.reset(new lib::NodeState(type, lib::State::UP)); + _current.reset(new lib::NodeState(type, lib::State::UP)); + } + + lib::NodeState::CSP getReportedNodeState() const { return _reported; } + lib::NodeState::CSP getCurrentNodeState() const { return _current; } + lib::ClusterState::CSP getSystemState() const { return _cluster; } + void addStateListener(StateListener& s) { + _listeners.push_back(&s); + } + void removeStateListener(StateListener&) {} + Lock::SP grabStateChangeLock() { return Lock::SP(new Lock); } + void setReportedNodeState(const lib::NodeState& state) + { _reported.reset(new lib::NodeState(state)); } + void setCurrentNodeState(const lib::NodeState& state) + { _current.reset(new lib::NodeState(state)); } + + void setClusterState(lib::ClusterState::CSP c) { + _cluster = c; + for (uint32_t i = 0; i < _listeners.size(); ++i) { + _listeners[i]->handleNewState(); + } + } +}; + +} // storage + diff --git a/storage/src/tests/common/teststorageapp.cpp b/storage/src/tests/common/teststorageapp.cpp new file mode 100644 index 00000000000..eb4c1c41c78 --- /dev/null +++ b/storage/src/tests/common/teststorageapp.cpp @@ -0,0 +1,292 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <tests/common/teststorageapp.h> + +#include <vespa/log/log.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <vespa/storage/bucketdb/storagebucketdbinitializer.h> +#include <vespa/storage/config/config-stor-server.h> +#include <vespa/storageframework/defaultimplementation/clock/realclock.h> +#include <vespa/storageframework/defaultimplementation/memory/nomemorymanager.h> +#include <vespa/config-fleetcontroller.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/io/fileutil.h> + +LOG_SETUP(".test.servicelayerapp"); + +using storage::framework::defaultimplementation::ComponentRegisterImpl; + +namespace storage { + +namespace { + template<typename T> + struct ConfigReader : public T::Subscriber, + public T + { + ConfigReader(const std::string& configId) { + T::subscribe(configId, *this); + } + void configure(const T& c) { dynamic_cast<T&>(*this) = c; } + }; +} + +TestStorageApp::TestStorageApp(StorageComponentRegisterImpl::UP compReg, + const lib::NodeType& type, NodeIndex index, + vespalib::stringref configId) + : TestComponentRegister(ComponentRegisterImpl::UP(std::move(compReg))), + _compReg(dynamic_cast<StorageComponentRegisterImpl&>( + TestComponentRegister::getComponentRegister())), + _docMan(), + _nodeStateUpdater(type), + _configId(configId), + _initialized(false) +{ + // Use config to adjust values + vespalib::string clusterName = "mycluster"; + uint32_t redundancy = 2; + uint32_t nodeCount = 10; + documentapi::LoadTypeSet::SP loadTypes; + if (!configId.empty()) { + config::ConfigUri uri(configId); + std::unique_ptr<vespa::config::content::core::StorServerConfig> serverConfig = config::ConfigGetter<vespa::config::content::core::StorServerConfig>::getConfig(uri.getConfigId(), uri.getContext()); + clusterName = serverConfig->clusterName; + if (index == 0xffff) index = serverConfig->nodeIndex; + redundancy = config::ConfigGetter<vespa::config::content::StorDistributionConfig>::getConfig(uri.getConfigId(), uri.getContext())->redundancy; + nodeCount = config::ConfigGetter<vespa::config::content::FleetcontrollerConfig>::getConfig(uri.getConfigId(), uri.getContext())->totalStorageCount; + _compReg.setPriorityConfig( + *config::ConfigGetter<StorageComponent::PriorityConfig> + ::getConfig(uri.getConfigId(), uri.getContext())); + loadTypes.reset(new documentapi::LoadTypeSet( + *config::ConfigGetter<vespa::config::content::LoadTypeConfig> + ::getConfig(uri.getConfigId(), uri.getContext()))); + } else { + if (index == 0xffff) index = 0; + loadTypes.reset(new documentapi::LoadTypeSet); + } + if (index >= nodeCount) nodeCount = index + 1; + if (redundancy > nodeCount) redundancy = nodeCount; + + _compReg.setNodeInfo(clusterName, type, index); + _compReg.setNodeStateUpdater(_nodeStateUpdater); + _compReg.setDocumentTypeRepo(_docMan.getTypeRepoSP()); + _compReg.setLoadTypes(loadTypes); + _compReg.setBucketIdFactory(document::BucketIdFactory()); + lib::Distribution::SP distr(new lib::Distribution( + lib::Distribution::getDefaultDistributionConfig( + redundancy, nodeCount))); + _compReg.setDistribution(distr); +} + +void +TestStorageApp::setDistribution(Redundancy redundancy, NodeCount nodeCount) +{ + lib::Distribution::SP distr(new lib::Distribution( + lib::Distribution::getDefaultDistributionConfig( + redundancy, nodeCount))); + _compReg.setDistribution(distr); +} + +void +TestStorageApp::setTypeRepo(document::DocumentTypeRepo::SP repo) +{ + _compReg.setDocumentTypeRepo(repo); +} + +void +TestStorageApp::setClusterState(const lib::ClusterState& c) +{ + _nodeStateUpdater.setClusterState( + lib::ClusterState::CSP(new lib::ClusterState(c))); +} + +void +TestStorageApp::waitUntilInitialized( + StorageBucketDBInitializer* initializer, framework::SecondTime timeout) +{ + // Always use real clock for wait timeouts. Component clock may be faked + // in tests + framework::defaultimplementation::RealClock clock; + framework::MilliSecTime endTime( + clock.getTimeInMillis() + timeout.getMillis()); + while (!isInitialized()) { + FastOS_Thread::Sleep(1); + framework::MilliSecTime currentTime(clock.getTimeInMillis()); + if (currentTime > endTime) { + std::ostringstream error; + error << "Failed to initialize service layer within timeout of " + << timeout << " seconds."; + if (initializer != 0) { + error << " "; + initializer->reportStatus(error, framework::HttpUrlPath("")); + LOG(error, "%s", error.str().c_str()); + CPPUNIT_FAIL(error.str().c_str()); + } + } + } +} + +namespace { + NodeIndex getIndexFromConfig(vespalib::stringref configId) { + if (!configId.empty()) { + config::ConfigUri uri(configId); + return NodeIndex( + config::ConfigGetter<vespa::config::content::core::StorServerConfig>::getConfig(uri.getConfigId(), uri.getContext())->nodeIndex); + } + return NodeIndex(0); + } +} + +TestServiceLayerApp::TestServiceLayerApp(vespalib::stringref configId) + : TestStorageApp( + StorageComponentRegisterImpl::UP( + new ServiceLayerComponentRegisterImpl), + lib::NodeType::STORAGE, getIndexFromConfig(configId), configId), + _compReg(dynamic_cast<ServiceLayerComponentRegisterImpl&>( + TestStorageApp::getComponentRegister())), + _persistenceProvider(), + _partitions(1) +{ + _compReg.setDiskCount(1); + lib::NodeState ns(*_nodeStateUpdater.getReportedNodeState()); + ns.setDiskCount(1); + _nodeStateUpdater.setReportedNodeState(ns); +} + +TestServiceLayerApp::TestServiceLayerApp(DiskCount dc, NodeIndex index, + vespalib::stringref configId) + : TestStorageApp( + StorageComponentRegisterImpl::UP( + new ServiceLayerComponentRegisterImpl), + lib::NodeType::STORAGE, index, configId), + _compReg(dynamic_cast<ServiceLayerComponentRegisterImpl&>( + TestStorageApp::getComponentRegister())), + _persistenceProvider(), + _partitions(dc) +{ + _compReg.setDiskCount(dc); + lib::NodeState ns(*_nodeStateUpdater.getReportedNodeState()); + ns.setDiskCount(dc); + _nodeStateUpdater.setReportedNodeState(ns); + // Tests should know how many disks they want to use. If testing auto + // detection, you should not need this utility. + CPPUNIT_ASSERT(dc > 0); +} + +void +TestServiceLayerApp::setupDummyPersistence() +{ + spi::PersistenceProvider::UP provider(new spi::dummy::DummyPersistence( + getTypeRepo(), _compReg.getDiskCount())); + setPersistenceProvider(std::move(provider)); +} + +void +TestServiceLayerApp::setPersistenceProvider( + spi::PersistenceProvider::UP provider) +{ + _partitions = provider->getPartitionStates().getList(); + CPPUNIT_ASSERT_EQUAL(spi::PartitionId(_compReg.getDiskCount()), + _partitions.size()); + _persistenceProvider = std::move(provider); +} + +spi::PersistenceProvider& +TestServiceLayerApp::getPersistenceProvider() +{ + if (_persistenceProvider.get() == 0) { + throw vespalib::IllegalStateException( + "Persistence provider requested but not initialized.", + VESPA_STRLOC); + } + return *_persistenceProvider; +} + +spi::PartitionStateList& +TestServiceLayerApp::getPartitions() +{ + if (_persistenceProvider.get() == 0) { + throw vespalib::IllegalStateException( + "Partition list requested but not initialized.", + VESPA_STRLOC); + } + return _partitions; +} + +uint16_t +TestServiceLayerApp::getPartition(const document::BucketId& bucket) +{ + lib::NodeState state(lib::NodeType::STORAGE, lib::State::UP); + state.setDiskCount(_compReg.getDiskCount()); + return getDistribution()->getIdealDisk( + state, _compReg.getIndex(), bucket.stripUnused(), + lib::Distribution::IDEAL_DISK_EVEN_IF_DOWN); +} + +namespace { + template<typename T> + const T getConfig(vespalib::stringref configId) { + config::ConfigUri uri(configId); + return *config::ConfigGetter<T>::getConfig( + uri.getConfigId(), uri.getContext()); + } +} + +void +TestDistributorApp::configure(vespalib::stringref id) +{ + if (id.empty()) return; + DistributorConfig dc(getConfig<vespa::config::content::core::StorDistributormanagerConfig>(id)); + _compReg.setDistributorConfig(dc); + VisitorConfig vc(getConfig<vespa::config::content::core::StorVisitordispatcherConfig>(id)); + _compReg.setVisitorConfig(vc); +} + +TestDistributorApp::TestDistributorApp(vespalib::stringref configId) + : TestStorageApp( + StorageComponentRegisterImpl::UP( + new DistributorComponentRegisterImpl), + lib::NodeType::DISTRIBUTOR, getIndexFromConfig(configId), configId), + _compReg(dynamic_cast<DistributorComponentRegisterImpl&>( + TestStorageApp::getComponentRegister())), + _lastUniqueTimestampRequested(0), + _uniqueTimestampCounter(0) +{ + _compReg.setTimeCalculator(*this); + configure(configId); +} + +TestDistributorApp::TestDistributorApp(NodeIndex index, + vespalib::stringref configId) + : TestStorageApp( + StorageComponentRegisterImpl::UP(new StorageComponentRegisterImpl), + lib::NodeType::DISTRIBUTOR, index, configId), + _compReg(dynamic_cast<DistributorComponentRegisterImpl&>( + TestStorageApp::getComponentRegister())), + _lastUniqueTimestampRequested(0), + _uniqueTimestampCounter(0) +{ + _compReg.setTimeCalculator(*this); + configure(configId); +} + +api::Timestamp +TestDistributorApp::getUniqueTimestamp() +{ + vespalib::Lock lock(_accessLock); + uint64_t timeNow(getClock().getTimeInSeconds().getTime()); + if (timeNow == _lastUniqueTimestampRequested) { + ++_uniqueTimestampCounter; + } else { + if (timeNow < _lastUniqueTimestampRequested) { + LOG(error, "Time has moved backwards, from %" PRIu64 " to %" PRIu64 ".", + _lastUniqueTimestampRequested, timeNow); + } + _lastUniqueTimestampRequested = timeNow; + _uniqueTimestampCounter = 0; + } + + return _lastUniqueTimestampRequested * 1000000ll + _uniqueTimestampCounter; +} + +} // storage diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h new file mode 100644 index 00000000000..e7da9178743 --- /dev/null +++ b/storage/src/tests/common/teststorageapp.h @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class storage::TestServiceLayerApp + * \ingroup common + * + * \brief Helper class for tests involving service layer. + * + * Some components need some dependencies injected in order to work correctly. + * This test class simplifies the process of creating these dependencies. + * + * Note that the interface between this class and the test class should be as + * clean as possible, such that we can change as little as possible when + * refactoring later. Also, advanced functionality should not be generated in + * here, but rather fixed by tests themselves. Functionality here should be + * needed by many tests, and we should avoid instantiating complex instances + * here that several tests + */ +#pragma once + +#include <vespa/document/base/testdocman.h> +#include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/storage/bucketdb/distrbucketdb.h> +#include <vespa/storage/bucketdb/storbucketdb.h> +#include <vespa/storage/common/doneinitializehandler.h> +#include <vespa/storage/common/nodestateupdater.h> +#include <vespa/storage/storageserver/framework.h> +#include <vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h> +#include <vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h> +#include <vespa/storageframework/generic/memory/memorymanagerinterface.h> +#include <vespa/storageframework/defaultimplementation/clock/realclock.h> +#include <vespa/storageframework/defaultimplementation/component/testcomponentregister.h> +#include <tests/common/testnodestateupdater.h> + +namespace storage { + +class StorageBucketDBInitializer; + +DEFINE_PRIMITIVE_WRAPPER(uint16_t, DiskCount); +DEFINE_PRIMITIVE_WRAPPER(uint16_t, NodeIndex); +DEFINE_PRIMITIVE_WRAPPER(uint16_t, NodeCount); +DEFINE_PRIMITIVE_WRAPPER(uint16_t, Redundancy); + +class TestStorageApp + : public framework::defaultimplementation::TestComponentRegister, + private DoneInitializeHandler +{ + StorageComponentRegisterImpl& _compReg; + +protected: + document::TestDocMan _docMan; + TestNodeStateUpdater _nodeStateUpdater; + vespalib::string _configId; + bool _initialized; + +public: + /** + * Set up a storage application. If node index is not set, it will be + * fetched from config if config id is given, otherwise it is set to 0. + * If configId is given, some critical values are taken from config. + * (node count, redundancy, node index etc). If configId set not set these + * will just have some default values. A non-default node index will + * override config, but be careful with this, as components may fetch index + * from config themselves. + */ + TestStorageApp(StorageComponentRegisterImpl::UP compReg, + const lib::NodeType&, NodeIndex = NodeIndex(0xffff), + vespalib::stringref configId = ""); + + // Set functions, to be able to modify content while running. + void setDistribution(Redundancy, NodeCount); + void setTypeRepo(document::DocumentTypeRepo::SP repo); + void setClusterState(const lib::ClusterState&); + + // Utility functions for getting a hold of currently used bits. Practical + // to avoid adding extra components in the tests. + StorageComponentRegisterImpl& getComponentRegister() { return _compReg; } + document::TestDocMan& getTestDocMan() { return _docMan; } + document::DocumentTypeRepo::SP getTypeRepo() + { return _compReg.getTypeRepo(); } + const document::BucketIdFactory& getBucketIdFactory() + { return _compReg.getBucketIdFactory(); } + TestNodeStateUpdater& getStateUpdater() { return _nodeStateUpdater; } + documentapi::LoadTypeSet::SP getLoadTypes() + { return _compReg.getLoadTypes(); } + lib::Distribution::SP getDistribution() + { return _compReg.getDistribution(); } + TestNodeStateUpdater& getNodeStateUpdater() { return _nodeStateUpdater; } + uint16_t getIndex() const { return _compReg.getIndex(); } + + // The storage app also implements the done initializer interface, so it can + // be sent to components needing this. + DoneInitializeHandler& getDoneInitializeHandler() { return *this; } + virtual void notifyDoneInitializing() { _initialized = true; } + bool isInitialized() const { return _initialized; } + void waitUntilInitialized( + StorageBucketDBInitializer* initializer = 0, + framework::SecondTime timeout = framework::SecondTime(30)); + +private: + // Storage server interface implementation (until we can remove it) + virtual api::Timestamp getUniqueTimestamp() { assert(0); throw; } + virtual StorBucketDatabase& getStorageBucketDatabase() { assert(0); throw; } + virtual distributor::BucketDatabase& getBucketDatabase() { assert(0); throw; } + virtual uint16_t getDiskCount() const { assert(0); throw; } +}; + +class TestServiceLayerApp : public TestStorageApp +{ + ServiceLayerComponentRegisterImpl& _compReg; + spi::PersistenceProvider::UP _persistenceProvider; + spi::PartitionStateList _partitions; + +public: + TestServiceLayerApp(vespalib::stringref configId = ""); + TestServiceLayerApp(DiskCount diskCount, NodeIndex = NodeIndex(0xffff), + vespalib::stringref configId = ""); + + void setupDummyPersistence(); + void setPersistenceProvider(spi::PersistenceProvider::UP); + + ServiceLayerComponentRegisterImpl& getComponentRegister() + { return _compReg; } + + spi::PersistenceProvider& getPersistenceProvider(); + spi::PartitionStateList& getPartitions(); + + uint16_t getPartition(const document::BucketId&); + + virtual StorBucketDatabase& getStorageBucketDatabase() + { return _compReg.getBucketDatabase(); } + +private: + // For storage server interface implementation we'll get rid of soon. + // Use getPartitions().size() instead. + virtual uint16_t getDiskCount() const { return _compReg.getDiskCount(); } +}; + +class TestDistributorApp : public TestStorageApp, + public UniqueTimeCalculator +{ + DistributorComponentRegisterImpl& _compReg; + vespalib::Lock _accessLock; + uint64_t _lastUniqueTimestampRequested; + uint32_t _uniqueTimestampCounter; + + void configure(vespalib::stringref configId); + +public: + TestDistributorApp(vespalib::stringref configId = ""); + TestDistributorApp(NodeIndex index, vespalib::stringref configId = ""); + + DistributorComponentRegisterImpl& getComponentRegister() + { return _compReg; } + virtual distributor::BucketDatabase& getBucketDatabase() + { return _compReg.getBucketDatabase(); } + + virtual api::Timestamp getUniqueTimestamp(); +}; + +} // storageo + diff --git a/storage/src/tests/config-doctypes.cfg b/storage/src/tests/config-doctypes.cfg new file mode 100644 index 00000000000..f41593ebfc3 --- /dev/null +++ b/storage/src/tests/config-doctypes.cfg @@ -0,0 +1,158 @@ +enablecompression false +documenttype[3] +documenttype[0].id -519202262 +documenttype[0].name "text/plain" +documenttype[0].version 0 +documenttype[0].headerstruct 160469461 +documenttype[0].bodystruct 749465898 +documenttype[0].inherits[0] +documenttype[0].datatype[2] +documenttype[0].datatype[0].id 160469461 +documenttype[0].datatype[0].type STRUCT +documenttype[0].datatype[0].array.element.id 0 +documenttype[0].datatype[0].map.key.id 0 +documenttype[0].datatype[0].map.value.id 0 +documenttype[0].datatype[0].wset.key.id 0 +documenttype[0].datatype[0].wset.createifnonexistent false +documenttype[0].datatype[0].wset.removeifzero false +documenttype[0].datatype[0].annotationref.annotation.id 0 +documenttype[0].datatype[0].sstruct.name "text/plain.header" +documenttype[0].datatype[0].sstruct.version 0 +documenttype[0].datatype[0].sstruct.compression.type NONE +documenttype[0].datatype[0].sstruct.compression.level 0 +documenttype[0].datatype[0].sstruct.compression.threshold 90 +documenttype[0].datatype[0].sstruct.compression.minsize 0 +documenttype[0].datatype[0].sstruct.field[3] +documenttype[0].datatype[0].sstruct.field[0].name "author" +documenttype[0].datatype[0].sstruct.field[0].id 644499292 +documenttype[0].datatype[0].sstruct.field[0].id_v6 177126295 +documenttype[0].datatype[0].sstruct.field[0].datatype 2 +documenttype[0].datatype[0].sstruct.field[1].name "date" +documenttype[0].datatype[0].sstruct.field[1].id 491786523 +documenttype[0].datatype[0].sstruct.field[1].id_v6 916979460 +documenttype[0].datatype[0].sstruct.field[1].datatype 0 +documenttype[0].datatype[0].sstruct.field[2].name "subject" +documenttype[0].datatype[0].sstruct.field[2].id 1797950813 +documenttype[0].datatype[0].sstruct.field[2].id_v6 943449689 +documenttype[0].datatype[0].sstruct.field[2].datatype 2 +documenttype[0].datatype[1].id 749465898 +documenttype[0].datatype[1].type STRUCT +documenttype[0].datatype[1].array.element.id 0 +documenttype[0].datatype[1].map.key.id 0 +documenttype[0].datatype[1].map.value.id 0 +documenttype[0].datatype[1].wset.key.id 0 +documenttype[0].datatype[1].wset.createifnonexistent false +documenttype[0].datatype[1].wset.removeifzero false +documenttype[0].datatype[1].annotationref.annotation.id 0 +documenttype[0].datatype[1].sstruct.name "text/plain.body" +documenttype[0].datatype[1].sstruct.version 0 +documenttype[0].datatype[1].sstruct.compression.type NONE +documenttype[0].datatype[1].sstruct.compression.level 0 +documenttype[0].datatype[1].sstruct.compression.threshold 90 +documenttype[0].datatype[1].sstruct.compression.minsize 0 +documenttype[0].datatype[1].sstruct.field[1] +documenttype[0].datatype[1].sstruct.field[0].name "content" +documenttype[0].datatype[1].sstruct.field[0].id 1721764358 +documenttype[0].datatype[1].sstruct.field[0].id_v6 1751481844 +documenttype[0].datatype[1].sstruct.field[0].datatype 3 +documenttype[0].annotationtype[0] +documenttype[1].id -653677105 +documenttype[1].name "text/html" +documenttype[1].version 0 +documenttype[1].headerstruct 143329936 +documenttype[1].bodystruct 1473469605 +documenttype[1].inherits[0] +documenttype[1].datatype[2] +documenttype[1].datatype[0].id 143329936 +documenttype[1].datatype[0].type STRUCT +documenttype[1].datatype[0].array.element.id 0 +documenttype[1].datatype[0].map.key.id 0 +documenttype[1].datatype[0].map.value.id 0 +documenttype[1].datatype[0].wset.key.id 0 +documenttype[1].datatype[0].wset.createifnonexistent false +documenttype[1].datatype[0].wset.removeifzero false +documenttype[1].datatype[0].annotationref.annotation.id 0 +documenttype[1].datatype[0].sstruct.name "text/html.header" +documenttype[1].datatype[0].sstruct.version 0 +documenttype[1].datatype[0].sstruct.compression.type NONE +documenttype[1].datatype[0].sstruct.compression.level 0 +documenttype[1].datatype[0].sstruct.compression.threshold 90 +documenttype[1].datatype[0].sstruct.compression.minsize 0 +documenttype[1].datatype[0].sstruct.field[3] +documenttype[1].datatype[0].sstruct.field[0].name "author" +documenttype[1].datatype[0].sstruct.field[0].id 644499292 +documenttype[1].datatype[0].sstruct.field[0].id_v6 177126295 +documenttype[1].datatype[0].sstruct.field[0].datatype 2 +documenttype[1].datatype[0].sstruct.field[1].name "date" +documenttype[1].datatype[0].sstruct.field[1].id 491786523 +documenttype[1].datatype[0].sstruct.field[1].id_v6 916979460 +documenttype[1].datatype[0].sstruct.field[1].datatype 0 +documenttype[1].datatype[0].sstruct.field[2].name "subject" +documenttype[1].datatype[0].sstruct.field[2].id 1797950813 +documenttype[1].datatype[0].sstruct.field[2].id_v6 943449689 +documenttype[1].datatype[0].sstruct.field[2].datatype 2 +documenttype[1].datatype[1].id 1473469605 +documenttype[1].datatype[1].type STRUCT +documenttype[1].datatype[1].array.element.id 0 +documenttype[1].datatype[1].map.key.id 0 +documenttype[1].datatype[1].map.value.id 0 +documenttype[1].datatype[1].wset.key.id 0 +documenttype[1].datatype[1].wset.createifnonexistent false +documenttype[1].datatype[1].wset.removeifzero false +documenttype[1].datatype[1].annotationref.annotation.id 0 +documenttype[1].datatype[1].sstruct.name "text/html.body" +documenttype[1].datatype[1].sstruct.version 0 +documenttype[1].datatype[1].sstruct.compression.type NONE +documenttype[1].datatype[1].sstruct.compression.level 0 +documenttype[1].datatype[1].sstruct.compression.threshold 90 +documenttype[1].datatype[1].sstruct.compression.minsize 0 +documenttype[1].datatype[1].sstruct.field[1] +documenttype[1].datatype[1].sstruct.field[0].name "content" +documenttype[1].datatype[1].sstruct.field[0].id 1721764358 +documenttype[1].datatype[1].sstruct.field[0].id_v6 1751481844 +documenttype[1].datatype[1].sstruct.field[0].datatype 3 +documenttype[1].annotationtype[0] +documenttype[2].id 238423572 +documenttype[2].name "testdoctype1" +documenttype[2].version 1 +documenttype[2].headerstruct -226322995 +documenttype[2].bodystruct -1016297758 +documenttype[2].inherits[0] +documenttype[2].datatype[2] +documenttype[2].datatype[0].id -226322995 +documenttype[2].datatype[0].type STRUCT +documenttype[2].datatype[0].array.element.id 0 +documenttype[2].datatype[0].map.key.id 0 +documenttype[2].datatype[0].map.value.id 0 +documenttype[2].datatype[0].wset.key.id 0 +documenttype[2].datatype[0].wset.createifnonexistent false +documenttype[2].datatype[0].wset.removeifzero false +documenttype[2].datatype[0].annotationref.annotation.id 0 +documenttype[2].datatype[0].sstruct.name "testdoctype1.header" +documenttype[2].datatype[0].sstruct.version 1 +documenttype[2].datatype[0].sstruct.compression.type NONE +documenttype[2].datatype[0].sstruct.compression.level 0 +documenttype[2].datatype[0].sstruct.compression.threshold 90 +documenttype[2].datatype[0].sstruct.compression.minsize 0 +documenttype[2].datatype[0].sstruct.field[0] +documenttype[2].datatype[1].id -1016297758 +documenttype[2].datatype[1].type STRUCT +documenttype[2].datatype[1].array.element.id 0 +documenttype[2].datatype[1].map.key.id 0 +documenttype[2].datatype[1].map.value.id 0 +documenttype[2].datatype[1].wset.key.id 0 +documenttype[2].datatype[1].wset.createifnonexistent false +documenttype[2].datatype[1].wset.removeifzero false +documenttype[2].datatype[1].annotationref.annotation.id 0 +documenttype[2].datatype[1].sstruct.name "testdoctype1.body" +documenttype[2].datatype[1].sstruct.version 1 +documenttype[2].datatype[1].sstruct.compression.type NONE +documenttype[2].datatype[1].sstruct.compression.level 0 +documenttype[2].datatype[1].sstruct.compression.threshold 90 +documenttype[2].datatype[1].sstruct.compression.minsize 0 +documenttype[2].datatype[1].sstruct.field[1] +documenttype[2].datatype[1].sstruct.field[0].name "content" +documenttype[2].datatype[1].sstruct.field[0].id 5 +documenttype[2].datatype[1].sstruct.field[0].id_v6 5 +documenttype[2].datatype[1].sstruct.field[0].datatype 2 +documenttype[2].annotationtype[0] diff --git a/storage/src/tests/config-document.cfg b/storage/src/tests/config-document.cfg new file mode 100644 index 00000000000..0ec7e881ddf --- /dev/null +++ b/storage/src/tests/config-document.cfg @@ -0,0 +1,78 @@ +enablecompression false +datatype[6] +datatype[0].id 143329936 +datatype[0].arraytype[0] +datatype[0].weightedsettype[0] +datatype[0].structtype[1] +datatype[0].structtype[0].name text/html.header +datatype[0].structtype[0].version 0 +datatype[0].structtype[0].field[3] +datatype[0].structtype[0].field[0].name author +datatype[0].structtype[0].field[0].id[0] +datatype[0].structtype[0].field[0].datatype 2 +datatype[0].structtype[0].field[1].name subject +datatype[0].structtype[0].field[1].id[0] +datatype[0].structtype[0].field[1].datatype 2 +datatype[0].structtype[0].field[2].name date +datatype[0].structtype[0].field[2].id[0] +datatype[0].structtype[0].field[2].datatype 0 +datatype[0].documenttype[0] +datatype[1].id 1473469605 +datatype[1].arraytype[0] +datatype[1].weightedsettype[0] +datatype[1].structtype[1] +datatype[1].structtype[0].name text/html.body +datatype[1].structtype[0].version 0 +datatype[1].structtype[0].field[1] +datatype[1].structtype[0].field[0].name content +datatype[1].structtype[0].field[0].id[0] +datatype[1].structtype[0].field[0].datatype 3 +datatype[1].documenttype[0] +datatype[2].id -653677105 +datatype[2].arraytype[0] +datatype[2].weightedsettype[0] +datatype[2].structtype[0] +datatype[2].documenttype[1] +datatype[2].documenttype[0].name text/html +datatype[2].documenttype[0].version 0 +datatype[2].documenttype[0].inherits[0] +datatype[2].documenttype[0].headerstruct 143329936 +datatype[2].documenttype[0].bodystruct 1473469605 +datatype[3].id 160469461 +datatype[3].arraytype[0] +datatype[3].weightedsettype[0] +datatype[3].structtype[1] +datatype[3].structtype[0].name text/plain.header +datatype[3].structtype[0].version 0 +datatype[3].structtype[0].field[3] +datatype[3].structtype[0].field[0].name author +datatype[3].structtype[0].field[0].id[0] +datatype[3].structtype[0].field[0].datatype 2 +datatype[3].structtype[0].field[1].name subject +datatype[3].structtype[0].field[1].id[0] +datatype[3].structtype[0].field[1].datatype 2 +datatype[3].structtype[0].field[2].name date +datatype[3].structtype[0].field[2].id[0] +datatype[3].structtype[0].field[2].datatype 0 +datatype[3].documenttype[0] +datatype[4].id 749465898 +datatype[4].arraytype[0] +datatype[4].weightedsettype[0] +datatype[4].structtype[1] +datatype[4].structtype[0].name text/plain.body +datatype[4].structtype[0].version 0 +datatype[4].structtype[0].field[1] +datatype[4].structtype[0].field[0].name content +datatype[4].structtype[0].field[0].id[0] +datatype[4].structtype[0].field[0].datatype 3 +datatype[4].documenttype[0] +datatype[5].id -519202262 +datatype[5].arraytype[0] +datatype[5].weightedsettype[0] +datatype[5].structtype[0] +datatype[5].documenttype[1] +datatype[5].documenttype[0].name text/plain +datatype[5].documenttype[0].version 0 +datatype[5].documenttype[0].inherits[0] +datatype[5].documenttype[0].headerstruct 160469461 +datatype[5].documenttype[0].bodystruct 749465898 diff --git a/storage/src/tests/config-testdocman-document.cfg b/storage/src/tests/config-testdocman-document.cfg new file mode 100644 index 00000000000..c4bf43d9e37 --- /dev/null +++ b/storage/src/tests/config-testdocman-document.cfg @@ -0,0 +1,138 @@ +datatype[14] +datatype[0].id 1001 +datatype[0].arraytype[1] +datatype[0].arraytype[0].datatype 2 +datatype[1].id 2001 +datatype[1].weightedsettype[1] +datatype[1].weightedsettype[0].datatype 2 +datatype[1].weightedsettype[0].createifnonexistant false +datatype[1].weightedsettype[0].removeifzero false +datatype[2].id -2092985851 +datatype[2].structtype[1] +datatype[2].structtype[0].name mystruct +datatype[2].structtype[0].version 2 +datatype[2].structtype[0].field[2] +datatype[2].structtype[0].field[0].name key +datatype[2].structtype[0].field[0].id[1] +datatype[2].structtype[0].field[0].id[0].id 1 +datatype[2].structtype[0].field[0].datatype 0 +datatype[2].structtype[0].field[1].name value +datatype[2].structtype[0].field[1].id[1] +datatype[2].structtype[0].field[1].id[0].id 2 +datatype[2].structtype[0].field[1].datatype 2 +datatype[3].id -1244861287 +datatype[3].arraytype[1] +datatype[3].arraytype[0].datatype 3 +datatype[4].id 759956026 +datatype[4].arraytype[1] +datatype[4].arraytype[0].datatype -2092985851 +datatype[5].id -226322995 +datatype[5].structtype[1] +datatype[5].structtype[0].name testdoctype1.header +datatype[5].structtype[0].version 1 +datatype[5].structtype[0].field[9] +datatype[5].structtype[0].field[0].name headerval +datatype[5].structtype[0].field[0].id[1] +datatype[5].structtype[0].field[0].id[0].id 2 +datatype[5].structtype[0].field[0].datatype 0 +datatype[5].structtype[0].field[1].name hfloatval +datatype[5].structtype[0].field[1].id[1] +datatype[5].structtype[0].field[1].id[0].id 3 +datatype[5].structtype[0].field[1].datatype 1 +datatype[5].structtype[0].field[2].name hstringval +datatype[5].structtype[0].field[2].id[1] +datatype[5].structtype[0].field[2].id[0].id 4 +datatype[5].structtype[0].field[2].datatype 2 +datatype[5].structtype[0].field[3].name mystruct +datatype[5].structtype[0].field[3].id[1] +datatype[5].structtype[0].field[3].id[0].id 513 +datatype[5].structtype[0].field[3].datatype -2092985851 +datatype[5].structtype[0].field[4].name stringweightedset +datatype[5].structtype[0].field[4].id[1] +datatype[5].structtype[0].field[4].id[0].id 7 +datatype[5].structtype[0].field[4].datatype 2001 +datatype[5].structtype[0].field[5].name stringweightedset2 +datatype[5].structtype[0].field[5].id[1] +datatype[5].structtype[0].field[5].id[0].id 8 +datatype[5].structtype[0].field[5].datatype 18 +datatype[5].structtype[0].field[6].name tags +datatype[5].structtype[0].field[6].id[1] +datatype[5].structtype[0].field[6].id[0].id 6 +datatype[5].structtype[0].field[6].datatype 1001 +datatype[5].structtype[0].field[7].name title +datatype[5].structtype[0].field[7].id[1] +datatype[5].structtype[0].field[7].id[0].id 12 +datatype[5].structtype[0].field[7].datatype 2 +datatype[5].structtype[0].field[8].name headerlongval +datatype[5].structtype[0].field[8].id[1] +datatype[5].structtype[0].field[8].id[0].id 9999 +datatype[5].structtype[0].field[8].datatype 4 +datatype[6].id -1016297758 +datatype[6].structtype[1] +datatype[6].structtype[0].name testdoctype1.body +datatype[6].structtype[0].version 1 +datatype[6].structtype[0].field[3] +datatype[6].structtype[0].field[0].name content +datatype[6].structtype[0].field[0].id[1] +datatype[6].structtype[0].field[0].id[0].id 5 +datatype[6].structtype[0].field[0].datatype 2 +datatype[6].structtype[0].field[1].name rawarray +datatype[6].structtype[0].field[1].id[1] +datatype[6].structtype[0].field[1].id[0].id 10 +datatype[6].structtype[0].field[1].datatype -1244861287 +datatype[6].structtype[0].field[2].name structarray +datatype[6].structtype[0].field[2].id[1] +datatype[6].structtype[0].field[2].id[0].id 7123 +datatype[6].structtype[0].field[2].datatype 759956026 +datatype[7].id 238423572 +datatype[7].documenttype[1] +datatype[7].documenttype[0].name testdoctype1 +datatype[7].documenttype[0].version 1 +datatype[7].documenttype[0].headerstruct -226322995 +datatype[7].documenttype[0].bodystruct -1016297758 +datatype[8].id -422836500 +datatype[8].structtype[1] +datatype[8].structtype[0].name testdoctype2.header +datatype[8].structtype[0].version 1 +datatype[8].structtype[0].field[1] +datatype[8].structtype[0].field[0].name onlyinchild +datatype[8].structtype[0].field[0].id[1] +datatype[8].structtype[0].field[0].id[0].id 9 +datatype[8].structtype[0].field[0].datatype 0 +datatype[9].id 726512577 +datatype[9].structtype[1] +datatype[9].structtype[0].name testdoctype2.body +datatype[9].structtype[0].version 1 +datatype[9].structtype[0].field[0] +datatype[10].id 238424533 +datatype[10].documenttype[1] +datatype[10].documenttype[0].name testdoctype2 +datatype[10].documenttype[0].version 1 +datatype[10].documenttype[0].inherits[1] +datatype[10].documenttype[0].inherits[0].name testdoctype1 +datatype[10].documenttype[0].inherits[0].version 1 +datatype[10].documenttype[0].headerstruct -422836500 +datatype[10].documenttype[0].bodystruct 726512577 +datatype[11].id -1301366770 +datatype[11].structtype[1] +datatype[11].structtype[0].name _test_doctype3_.header +datatype[11].structtype[0].version 1 +datatype[11].structtype[0].field[1] +datatype[11].structtype[0].field[0].name _only_in_child_ +datatype[11].structtype[0].field[0].id[1] +datatype[11].structtype[0].field[0].id[0].id 9 +datatype[11].structtype[0].field[0].datatype 0 +datatype[12].id 1422804323 +datatype[12].structtype[1] +datatype[12].structtype[0].name _test_doctype3_.body +datatype[12].structtype[0].version 1 +datatype[12].structtype[0].field[0] +datatype[13].id 1088783091 +datatype[13].documenttype[1] +datatype[13].documenttype[0].name _test_doctype3_ +datatype[13].documenttype[0].version 1 +datatype[13].documenttype[0].inherits[1] +datatype[13].documenttype[0].inherits[0].name testdoctype1 +datatype[13].documenttype[0].inherits[0].version 1 +datatype[13].documenttype[0].headerstruct -1301366770 +datatype[13].documenttype[0].bodystruct 1422804323 diff --git a/storage/src/tests/distributor/.gitignore b/storage/src/tests/distributor/.gitignore new file mode 100644 index 00000000000..333f254ba10 --- /dev/null +++ b/storage/src/tests/distributor/.gitignore @@ -0,0 +1,8 @@ +*.So +*.lo +.*.swp +.depend +.depend.NEW +.deps +.libs +Makefile diff --git a/storage/src/tests/distributor/CMakeLists.txt b/storage/src/tests/distributor/CMakeLists.txt new file mode 100644 index 00000000000..6c6ba62ba6e --- /dev/null +++ b/storage/src/tests/distributor/CMakeLists.txt @@ -0,0 +1,44 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storage_testdistributor + SOURCES + mergelimitertest.cpp + bucketdatabasetest.cpp + messagesenderstub.cpp + externaloperationhandlertest.cpp + getoperationtest.cpp + idealstatemanagertest.cpp + putoperationtest.cpp + removeoperationtest.cpp + removebucketoperationtest.cpp + mergeoperationtest.cpp + splitbuckettest.cpp + joinbuckettest.cpp + visitoroperationtest.cpp + twophaseupdateoperationtest.cpp + removelocationtest.cpp + bucketdbupdatertest.cpp + statoperationtest.cpp + pendingmessagetrackertest.cpp + distributortestutil.cpp + simplebucketprioritydatabasetest.cpp + simplemaintenancescannertest.cpp + maintenanceschedulertest.cpp + throttlingoperationstartertest.cpp + blockingoperationstartertest.cpp + nodeinfotest.cpp + updateoperationtest.cpp + bucketstateoperationtest.cpp + distributortest.cpp + mapbucketdatabasetest.cpp + operationtargetresolvertest.cpp + garbagecollectiontest.cpp + statecheckerstest.cpp + statusreporterdelegatetest.cpp + bucketdbmetricupdatertest.cpp + bucketgctimecalculatortest.cpp + nodemaintenancestatstrackertest.cpp + distributor_host_info_reporter_test.cpp + DEPENDS + AFTER + storage_storageconfig +) diff --git a/storage/src/tests/distributor/blockingoperationstartertest.cpp b/storage/src/tests/distributor/blockingoperationstartertest.cpp new file mode 100644 index 00000000000..ee0058643d9 --- /dev/null +++ b/storage/src/tests/distributor/blockingoperationstartertest.cpp @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <string> +#include <sstream> +#include <memory> +#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> +#include <vespa/storage/distributor/blockingoperationstarter.h> +#include <vespa/storage/distributor/pendingmessagetracker.h> +#include <tests/distributor/maintenancemocks.h> + +namespace storage { + +namespace distributor { + +using document::BucketId; + +class BlockingOperationStarterTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(BlockingOperationStarterTest); + CPPUNIT_TEST(testOperationNotBlockedWhenNoMessagesPending); + CPPUNIT_TEST(testOperationBlockedWhenMessagesPending); + CPPUNIT_TEST_SUITE_END(); + + std::shared_ptr<Operation> createMockOperation() { + return std::shared_ptr<Operation>(new MockOperation(BucketId(16, 1))); + } + std::shared_ptr<Operation> createBlockingMockOperation() { + std::shared_ptr<MockOperation> op(new MockOperation(BucketId(16, 1))); + op->setShouldBlock(true); + return op; + } + + framework::defaultimplementation::FakeClock _clock; + std::unique_ptr<MockOperationStarter> _starterImpl; + std::unique_ptr<StorageComponentRegisterImpl> _compReg; + std::unique_ptr<PendingMessageTracker> _messageTracker; + std::unique_ptr<BlockingOperationStarter> _operationStarter; + +public: + void testOperationNotBlockedWhenNoMessagesPending(); + void testOperationBlockedWhenMessagesPending(); + + void setUp(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(BlockingOperationStarterTest); + +void +BlockingOperationStarterTest::setUp() +{ + _starterImpl.reset(new MockOperationStarter()); + _compReg.reset(new StorageComponentRegisterImpl()); + _compReg->setClock(_clock); + _clock.setAbsoluteTimeInSeconds(1); + _messageTracker.reset(new PendingMessageTracker(*_compReg)); + _operationStarter.reset(new BlockingOperationStarter(*_messageTracker, *_starterImpl)); +} + +void +BlockingOperationStarterTest::testOperationNotBlockedWhenNoMessagesPending() +{ + CPPUNIT_ASSERT(_operationStarter->start(createMockOperation(), + OperationStarter::Priority(0))); + CPPUNIT_ASSERT_EQUAL(std::string("BucketId(0x4000000000000001), pri 0\n"), + _starterImpl->toString()); +} + +void +BlockingOperationStarterTest::testOperationBlockedWhenMessagesPending() +{ + // start should return true but not forward message to underlying starter. + CPPUNIT_ASSERT(_operationStarter->start(createBlockingMockOperation(), + OperationStarter::Priority(0))); + CPPUNIT_ASSERT_EQUAL(std::string(""), _starterImpl->toString()); +} + +} +} diff --git a/storage/src/tests/distributor/bucketdatabasetest.cpp b/storage/src/tests/distributor/bucketdatabasetest.cpp new file mode 100644 index 00000000000..011b02c8f89 --- /dev/null +++ b/storage/src/tests/distributor/bucketdatabasetest.cpp @@ -0,0 +1,550 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <tests/distributor/bucketdatabasetest.h> +#include <vespa/storageframework/defaultimplementation/clock/realclock.h> +#include <iostream> +#include <fstream> +#include <iomanip> + +namespace storage { +namespace distributor { + +using document::BucketId; + +void +BucketDatabaseTest::setUp() +{ + db().clear(); +} + +namespace { + BucketCopy BC(uint32_t nodeIdx) { + return BucketCopy(0, nodeIdx, api::BucketInfo()); + } + + BucketInfo BI(uint32_t nodeIdx) { + BucketInfo bi; + bi.addNode(BC(nodeIdx), toVector<uint16_t>(0)); + return bi; + } +} + +void +BucketDatabaseTest::testClear() { + db().update(BucketDatabase::Entry(document::BucketId(16, 16), BI(1))); + db().update(BucketDatabase::Entry(document::BucketId(16, 11), BI(2))); + db().clear(); + CPPUNIT_ASSERT_EQUAL(size_t(0), db().size()); +} + +void +BucketDatabaseTest::testUpdateGetAndRemove() { + // Do some insertions + CPPUNIT_ASSERT_EQUAL(0, (int)db().size()); + db().update(BucketDatabase::Entry(document::BucketId(16, 16), BI(1))); + db().update(BucketDatabase::Entry(document::BucketId(16, 11), BI(2))); + db().update(BucketDatabase::Entry(document::BucketId(16, 42), BI(3))); + CPPUNIT_ASSERT_EQUAL(3, (int)db().size()); + + db().update(BucketDatabase::Entry(document::BucketId(16, 11), BI(4))); + CPPUNIT_ASSERT_EQUAL(3, (int)db().size()); + + // Access some elements + CPPUNIT_ASSERT_EQUAL(BI(4), db().get(document::BucketId(16, 11)).getBucketInfo()); + CPPUNIT_ASSERT_EQUAL(BI(1), db().get(document::BucketId(16, 16)).getBucketInfo()); + CPPUNIT_ASSERT_EQUAL(BI(3), db().get(document::BucketId(16, 42)).getBucketInfo()); + + // Do removes + db().remove(document::BucketId(16, 12)); + + CPPUNIT_ASSERT_EQUAL(3, (int)db().size()); + + db().remove(document::BucketId(16, 11)); + + CPPUNIT_ASSERT_EQUAL(2, (int)db().size()); + + db().remove(document::BucketId(16, 16)); + db().remove(document::BucketId(16, 42)); + + CPPUNIT_ASSERT_EQUAL(0, (int)db().size()); +} + +namespace { + +struct ModifyProcessor : public BucketDatabase::MutableEntryProcessor +{ + bool process(BucketDatabase::Entry& e) { + if (e.getBucketId() == document::BucketId(16, 0x0b)) { + e.getBucketInfo() = BI(7); + } else if (e.getBucketId() == document::BucketId(16, 0x2a)) { + e->clear(); + e->addNode(BC(4), toVector<uint16_t>(0)); + e->addNode(BC(5), toVector<uint16_t>(0)); + } + + return true; + } +}; + +struct ListAllProcessor : public BucketDatabase::EntryProcessor +{ + std::ostringstream ost; + + bool process(const BucketDatabase::Entry& e) { + ost << e << "\n"; + return true; + } +}; + +struct DummyProcessor : public BucketDatabase::EntryProcessor +{ + std::ostringstream ost; + + bool process(const BucketDatabase::Entry&) { + return true; + } +}; + + +struct StoppingProcessor : public BucketDatabase::EntryProcessor +{ + std::ostringstream ost; + + bool process(const BucketDatabase::Entry& e) { + ost << e << "\n"; + + if (e.getBucketId() == document::BucketId(16, 0x2a)) { + return false; + } + + return true; + } +}; + +} + +void +BucketDatabaseTest::testIterating() { + // Do some insertions + db().update(BucketDatabase::Entry(document::BucketId(16, 0x10), BI(1))); + db().update(BucketDatabase::Entry(document::BucketId(16, 0x0b), BI(2))); + db().update(BucketDatabase::Entry(document::BucketId(16, 0x2a), BI(3))); + + { + ListAllProcessor proc; + db().forEach(proc, document::BucketId()); + + CPPUNIT_ASSERT_EQUAL( + std::string( + "BucketId(0x4000000000000010) : " + "node(idx=1,crc=0x0,docs=0/0,bytes=1/1,trusted=false,active=false)\n" + "BucketId(0x400000000000002a) : " + "node(idx=3,crc=0x0,docs=0/0,bytes=1/1,trusted=false,active=false)\n" + "BucketId(0x400000000000000b) : " + "node(idx=2,crc=0x0,docs=0/0,bytes=1/1,trusted=false,active=false)\n"), + proc.ost.str()); + } + + { + ListAllProcessor proc; + db().forEach(proc, document::BucketId(16, 0x2a)); + + CPPUNIT_ASSERT_EQUAL( + std::string( + "BucketId(0x400000000000000b) : " + "node(idx=2,crc=0x0,docs=0/0,bytes=1/1,trusted=false,active=false)\n"), + proc.ost.str()); + } + + { + StoppingProcessor proc; + db().forEach(proc, document::BucketId()); + + CPPUNIT_ASSERT_EQUAL( + std::string( + "BucketId(0x4000000000000010) : " + "node(idx=1,crc=0x0,docs=0/0,bytes=1/1,trusted=false,active=false)\n" + "BucketId(0x400000000000002a) : " + "node(idx=3,crc=0x0,docs=0/0,bytes=1/1,trusted=false,active=false)\n"), + proc.ost.str()); + } + + { + ModifyProcessor alterProc; + db().forEach(alterProc, document::BucketId()); + // Verify content after altering + ListAllProcessor proc; + db().forEach(proc); + + CPPUNIT_ASSERT_EQUAL( + std::string( + "BucketId(0x4000000000000010) : " + "node(idx=1,crc=0x0,docs=0/0,bytes=1/1,trusted=false,active=false)\n" + "BucketId(0x400000000000002a) : " + "node(idx=4,crc=0x0,docs=0/0,bytes=1/1,trusted=false,active=false), " + "node(idx=5,crc=0x0,docs=0/0,bytes=1/1,trusted=false,active=false)\n" + "BucketId(0x400000000000000b) : " + "node(idx=7,crc=0x0,docs=0/0,bytes=1/1,trusted=false,active=false)\n"), + proc.ost.str()); + } +} + +std::string +BucketDatabaseTest::doFindParents(const std::vector<document::BucketId>& ids, + const document::BucketId& searchId) +{ + db().clear(); + + for (uint32_t i = 0; i < ids.size(); ++i) { + db().update(BucketDatabase::Entry(ids[i], BI(i))); + } + + std::vector<BucketDatabase::Entry> entries; + db().getParents(searchId, entries); + + std::ostringstream ost; + for (uint32_t i = 0; i < ids.size(); ++i) { + if (std::find(entries.begin(), entries.end(), + BucketDatabase::Entry(ids[i], BI(i))) != entries.end()) { + if (!ost.str().empty()) { + ost << ","; + } + ost << i; + } + } + + return ost.str(); +} + +void +BucketDatabaseTest::testFindParents() { + // test what parents in the DB (specified in vector) are parents of the + // specified bucket. Result is a list of indexes into the vector. + CPPUNIT_ASSERT_EQUAL( + std::string("2"), + doFindParents(toVector(document::BucketId(17, 0x0ffff), + document::BucketId(18, 0x1ffff), + document::BucketId(18, 0x3ffff)), + document::BucketId(22, 0xfffff))); + + CPPUNIT_ASSERT_EQUAL( + std::string("0,2,3"), + doFindParents(toVector(document::BucketId(16, 0x0ffff), + document::BucketId(17, 0x0ffff), + document::BucketId(17, 0x1ffff), + document::BucketId(19, 0xfffff)), + document::BucketId(22, 0xfffff))); + + CPPUNIT_ASSERT_EQUAL( + std::string("0,2,3"), + doFindParents(toVector(document::BucketId(16, 0x0ffff), + document::BucketId(17, 0x0ffff), + document::BucketId(17, 0x1ffff), + document::BucketId(18, 0x1ffff)), + document::BucketId(22, 0x1ffff))); + + CPPUNIT_ASSERT_EQUAL( + std::string("0"), + doFindParents(toVector(document::BucketId(16, 0x0ffff), + document::BucketId(17, 0x0ffff)), + document::BucketId(22, 0x1ffff))); + + CPPUNIT_ASSERT_EQUAL( // ticket 3121525 + std::string("0"), + doFindParents(toVector(document::BucketId(16, 0x0ffff), + document::BucketId(17, 0x0ffff), + document::BucketId(19, 0x1ffff)), + document::BucketId(18, 0x1ffff))); + + CPPUNIT_ASSERT_EQUAL( // ticket 3121525 + std::string("0"), + doFindParents(toVector(document::BucketId(16, 0x0ffff), + document::BucketId(17, 0x0ffff), + document::BucketId(19, 0x5ffff)), + document::BucketId(18, 0x1ffff))); +} + +std::string +BucketDatabaseTest::doFindAll(const std::vector<document::BucketId>& ids, + const document::BucketId& searchId) +{ + db().clear(); + + for (uint32_t i = 0; i < ids.size(); ++i) { + db().update(BucketDatabase::Entry(ids[i], BI(i))); + } + + std::vector<BucketDatabase::Entry> entries; + db().getAll(searchId, entries); + + std::ostringstream ost; + for (uint32_t i = 0; i < ids.size(); ++i) { + if (std::find(entries.begin(), entries.end(), + BucketDatabase::Entry(ids[i], BI(i))) != entries.end()) { + if (!ost.str().empty()) { + ost << ","; + } + ost << i; + } + } + + return ost.str(); +} + +void +BucketDatabaseTest::testFindAll() +{ + std::vector<document::BucketId> buckets; + CPPUNIT_ASSERT_EQUAL( + std::string(""), + doFindAll(buckets, document::BucketId(18, 0x1ffff))); + + buckets.push_back(document::BucketId(16, 0x0aaaa)); // contains bucket 2-7 + buckets.push_back(document::BucketId(17, 0x0aaaa)); // contains bucket 3-4 + buckets.push_back(document::BucketId(20, 0xcaaaa)); + buckets.push_back(document::BucketId(20, 0xeaaaa)); + buckets.push_back(document::BucketId(17, 0x1aaaa)); // contains bucket 6-7 + buckets.push_back(document::BucketId(20, 0xdaaaa)); + buckets.push_back(document::BucketId(20, 0xfaaaa)); + buckets.push_back(document::BucketId(20, 0xceaaa)); + buckets.push_back(document::BucketId(17, 0x1ffff)); + + CPPUNIT_ASSERT_EQUAL( + std::string("0,4,5,6"), + doFindAll(buckets, document::BucketId(17, 0x1aaaa))); + + CPPUNIT_ASSERT_EQUAL( + std::string("8"), + doFindAll(buckets, document::BucketId(16, 0xffff))); + + CPPUNIT_ASSERT_EQUAL( + std::string("0,1"), + doFindAll(toVector(document::BucketId(17, 0x00001), + document::BucketId(17, 0x10001)), + document::BucketId(16, 0x00001))); + + document::BucketId id(33, 0x1053c7089); // Bit 32 is set, but unused. + id.setUsedBits(32); + CPPUNIT_ASSERT_EQUAL( + std::string("1,2"), + doFindAll(toVector(document::BucketId(24, 0x000dc7089), + document::BucketId(33, 0x0053c7089), + document::BucketId(33, 0x1053c7089), + document::BucketId(24, 0x000bc7089)), + id)); + + CPPUNIT_ASSERT_EQUAL( // Inconsistent split + std::string("0,1,2"), + doFindAll(toVector( + document::BucketId(16, 0x00001), // contains 2-3 + document::BucketId(17, 0x00001), + document::BucketId(17, 0x10001)), + document::BucketId(16, 0x00001))); + + CPPUNIT_ASSERT_EQUAL( // Inconsistent split + std::string("1,2"), + doFindAll(toVector( + document::BucketId(17, 0x10000), + document::BucketId(27, 0x007228034), // contains 3 + document::BucketId(29, 0x007228034), + document::BucketId(17, 0x1ffff)), + document::BucketId(32, 0x027228034))); + + CPPUNIT_ASSERT_EQUAL( // Inconsistent split + std::string("0"), + doFindAll(toVector( + document::BucketId(16, 0x0ffff), + document::BucketId(17, 0x0ffff)), + document::BucketId(22, 0x1ffff))); + + CPPUNIT_ASSERT_EQUAL( // Inconsistent split + std::string("0,2"), + doFindAll(toVector( + document::BucketId(16, 0x0ffff), + document::BucketId(17, 0x0ffff), + document::BucketId(19, 0x1ffff)), + document::BucketId(18, 0x1ffff))); + + CPPUNIT_ASSERT_EQUAL( // Inconsistent split, ticket 3121525 + std::string("0,2"), + doFindAll(toVector( + document::BucketId(16, 0x0ffff), + document::BucketId(17, 0x0ffff), + document::BucketId(19, 0x5ffff)), + document::BucketId(18, 0x1ffff))); +} + +document::BucketId +BucketDatabaseTest::doCreate(const std::vector<document::BucketId>& ids, + uint32_t minBits, + const document::BucketId& wantedId) +{ + db().clear(); + + for (uint32_t i = 0; i < ids.size(); ++i) { + db().update(BucketDatabase::Entry(ids[i], BI(i))); + } + + BucketDatabase::Entry entry = db().createAppropriateBucket(minBits, wantedId); + return entry.getBucketId(); +} + +void +BucketDatabaseTest::testCreateAppropriateBucket() { + // Use min split bits when no relevant bucket exist. + CPPUNIT_ASSERT_EQUAL( + document::BucketId(36,0x0000004d2), + doCreate(toVector(document::BucketId(58, 0x43d6c878000004d2ull)), 36, + document::BucketId(58, 0x423bf1e0000004d2ull))); + // New bucket has bits in common with existing bucket. + // Create bucket with min amount of bits while not being overlapping + CPPUNIT_ASSERT_EQUAL( + document::BucketId(34,0x0000004d2), + doCreate(toVector(document::BucketId(58, 0xeaf77782000004d2)), + 16, + document::BucketId(58, 0x00000000000004d2))); + // Create sibling of existing bucket with most LSB bits in common. + CPPUNIT_ASSERT_EQUAL( + document::BucketId(40, 0x0000004d2), + doCreate(toVector(document::BucketId(58, 0xeaf77780000004d2), + document::BucketId(58, 0xeaf77782000004d2)), + 16, + document::BucketId(58, 0x00000000000004d2))); + // Create sibling of existing bucket with most LSB bits in common. + CPPUNIT_ASSERT_EQUAL( + document::BucketId(25, 0x0010004d2), + doCreate(toVector(document::BucketId(16, 0x00000000000004d1), + document::BucketId(40, 0x00000000000004d2)), + 16, + document::BucketId(58, 0x00000000010004d2))); + + CPPUNIT_ASSERT_EQUAL( + document::BucketId(36, 0x10000004000004d2), + doCreate(toVector(document::BucketId(0x8c000000000004d2), + document::BucketId(0xeb54b3ac000004d2), + document::BucketId(0x88000002000004d2), + document::BucketId(0x84000001000004d2)), + 16, + document::BucketId(58, 0x1944a44000004d2))); + CPPUNIT_ASSERT_EQUAL( + document::BucketId(25, 0x0010004d2), + doCreate(toVector(document::BucketId(58, 0xeaf77780000004d2), + document::BucketId(40, 0x00000000000004d1)), + 16, + document::BucketId(58,0x00000000010004d2))); + // Test empty bucket database case. (Use min split bits) + std::vector<document::BucketId> buckets; + CPPUNIT_ASSERT_EQUAL( + document::BucketId(16, 0x0000004d2ull), + doCreate(buckets, 16, + document::BucketId(58, 0x00000000010004d2))); +} + +void +BucketDatabaseTest::testGetNext() +{ + db().clear(); + db().update(BucketDatabase::Entry(document::BucketId(16, 16), BI(1))); + db().update(BucketDatabase::Entry(document::BucketId(16, 11), BI(2))); + db().update(BucketDatabase::Entry(document::BucketId(16, 42), BI(3))); + + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 16), + db().getNext(document::BucketId()).getBucketId()); + + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 42), + db().getNext(document::BucketId(16, 16)).getBucketId()); + + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 11), + db().getNext(document::BucketId(16, 42)).getBucketId()); +} + +void +BucketDatabaseTest::doTestUpperBound(const UBoundFunc& f) +{ + db().clear(); + // Tree is rooted at the LSB bit, so the following buckets are in iteration + // order based on the reverse of their "normal" bitstring: + // 0010:3 + db().update(BucketDatabase::Entry(document::BucketId(3, 4), BI(2))); + // 1000:3 + db().update(BucketDatabase::Entry(document::BucketId(3, 1), BI(2))); + // 1001:4 + db().update(BucketDatabase::Entry(document::BucketId(4, 9), BI(1))); + // 10010:5 + db().update(BucketDatabase::Entry(document::BucketId(5, 9), BI(1))); + // 1100:3 + db().update(BucketDatabase::Entry(document::BucketId(3, 3), BI(3))); + + // 0000:0 (default constructed) has ubound of 0010:3 + CPPUNIT_ASSERT_EQUAL(BucketId(3, 4), f(db(), BucketId())); + // 0011:4 has ubound of 1000:3 + CPPUNIT_ASSERT_EQUAL(document::BucketId(3, 1), f(db(), BucketId(4, 12))); + // 1000:1 has ubound of 1000:3 + CPPUNIT_ASSERT_EQUAL(BucketId(3, 4), f(db(), BucketId(1, 0))); + CPPUNIT_ASSERT_EQUAL(BucketId(3, 1), f(db(), BucketId(3, 4))); + CPPUNIT_ASSERT_EQUAL(BucketId(4, 9), f(db(), BucketId(3, 1))); + CPPUNIT_ASSERT_EQUAL(BucketId(5, 9), f(db(), BucketId(4, 9))); + CPPUNIT_ASSERT_EQUAL(BucketId(3, 3), f(db(), BucketId(5, 9))); + // 100101:6 does not exist, should also return 1100:3 + CPPUNIT_ASSERT_EQUAL(BucketId(3, 3), f(db(), BucketId(6, 41))); + + // Test extremes. + db().clear(); + db().update(BucketDatabase::Entry(document::BucketId(8, 0), BI(2))); + db().update(BucketDatabase::Entry(document::BucketId(8, 0xff), BI(2))); + + CPPUNIT_ASSERT_EQUAL(BucketId(8, 0), f(db(), BucketId())); + CPPUNIT_ASSERT_EQUAL(BucketId(8, 0xff), f(db(), BucketId(8, 0))); +} + +void +BucketDatabaseTest::testUpperBoundReturnsNextInOrderGreaterBucket() +{ + doTestUpperBound([](const BucketDatabase& bucketDb, + const document::BucketId& id) + { + return bucketDb.upperBound(id).getBucketId(); + }); +} + +void +BucketDatabaseTest::testGetNextReturnsUpperBoundBucket() +{ + // getNext() would generally be implemented in terms of upperBound(), but + // make sure it conforms to the same contract in case this changes. + doTestUpperBound([](const BucketDatabase& bucketDb, + const document::BucketId& id) + { + return bucketDb.getNext(id).getBucketId(); + }); +} + +void +BucketDatabaseTest::testChildCount() +{ + db().clear(); + // Empty tree; inserts cannot create inconsistencies. + CPPUNIT_ASSERT_EQUAL(0u, db().childCount(BucketId(3, 1))); + + // Same bucket; cannot be inconsistent with itself. + db().update(BucketDatabase::Entry(document::BucketId(3, 1), BI(1))); + CPPUNIT_ASSERT_EQUAL(0u, db().childCount(BucketId(3, 1))); + + // (2, 1) has one subtree. + CPPUNIT_ASSERT_EQUAL(1u, db().childCount(BucketId(2, 1))); + + // Bucket exists in another subtree from (1, 1); inconsistency would + // result if we tried inserting it. + db().update(BucketDatabase::Entry(document::BucketId(3, 3), BI(2))); + CPPUNIT_ASSERT_EQUAL(2u, db().childCount(BucketId(1, 1))); + + // Inner node with 1 subtree. + CPPUNIT_ASSERT_EQUAL(1u, db().childCount(BucketId(2, 3))); + + // Leaves have no subtrees. + CPPUNIT_ASSERT_EQUAL(0u, db().childCount(BucketId(3, 1))); + CPPUNIT_ASSERT_EQUAL(0u, db().childCount(BucketId(3, 5))); +} + +} +} // storage diff --git a/storage/src/tests/distributor/bucketdatabasetest.h b/storage/src/tests/distributor/bucketdatabasetest.h new file mode 100644 index 00000000000..1eb8bf86add --- /dev/null +++ b/storage/src/tests/distributor/bucketdatabasetest.h @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/document_runnable.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <cppunit/extensions/HelperMacros.h> +#include <vespa/storage/distributor/bucketdb/judybucketdatabase.h> +#include <vespa/storage/storageutil/utils.h> + +#define SETUP_DATABASE_TESTS() \ + CPPUNIT_TEST(testUpdateGetAndRemove); \ + CPPUNIT_TEST(testClear); \ + CPPUNIT_TEST(testIterating); \ + CPPUNIT_TEST(testFindParents); \ + CPPUNIT_TEST(testFindAll); \ + CPPUNIT_TEST(testCreateAppropriateBucket); \ + CPPUNIT_TEST(testGetNext); \ + CPPUNIT_TEST(testGetNextReturnsUpperBoundBucket); \ + CPPUNIT_TEST(testUpperBoundReturnsNextInOrderGreaterBucket); \ + CPPUNIT_TEST(testChildCount); + +namespace storage { +namespace distributor { + +struct BucketDatabaseTest : public CppUnit::TestFixture { + void setUp(); + + void testUpdateGetAndRemove(); + void testClear(); + void testIterating(); + void testFindParents(); + void testFindAll(); + void testCreateAppropriateBucket(); + void testGetNext(); + void testGetNextReturnsUpperBoundBucket(); + void testUpperBoundReturnsNextInOrderGreaterBucket(); + void testChildCount(); + + void testBenchmark(); + + std::string doFindParents(const std::vector<document::BucketId>& ids, + const document::BucketId& searchId); + std::string doFindAll(const std::vector<document::BucketId>& ids, + const document::BucketId& searchId); + document::BucketId doCreate(const std::vector<document::BucketId>& ids, + uint32_t minBits, + const document::BucketId& wantedId); + + virtual BucketDatabase& db() = 0; + +private: + using UBoundFunc = std::function< + document::BucketId(const BucketDatabase&, + const document::BucketId&)>; + + void doTestUpperBound(const UBoundFunc& f); +}; + +} + +} + diff --git a/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp b/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp new file mode 100644 index 00000000000..6aa9ef3a844 --- /dev/null +++ b/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp @@ -0,0 +1,361 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <string> +#include <sstream> +#include <vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h> +#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/distributor/distributormetricsset.h> +#include <vespa/storage/distributor/idealstatemetricsset.h> +#include <vespa/storage/config/config-stor-distributormanager.h> + +namespace storage { +namespace distributor { + +using document::BucketId; + +class BucketDBMetricUpdaterTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(BucketDBMetricUpdaterTest); + CPPUNIT_TEST(testDocAndByteCountsAreUpdated); + CPPUNIT_TEST(testBucketsWithTooFewAndTooManyCopies); + CPPUNIT_TEST(testBucketsWithVaryingTrustedness); + CPPUNIT_TEST(testPickCountsFromTrustedCopy); + CPPUNIT_TEST(testPickLargestCopyIfNoTrusted); + CPPUNIT_TEST(testCompleteRoundClearsWorkingState); + CPPUNIT_TEST(testMinBucketReplicaTrackedAndReportedPerNode); + CPPUNIT_TEST(nonTrustedReplicasAlsoCountedInModeAny); + CPPUNIT_TEST(minimumReplicaCountReturnedForNodeInModeAny); + CPPUNIT_TEST_SUITE_END(); + + void visitBucketWith2Copies1Trusted(BucketDBMetricUpdater& metricUpdater); + void visitBucketWith2CopiesBothTrusted( + BucketDBMetricUpdater& metricUpdater); + void visitBucketWith1Copy(BucketDBMetricUpdater& metricUpdater); + + + using NodeToReplicasMap = std::unordered_map<uint16_t, uint32_t>; + NodeToReplicasMap replicaStatsOf(BucketDBMetricUpdater& metricUpdater); + + metrics::LoadTypeSet _loadTypes; +public: + BucketDBMetricUpdaterTest(); + + void testDocAndByteCountsAreUpdated(); + void testBucketsWithTooFewAndTooManyCopies(); + void testBucketsWithVaryingTrustedness(); + void testPickCountsFromTrustedCopy(); + void testPickLargestCopyIfNoTrusted(); + void testCompleteRoundClearsWorkingState(); + void testMinBucketReplicaTrackedAndReportedPerNode(); + void nonTrustedReplicasAlsoCountedInModeAny(); + void minimumReplicaCountReturnedForNodeInModeAny(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(BucketDBMetricUpdaterTest); + +BucketDBMetricUpdaterTest::BucketDBMetricUpdaterTest() +{ + _loadTypes.push_back(metrics::LoadType(0, "foo")); +} + +namespace { + +void addNode(BucketInfo& info, uint16_t node, uint32_t crc) { + auto apiInfo = api::BucketInfo(crc, crc + 1, crc + 2); + std::vector<uint16_t> order; + info.addNode(BucketCopy(1234, node, apiInfo), order); +} + +typedef bool Trusted; + +BucketInfo +makeInfo(uint32_t copy0Crc) +{ + BucketInfo info; + addNode(info, 0, copy0Crc); + return info; +} + +BucketInfo +makeInfo(uint32_t copy0Crc, uint32_t copy1Crc) +{ + BucketInfo info; + addNode(info, 0, copy0Crc); + addNode(info, 1, copy1Crc); + return info; +} + +} // anonymous namespace + +void +BucketDBMetricUpdaterTest::testDocAndByteCountsAreUpdated() +{ + BucketDBMetricUpdater metricUpdater; + IdealStateMetricSet ims; + DistributorMetricSet dms(_loadTypes); + + CPPUNIT_ASSERT_EQUAL(false, metricUpdater.hasCompletedRound()); + + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + metricUpdater.completeRound(false); + + CPPUNIT_ASSERT_EQUAL(true, metricUpdater.hasCompletedRound()); + + CPPUNIT_ASSERT_EQUAL(int64_t(0), dms.docsStored.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(0), dms.bytesStored.getLast()); + { + BucketDatabase::Entry e(document::BucketId(16, 1), makeInfo(10)); + metricUpdater.visit(e, 1); + } + + metricUpdater.completeRound(false); + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + + CPPUNIT_ASSERT_EQUAL(true, metricUpdater.hasCompletedRound()); + + CPPUNIT_ASSERT_EQUAL(int64_t(11), dms.docsStored.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(12), dms.bytesStored.getLast()); + + { + BucketDatabase::Entry e(document::BucketId(16, 1), makeInfo(20)); + metricUpdater.visit(e, 1); + } + + metricUpdater.completeRound(false); + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + + CPPUNIT_ASSERT_EQUAL(int64_t(32), dms.docsStored.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(34), dms.bytesStored.getLast()); +} + +void +BucketDBMetricUpdaterTest::testBucketsWithTooFewAndTooManyCopies() +{ + BucketDBMetricUpdater metricUpdater; + IdealStateMetricSet ims; + DistributorMetricSet dms(_loadTypes); + + metricUpdater.completeRound(); + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + CPPUNIT_ASSERT_EQUAL(int64_t(0), ims.buckets_toofewcopies.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(0), ims.buckets_toomanycopies.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(0), ims.buckets.getLast()); + + // 1 copy too little + { + BucketDatabase::Entry e(document::BucketId(16, 1), makeInfo(10)); + metricUpdater.visit(e, 2); + } + metricUpdater.completeRound(false); + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + + CPPUNIT_ASSERT_EQUAL(int64_t(1), ims.buckets_toofewcopies.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(0), ims.buckets_toomanycopies.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(1), ims.buckets.getLast()); + + // 1 copy too many + { + BucketDatabase::Entry e(document::BucketId(16, 1), makeInfo(40, 40)); + metricUpdater.visit(e, 1); + } + metricUpdater.completeRound(false); + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + + CPPUNIT_ASSERT_EQUAL(int64_t(1), ims.buckets_toofewcopies.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(1), ims.buckets_toomanycopies.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(2), ims.buckets.getLast()); + + // Right amount of copies, just inc bucket counter. + { + BucketDatabase::Entry e(document::BucketId(16, 1), makeInfo(40, 40)); + metricUpdater.visit(e, 2); + } + metricUpdater.completeRound(false); + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + + CPPUNIT_ASSERT_EQUAL(int64_t(1), ims.buckets_toofewcopies.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(1), ims.buckets_toomanycopies.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(3), ims.buckets.getLast()); +} + +void +BucketDBMetricUpdaterTest::testBucketsWithVaryingTrustedness() +{ + BucketDBMetricUpdater metricUpdater; + IdealStateMetricSet ims; + DistributorMetricSet dms(_loadTypes); + + metricUpdater.completeRound(false); + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + CPPUNIT_ASSERT_EQUAL(int64_t(0), ims.buckets_notrusted.getLast()); + // Has only trusted (implicit for first added) + { + BucketDatabase::Entry e(document::BucketId(16, 1), makeInfo(100)); + metricUpdater.visit(e, 2); + } + metricUpdater.completeRound(false); + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + CPPUNIT_ASSERT_EQUAL(int64_t(0), ims.buckets_notrusted.getLast()); + // Has at least one trusted (implicit for first added) + { + BucketDatabase::Entry e(document::BucketId(16, 2), makeInfo(100, 200)); + metricUpdater.visit(e, 2); + } + metricUpdater.completeRound(false); + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + CPPUNIT_ASSERT_EQUAL(int64_t(0), ims.buckets_notrusted.getLast()); + // Has no trusted + { + BucketInfo info(makeInfo(100, 200)); + info.resetTrusted(); + BucketDatabase::Entry e(document::BucketId(16, 3), info); + metricUpdater.visit(e, 2); + } + metricUpdater.completeRound(false); + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + CPPUNIT_ASSERT_EQUAL(int64_t(1), ims.buckets_notrusted.getLast()); +} + +void +BucketDBMetricUpdaterTest::testPickCountsFromTrustedCopy() +{ + BucketDBMetricUpdater metricUpdater; + IdealStateMetricSet ims; + DistributorMetricSet dms(_loadTypes); + + // First copy added is implicitly trusted, but it is not the largest. + BucketDatabase::Entry e(document::BucketId(16, 2), makeInfo(100, 200)); + metricUpdater.visit(e, 2); + metricUpdater.completeRound(false); + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + + CPPUNIT_ASSERT_EQUAL(int64_t(101), dms.docsStored.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(102), dms.bytesStored.getLast()); +} + +void +BucketDBMetricUpdaterTest::testPickLargestCopyIfNoTrusted() +{ + BucketDBMetricUpdater metricUpdater; + IdealStateMetricSet ims; + DistributorMetricSet dms(_loadTypes); + + // No trusted copies, so must pick second copy. + BucketInfo info(makeInfo(100, 200)); + info.resetTrusted(); + BucketDatabase::Entry e(document::BucketId(16, 2), info); + metricUpdater.visit(e, 2); + metricUpdater.completeRound(false); + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + + CPPUNIT_ASSERT_EQUAL(int64_t(201), dms.docsStored.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(202), dms.bytesStored.getLast()); +} + +void +BucketDBMetricUpdaterTest::testCompleteRoundClearsWorkingState() +{ + BucketDBMetricUpdater metricUpdater; + IdealStateMetricSet ims; + DistributorMetricSet dms(_loadTypes); + + { + BucketDatabase::Entry e(document::BucketId(16, 1), makeInfo(10)); + metricUpdater.visit(e, 1); + } + metricUpdater.completeRound(); + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + + CPPUNIT_ASSERT_EQUAL(int64_t(11), dms.docsStored.getLast()); + // Completing the round again with no visits having been done will + // propagate an empty working state to the complete state. + metricUpdater.completeRound(); + metricUpdater.getLastCompleteStats().propagateMetrics(ims, dms); + + CPPUNIT_ASSERT_EQUAL(int64_t(0), dms.docsStored.getLast()); +} + +// Replicas on nodes 0 and 1. +void +BucketDBMetricUpdaterTest::visitBucketWith2Copies1Trusted( + BucketDBMetricUpdater& metricUpdater) +{ + BucketInfo info; + addNode(info, 0, 100); + addNode(info, 1, 101); // Note different checksums => #trusted = 1 + BucketDatabase::Entry e(document::BucketId(16, 1), info); + metricUpdater.visit(e, 2); +} + +// Replicas on nodes 0 and 2. +void +BucketDBMetricUpdaterTest::visitBucketWith2CopiesBothTrusted( + BucketDBMetricUpdater& metricUpdater) +{ + BucketInfo info; + addNode(info, 0, 200); + addNode(info, 2, 200); + BucketDatabase::Entry e(document::BucketId(16, 2), info); + metricUpdater.visit(e, 2); +} + +// Single replica on node 2. +void +BucketDBMetricUpdaterTest::visitBucketWith1Copy( + BucketDBMetricUpdater& metricUpdater) +{ + BucketInfo info; + addNode(info, 2, 100); + BucketDatabase::Entry e(document::BucketId(16, 1), info); + metricUpdater.visit(e, 2); +} + +BucketDBMetricUpdaterTest::NodeToReplicasMap +BucketDBMetricUpdaterTest::replicaStatsOf(BucketDBMetricUpdater& metricUpdater) +{ + metricUpdater.completeRound(true); + return metricUpdater.getLastCompleteStats()._minBucketReplica; +} + +void BucketDBMetricUpdaterTest::testMinBucketReplicaTrackedAndReportedPerNode() +{ + BucketDBMetricUpdater metricUpdater; + + // Node 0 and 1 should have min replica 1, while node 2 should have min + // replica 2. + visitBucketWith2Copies1Trusted(metricUpdater); + visitBucketWith2CopiesBothTrusted(metricUpdater); + + CPPUNIT_ASSERT_EQUAL(NodeToReplicasMap({{0, 1}, {1, 1}, {2, 2}}), + replicaStatsOf(metricUpdater)); +} + +void +BucketDBMetricUpdaterTest::nonTrustedReplicasAlsoCountedInModeAny() +{ + BucketDBMetricUpdater metricUpdater; + using CountingMode = BucketDBMetricUpdater::ReplicaCountingMode; + metricUpdater.setMinimumReplicaCountingMode(CountingMode::ANY); + visitBucketWith2Copies1Trusted(metricUpdater); + visitBucketWith2CopiesBothTrusted(metricUpdater); + + CPPUNIT_ASSERT_EQUAL(NodeToReplicasMap({{0, 2}, {1, 2}, {2, 2}}), + replicaStatsOf(metricUpdater)); +} + +void +BucketDBMetricUpdaterTest::minimumReplicaCountReturnedForNodeInModeAny() +{ + BucketDBMetricUpdater metricUpdater; + using CountingMode = BucketDBMetricUpdater::ReplicaCountingMode; + metricUpdater.setMinimumReplicaCountingMode(CountingMode::ANY); + visitBucketWith2CopiesBothTrusted(metricUpdater); + visitBucketWith1Copy(metricUpdater); + + // Node 2 has a bucket with only 1 replica. + CPPUNIT_ASSERT_EQUAL(NodeToReplicasMap({{0, 2}, {2, 1}}), + replicaStatsOf(metricUpdater)); +} + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/bucketdbupdatertest.cpp b/storage/src/tests/distributor/bucketdbupdatertest.cpp new file mode 100644 index 00000000000..a1c933d2606 --- /dev/null +++ b/storage/src/tests/distributor/bucketdbupdatertest.cpp @@ -0,0 +1,2296 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <iomanip> +#include <iostream> +#include <memory> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storage/distributor/bucketdbupdater.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/vdslib/state/random.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storage/distributor/pendingclusterstate.h> +#include <vespa/storageframework/defaultimplementation/clock/realclock.h> +#include <vespa/storageframework/storageframework.h> +#include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/storage/storageutil/distributorstatecache.h> +#include <tests/distributor/distributortestutil.h> +#include <tests/distributor/messagesenderstub.h> +#include <vespa/storage/distributor/simpleclusterinformation.h> + +#include <iostream> +#include <fstream> +#include <string> + +using namespace storage::api; +using namespace storage::lib; + +namespace storage { +namespace distributor { + +class BucketDBUpdaterTest : public CppUnit::TestFixture, + public DistributorTestUtil +{ + CPPUNIT_TEST_SUITE(BucketDBUpdaterTest); + CPPUNIT_TEST(testNormalUsage); // Make sure that bucketdbupdater sends requests to nodes, send responses back for 3 nodes, check that bucketdb is in correct state + CPPUNIT_TEST(testDistributorChange); + CPPUNIT_TEST(testDistributorChangeWithGrouping); + CPPUNIT_TEST(testNormalUsageInitializing); // Check that we send request bucket info when storage node is initializing, and send another when it's up. + CPPUNIT_TEST(testFailedRequestBucketInfo); + CPPUNIT_TEST(testBitChange); // Check what happens when distribution bits change + CPPUNIT_TEST(testNodeDown); + CPPUNIT_TEST(testStorageNodeInMaintenanceClearsBucketsForNode); + CPPUNIT_TEST(testNodeDownCopiesGetInSync); + CPPUNIT_TEST(testDownWhileInit); + CPPUNIT_TEST(testInitializingWhileRecheck); + CPPUNIT_TEST(testRecheckNode); + CPPUNIT_TEST(testRecheckNodeWithFailure); + CPPUNIT_TEST(testNotifyBucketChange); + CPPUNIT_TEST(testNotifyBucketChangeFromNodeDown); + CPPUNIT_TEST(testNotifyChangeWithPendingStateQueuesBucketInfoRequests); + CPPUNIT_TEST(testMergeReply); + CPPUNIT_TEST(testMergeReplyNodeDown); + CPPUNIT_TEST(testMergeReplyNodeDownAfterRequestSent); + CPPUNIT_TEST(testFlush); + CPPUNIT_TEST(testPendingClusterStateSendMessages); + CPPUNIT_TEST(testPendingClusterStateReceive); + CPPUNIT_TEST(testPendingClusterStateMerge); + CPPUNIT_TEST(testPendingClusterStateMergeReplicaChanged); + CPPUNIT_TEST(testPendingClusterStateWithGroupDown); + CPPUNIT_TEST(testPendingClusterStateWithGroupDownAndNoHandover); + CPPUNIT_TEST(testNoDbResurrectionForBucketNotOwnedInCurrentState); + CPPUNIT_TEST(testNoDbResurrectionForBucketNotOwnedInPendingState); + CPPUNIT_TEST(testClusterStateAlwaysSendsFullFetchWhenDistributionChangePending); + CPPUNIT_TEST(testChangedDistributionConfigTriggersRecoveryMode); + CPPUNIT_TEST(testNewlyAddedBucketsHaveCurrentTimeAsGcTimestamp); + CPPUNIT_TEST(testNewerMutationsNotOverwrittenByEarlierBucketFetch); + CPPUNIT_TEST(preemptedDistrChangeCarriesNodeSetOverToNextStateFetch); + CPPUNIT_TEST(preemptedStorChangeCarriesNodeSetOverToNextStateFetch); + CPPUNIT_TEST(preemptedStorageNodeDownMustBeReFetched); + CPPUNIT_TEST(outdatedNodeSetClearedAfterSuccessfulStateCompletion); + CPPUNIT_TEST(doNotSendToPreemptedNodeNowInDownState); + CPPUNIT_TEST(doNotSendToPreemptedNodeNotPartOfNewState); + CPPUNIT_TEST_DISABLED(clusterConfigDownsizeOnlySendsToAvailableNodes); + CPPUNIT_TEST(changedDiskSetTriggersReFetch); + CPPUNIT_TEST(nodeMissingFromConfigIsTreatedAsNeedingOwnershipTransfer); + CPPUNIT_TEST_SUITE_END(); + +protected: + void testNormalUsage(); + void testDistributorChange(); + void testDistributorChangeWithGrouping(); + void testNormalUsageInitializing(); + void testFailedRequestBucketInfo(); + void testNoResponses(); + void testBitChange(); + void testInconsistentChecksum(); + void testAddEmptyNode(); + void testNodeDown(); + void testStorageNodeInMaintenanceClearsBucketsForNode(); + void testNodeDownCopiesGetInSync(); + void testDownWhileInit(); + void testInitializingWhileRecheck(); + void testRecheckNode(); + void testRecheckNodeWithFailure(); + void testNotifyBucketChange(); + void testNotifyBucketChangeFromNodeDown(); + void testNotifyChangeWithPendingStateQueuesBucketInfoRequests(); + void testMergeReply(); + void testMergeReplyNodeDown(); + void testMergeReplyNodeDownAfterRequestSent(); + void testFlush(); + void testPendingClusterStateSendMessages(); + void testPendingClusterStateReceive(); + void testPendingClusterStateMerge(); + void testPendingClusterStateMergeReplicaChanged(); + void testPendingClusterStateWithGroupDown(); + void testPendingClusterStateWithGroupDownAndNoHandover(); + void testNoDbResurrectionForBucketNotOwnedInCurrentState(); + void testNoDbResurrectionForBucketNotOwnedInPendingState(); + void testClusterStateAlwaysSendsFullFetchWhenDistributionChangePending(); + void testChangedDistributionConfigTriggersRecoveryMode(); + void testNewlyAddedBucketsHaveCurrentTimeAsGcTimestamp(); + void testNewerMutationsNotOverwrittenByEarlierBucketFetch(); + void preemptedDistrChangeCarriesNodeSetOverToNextStateFetch(); + void preemptedStorChangeCarriesNodeSetOverToNextStateFetch(); + void preemptedStorageNodeDownMustBeReFetched(); + void outdatedNodeSetClearedAfterSuccessfulStateCompletion(); + void doNotSendToPreemptedNodeNowInDownState(); + void doNotSendToPreemptedNodeNotPartOfNewState(); + void clusterConfigDownsizeOnlySendsToAvailableNodes(); + void changedDiskSetTriggersReFetch(); + void nodeMissingFromConfigIsTreatedAsNeedingOwnershipTransfer(); + + bool bucketExistsThatHasNode(int bucketCount, uint16_t node) const; + + ClusterInformation::CSP createClusterInfo(const std::string& clusterState) { + ClusterInformation::CSP clusterInfo( + new SimpleClusterInformation( + getBucketDBUpdater().getDistributorComponent().getIndex(), + getBucketDBUpdater().getDistributorComponent().getDistribution(), + lib::ClusterState(clusterState), + "ui")); + return clusterInfo; + } + +public: + void setUp() { + createLinks(); + }; + + void tearDown() { + close(); + } + + std::shared_ptr<RequestBucketInfoReply> getFakeBucketReply( + const lib::ClusterState& state, + RequestBucketInfoCommand& cmd, + int storageIndex, + int bucketCount, + int invalidBucketCount = 0) + { + RequestBucketInfoReply* sreply = new RequestBucketInfoReply(cmd); + sreply->setAddress(storageAddress(storageIndex)); + + api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); + + for (int i=0; i<bucketCount + invalidBucketCount; i++) { + if (!getBucketDBUpdater().getDistributorComponent() + .ownsBucketInState(state, document::BucketId(16, i))) { + continue; + } + + std::vector<uint16_t> nodes; + getBucketDBUpdater().getDistributorComponent() + .getDistribution().getIdealNodes( + lib::NodeType::STORAGE, + state, + document::BucketId(16, i), + nodes); + + for (uint32_t j=0; j<nodes.size(); j++) { + if (nodes[j] == storageIndex) { + if (i >= bucketCount) { + vec.push_back(api::RequestBucketInfoReply::Entry( + document::BucketId(16, i), + api::BucketInfo())); + } else { + vec.push_back(api::RequestBucketInfoReply::Entry( + document::BucketId(16, i), + api::BucketInfo(10,1,1))); + } + } + } + } + + return std::shared_ptr<api::RequestBucketInfoReply>(sreply); + } + + void fakeBucketReply( + const lib::ClusterState& state, + RequestBucketInfoCommand& cmd, + int storageIndex, + int bucketCount, + int invalidBucketCount = 0) + { + getBucketDBUpdater().onRequestBucketInfoReply( + getFakeBucketReply(state, + cmd, + storageIndex, + bucketCount, + invalidBucketCount)); + } + + void sendFakeReplyForSingleBucketRequest( + const api::RequestBucketInfoCommand& rbi) + { + CPPUNIT_ASSERT_EQUAL(size_t(1), rbi.getBuckets().size()); + const document::BucketId& bucket(rbi.getBuckets()[0]); + + std::shared_ptr<api::RequestBucketInfoReply> reply( + new api::RequestBucketInfoReply(rbi)); + reply->getBucketInfo().push_back( + api::RequestBucketInfoReply::Entry(bucket, + api::BucketInfo(20, 10, 12, 50, 60, true, true))); + getBucketDBUpdater().onRequestBucketInfoReply(reply); + } + + std::string verifyBucket(document::BucketId id, const lib::ClusterState& state) { + BucketDatabase::Entry entry = getBucketDatabase().get(id); + if (!entry.valid()) { + return vespalib::make_string("%s doesn't exist in DB", + id.toString().c_str()); + } + + std::vector<uint16_t> nodes; + getBucketDBUpdater().getDistributorComponent().getDistribution().getIdealNodes( + lib::NodeType::STORAGE, + state, + document::BucketId(id), + nodes); + + if (nodes.size() != entry->getNodeCount()) { + return vespalib::make_string("Bucket Id %s has %d nodes in " + "ideal state, but has only %d in DB", + id.toString().c_str(), + (int)nodes.size(), + (int)entry->getNodeCount()); + } + + for (uint32_t i = 0; i<nodes.size(); i++) { + bool found = false; + + for (uint32_t j = 0; j<entry->getNodeCount(); j++) { + if (nodes[i] == entry->getNodeRef(j).getNode()) { + found = true; + } + } + + if (!found) { + return vespalib::make_string( + "Bucket Id %s has no copy from node %d", + id.toString().c_str(), + nodes[i]); + } + } + + return ""; + } + + + void verifyInvalid(document::BucketId id, int storageNode) { + BucketDatabase::Entry entry = getBucketDatabase().get(id); + + CPPUNIT_ASSERT(entry.valid()); + + bool found = false; + for (uint32_t j = 0; j<entry->getNodeCount(); j++) { + if (entry->getNodeRef(j).getNode() == storageNode) { + CPPUNIT_ASSERT(!entry->getNodeRef(j).valid()); + found = true; + } + } + + CPPUNIT_ASSERT(found); + } + + struct OrderByIncreasingNodeIndex { + template <typename T> + bool operator()(const T& lhs, const T& rhs) { + return (lhs->getAddress()->getIndex() + < rhs->getAddress()->getIndex()); + } + }; + + void sortSentMessagesByIndex(MessageSenderStub& sender, + size_t sortFromOffset = 0) + { + std::sort(sender.commands.begin() + sortFromOffset, + sender.commands.end(), + OrderByIncreasingNodeIndex()); + } + + void setSystemState(const lib::ClusterState& state) { + const size_t sizeBeforeState = _sender.commands.size(); + getBucketDBUpdater().onSetSystemState( + std::shared_ptr<api::SetSystemStateCommand>( + new api::SetSystemStateCommand(state))); + // A lot of test logic has the assumption that all messages sent as a + // result of cluster state changes will be in increasing index order + // (for simplicity, not because this is required for correctness). + // Only sort the messages that arrived as a result of the state, don't + // jumble the sorting with any existing messages. + sortSentMessagesByIndex(_sender, sizeBeforeState); + } + + void setAndEnableClusterState(const lib::ClusterState& state, + uint32_t expectedMsgs, + uint32_t nBuckets) { + _sender.clear(); + setSystemState(state); + CPPUNIT_ASSERT_EQUAL(size_t(expectedMsgs), _sender.commands.size()); + + for (uint32_t i = 0; i < _sender.commands.size(); i++) { + CPPUNIT_ASSERT(_sender.commands[i]->getType() == + MessageType::REQUESTBUCKETINFO); + + const api::StorageMessageAddress& address( + *_sender.commands[i]->getAddress()); + fakeBucketReply( + state, + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), + address.getIndex(), + nBuckets); + } + } + + + void setStorageNodes(uint32_t numStorageNodes) { + _sender.clear(); + + lib::ClusterState newState( + vespalib::make_string("distributor:1 storage:%d", numStorageNodes)); + + setSystemState(newState); + + for (uint32_t i=0; i<numStorageNodes; i++) { + CPPUNIT_ASSERT(_sender.commands[i]->getType() == + MessageType::REQUESTBUCKETINFO); + + const api::StorageMessageAddress *address = _sender.commands[i]->getAddress(); + CPPUNIT_ASSERT_EQUAL(i, (uint32_t)address->getIndex()); + } + } + + void initializeNodesAndBuckets(uint32_t numStorageNodes, + uint32_t numBuckets) + { + setStorageNodes(numStorageNodes); + + vespalib::string state(vespalib::make_string( + "distributor:1 storage:%d", numStorageNodes)); + lib::ClusterState newState(state); + + for (uint32_t i=0; i<numStorageNodes; i++) { + fakeBucketReply(newState, + *((RequestBucketInfoCommand*)_sender.commands[i].get()), + i, + numBuckets); + } + assertCorrectBuckets(numBuckets, state); + } + + bool bucketHasNode(document::BucketId id, uint16_t node) const { + BucketDatabase::Entry entry = getBucket(id); + CPPUNIT_ASSERT(entry.valid()); + + for (uint32_t j=0; j<entry->getNodeCount(); j++) { + if (entry->getNodeRef(j).getNode() == node) { + return true; + } + } + + return false; + } + + api::StorageMessageAddress storageAddress(uint16_t node) { + return api::StorageMessageAddress("storage", lib::NodeType::STORAGE, node); + } + + std::string getSentNodes(const std::string& oldClusterState, + const std::string& newClusterState); + + std::string getSentNodesDistributionChanged( + const std::string& oldClusterState); + + std::vector<uint16_t> getSentNodesWithPreemption( + const std::string& oldClusterState, + uint32_t expectedOldStateMessages, + const std::string& preemptedClusterState, + const std::string& newClusterState); + + std::vector<uint16_t> getSendSet() const; + + std::string mergeBucketLists( + const lib::ClusterState& oldState, + const std::string& existingData, + const lib::ClusterState& newState, + const std::string& newData, + bool includeBucketInfo = false); + + std::string mergeBucketLists( + const std::string& existingData, + const std::string& newData, + bool includeBucketInfo = false); + + void assertCorrectBuckets(int numBuckets, const std::string& stateStr) { + lib::ClusterState state(stateStr); + for (int i=0; i<numBuckets; i++) { + CPPUNIT_ASSERT_EQUAL( + getIdealStr(document::BucketId(16, i), state), + getNodes(document::BucketId(16, i))); + } + } + + void setDistribution(const std::string& distConfig) { + lib::Distribution* distribution = new lib::Distribution(distConfig); + _node->getComponentRegister().setDistribution( + lib::Distribution::SP(distribution)); + } + + std::string getDistConfig6Nodes3Groups() const { + return ("redundancy 2\n" + "group[3]\n" + "group[0].name \"invalid\"\n" + "group[0].index \"invalid\"\n" + "group[0].partitions 1|*\n" + "group[0].nodes[0]\n" + "group[1].name rack0\n" + "group[1].index 0\n" + "group[1].nodes[3]\n" + "group[1].nodes[0].index 0\n" + "group[1].nodes[1].index 1\n" + "group[1].nodes[2].index 2\n" + "group[2].name rack1\n" + "group[2].index 1\n" + "group[2].nodes[3]\n" + "group[2].nodes[0].index 3\n" + "group[2].nodes[1].index 4\n" + "group[2].nodes[2].index 5\n"); + } + + std::string getDistConfig6Nodes4Groups() const { + return ("redundancy 2\n" + "group[4]\n" + "group[0].name \"invalid\"\n" + "group[0].index \"invalid\"\n" + "group[0].partitions 1|*\n" + "group[0].nodes[0]\n" + "group[1].name rack0\n" + "group[1].index 0\n" + "group[1].nodes[2]\n" + "group[1].nodes[0].index 0\n" + "group[1].nodes[1].index 1\n" + "group[2].name rack1\n" + "group[2].index 1\n" + "group[2].nodes[2]\n" + "group[2].nodes[0].index 2\n" + "group[2].nodes[1].index 3\n" + "group[3].name rack2\n" + "group[3].index 2\n" + "group[3].nodes[2]\n" + "group[3].nodes[0].index 4\n" + "group[3].nodes[1].index 5\n"); + } + + std::string getDistConfig3Nodes1Group() const { + return ("redundancy 2\n" + "group[2]\n" + "group[0].name \"invalid\"\n" + "group[0].index \"invalid\"\n" + "group[0].partitions 1|*\n" + "group[0].nodes[0]\n" + "group[1].name rack0\n" + "group[1].index 0\n" + "group[1].nodes[3]\n" + "group[1].nodes[0].index 0\n" + "group[1].nodes[1].index 1\n" + "group[1].nodes[2].index 2\n"); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(BucketDBUpdaterTest); + +void +BucketDBUpdaterTest::testNormalUsage() +{ + setSystemState(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3")); + + CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); + + // Ensure distribution hash is set correctly + CPPUNIT_ASSERT_EQUAL( + getBucketDBUpdater().getDistributorComponent().getDistribution() + .getNodeGraph().getDistributionConfigHash(), + dynamic_cast<const RequestBucketInfoCommand&>( + *_sender.commands[0]).getDistributionHash()); + + fakeBucketReply( + lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0]), + 0, 10); + + _sender.clear(); + + // Optimization for not refetching unneeded data after cluster state + // change is only implemented after completion of previous cluster state + setSystemState(lib::ClusterState("distributor:2 .0.s:i storage:3")); + + CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); + // Expect reply of first set SystemState request. + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.replies.size()); + + for (uint32_t i = 0; i < 3; ++i) { + fakeBucketReply( + lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), + i, 10); + } + + assertCorrectBuckets(10, "distributor:2 storage:3"); +} + +void +BucketDBUpdaterTest::testDistributorChange() +{ + int numBuckets = 100; + + // First sends request + setSystemState(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3")); + CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); + for (uint32_t i = 0; i < 3; ++i) { + fakeBucketReply( + lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), + i, numBuckets); + } + _sender.clear(); + + // No change from initializing to up (when done with last job) + setSystemState(lib::ClusterState("distributor:2 storage:3")); + CPPUNIT_ASSERT_EQUAL(size_t(0), _sender.commands.size()); + _sender.clear(); + + // Adding node. No new read requests, but buckets thrown + setSystemState(lib::ClusterState("distributor:3 storage:3")); + CPPUNIT_ASSERT_EQUAL(size_t(0), _sender.commands.size()); + assertCorrectBuckets(numBuckets, "distributor:3 storage:3"); + _sender.clear(); + + // Removing distributor. Need to refetch new data from all nodes. + setSystemState(lib::ClusterState("distributor:2 storage:3")); + CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); + for (uint32_t i = 0; i < 3; ++i) { + fakeBucketReply( + lib::ClusterState("distributor:2 storage:3"), + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), + i, numBuckets); + } + _sender.clear(); + assertCorrectBuckets(numBuckets, "distributor:2 storage:3"); +} + +void +BucketDBUpdaterTest::testDistributorChangeWithGrouping() +{ + std::string distConfig(getDistConfig6Nodes3Groups()); + setDistribution(distConfig); + _distributor->enableNextDistribution(); + int numBuckets = 100; + + setSystemState(lib::ClusterState("distributor:6 storage:6")); + CPPUNIT_ASSERT_EQUAL(size_t(6), _sender.commands.size()); + for (uint32_t i = 0; i < 6; ++i) { + fakeBucketReply( + lib::ClusterState("distributor:6 storage:6"), + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), + i, numBuckets); + } + _sender.clear(); + + // Distributor going down in other group, no change + setSystemState(lib::ClusterState("distributor:6 .5.s:d storage:6")); + CPPUNIT_ASSERT_EQUAL(size_t(0), _sender.commands.size()); + _sender.clear(); + + setSystemState(lib::ClusterState("distributor:6 storage:6")); + CPPUNIT_ASSERT_EQUAL(size_t(0), _sender.commands.size()); + assertCorrectBuckets(numBuckets, "distributor:6 storage:6"); + _sender.clear(); + + // Unchanged grouping cause no change. + setDistribution(distConfig); + _distributor->storageDistributionChanged(); + _distributor->enableNextDistribution(); + CPPUNIT_ASSERT_EQUAL(size_t(0), _sender.commands.size()); + + // Changed grouping cause change + setDistribution(getDistConfig6Nodes4Groups()); + _distributor->storageDistributionChanged(); + _distributor->enableNextDistribution(); + + CPPUNIT_ASSERT_EQUAL(size_t(6), _sender.commands.size()); +} + +void +BucketDBUpdaterTest::testNormalUsageInitializing() +{ + setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1 .0.s:i")); + + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + + // Not yet passing on system state. + CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.commands.size()); + + fakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), + *((RequestBucketInfoCommand*)_sender.commands[0].get()), + 0, + 10, + 10); + + assertCorrectBuckets(10, "distributor:1 storage:1"); + + for (int i=10; i<20; i++) { + verifyInvalid(document::BucketId(16, i), 0); + } + + // Pass on cluster state and recheck buckets now. + CPPUNIT_ASSERT_EQUAL(size_t(1), _senderDown.commands.size()); + + _sender.clear(); + _senderDown.clear(); + + setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1")); + + // Send a new request bucket info up. + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + + fakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), + *((RequestBucketInfoCommand*)_sender.commands[0].get()), + 0, + 20); + + // Pass on cluster state and recheck buckets now. + CPPUNIT_ASSERT_EQUAL(size_t(1), _senderDown.commands.size()); + + assertCorrectBuckets(20, "distributor:1 storage:1"); +} + +void +BucketDBUpdaterTest::testFailedRequestBucketInfo() +{ + setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1")); + + // 2 messages sent up: 1 to the nodes, and one reply to the setsystemstate. + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + + { + std::shared_ptr<api::RequestBucketInfoReply> reply = + getFakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), + *((RequestBucketInfoCommand*)_sender.commands[0].get()), + 0, + 10); + + reply->setResult(api::ReturnCode::NOT_CONNECTED); + getBucketDBUpdater().onRequestBucketInfoReply(reply); + // Trigger that delayed message is sent + getClock().addSecondsToTime(10); + getBucketDBUpdater().resendDelayedMessages(); + } + + // Should be resent. + CPPUNIT_ASSERT_EQUAL(std::string("Request bucket info," + "Request bucket info"), + _sender.getCommands()); + + CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.commands.size()); + + fakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), + *((RequestBucketInfoCommand*)_sender.commands[1].get()), + 0, + 10); + + for (int i=0; i<10; i++) { + CPPUNIT_ASSERT_EQUAL( + std::string(""), + verifyBucket(document::BucketId(16, i), + lib::ClusterState("distributor:1 storage:1"))); + } + + // Set system state should now be passed on + CPPUNIT_ASSERT_EQUAL(std::string("Set system state"), + _senderDown.getCommands()); +} + +void +BucketDBUpdaterTest::testDownWhileInit() +{ + setStorageNodes(3); + + fakeBucketReply(lib::ClusterState("distributor:1 storage:3"), + *((RequestBucketInfoCommand*)_sender.commands[0].get()), + 0, + 5); + + setSystemState(lib::ClusterState("distributor:1 storage:3 .1.s:d")); + + fakeBucketReply(lib::ClusterState("distributor:1 storage:3"), + *((RequestBucketInfoCommand*)_sender.commands[2].get()), + 2, + 5); + + fakeBucketReply(lib::ClusterState("distributor:1 storage:3"), + *((RequestBucketInfoCommand*)_sender.commands[1].get()), + 1, + 5); +} + +bool +BucketDBUpdaterTest::bucketExistsThatHasNode(int bucketCount, uint16_t node) const +{ + for (int i=1; i<bucketCount; i++) { + if (bucketHasNode(document::BucketId(16, i), node)) { + return true; + } + } + + return false; +} + +void +BucketDBUpdaterTest::testNodeDown() +{ + setStorageNodes(3); + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:3")); + + for (int i=1; i<100; i++) { + addIdealNodes(document::BucketId(16, i)); + } + + CPPUNIT_ASSERT(bucketExistsThatHasNode(100, 1)); + + setSystemState(lib::ClusterState("distributor:1 storage:3 .1.s:d")); + + CPPUNIT_ASSERT(!bucketExistsThatHasNode(100, 1)); +} + +void +BucketDBUpdaterTest::testStorageNodeInMaintenanceClearsBucketsForNode() +{ + setStorageNodes(3); + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:3")); + + for (int i=1; i<100; i++) { + addIdealNodes(document::BucketId(16, i)); + } + + CPPUNIT_ASSERT(bucketExistsThatHasNode(100, 1)); + + setSystemState(lib::ClusterState("distributor:1 storage:3 .1.s:m")); + + CPPUNIT_ASSERT(!bucketExistsThatHasNode(100, 1)); +} + +void +BucketDBUpdaterTest::testNodeDownCopiesGetInSync() +{ + setStorageNodes(3); + + lib::ClusterState systemState("distributor:1 storage:3"); + document::BucketId bid(16, 1); + + addNodesToBucketDB(bid, "0=3,1=2,2=3"); + + setSystemState(lib::ClusterState("distributor:1 storage:3 .1.s:d")); + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001) : " + "node(idx=0,crc=0x3,docs=3/3,bytes=3/3,trusted=true,active=false), " + "node(idx=2,crc=0x3,docs=3/3,bytes=3/3,trusted=true,active=false)"), + dumpBucket(bid)); +} + +void +BucketDBUpdaterTest::testInitializingWhileRecheck() +{ + lib::ClusterState systemState("distributor:1 storage:2 .0.s:i .0.i:0.1"); + setSystemState(systemState); + + CPPUNIT_ASSERT_EQUAL(size_t(2), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.commands.size()); + + getBucketDBUpdater().recheckBucketInfo(1, document::BucketId(16, 3)); + + for (int i=0; i<2; i++) { + fakeBucketReply(systemState, + *((RequestBucketInfoCommand*)_sender.commands[i].get()), + i, + 100); + } + + // Now we can pass on system state. + CPPUNIT_ASSERT_EQUAL(size_t(1), _senderDown.commands.size()); + + CPPUNIT_ASSERT_EQUAL(MessageType::SETSYSTEMSTATE, + _senderDown.commands[0]->getType()); +} + +void +BucketDBUpdaterTest::testBitChange() +{ + + std::vector<document::BucketId> bucketlist; + + { + setSystemState(lib::ClusterState("bits:14 storage:1 distributor:2")); + + CPPUNIT_ASSERT_EQUAL(1, (int)_sender.commands.size()); + + CPPUNIT_ASSERT(_sender.commands[0]->getType() == MessageType::REQUESTBUCKETINFO); + + RequestBucketInfoReply* sreply = + new RequestBucketInfoReply(*((RequestBucketInfoCommand*)_sender.commands[0].get())); + sreply->setAddress(storageAddress(0)); + api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); + + + int cnt=0; + for (int i=0; cnt < 2; i++) { + lib::Distribution distribution = getBucketDBUpdater().getDistributorComponent() + .getDistribution(); + std::vector<uint16_t> distributors; + if (distribution.getIdealDistributorNode( + lib::ClusterState("redundancy:1 bits:14 storage:1 distributor:2"), + document::BucketId(16, i)) + == 0) + { + vec.push_back(api::RequestBucketInfoReply::Entry( + document::BucketId(16, i), + api::BucketInfo(10,1,1))); + + bucketlist.push_back(document::BucketId(16, i)); + cnt++; + } + } + + getBucketDBUpdater().onRequestBucketInfoReply(std::shared_ptr<RequestBucketInfoReply>(sreply)); + } + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001) : " + "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false)"), + dumpBucket(bucketlist[0])); + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000002) : " + "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false)"), + dumpBucket(bucketlist[1])); + + { + _sender.clear(); + setSystemState(lib::ClusterState("bits:16 storage:1 distributor:2")); + + CPPUNIT_ASSERT_EQUAL(1, (int)_sender.commands.size()); + + CPPUNIT_ASSERT(_sender.commands[0]->getType() == MessageType::REQUESTBUCKETINFO); + + RequestBucketInfoReply* sreply = + new RequestBucketInfoReply( + *((RequestBucketInfoCommand*)_sender.commands[0].get())); + sreply->setAddress(storageAddress(0)); + sreply->setResult(api::ReturnCode::OK); + api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); + + for (uint32_t i = 0; i < 3; ++i) { + vec.push_back(api::RequestBucketInfoReply::Entry( + document::BucketId(16, i), + api::BucketInfo(10,1,1))); + } + + vec.push_back(api::RequestBucketInfoReply::Entry( + document::BucketId(16, 4), + api::BucketInfo(10,1,1))); + + getBucketDBUpdater().onRequestBucketInfoReply( + std::shared_ptr<RequestBucketInfoReply>(sreply)); + } + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000000) : " + "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false)"), + dumpBucket(document::BucketId(16, 0))); + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001) : " + "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false)"), + dumpBucket(document::BucketId(16, 1))); + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000002) : " + "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false)"), + dumpBucket(document::BucketId(16, 2))); + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000004) : " + "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false)"), + dumpBucket(document::BucketId(16, 4))); + + { + _sender.clear(); + setSystemState(lib::ClusterState("storage:1 distributor:2 .1.s:i")); + } + + { + _sender.clear(); + setSystemState(lib::ClusterState("storage:1 distributor:2")); + } +}; + +void +BucketDBUpdaterTest::testRecheckNodeWithFailure() +{ + initializeNodesAndBuckets(3, 5); + + _sender.clear(); + + getBucketDBUpdater().recheckBucketInfo(1, document::BucketId(16, 3)); + + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + + + uint16_t index = 0; + { + api::RequestBucketInfoCommand& rbi( + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0])); + CPPUNIT_ASSERT_EQUAL(size_t(1), rbi.getBuckets().size()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 3), rbi.getBuckets()[0]); + auto reply(std::make_shared<api::RequestBucketInfoReply>(rbi)); + + const api::StorageMessageAddress *address = _sender.commands[0]->getAddress(); + index = address->getIndex(); + + reply->setResult(api::ReturnCode::NOT_CONNECTED); + getBucketDBUpdater().onRequestBucketInfoReply(reply); + // Trigger that delayed message is sent + getClock().addSecondsToTime(10); + getBucketDBUpdater().resendDelayedMessages(); + } + + CPPUNIT_ASSERT_EQUAL(size_t(2), _sender.commands.size()); + + setSystemState( + lib::ClusterState(vespalib::make_string("distributor:1 storage:3 .%d.s:d", index))); + + // Recheck bucket. + { + api::RequestBucketInfoCommand& rbi(dynamic_cast<RequestBucketInfoCommand&> + (*_sender.commands[1])); + CPPUNIT_ASSERT_EQUAL(size_t(1), rbi.getBuckets().size()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 3), rbi.getBuckets()[0]); + auto reply(std::make_shared<api::RequestBucketInfoReply>(rbi)); + reply->setResult(api::ReturnCode::NOT_CONNECTED); + getBucketDBUpdater().onRequestBucketInfoReply(reply); + } + + // Should not retry since node is down. + CPPUNIT_ASSERT_EQUAL(size_t(2), _sender.commands.size()); +} + +void +BucketDBUpdaterTest::testRecheckNode() +{ + initializeNodesAndBuckets(3, 5); + + _sender.clear(); + + getBucketDBUpdater().recheckBucketInfo(1, document::BucketId(16, 3)); + + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + + api::RequestBucketInfoCommand& rbi( + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0])); + CPPUNIT_ASSERT_EQUAL(size_t(1), rbi.getBuckets().size()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 3), rbi.getBuckets()[0]); + + auto reply(std::make_shared<api::RequestBucketInfoReply>(rbi)); + reply->getBucketInfo().push_back( + api::RequestBucketInfoReply::Entry(document::BucketId(16, 3), + api::BucketInfo(20, 10, 12, 50, 60, true, true))); + getBucketDBUpdater().onRequestBucketInfoReply(reply); + + lib::ClusterState state("distributor:1 storage:3"); + for (uint32_t i = 0; i < 3; i++) { + CPPUNIT_ASSERT_EQUAL( + getIdealStr(document::BucketId(16, i), state), + getNodes(document::BucketId(16, i))); + } + + for (uint32_t i = 4; i < 5; i++) { + CPPUNIT_ASSERT_EQUAL( + getIdealStr(document::BucketId(16, i), state), + getNodes(document::BucketId(16, i))); + } + + BucketDatabase::Entry entry = getBucketDatabase().get(document::BucketId(16, 3)); + CPPUNIT_ASSERT(entry.valid()); + + const BucketCopy* copy = entry->getNode(1); + CPPUNIT_ASSERT(copy != 0); + CPPUNIT_ASSERT_EQUAL(api::BucketInfo(20,10,12, 50, 60, true, true), + copy->getBucketInfo()); +} + +void +BucketDBUpdaterTest::testNotifyBucketChange() +{ + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:1")); + + addNodesToBucketDB(document::BucketId(16, 1), "0=1234"); + _sender.replies.clear(); + + { + api::BucketInfo info(1, 2, 3, 4, 5, true, true); + auto cmd(std::make_shared<api::NotifyBucketChangeCommand>( + document::BucketId(16, 1), info)); + cmd->setSourceIndex(0); + getBucketDBUpdater().onNotifyBucketChange(cmd); + } + + { + api::BucketInfo info(10, 11, 12, 13, 14, false, false); + auto cmd(std::make_shared<api::NotifyBucketChangeCommand>( + document::BucketId(16, 2), info)); + cmd->setSourceIndex(0); + getBucketDBUpdater().onNotifyBucketChange(cmd); + } + + // Must receive reply + CPPUNIT_ASSERT_EQUAL(size_t(2), _sender.replies.size()); + + for (int i = 0; i < 2; ++i) { + CPPUNIT_ASSERT_EQUAL(MessageType::NOTIFYBUCKETCHANGE_REPLY, + _sender.replies[i]->getType()); + } + + // No database update until request bucket info replies have been received. + CPPUNIT_ASSERT_EQUAL(std::string("BucketId(0x4000000000000001) : " + "node(idx=0,crc=0x4d2,docs=1234/1234,bytes=1234/1234," + "trusted=false,active=false)"), + dumpBucket(document::BucketId(16, 1))); + CPPUNIT_ASSERT_EQUAL(std::string("NONEXISTING"), + dumpBucket(document::BucketId(16, 2))); + + CPPUNIT_ASSERT_EQUAL(size_t(2), _sender.commands.size()); + + std::vector<api::BucketInfo> infos; + infos.push_back(api::BucketInfo(4567, 200, 2000, 400, 4000, true, true)); + infos.push_back(api::BucketInfo(8999, 300, 3000, 500, 5000, false, false)); + + for (int i = 0; i < 2; ++i) { + api::RequestBucketInfoCommand& rbi( + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i])); + CPPUNIT_ASSERT_EQUAL(size_t(1), rbi.getBuckets().size()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, i + 1), rbi.getBuckets()[0]); + + auto reply(std::make_shared<api::RequestBucketInfoReply>(rbi)); + reply->getBucketInfo().push_back( + api::RequestBucketInfoReply::Entry(document::BucketId(16, i + 1), + infos[i])); + getBucketDBUpdater().onRequestBucketInfoReply(reply); + } + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001) : " + "node(idx=0,crc=0x11d7,docs=200/400,bytes=2000/4000,trusted=true,active=true)"), + dumpBucket(document::BucketId(16, 1))); + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000002) : " + "node(idx=0,crc=0x2327,docs=300/500,bytes=3000/5000,trusted=true,active=false)"), + dumpBucket(document::BucketId(16, 2))); + +} + +void +BucketDBUpdaterTest::testNotifyBucketChangeFromNodeDown() +{ + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:2")); + + addNodesToBucketDB(document::BucketId(16, 1), "1=1234"); + + _sender.replies.clear(); + + { + api::BucketInfo info(8999, 300, 3000, 500, 5000, false, false); + auto cmd(std::make_shared<api::NotifyBucketChangeCommand>( + document::BucketId(16, 1), info)); + cmd->setSourceIndex(0); + getBucketDBUpdater().onNotifyBucketChange(cmd); + } + // Enable here to avoid having request bucket info be silently swallowed + // (sendRequestBucketInfo drops message if node is down). + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:2 .0.s:d")); + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001) : " + "node(idx=1,crc=0x4d2,docs=1234/1234,bytes=1234/1234,trusted=false,active=false)"), + dumpBucket(document::BucketId(16, 1))); + + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.replies.size()); + CPPUNIT_ASSERT_EQUAL(MessageType::NOTIFYBUCKETCHANGE_REPLY, + _sender.replies[0]->getType()); + + // Currently, this pending operation will be auto-flushed when the cluster state + // changes so the behavior is still correct. Keep this test around to prevent + // regressions here. + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + api::RequestBucketInfoCommand& rbi( + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0])); + CPPUNIT_ASSERT_EQUAL(size_t(1), rbi.getBuckets().size()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1), rbi.getBuckets()[0]); + + auto reply(std::make_shared<api::RequestBucketInfoReply>(rbi)); + reply->getBucketInfo().push_back( + api::RequestBucketInfoReply::Entry( + document::BucketId(16, 1), + api::BucketInfo(8999, 300, 3000, 500, 5000, false, false))); + getBucketDBUpdater().onRequestBucketInfoReply(reply); + + // No change + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001) : " + "node(idx=1,crc=0x4d2,docs=1234/1234,bytes=1234/1234,trusted=false,active=false)"), + dumpBucket(document::BucketId(16, 1))); +} + +/** + * Test that NotifyBucketChange received while there's a pending cluster state + * waits until the cluster state has been enabled as current before it sends off + * the single bucket info requests. This is to prevent a race condition where + * the replies to bucket info requests for buckets that would be owned by the + * distributor in the pending state but not by the current state would be + * discarded when attempted inserted into the bucket database. + */ +void +BucketDBUpdaterTest::testNotifyChangeWithPendingStateQueuesBucketInfoRequests() +{ + setSystemState(lib::ClusterState("distributor:1 storage:1")); + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + + { + api::BucketInfo info(8999, 300, 3000, 500, 5000, false, false); + auto cmd(std::make_shared<api::NotifyBucketChangeCommand>( + document::BucketId(16, 1), info)); + cmd->setSourceIndex(0); + getBucketDBUpdater().onNotifyBucketChange(cmd); + } + + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + + fakeBucketReply( + lib::ClusterState("distributor:1 storage:1"), + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0]), + 0, 10); + + CPPUNIT_ASSERT_EQUAL(size_t(2), _sender.commands.size()); + + { + api::RequestBucketInfoCommand& rbi( + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[1])); + CPPUNIT_ASSERT_EQUAL(size_t(1), rbi.getBuckets().size()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1), rbi.getBuckets()[0]); + } + _sender.clear(); + + // Queue must be cleared once pending state is enabled. + { + lib::ClusterState state("distributor:1 storage:2"); + uint32_t expectedMsgs = 1, dummyBucketsToReturn = 1; + setAndEnableClusterState(state, expectedMsgs, dummyBucketsToReturn); + } + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + { + api::RequestBucketInfoCommand& rbi( + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0])); + CPPUNIT_ASSERT_EQUAL(size_t(0), rbi.getBuckets().size()); + } +} + +void +BucketDBUpdaterTest::testMergeReply() +{ + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:3")); + + addNodesToBucketDB(document::BucketId(16, 1234), + "0=1234,1=1234,2=1234"); + + std::vector<api::MergeBucketCommand::Node> nodes; + nodes.push_back(api::MergeBucketCommand::Node(0)); + nodes.push_back(api::MergeBucketCommand::Node(1)); + nodes.push_back(api::MergeBucketCommand::Node(2)); + + api::MergeBucketCommand cmd(document::BucketId(16, 1234), nodes, 0); + + auto reply(std::make_shared<api::MergeBucketReply>(cmd)); + + _sender.clear(); + getBucketDBUpdater().onMergeBucketReply(reply); + + CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); + + for (uint32_t i = 0; i < 3; i++) { + std::shared_ptr<api::RequestBucketInfoCommand> + req(std::dynamic_pointer_cast<api::RequestBucketInfoCommand>( + _sender.commands[i])); + + CPPUNIT_ASSERT(req.get()); + CPPUNIT_ASSERT_EQUAL(size_t(1), req->getBuckets().size()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1234), req->getBuckets()[0]); + + auto reqreply(std::make_shared<api::RequestBucketInfoReply>(*req)); + reqreply->getBucketInfo().push_back( + api::RequestBucketInfoReply::Entry(document::BucketId(16, 1234), + api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1)))); + + getBucketDBUpdater().onRequestBucketInfoReply(reqreply); + } + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x40000000000004d2) : " + "node(idx=0,crc=0xa,docs=100/100,bytes=1000/1000,trusted=false,active=false), " + "node(idx=1,crc=0x14,docs=200/200,bytes=2000/2000,trusted=false,active=false), " + "node(idx=2,crc=0x1e,docs=300/300,bytes=3000/3000,trusted=false,active=false)"), + dumpBucket(document::BucketId(16, 1234))); +}; + +void +BucketDBUpdaterTest::testMergeReplyNodeDown() +{ + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:3")); + std::vector<api::MergeBucketCommand::Node> nodes; + + addNodesToBucketDB(document::BucketId(16, 1234), "0=1234,1=1234,2=1234"); + + for (uint32_t i = 0; i < 3; ++i) { + nodes.push_back(api::MergeBucketCommand::Node(i)); + } + + api::MergeBucketCommand cmd(document::BucketId(16, 1234), nodes, 0); + + auto reply(std::make_shared<api::MergeBucketReply>(cmd)); + + setSystemState(lib::ClusterState("distributor:1 storage:2")); + + _sender.clear(); + getBucketDBUpdater().onMergeBucketReply(reply); + + CPPUNIT_ASSERT_EQUAL(size_t(2), _sender.commands.size()); + + for (uint32_t i = 0; i < 2; i++) { + std::shared_ptr<api::RequestBucketInfoCommand> req( + std::dynamic_pointer_cast<api::RequestBucketInfoCommand>( + _sender.commands[i])); + + CPPUNIT_ASSERT(req.get()); + CPPUNIT_ASSERT_EQUAL(size_t(1), req->getBuckets().size()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1234), req->getBuckets()[0]); + + auto reqreply(std::make_shared<api::RequestBucketInfoReply>(*req)); + reqreply->getBucketInfo().push_back( + api::RequestBucketInfoReply::Entry( + document::BucketId(16, 1234), + api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1)))); + getBucketDBUpdater().onRequestBucketInfoReply(reqreply); + } + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x40000000000004d2) : " + "node(idx=0,crc=0xa,docs=100/100,bytes=1000/1000,trusted=false,active=false), " + "node(idx=1,crc=0x14,docs=200/200,bytes=2000/2000,trusted=false,active=false)"), + dumpBucket(document::BucketId(16, 1234))); +}; + +void +BucketDBUpdaterTest::testMergeReplyNodeDownAfterRequestSent() +{ + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:3")); + std::vector<api::MergeBucketCommand::Node> nodes; + + addNodesToBucketDB(document::BucketId(16, 1234), "0=1234,1=1234,2=1234"); + + for (uint32_t i = 0; i < 3; ++i) { + nodes.push_back(api::MergeBucketCommand::Node(i)); + } + + api::MergeBucketCommand cmd(document::BucketId(16, 1234), nodes, 0); + + auto reply(std::make_shared<api::MergeBucketReply>(cmd)); + + _sender.clear(); + getBucketDBUpdater().onMergeBucketReply(reply); + + CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); + + setSystemState(lib::ClusterState("distributor:1 storage:2")); + + for (uint32_t i = 0; i < 3; i++) { + std::shared_ptr<api::RequestBucketInfoCommand> req( + std::dynamic_pointer_cast<api::RequestBucketInfoCommand>( + _sender.commands[i])); + + CPPUNIT_ASSERT(req.get()); + CPPUNIT_ASSERT_EQUAL(size_t(1), req->getBuckets().size()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1234), req->getBuckets()[0]); + + auto reqreply(std::make_shared<api::RequestBucketInfoReply>(*req)); + reqreply->getBucketInfo().push_back( + api::RequestBucketInfoReply::Entry( + document::BucketId(16, 1234), + api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1)))); + getBucketDBUpdater().onRequestBucketInfoReply(reqreply); + } + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x40000000000004d2) : " + "node(idx=0,crc=0xa,docs=100/100,bytes=1000/1000,trusted=false,active=false), " + "node(idx=1,crc=0x14,docs=200/200,bytes=2000/2000,trusted=false,active=false)"), + dumpBucket(document::BucketId(16, 1234))); +}; + + +void +BucketDBUpdaterTest::testFlush() +{ + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:3")); + _sender.clear(); + + addNodesToBucketDB(document::BucketId(16, 1234), "0=1234,1=1234,2=1234"); + + std::vector<api::MergeBucketCommand::Node> nodes; + for (uint32_t i = 0; i < 3; ++i) { + nodes.push_back(api::MergeBucketCommand::Node(i)); + } + + api::MergeBucketCommand cmd(document::BucketId(16, 1234), + nodes, + 0); + + auto reply(std::make_shared<api::MergeBucketReply>(cmd)); + + _sender.clear(); + getBucketDBUpdater().onMergeBucketReply(reply); + + CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); + CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.replies.size()); + + getBucketDBUpdater().flush(); + // Flushing should drop all merge bucket replies + CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.commands.size()); +} + +std::string +BucketDBUpdaterTest::getSentNodes( + const std::string& oldClusterState, + const std::string& newClusterState) +{ + MessageSenderStub sender; + + std::shared_ptr<api::SetSystemStateCommand> cmd( + new api::SetSystemStateCommand( + lib::ClusterState(newClusterState))); + + framework::defaultimplementation::FakeClock clock; + ClusterInformation::CSP clusterInfo(createClusterInfo(oldClusterState)); + + std::unordered_set<uint16_t> outdatedNodes; + std::unique_ptr<PendingClusterState> state( + PendingClusterState::createForClusterStateChange( + clock, clusterInfo, sender, cmd, outdatedNodes, + api::Timestamp(1))); + + sortSentMessagesByIndex(sender); + + std::ostringstream ost; + for (uint32_t i = 0; i < sender.commands.size(); i++) { + RequestBucketInfoCommand* req = + dynamic_cast<RequestBucketInfoCommand*>(sender.commands[i].get()); + + if (i > 0) { + ost << ","; + } + + ost << req->getAddress()->getIndex(); + } + + return ost.str(); +} + +std::string +BucketDBUpdaterTest::getSentNodesDistributionChanged( + const std::string& oldClusterState) +{ + MessageSenderStub sender; + + framework::defaultimplementation::FakeClock clock; + ClusterInformation::CSP clusterInfo(createClusterInfo(oldClusterState)); + std::unique_ptr<PendingClusterState> state( + PendingClusterState::createForDistributionChange( + clock, clusterInfo, sender, api::Timestamp(1))); + + sortSentMessagesByIndex(sender); + + std::ostringstream ost; + for (uint32_t i = 0; i < sender.commands.size(); i++) { + RequestBucketInfoCommand* req = + dynamic_cast<RequestBucketInfoCommand*>(sender.commands[i].get()); + + if (i > 0) { + ost << ","; + } + + ost << req->getAddress()->getIndex(); + } + + return ost.str(); +} + +void +BucketDBUpdaterTest::testPendingClusterStateSendMessages() +{ + CPPUNIT_ASSERT_EQUAL( + std::string("0,1,2"), + getSentNodes("cluster:d", + "distributor:1 storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string("0,1"), + getSentNodes("cluster:d", + "distributor:1 storage:3 .2.s:m")); + + CPPUNIT_ASSERT_EQUAL( + std::string("2"), + getSentNodes("distributor:1 storage:2", + "distributor:1 storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string("2,3,4,5"), + getSentNodes("distributor:1 storage:2", + "distributor:1 storage:6")); + + CPPUNIT_ASSERT_EQUAL( + std::string("0,1,2"), + getSentNodes("distributor:4 storage:3", + "distributor:3 storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string("0,1,2,3"), + getSentNodes("distributor:4 storage:3", + "distributor:4 .2.s:d storage:4")); + + CPPUNIT_ASSERT_EQUAL( + std::string(""), + getSentNodes("distributor:4 storage:3", + "distributor:4 .0.s:d storage:4")); + + CPPUNIT_ASSERT_EQUAL( + std::string(""), + getSentNodes("distributor:3 storage:3", + "distributor:4 storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string("2"), + getSentNodes("distributor:3 storage:3 .2.s:i", + "distributor:3 storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string("1"), + getSentNodes("distributor:3 storage:3 .1.s:d", + "distributor:3 storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string("1,2,4"), + getSentNodes("distributor:3 storage:4 .1.s:d .2.s:i", + "distributor:3 storage:5")); + + CPPUNIT_ASSERT_EQUAL( + std::string(""), + getSentNodes("distributor:1 storage:3", + "cluster:d")); + + CPPUNIT_ASSERT_EQUAL( + std::string(""), + getSentNodes("distributor:1 storage:3", + "distributor:1 storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string(""), + getSentNodes("distributor:1 storage:3", + "cluster:d distributor:1 storage:6")); + + CPPUNIT_ASSERT_EQUAL( + std::string(""), + getSentNodes("distributor:3 storage:3", + "distributor:3 .2.s:m storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string("0,1,2"), + getSentNodes("distributor:3 .2.s:m storage:3", + "distributor:3 .2.s:d storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string(""), + getSentNodes("distributor:3 .2.s:m storage:3", + "distributor:3 storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string("0,1,2"), + getSentNodesDistributionChanged("distributor:3 storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string("0,1"), + getSentNodes("distributor:10 storage:2", + "distributor:10 .1.s:d storage:2")); + + CPPUNIT_ASSERT_EQUAL( + std::string("1"), + getSentNodes("distributor:2 storage:2", + "distributor:2 storage:2 .1.d:3 .1.d.1.s:d")); + + CPPUNIT_ASSERT_EQUAL( + std::string("1"), + getSentNodes("distributor:2 storage:2 .1.s:d", + "distributor:2 storage:2 .1.d:3 .1.d.1.s:d")); + + CPPUNIT_ASSERT_EQUAL( + std::string(""), + getSentNodes("distributor:2 storage:2", + "distributor:3 .2.s:i storage:2")); + + CPPUNIT_ASSERT_EQUAL( + std::string("0,1,2"), + getSentNodes("distributor:3 storage:3", + "distributor:3 .2.s:s storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string(""), + getSentNodes("distributor:3 .2.s:s storage:3", + "distributor:3 .2.s:d storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string("1"), + getSentNodes("distributor:3 storage:3 .1.s:m", + "distributor:3 storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string(""), + getSentNodes("distributor:3 storage:3", + "distributor:3 storage:3 .1.s:m")); +}; + +void +BucketDBUpdaterTest::testPendingClusterStateReceive() +{ + MessageSenderStub sender; + + auto cmd(std::make_shared<api::SetSystemStateCommand>( + lib::ClusterState("distributor:1 storage:3"))); + + framework::defaultimplementation::FakeClock clock; + ClusterInformation::CSP clusterInfo(createClusterInfo("cluster:d")); + std::unordered_set<uint16_t> outdatedNodes; + std::unique_ptr<PendingClusterState> state( + PendingClusterState::createForClusterStateChange( + clock, clusterInfo, sender, cmd, outdatedNodes, + api::Timestamp(1))); + + CPPUNIT_ASSERT_EQUAL(3, (int)sender.commands.size()); + + sortSentMessagesByIndex(sender); + + std::ostringstream ost; + for (uint32_t i = 0; i < sender.commands.size(); i++) { + RequestBucketInfoCommand* req = + dynamic_cast<RequestBucketInfoCommand*>(sender.commands[i].get()); + + RequestBucketInfoReply* rep = + new RequestBucketInfoReply(*req); + + rep->getBucketInfo().push_back( + RequestBucketInfoReply::Entry( + document::BucketId(16, i), + api::BucketInfo(i, i, i, i, i))); + + CPPUNIT_ASSERT( + state->onRequestBucketInfoReply( + std::shared_ptr<api::RequestBucketInfoReply>(rep))); + + CPPUNIT_ASSERT_EQUAL(i == sender.commands.size() - 1 ? true : false, + state->done()); + } + + CPPUNIT_ASSERT_EQUAL(3, (int)state->results().size()); +} + +void +BucketDBUpdaterTest::testPendingClusterStateWithGroupDown() +{ + std::string config(getDistConfig6Nodes4Groups()); + config += "distributor_auto_ownership_transfer_on_whole_group_down true\n"; + setDistribution(config); + + // Group config has nodes {0, 1}, {2, 3}, {4, 5} + // We're node index 0. + + // Entire group 1 goes down. Must refetch from all nodes. + CPPUNIT_ASSERT_EQUAL( + std::string("0,1,2,3,4,5"), + getSentNodes("distributor:6 storage:6", + "distributor:6 .2.s:d .3.s:d storage:6")); + + // But don't fetch if not the entire group is down. + CPPUNIT_ASSERT_EQUAL( + std::string(""), + getSentNodes("distributor:6 storage:6", + "distributor:6 .2.s:d storage:6")); +} + +void +BucketDBUpdaterTest::testPendingClusterStateWithGroupDownAndNoHandover() +{ + std::string config(getDistConfig6Nodes4Groups()); + config += "distributor_auto_ownership_transfer_on_whole_group_down false\n"; + setDistribution(config); + + // Group is down, but config says to not do anything about it. + CPPUNIT_ASSERT_EQUAL( + std::string(""), + getSentNodes("distributor:6 storage:6", + "distributor:6 .2.s:d .3.s:d storage:6")); +} + +void +parseInputData(const std::string& data, + uint64_t timestamp, + PendingClusterState& state, + bool includeBucketInfo) +{ + vespalib::StringTokenizer tokenizer(data, "|"); + for (uint32_t i = 0; i < tokenizer.size(); i++) { + vespalib::StringTokenizer tok2(tokenizer[i], ":"); + + uint16_t node = atoi(tok2[0].c_str()); + + state.setNodeReplied(node); + + vespalib::StringTokenizer tok3(tok2[1], ","); + for (uint32_t j = 0; j < tok3.size(); j++) { + if (includeBucketInfo) { + vespalib::StringTokenizer tok4(tok3[j], "/"); + + state.addNodeInfo( + document::BucketId(16, atoi(tok4[0].c_str())), + BucketCopy( + timestamp, + node, + api::BucketInfo( + atoi(tok4[1].c_str()), + atoi(tok4[2].c_str()), + atoi(tok4[3].c_str()), + atoi(tok4[2].c_str()), + atoi(tok4[3].c_str())))); + } else { + state.addNodeInfo( + document::BucketId(16, atoi(tok3[j].c_str())), + BucketCopy(timestamp, + node, + api::BucketInfo(3, 3, 3, 3, 3))); + } + } + } +} + +struct BucketDumper : public BucketDatabase::EntryProcessor +{ + std::ostringstream ost; + bool _includeBucketInfo; + + BucketDumper(bool includeBucketInfo) + : _includeBucketInfo(includeBucketInfo) + { + } + + bool process(const BucketDatabase::Entry& e) { + document::BucketId bucketId(e.getBucketId()); + + ost << (uint32_t)bucketId.getRawId() << ":"; + for (uint32_t i = 0; i < e->getNodeCount(); ++i) { + if (i > 0) { + ost << ","; + } + const BucketCopy& copy(e->getNodeRef(i)); + ost << copy.getNode(); + if (_includeBucketInfo) { + ost << "/" << copy.getChecksum() + << "/" << copy.getDocumentCount() + << "/" << copy.getTotalDocumentSize() + << "/" << (copy.trusted() ? "t" : "u"); + } + } + ost << "|"; + return true; + } +}; + +std::string +BucketDBUpdaterTest::mergeBucketLists( + const lib::ClusterState& oldState, + const std::string& existingData, + const lib::ClusterState& newState, + const std::string& newData, + bool includeBucketInfo) +{ + framework::defaultimplementation::FakeClock clock; + framework::MilliSecTimer timer(clock); + + MessageSenderStub sender; + std::unordered_set<uint16_t> outdatedNodes; + + { + auto cmd(std::make_shared<api::SetSystemStateCommand>(oldState)); + + api::Timestamp beforeTime(1); + + ClusterInformation::CSP clusterInfo(createClusterInfo("cluster:d")); + std::unique_ptr<PendingClusterState> state( + PendingClusterState::createForClusterStateChange( + clock, clusterInfo, sender, cmd, outdatedNodes, + beforeTime)); + + parseInputData(existingData, beforeTime, *state, includeBucketInfo); + state->mergeInto(getBucketDBUpdater().getDistributorComponent().getBucketDatabase()); + } + + BucketDumper dumper_tmp(true); + getBucketDatabase().forEach(dumper_tmp); + + { + auto cmd(std::make_shared<api::SetSystemStateCommand>( + lib::ClusterState(newState))); + + api::Timestamp afterTime(2); + + ClusterInformation::CSP clusterInfo(createClusterInfo(oldState.toString())); + std::unique_ptr<PendingClusterState> state( + PendingClusterState::createForClusterStateChange( + clock, clusterInfo, sender, cmd, outdatedNodes, + afterTime)); + + parseInputData(newData, afterTime, *state, includeBucketInfo); + state->mergeInto(getBucketDBUpdater().getDistributorComponent() + .getBucketDatabase()); + } + + BucketDumper dumper(includeBucketInfo); + getBucketDBUpdater().getDistributorComponent() + .getBucketDatabase().forEach(dumper); + getBucketDBUpdater().getDistributorComponent() + .getBucketDatabase().clear(); + return dumper.ost.str(); +} + +std::string +BucketDBUpdaterTest::mergeBucketLists(const std::string& existingData, + const std::string& newData, + bool includeBucketInfo) +{ + return mergeBucketLists( + lib::ClusterState("distributor:1 storage:3"), + existingData, + lib::ClusterState("distributor:1 storage:3"), + newData, + includeBucketInfo); +} + +void +BucketDBUpdaterTest::testPendingClusterStateMerge() +{ + // Simple initializing case - ask all nodes for info + CPPUNIT_ASSERT_EQUAL( + // Result is on the form: [bucket w/o count bits]:[node indexes]|.. + std::string("4:0,1|2:0,1|6:1,2|1:0,2|5:2,0|3:2,1|"), + // Input is on the form: [node]:[bucket w/o count bits]|... + mergeBucketLists( + "", + "0:1,2,4,5|1:2,3,4,6|2:1,3,5,6")); + + // Node came up with fewer buckets (lost disk) + CPPUNIT_ASSERT_EQUAL( + std::string("4:1|2:0,1|6:1,2|1:2,0|5:2|3:2,1|"), + mergeBucketLists( + lib::ClusterState("distributor:1 storage:3"), + "0:1,2,4,5|1:2,3,4,6|2:1,3,5,6", + lib::ClusterState("distributor:1 storage:3 .0.d:3 .0.d.1.s:d"), + "0:1,2") + ); + + // New node came up + CPPUNIT_ASSERT_EQUAL( + std::string("4:0,1|2:0,1|6:1,2,3|1:0,2,3|5:2,0,3|3:2,1,3|"), + mergeBucketLists( + "0:1,2,4,5|1:2,3,4,6|2:1,3,5,6", + "3:1,3,5,6")); + + // Node came up with some buckets removed and some added + // Buckets that were removed should not be removed as the node + // didn't lose a disk. + CPPUNIT_ASSERT_EQUAL( + std::string("8:0|4:0,1|2:0,1|6:1,0,2|1:0,2|5:2,0|3:2,1|"), + mergeBucketLists( + "0:1,2,4,5|1:2,3,4,6|2:1,3,5,6", + "0:1,2,6,8")); + + // Node came up with no buckets + CPPUNIT_ASSERT_EQUAL( + std::string("4:1|2:1|6:1,2|1:2|5:2|3:2,1|"), + mergeBucketLists( + lib::ClusterState("distributor:1 storage:3"), + "0:1,2,4,5|1:2,3,4,6|2:1,3,5,6", + lib::ClusterState("distributor:1 storage:3 .0.d:3 .0.d.1.s:d"), + "0:") + ); + + // One node lost a disk, another was just reasked (distributor + // change) + CPPUNIT_ASSERT_EQUAL( + std::string("2:0,1|6:1,2|1:2,0|5:2|3:2,1|"), + mergeBucketLists( + lib::ClusterState("distributor:1 storage:3"), + "0:1,2,4,5|1:2,3,6|2:1,3,5,6", + lib::ClusterState("distributor:1 storage:3 .0.d:3 .0.d.1.s:d"), + "0:1,2|1:2,3") + ); + + // Bucket info format is "bucketid/checksum/count/size" + // Node went from initializing to up and invalid bucket went to empty. + CPPUNIT_ASSERT_EQUAL( + std::string("2:0/0/0/0/t|"), + mergeBucketLists( + "0:2/0/0/1", + "0:2/0/0/0", + true)); + + CPPUNIT_ASSERT_EQUAL( + std::string("5:1/2/3/4/u,0/0/0/0/u|"), + mergeBucketLists("", "0:5/0/0/0|1:5/2/3/4", true)); +} + +void +BucketDBUpdaterTest::testPendingClusterStateMergeReplicaChanged() +{ + // Node went from initializing to up and non-invalid bucket changed. + CPPUNIT_ASSERT_EQUAL( + std::string("2:0/2/3/4/t|3:0/2/4/6/t|"), + mergeBucketLists( + lib::ClusterState("distributor:1 storage:1 .0.s:i"), + "0:2/1/2/3,3/2/4/6", + lib::ClusterState("distributor:1 storage:1"), + "0:2/2/3/4,3/2/4/6", + true)); +} + +void +BucketDBUpdaterTest::testNoDbResurrectionForBucketNotOwnedInCurrentState() +{ + document::BucketId bucket(16, 3); + lib::ClusterState stateBefore("distributor:1 storage:1"); + { + uint32_t expectedMsgs = 1, dummyBucketsToReturn = 1; + setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); + } + _sender.clear(); + + getBucketDBUpdater().recheckBucketInfo(0, bucket); + + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + std::shared_ptr<api::RequestBucketInfoCommand> rbi( + std::dynamic_pointer_cast<RequestBucketInfoCommand>( + _sender.commands[0])); + + lib::ClusterState stateAfter("distributor:3 storage:3"); + + { + uint32_t expectedMsgs = 2, dummyBucketsToReturn = 1; + setAndEnableClusterState(stateAfter, expectedMsgs, dummyBucketsToReturn); + } + CPPUNIT_ASSERT(!getBucketDBUpdater().getDistributorComponent() + .ownsBucketInCurrentState(bucket)); + + sendFakeReplyForSingleBucketRequest(*rbi); + + CPPUNIT_ASSERT_EQUAL(std::string("NONEXISTING"), dumpBucket(bucket)); +} + +void +BucketDBUpdaterTest::testNoDbResurrectionForBucketNotOwnedInPendingState() +{ + document::BucketId bucket(16, 3); + lib::ClusterState stateBefore("distributor:1 storage:1"); + { + uint32_t expectedMsgs = 1, dummyBucketsToReturn = 1; + setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); + } + _sender.clear(); + + getBucketDBUpdater().recheckBucketInfo(0, bucket); + + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + std::shared_ptr<api::RequestBucketInfoCommand> rbi( + std::dynamic_pointer_cast<RequestBucketInfoCommand>( + _sender.commands[0])); + + lib::ClusterState stateAfter("distributor:3 storage:3"); + // Set, but _don't_ enable cluster state. We want it to be pending. + setSystemState(stateAfter); + CPPUNIT_ASSERT(getBucketDBUpdater().getDistributorComponent() + .ownsBucketInCurrentState(bucket)); + CPPUNIT_ASSERT(!getBucketDBUpdater() + .checkOwnershipInPendingState(bucket).isOwned()); + + sendFakeReplyForSingleBucketRequest(*rbi); + + CPPUNIT_ASSERT_EQUAL(std::string("NONEXISTING"), dumpBucket(bucket)); +} + +/* + * If we get a distribution config change, it's important that cluster states that + * arrive after this--but _before_ the pending cluster state has finished--must trigger + * a full bucket info fetch no matter what the cluster state change was! Otherwise, we + * will with a high likelihood end up not getting the complete view of the buckets in + * the cluster. + */ +void +BucketDBUpdaterTest::testClusterStateAlwaysSendsFullFetchWhenDistributionChangePending() +{ + lib::ClusterState stateBefore("distributor:6 storage:6"); + { + uint32_t expectedMsgs = 6, dummyBucketsToReturn = 1; + setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); + } + _sender.clear(); + std::string distConfig(getDistConfig6Nodes3Groups()); + { + _node->getComponentRegister().setDistribution( + std::make_shared<lib::Distribution>(distConfig)); + _distributor->storageDistributionChanged(); + _distributor->enableNextDistribution(); + } + sortSentMessagesByIndex(_sender); + CPPUNIT_ASSERT_EQUAL(size_t(6), _sender.commands.size()); + // Suddenly, a wild cluster state change appears! Even though this state + // does not in itself imply any bucket changes, it will still overwrite the + // pending cluster state and thus its state of pending bucket info requests. + setSystemState(lib::ClusterState("distributor:6 .2.t:12345 storage:6")); + + CPPUNIT_ASSERT_EQUAL(size_t(12), _sender.commands.size()); + + // Send replies for first 6 (outdated requests). + int numBuckets = 10; + for (uint32_t i = 0; i < 6; ++i) { + fakeBucketReply( + lib::ClusterState("distributor:6 storage:6"), + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), + i, numBuckets); + } + // No change from these. + assertCorrectBuckets(1, "distributor:6 storage:6"); + + // Send for current pending. + for (uint32_t i = 0; i < 6; ++i) { + fakeBucketReply( + lib::ClusterState("distributor:6 .2.t:12345 storage:6"), + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i + 6]), + i, numBuckets); + } + assertCorrectBuckets(numBuckets, "distributor:6 storage:6"); + _sender.clear(); + + // No more pending global fetch; this should be a no-op state. + setSystemState(lib::ClusterState("distributor:6 .3.t:12345 storage:6")); + CPPUNIT_ASSERT_EQUAL(size_t(0), _sender.commands.size()); +} + +void +BucketDBUpdaterTest::testChangedDistributionConfigTriggersRecoveryMode() +{ + setAndEnableClusterState(lib::ClusterState("distributor:6 storage:6"), 6, 20); + _sender.clear(); + // First cluster state; implicit scan of all buckets which does not + // use normal recovery mode ticking-path. + CPPUNIT_ASSERT(!_distributor->isInRecoveryMode()); + + std::string distConfig(getDistConfig6Nodes4Groups()); + _node->getComponentRegister().setDistribution( + std::make_shared<lib::Distribution>(distConfig)); + _distributor->storageDistributionChanged(); + _distributor->enableNextDistribution(); + sortSentMessagesByIndex(_sender); + // No replies received yet, still no recovery mode. + CPPUNIT_ASSERT(!_distributor->isInRecoveryMode()); + + CPPUNIT_ASSERT_EQUAL(size_t(6), _sender.commands.size()); + uint32_t numBuckets = 10; + for (uint32_t i = 0; i < 6; ++i) { + fakeBucketReply( + lib::ClusterState("distributor:6 storage:6"), + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[i]), + i, numBuckets); + } + + // Pending cluster state (i.e. distribution) has been enabled, which should + // cause recovery mode to be entered. + CPPUNIT_ASSERT(_distributor->isInRecoveryMode()); +} + +void +BucketDBUpdaterTest::testNewlyAddedBucketsHaveCurrentTimeAsGcTimestamp() +{ + getClock().setAbsoluteTimeInSeconds(101234); + lib::ClusterState stateBefore("distributor:1 storage:1"); + { + uint32_t expectedMsgs = 1, dummyBucketsToReturn = 1; + setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); + } + + // setAndEnableClusterState adds n buckets with id (16, i) + document::BucketId bucket(16, 0); + BucketDatabase::Entry e(getBucket(bucket)); + CPPUNIT_ASSERT(e.valid()); + CPPUNIT_ASSERT_EQUAL(uint32_t(101234), e->getLastGarbageCollectionTime()); +} + +void +BucketDBUpdaterTest::testNewerMutationsNotOverwrittenByEarlierBucketFetch() +{ + { + lib::ClusterState stateBefore("distributor:1 storage:1 .0.s:i"); + uint32_t expectedMsgs = 1, dummyBucketsToReturn = 0; + // This step is required to make the distributor ready for accepting + // the below explicit database insertion towards node 0. + setAndEnableClusterState(stateBefore, expectedMsgs, + dummyBucketsToReturn); + } + _sender.clear(); + getClock().setAbsoluteTimeInSeconds(1000); + lib::ClusterState state("distributor:1 storage:1"); + setSystemState(state); + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + + // Before replying with the bucket info, simulate the arrival of a mutation + // reply that alters the state of the bucket with information that will be + // more recent that what is returned by the bucket info. This information + // must not be lost when the bucket info is later merged into the database. + document::BucketId bucket(16, 1); + constexpr uint64_t insertionTimestamp = 1001ULL * 1000000; + api::BucketInfo wantedInfo(5, 6, 7); + getBucketDBUpdater().getDistributorComponent().updateBucketDatabase( + bucket, + BucketCopy(insertionTimestamp, 0, wantedInfo), + DatabaseUpdate::CREATE_IF_NONEXISTING); + + getClock().setAbsoluteTimeInSeconds(1002); + constexpr uint32_t bucketsReturned = 10; // Buckets (16, 0) ... (16, 9) + // Return bucket information which on the timeline might originate from + // anywhere between [1000, 1002]. Our assumption is that any mutations + // taking place after t=1000 must have its reply received and processed + // by this distributor and timestamped strictly higher than t=1000 (modulo + // clock skew, of course, but that is outside the scope of this). A mutation + // happening before t=1000 but receiving a reply at t>1000 does not affect + // correctness, as this should contain the same bucket info as that + // contained in the full bucket reply and the DB update is thus idempotent. + fakeBucketReply( + state, + dynamic_cast<RequestBucketInfoCommand&>(*_sender.commands[0]), + 0, + bucketsReturned); + + BucketDatabase::Entry e(getBucket(bucket)); + CPPUNIT_ASSERT_EQUAL(uint32_t(1), e->getNodeCount()); + CPPUNIT_ASSERT_EQUAL(wantedInfo, e->getNodeRef(0).getBucketInfo()); + +} + +std::vector<uint16_t> +BucketDBUpdaterTest::getSendSet() const +{ + std::vector<uint16_t> nodes; + std::transform(_sender.commands.begin(), + _sender.commands.end(), + std::back_inserter(nodes), + [](auto& cmd) + { + auto& req(dynamic_cast<const api::RequestBucketInfoCommand&>(*cmd)); + return req.getAddress()->getIndex(); + }); + return nodes; +} + +std::vector<uint16_t> +BucketDBUpdaterTest::getSentNodesWithPreemption( + const std::string& oldClusterState, + uint32_t expectedOldStateMessages, + const std::string& preemptedClusterState, + const std::string& newClusterState) +{ + lib::ClusterState stateBefore(oldClusterState); + uint32_t dummyBucketsToReturn = 10; + setAndEnableClusterState(lib::ClusterState(oldClusterState), + expectedOldStateMessages, + dummyBucketsToReturn); + _sender.clear(); + + setSystemState(lib::ClusterState(preemptedClusterState)); + _sender.clear(); + // Do not allow the pending state to become the active state; trigger a + // new transition without ACKing the info requests first. This will + // overwrite the pending state entirely. + setSystemState(lib::ClusterState(newClusterState)); + return getSendSet(); +} + +using nodeVec = std::vector<uint16_t>; + +/* + * If we don't carry over the set of nodes that we need to fetch from, + * a naive comparison between the active state and the new state will + * make it appear to the distributor that nothing has changed, as any + * database modifications caused by intermediate states will not be + * accounted for (basically the ABA problem in a distributed setting). + */ +void +BucketDBUpdaterTest::preemptedDistrChangeCarriesNodeSetOverToNextStateFetch() +{ + CPPUNIT_ASSERT_EQUAL( + (nodeVec{0, 1, 2, 3, 4, 5}), + getSentNodesWithPreemption("version:1 distributor:6 storage:6", 6, + "version:2 distributor:6 .5.s:d storage:6", + "version:3 distributor:6 storage:6")); +} + +void +BucketDBUpdaterTest::preemptedStorChangeCarriesNodeSetOverToNextStateFetch() +{ + CPPUNIT_ASSERT_EQUAL( + (nodeVec{2, 3}), + getSentNodesWithPreemption( + "version:1 distributor:6 storage:6 .2.s:d", 5, + "version:2 distributor:6 storage:6 .2.s:d .3.s:d", + "version:3 distributor:6 storage:6")); +} + +void +BucketDBUpdaterTest::preemptedStorageNodeDownMustBeReFetched() +{ + CPPUNIT_ASSERT_EQUAL( + (nodeVec{2}), + getSentNodesWithPreemption( + "version:1 distributor:6 storage:6", 6, + "version:2 distributor:6 storage:6 .2.s:d", + "version:3 distributor:6 storage:6")); +} + +void +BucketDBUpdaterTest::doNotSendToPreemptedNodeNowInDownState() +{ + CPPUNIT_ASSERT_EQUAL( + nodeVec{}, + getSentNodesWithPreemption( + "version:1 distributor:6 storage:6 .2.s:d", 5, + "version:2 distributor:6 storage:6", // Sends to 2. + "version:3 distributor:6 storage:6 .2.s:d")); // 2 down again. +} + +void +BucketDBUpdaterTest::doNotSendToPreemptedNodeNotPartOfNewState() +{ + // Even though 100 nodes are preempted, not all of these should be part + // of the request afterwards when only 6 are part of the state. + CPPUNIT_ASSERT_EQUAL( + (nodeVec{0, 1, 2, 3, 4, 5}), + getSentNodesWithPreemption( + "version:1 distributor:6 storage:100", 100, + "version:2 distributor:5 .4.s:d storage:100", + "version:3 distributor:6 storage:6")); +} + +void +BucketDBUpdaterTest::outdatedNodeSetClearedAfterSuccessfulStateCompletion() +{ + lib::ClusterState stateBefore( + "version:1 distributor:6 storage:6 .1.t:1234"); + uint32_t expectedMsgs = 6, dummyBucketsToReturn = 10; + setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn); + _sender.clear(); + // New cluster state that should not by itself trigger any new fetches, + // unless outdated node set is somehow not cleared after an enabled + // (completed) cluster state has been set. + lib::ClusterState stateAfter("version:3 distributor:6 storage:6"); + setSystemState(stateAfter); + CPPUNIT_ASSERT_EQUAL(size_t(0), _sender.commands.size()); +} + +// XXX test currently disabled since distribution config currently isn't used +// at all in order to deduce the set of nodes to send to. This might not matter +// in practice since it is assumed that the cluster state matching the new +// distribution config will follow very shortly after the config has been +// applied to the node. The new cluster state will then send out requests to +// the correct node set. +void +BucketDBUpdaterTest::clusterConfigDownsizeOnlySendsToAvailableNodes() +{ + uint32_t expectedMsgs = 6, dummyBucketsToReturn = 20; + setAndEnableClusterState(lib::ClusterState("distributor:6 storage:6"), + expectedMsgs, dummyBucketsToReturn); + _sender.clear(); + + // Intentionally trigger a racing config change which arrives before the + // new cluster state representing it. + std::string distConfig(getDistConfig3Nodes1Group()); + _node->getComponentRegister().setDistribution( + std::make_shared<lib::Distribution>(distConfig)); + _distributor->storageDistributionChanged(); + _distributor->enableNextDistribution(); + sortSentMessagesByIndex(_sender); + + CPPUNIT_ASSERT_EQUAL((nodeVec{0, 1, 2}), getSendSet()); +} + +void +BucketDBUpdaterTest::changedDiskSetTriggersReFetch() +{ + // Same number of online disks, but the set of disks has changed. + CPPUNIT_ASSERT_EQUAL( + std::string("1"), + getSentNodes("distributor:2 storage:2 .1.d:3 .1.d.2.s:d", + "distributor:2 storage:2 .1.d:3 .1.d.1.s:d")); +} + +/** + * Test scenario where a cluster is downsized by removing a subset of the nodes + * from the distribution configuration. The system must be able to deal with + * a scenario where the set of nodes between two cluster states across a config + * change may differ. + * + * See VESPA-790 for details. + */ +void +BucketDBUpdaterTest::nodeMissingFromConfigIsTreatedAsNeedingOwnershipTransfer() +{ + uint32_t expectedMsgs = 3, dummyBucketsToReturn = 1; + setAndEnableClusterState(lib::ClusterState("distributor:3 storage:3"), + expectedMsgs, dummyBucketsToReturn); + _sender.clear(); + + // Cluster goes from {0, 1, 2} -> {0, 1}. This leaves us with a config + // that does not contain node 2 while the _active_ cluster state still + // contains this node. + const char* downsizeCfg = + "redundancy 2\n" + "distributor_auto_ownership_transfer_on_whole_group_down true\n" + "group[2]\n" + "group[0].name \"invalid\"\n" + "group[0].index \"invalid\"\n" + "group[0].partitions 1|*\n" + "group[0].nodes[0]\n" + "group[1].name rack0\n" + "group[1].index 0\n" + "group[1].nodes[2]\n" + "group[1].nodes[0].index 0\n" + "group[1].nodes[1].index 1\n"; + + _node->getComponentRegister().setDistribution( + std::make_shared<lib::Distribution>(downsizeCfg)); + _distributor->storageDistributionChanged(); + _distributor->enableNextDistribution(); + sortSentMessagesByIndex(_sender); + _sender.clear(); + + // Attempt to apply state with {0, 1} set. This will compare the new state + // with the previous state, which still has node 2. + expectedMsgs = 2; + setAndEnableClusterState(lib::ClusterState("distributor:2 storage:2"), + expectedMsgs, dummyBucketsToReturn); + + CPPUNIT_ASSERT_EQUAL((nodeVec{0, 1}), getSendSet()); +} + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/bucketgctimecalculatortest.cpp b/storage/src/tests/distributor/bucketgctimecalculatortest.cpp new file mode 100644 index 00000000000..39bef3ec395 --- /dev/null +++ b/storage/src/tests/distributor/bucketgctimecalculatortest.cpp @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <chrono> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/distributor/bucketgctimecalculator.h> + +namespace storage { +namespace distributor { + +struct MockBucketIdHasher : public BucketGcTimeCalculator::BucketIdHasher +{ + size_t nextGeneratedHash {0}; + + size_t doHash(const document::BucketId&) const override { + return nextGeneratedHash; + } +}; + +struct BucketGcTimeCalculatorTest : public CppUnit::TestFixture +{ + void noGcIfAlreadyCheckedAfterStartPoint(); + void gcIfNotRunInCurrentPeriodAndCheckPeriodPassed(); + void noGcIfNotRunInCurrentPeriodAndCheckPeriodNotPassed(); + void noGcIfCheckIntervalIsZero(); + void identityHasherReturnsBucketId(); + + BucketGcTimeCalculatorTest(); + + CPPUNIT_TEST_SUITE(BucketGcTimeCalculatorTest); + CPPUNIT_TEST(noGcIfAlreadyCheckedAfterStartPoint); + CPPUNIT_TEST(gcIfNotRunInCurrentPeriodAndCheckPeriodPassed); + CPPUNIT_TEST(noGcIfNotRunInCurrentPeriodAndCheckPeriodNotPassed); + CPPUNIT_TEST(noGcIfCheckIntervalIsZero); + CPPUNIT_TEST(identityHasherReturnsBucketId); + CPPUNIT_TEST_SUITE_END(); + +private: + // Ease of reading aliases + using CurrentTime = std::chrono::seconds; + using LastRunAt = std::chrono::seconds; + + MockBucketIdHasher hasher; + std::chrono::seconds checkInterval; + BucketGcTimeCalculator calc; + document::BucketId b; +}; + +BucketGcTimeCalculatorTest::BucketGcTimeCalculatorTest() + : checkInterval(1000), + calc(hasher, checkInterval), + b(16, 1) +{ + hasher.nextGeneratedHash = 500; +} + +CPPUNIT_TEST_SUITE_REGISTRATION(BucketGcTimeCalculatorTest); + +void +BucketGcTimeCalculatorTest::noGcIfAlreadyCheckedAfterStartPoint() +{ + // Note: LastRun(0) is considered to be within the current period. + CPPUNIT_ASSERT(!calc.shouldGc(b, CurrentTime(0), LastRunAt(0))); + CPPUNIT_ASSERT(!calc.shouldGc(b, CurrentTime(499), LastRunAt(0))); + CPPUNIT_ASSERT(!calc.shouldGc(b, CurrentTime(999), LastRunAt(500))); + + CPPUNIT_ASSERT(!calc.shouldGc(b, CurrentTime(1000), LastRunAt(1000))); + CPPUNIT_ASSERT(!calc.shouldGc(b, CurrentTime(1234), LastRunAt(1100))); + CPPUNIT_ASSERT(!calc.shouldGc(b, CurrentTime(1600), LastRunAt(1500))); +} + +void +BucketGcTimeCalculatorTest::gcIfNotRunInCurrentPeriodAndCheckPeriodPassed() +{ + CPPUNIT_ASSERT(calc.shouldGc(b, CurrentTime(500), LastRunAt(0))); + CPPUNIT_ASSERT(calc.shouldGc(b, CurrentTime(1600), LastRunAt(500))); + // Note: this may look wrong, but is correct since GC should have been + // scheduled _after_ 1499 so this is most likely the case where a bucket + // has been added to the database at this point in time. Not treating + // this as a valid GC scenario would mean newly added buckets would have to + // wait until the next period to be considered. If the period is long and + // the system is unstable (causing many bucket handoffs), we'd risk not + // being able to scheduled many buckets at all. + CPPUNIT_ASSERT(calc.shouldGc(b, CurrentTime(1600), LastRunAt(1499))); + + CPPUNIT_ASSERT(calc.shouldGc(b, CurrentTime(2000), LastRunAt(500))); + CPPUNIT_ASSERT(calc.shouldGc(b, CurrentTime(2600), LastRunAt(1500))); +} + +void +BucketGcTimeCalculatorTest::noGcIfNotRunInCurrentPeriodAndCheckPeriodNotPassed() +{ + CPPUNIT_ASSERT(!calc.shouldGc(b, CurrentTime(1000), LastRunAt(500))); +} + +void +BucketGcTimeCalculatorTest::noGcIfCheckIntervalIsZero() +{ + BucketGcTimeCalculator calc2(hasher, std::chrono::seconds(0)); + CPPUNIT_ASSERT(!calc2.shouldGc(b, CurrentTime(5000), LastRunAt(0))); +} + +void +BucketGcTimeCalculatorTest::identityHasherReturnsBucketId() +{ + BucketGcTimeCalculator::BucketIdIdentityHasher hasher2; + document::BucketId bucket(36, 1234); + + CPPUNIT_ASSERT_EQUAL(bucket.getId(), hasher2.hash(bucket)); +} + +} // distributor +} // storage + diff --git a/storage/src/tests/distributor/bucketstateoperationtest.cpp b/storage/src/tests/distributor/bucketstateoperationtest.cpp new file mode 100644 index 00000000000..1477f1d6ed0 --- /dev/null +++ b/storage/src/tests/distributor/bucketstateoperationtest.cpp @@ -0,0 +1,251 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <tests/distributor/distributortestutil.h> +#include <vespa/storage/distributor/operations/idealstate/setbucketstateoperation.h> +#include <vespa/storageapi/messageapi/storagemessage.h> + +namespace storage { + +namespace distributor { + +class BucketStateOperationTest : public CppUnit::TestFixture, + public DistributorTestUtil +{ + CPPUNIT_TEST_SUITE(BucketStateOperationTest); + CPPUNIT_TEST(testActiveStateSupportedInBucketDb); + CPPUNIT_TEST(testActivateSingleNode); + CPPUNIT_TEST(testActivateAndDeactivateNodes); + CPPUNIT_TEST(testDoNotDeactivateIfActivateFails); + CPPUNIT_TEST(testBucketDbNotUpdatedOnFailure); + CPPUNIT_TEST_SUITE_END(); + +private: + void testActiveStateSupportedInBucketDb(); + void testActivateSingleNode(); + void testActivateAndDeactivateNodes(); + void testDoNotDeactivateIfActivateFails(); + void testBucketDbNotUpdatedOnFailure(); + +public: + void setUp() + { + createLinks(); + } + + void tearDown() + { + close(); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(BucketStateOperationTest); + +void +BucketStateOperationTest::testActiveStateSupportedInBucketDb() +{ + document::BucketId bid(16, 1); + insertBucketInfo(bid, 0, 0xabc, 10, 1100, true, true); + + BucketDatabase::Entry entry = getBucket(bid); + CPPUNIT_ASSERT(entry.valid()); + CPPUNIT_ASSERT(entry->getNode(0)->active()); + CPPUNIT_ASSERT_EQUAL( + std::string("node(idx=0,crc=0xabc,docs=10/10,bytes=1100/1100," + "trusted=true,active=true)"), + entry->getNode(0)->toString()); +} + +void +BucketStateOperationTest::testActivateSingleNode() +{ + document::BucketId bid(16, 1); + insertBucketInfo(bid, 0, 0xabc, 10, 1100, true, false); + + BucketAndNodes bucketAndNodes(bid, toVector<uint16_t>(0)); + std::vector<uint16_t> active; + active.push_back(0); + SetBucketStateOperation op("storage", bucketAndNodes, active); + + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL((size_t)1, _sender.commands.size()); + + std::shared_ptr<api::StorageCommand> msg = _sender.commands[0]; + CPPUNIT_ASSERT(msg->getType() == api::MessageType::SETBUCKETSTATE); + CPPUNIT_ASSERT_EQUAL( + api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 0).toString(), + msg->getAddress()->toString()); + + const api::SetBucketStateCommand& cmd( + dynamic_cast<const api::SetBucketStateCommand&>(*msg)); + CPPUNIT_ASSERT_EQUAL(bid, cmd.getBucketId()); + CPPUNIT_ASSERT_EQUAL(api::SetBucketStateCommand::ACTIVE, cmd.getState()); + + std::shared_ptr<api::StorageReply> reply(msg->makeReply().release()); + op.receive(_sender, reply); + + BucketDatabase::Entry entry = getBucket(bid); + CPPUNIT_ASSERT(entry.valid()); + CPPUNIT_ASSERT(entry->getNodeRef(0).active()); + + CPPUNIT_ASSERT(op.ok()); + + // TODO: check that it's done +} + +void +BucketStateOperationTest::testActivateAndDeactivateNodes() +{ + document::BucketId bid(16, 1); + insertBucketInfo(bid, 0, 0xabc, 10, 1100, false, true); + insertBucketInfo(bid, 1, 0xdef, 15, 1500, false, false); + + BucketAndNodes bucketAndNodes(bid, toVector<uint16_t>(0, 1)); + std::vector<uint16_t> active; + active.push_back(1); + SetBucketStateOperation op("storage", bucketAndNodes, active); + + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL((size_t)1, _sender.commands.size()); + { + std::shared_ptr<api::StorageCommand> msg = _sender.commands[0]; + CPPUNIT_ASSERT(msg->getType() == api::MessageType::SETBUCKETSTATE); + CPPUNIT_ASSERT_EQUAL( + api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 1).toString(), + msg->getAddress()->toString()); + + const api::SetBucketStateCommand& cmd( + dynamic_cast<const api::SetBucketStateCommand&>(*msg)); + CPPUNIT_ASSERT_EQUAL(bid, cmd.getBucketId()); + CPPUNIT_ASSERT_EQUAL(api::SetBucketStateCommand::ACTIVE, cmd.getState()); + + std::shared_ptr<api::StorageReply> reply(msg->makeReply().release()); + op.receive(_sender, reply); + } + + CPPUNIT_ASSERT_EQUAL((size_t)2, _sender.commands.size()); + { + std::shared_ptr<api::StorageCommand> msg = _sender.commands[1]; + CPPUNIT_ASSERT(msg->getType() == api::MessageType::SETBUCKETSTATE); + CPPUNIT_ASSERT_EQUAL( + api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 0).toString(), + msg->getAddress()->toString()); + + const api::SetBucketStateCommand& cmd( + dynamic_cast<const api::SetBucketStateCommand&>(*msg)); + CPPUNIT_ASSERT_EQUAL(bid, cmd.getBucketId()); + CPPUNIT_ASSERT_EQUAL(api::SetBucketStateCommand::INACTIVE, cmd.getState()); + + std::shared_ptr<api::StorageReply> reply(msg->makeReply().release()); + op.receive(_sender, reply); + } + + BucketDatabase::Entry entry = getBucket(bid); + CPPUNIT_ASSERT(entry.valid()); + CPPUNIT_ASSERT_EQUAL( + std::string("node(idx=0,crc=0xabc,docs=10/10,bytes=1100/1100," + "trusted=true,active=false)"), + entry->getNodeRef(0).toString()); + CPPUNIT_ASSERT_EQUAL( + std::string("node(idx=1,crc=0xdef,docs=15/15,bytes=1500/1500," + "trusted=false,active=true)"), + entry->getNodeRef(1).toString()); + + CPPUNIT_ASSERT(op.ok()); +} + +void +BucketStateOperationTest::testDoNotDeactivateIfActivateFails() +{ + document::BucketId bid(16, 1); + insertBucketInfo(bid, 0, 0xabc, 10, 1100, false, true); + insertBucketInfo(bid, 1, 0xdef, 15, 1500, false, false); + + BucketAndNodes bucketAndNodes(bid, toVector<uint16_t>(0, 1)); + std::vector<uint16_t> active; + active.push_back(1); + SetBucketStateOperation op("storage", bucketAndNodes, active); + + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL((size_t)1, _sender.commands.size()); + { + std::shared_ptr<api::StorageCommand> msg = _sender.commands[0]; + CPPUNIT_ASSERT(msg->getType() == api::MessageType::SETBUCKETSTATE); + CPPUNIT_ASSERT_EQUAL( + api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 1).toString(), + msg->getAddress()->toString()); + + const api::SetBucketStateCommand& cmd( + dynamic_cast<const api::SetBucketStateCommand&>(*msg)); + CPPUNIT_ASSERT_EQUAL(bid, cmd.getBucketId()); + CPPUNIT_ASSERT_EQUAL(api::SetBucketStateCommand::ACTIVE, cmd.getState()); + + std::shared_ptr<api::StorageReply> reply(msg->makeReply().release()); + reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, "aaarg!")); + op.receive(_sender, reply); + } + + CPPUNIT_ASSERT_EQUAL((size_t)1, _sender.commands.size()); + + BucketDatabase::Entry entry = getBucket(bid); + CPPUNIT_ASSERT(entry.valid()); + CPPUNIT_ASSERT_EQUAL( + std::string("node(idx=0,crc=0xabc,docs=10/10,bytes=1100/1100," + "trusted=true,active=true)"), + entry->getNodeRef(0).toString()); + CPPUNIT_ASSERT_EQUAL( + std::string("node(idx=1,crc=0xdef,docs=15/15,bytes=1500/1500," + "trusted=false,active=false)"), + entry->getNodeRef(1).toString()); + + CPPUNIT_ASSERT(!op.ok()); +} + +void +BucketStateOperationTest::testBucketDbNotUpdatedOnFailure() +{ + document::BucketId bid(16, 1); + insertBucketInfo(bid, 0, 0xabc, 10, 1100, true, false); + + BucketAndNodes bucketAndNodes(bid, toVector<uint16_t>(0)); + std::vector<uint16_t> active; + active.push_back(0); + SetBucketStateOperation op("storage", bucketAndNodes, active); + + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL((size_t)1, _sender.commands.size()); + + std::shared_ptr<api::StorageCommand> msg = _sender.commands[0]; + CPPUNIT_ASSERT(msg->getType() == api::MessageType::SETBUCKETSTATE); + CPPUNIT_ASSERT_EQUAL( + api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 0).toString(), + msg->getAddress()->toString()); + + std::shared_ptr<api::StorageReply> reply(msg->makeReply().release()); + reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, "aaarg!")); + op.receive(_sender, reply); + + BucketDatabase::Entry entry = getBucket(bid); + CPPUNIT_ASSERT(entry.valid()); + // Should not be updated + CPPUNIT_ASSERT(!entry->getNodeRef(0).active()); + + CPPUNIT_ASSERT(!op.ok()); +} + +} // namespace distributor + +} // namespace storage diff --git a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp new file mode 100644 index 00000000000..65ccc65bdcf --- /dev/null +++ b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp @@ -0,0 +1,225 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/distributor/distributor_host_info_reporter.h> +#include <vespa/storage/distributor/latency_statistics_provider.h> +#include <vespa/storage/distributor/min_replica_provider.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/util/jsonstream.h> +#include <tests/common/hostreporter/util.h> + +namespace storage { +namespace distributor { + +using End = vespalib::JsonStream::End; +using File = vespalib::File; +using Object = vespalib::JsonStream::Object; + +class DistributorHostInfoReporterTest : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE(DistributorHostInfoReporterTest); + CPPUNIT_TEST(hostInfoWithPutLatenciesOnly); + CPPUNIT_TEST(hostInfoAllInfo); + CPPUNIT_TEST(generateExampleJson); + CPPUNIT_TEST(noReportGeneratedIfDisabled); + CPPUNIT_TEST_SUITE_END(); + + void hostInfoWithPutLatenciesOnly(); + void hostInfoAllInfo(); + void verifyReportedNodeLatencies( + const vespalib::Slime& root, + uint16_t node, + int64_t latencySum, + int64_t count); + void generateExampleJson(); + void noReportGeneratedIfDisabled(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(DistributorHostInfoReporterTest); + +using ms = std::chrono::milliseconds; + +namespace { + +OperationStats +makeOpStats(std::chrono::milliseconds totalLatency, uint64_t numRequests) +{ + OperationStats stats; + stats.totalLatency = totalLatency; + stats.numRequests = numRequests; + return stats; +} + +// My kingdom for GoogleMock! +struct MockedLatencyStatisticsProvider : LatencyStatisticsProvider +{ + NodeStatsSnapshot returnedSnapshot; + + NodeStatsSnapshot doGetLatencyStatistics() const { + return returnedSnapshot; + } +}; + +struct MockedMinReplicaProvider : MinReplicaProvider +{ + std::unordered_map<uint16_t, uint32_t> minReplica; + + std::unordered_map<uint16_t, uint32_t> getMinReplica() const override { + return minReplica; + } +}; + +const vespalib::slime::Inspector& +getNode(const vespalib::Slime& root, uint16_t nodeIndex) +{ + auto& storage_nodes = root.get()["distributor"]["storage-nodes"]; + const size_t n = storage_nodes.entries(); + for (size_t i = 0; i < n; ++i) { + if (storage_nodes[i]["node-index"].asLong() == nodeIndex) { + return storage_nodes[i]; + } + } + throw std::runtime_error("No node found with index " + + std::to_string(nodeIndex)); +} + +int +getMinReplica(const vespalib::Slime& root, uint16_t nodeIndex) +{ + return getNode(root, nodeIndex)["min-current-replication-factor"].asLong(); +} + +const vespalib::slime::Inspector& +getLatenciesForNode(const vespalib::Slime& root, uint16_t nodeIndex) +{ + return getNode(root, nodeIndex)["ops-latency"]; +} + +} // anon ns + +void +DistributorHostInfoReporterTest::verifyReportedNodeLatencies( + const vespalib::Slime& root, + uint16_t node, + int64_t latencySum, + int64_t count) +{ + auto& latencies = getLatenciesForNode(root, node); + CPPUNIT_ASSERT_EQUAL(latencySum, + latencies["put"]["latency-ms-sum"].asLong()); + CPPUNIT_ASSERT_EQUAL(count, latencies["put"]["count"].asLong()); +} + +void +DistributorHostInfoReporterTest::hostInfoWithPutLatenciesOnly() +{ + MockedLatencyStatisticsProvider latencyStatsProvider; + MockedMinReplicaProvider minReplicaProvider; + DistributorHostInfoReporter reporter(latencyStatsProvider, + minReplicaProvider); + + NodeStatsSnapshot snapshot; + snapshot.nodeToStats[0] = { makeOpStats(ms(10000), 3) }; + snapshot.nodeToStats[5] = { makeOpStats(ms(25000), 7) }; + + latencyStatsProvider.returnedSnapshot = snapshot; + + vespalib::Slime root; + util::reporterToSlime(reporter, root); + verifyReportedNodeLatencies(root, 0, 10000, 3); + verifyReportedNodeLatencies(root, 5, 25000, 7); +} + +void +DistributorHostInfoReporterTest::hostInfoAllInfo() +{ + MockedLatencyStatisticsProvider latencyStatsProvider; + MockedMinReplicaProvider minReplicaProvider; + DistributorHostInfoReporter reporter(latencyStatsProvider, + minReplicaProvider); + + NodeStatsSnapshot latencySnapshot; + latencySnapshot.nodeToStats[0] = { makeOpStats(ms(10000), 3) }; + latencySnapshot.nodeToStats[5] = { makeOpStats(ms(25000), 7) }; + latencyStatsProvider.returnedSnapshot = latencySnapshot; + + std::unordered_map<uint16_t, uint32_t> minReplica; + minReplica[0] = 2; + minReplica[5] = 9; + minReplicaProvider.minReplica = minReplica; + + vespalib::Slime root; + util::reporterToSlime(reporter, root); + verifyReportedNodeLatencies(root, 0, 10000, 3); + verifyReportedNodeLatencies(root, 5, 25000, 7); + + CPPUNIT_ASSERT_EQUAL(2, getMinReplica(root, 0)); + CPPUNIT_ASSERT_EQUAL(9, getMinReplica(root, 5)); +} + +void +DistributorHostInfoReporterTest::generateExampleJson() +{ + MockedLatencyStatisticsProvider latencyStatsProvider; + MockedMinReplicaProvider minReplicaProvider; + DistributorHostInfoReporter reporter(latencyStatsProvider, + minReplicaProvider); + + NodeStatsSnapshot snapshot; + snapshot.nodeToStats[0] = { makeOpStats(ms(10000), 3) }; + snapshot.nodeToStats[5] = { makeOpStats(ms(25000), 7) }; + latencyStatsProvider.returnedSnapshot = snapshot; + + std::unordered_map<uint16_t, uint32_t> minReplica; + minReplica[0] = 2; + minReplica[5] = 9; + minReplicaProvider.minReplica = minReplica; + + vespalib::asciistream json; + vespalib::JsonStream stream(json, true); + + stream << Object(); + reporter.report(stream); + stream << End(); + stream.finalize(); + + std::string jsonString = json.str(); + + std::string path = "../../../protocols/getnodestate/distributor.json"; + std::string goldenString = File::readAll(path); + + vespalib::slime::Memory goldenMemory(goldenString); + vespalib::Slime goldenSlime; + vespalib::slime::JsonFormat::decode(goldenMemory, goldenSlime); + + vespalib::slime::Memory jsonMemory(jsonString); + vespalib::Slime jsonSlime; + vespalib::slime::JsonFormat::decode(jsonMemory, jsonSlime); + + CPPUNIT_ASSERT_EQUAL(goldenSlime, jsonSlime); +} + +void +DistributorHostInfoReporterTest::noReportGeneratedIfDisabled() +{ + MockedLatencyStatisticsProvider latencyStatsProvider; + MockedMinReplicaProvider minReplicaProvider; + DistributorHostInfoReporter reporter(latencyStatsProvider, + minReplicaProvider); + reporter.enableReporting(false); + + NodeStatsSnapshot snapshot; + snapshot.nodeToStats[0] = { makeOpStats(ms(10000), 3) }; + snapshot.nodeToStats[5] = { makeOpStats(ms(25000), 7) }; + + latencyStatsProvider.returnedSnapshot = snapshot; + + vespalib::Slime root; + util::reporterToSlime(reporter, root); + CPPUNIT_ASSERT_EQUAL(size_t(0), root.get().children()); +} + +} // distributor +} // storage + diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp new file mode 100644 index 00000000000..b51c8dd3873 --- /dev/null +++ b/storage/src/tests/distributor/distributortest.cpp @@ -0,0 +1,691 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <iomanip> +#include <iostream> +#include <memory> +#include <boost/assign/std/vector.hpp> // for 'operator+=()' +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/bucketdb/distrbucketdb.h> +#include <vespa/storage/distributor/idealstatemetricsset.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storageapi/message/visitor.h> +#include <vespa/storageapi/message/removelocation.h> +#include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> +#include <tests/distributor/distributortestutil.h> +#include <vespa/storage/config/config-stor-distributormanager.h> +#include <tests/common/dummystoragelink.h> + +namespace storage { + +namespace distributor { + +class Distributor_Test : public CppUnit::TestFixture, + public DistributorTestUtil +{ + CPPUNIT_TEST_SUITE(Distributor_Test); + CPPUNIT_TEST(testOperationGeneration); + CPPUNIT_TEST(testOperationsGeneratedAndStartedWithoutDuplicates); + CPPUNIT_TEST(testRecoveryModeOnClusterStateChange); + CPPUNIT_TEST(testOperationsAreThrottled); + CPPUNIT_TEST_IGNORED(testRecoveryModeEntryResetsScanner); + CPPUNIT_TEST_IGNORED(testReprioritizeBucketOnMaintenanceReply); + CPPUNIT_TEST(testHandleUnknownMaintenanceReply); + CPPUNIT_TEST(testContainsTimeStatement); + CPPUNIT_TEST(testUpdateBucketDatabase); + CPPUNIT_TEST(testTickProcessesStatusRequests); + CPPUNIT_TEST(testMetricUpdateHookUpdatesPendingMaintenanceMetrics); + CPPUNIT_TEST(testPriorityConfigIsPropagatedToDistributorConfiguration); + CPPUNIT_TEST(testNoDbResurrectionForBucketNotOwnedInPendingState); + CPPUNIT_TEST(testAddedDbBucketsWithoutGcTimestampImplicitlyGetCurrentTime); + CPPUNIT_TEST(mergeStatsAreAccumulatedDuringDatabaseIteration); + CPPUNIT_TEST(statsGeneratedForPreemptedOperations); + CPPUNIT_TEST(hostInfoReporterConfigIsPropagatedToReporter); + CPPUNIT_TEST(replicaCountingModeIsConfiguredToTrustedByDefault); + CPPUNIT_TEST(replicaCountingModeConfigIsPropagatedToMetricUpdater); + CPPUNIT_TEST(bucketActivationIsEnabledByDefault); + CPPUNIT_TEST(bucketActivationConfigIsPropagatedToDistributorConfiguration); + CPPUNIT_TEST_SUITE_END(); + +protected: + void testOperationGeneration(); + void testOperationsGeneratedAndStartedWithoutDuplicates(); + void testRecoveryModeOnClusterStateChange(); + void testOperationsAreThrottled(); + void testRecoveryModeEntryResetsScanner(); + void testReprioritizeBucketOnMaintenanceReply(); + void testHandleUnknownMaintenanceReply(); + void testContainsTimeStatement(); + void testUpdateBucketDatabase(); + void testTickProcessesStatusRequests(); + void testMetricUpdateHookUpdatesPendingMaintenanceMetrics(); + void testPriorityConfigIsPropagatedToDistributorConfiguration(); + void testNoDbResurrectionForBucketNotOwnedInPendingState(); + void testAddedDbBucketsWithoutGcTimestampImplicitlyGetCurrentTime(); + void mergeStatsAreAccumulatedDuringDatabaseIteration(); + void statsGeneratedForPreemptedOperations(); + void hostInfoReporterConfigIsPropagatedToReporter(); + void replicaCountingModeIsConfiguredToTrustedByDefault(); + void replicaCountingModeConfigIsPropagatedToMetricUpdater(); + void bucketActivationIsEnabledByDefault(); + void bucketActivationConfigIsPropagatedToDistributorConfiguration(); + +public: + void setUp() { + createLinks(); + }; + + void tearDown() { + close(); + } + +private: + // Simple type aliases to make interfacing with certain utility functions + // easier. Note that this is only for readability and does not provide any + // added type safety. + using NodeCount = int; + using Redundancy = int; + + using ConfigBuilder = vespa::config::content::core::StorDistributormanagerConfigBuilder; + + void configureDistributor(const ConfigBuilder& config) { + getConfig().configure(config); + _distributor->enableNextConfig(); + } + + auto currentReplicaCountingMode() const noexcept { + return _distributor->_bucketDBMetricUpdater + .getMinimumReplicaCountingMode(); + } + + std::string testOp(api::StorageMessage* msg) + { + api::StorageMessage::SP msgPtr(msg); + _distributor->handleMessage(msgPtr); + + std::string tmp = _sender.getCommands(); + _sender.clear(); + return tmp; + } + + void tickDistributorNTimes(uint32_t n) { + for (uint32_t i = 0; i < n; ++i) { + tick(); + } + } + + typedef bool ResetTrusted; + + std::string updateBucketDB(const std::string& firstState, + const std::string& secondState, + bool resetTrusted = false) + { + std::vector<std::string> states(toVector<std::string>(firstState, secondState)); + + for (uint32_t i = 0; i < states.size(); ++i) { + std::vector<uint16_t> removedNodes; + std::vector<BucketCopy> changedNodes; + + vespalib::StringTokenizer tokenizer(states[i], ","); + for (uint32_t j = 0; j < tokenizer.size(); ++j) { + vespalib::StringTokenizer tokenizer2(tokenizer[j], ":"); + + bool trusted = false; + if (tokenizer2.size() > 2) { + trusted = true; + } + + uint16_t node = atoi(tokenizer2[0].c_str()); + if (tokenizer2[1] == "r") { + removedNodes.push_back(node); + } else { + uint32_t checksum = atoi(tokenizer2[1].c_str()); + changedNodes.push_back( + BucketCopy( + i + 1, + node, + api::BucketInfo( + checksum, + checksum / 2, + checksum / 4)).setTrusted(trusted)); + } + } + + getExternalOperationHandler().removeNodesFromDB(document::BucketId(16, 1), removedNodes); + + uint32_t flags(DatabaseUpdate::CREATE_IF_NONEXISTING + | (resetTrusted ? DatabaseUpdate::RESET_TRUSTED : 0)); + + getExternalOperationHandler().updateBucketDatabase(document::BucketId(16, 1), + changedNodes, + flags); + } + + std::string retVal = dumpBucket(document::BucketId(16, 1)); + getBucketDatabase().clear(); + return retVal; + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(Distributor_Test); + +void +Distributor_Test::testOperationGeneration() +{ + setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); + + document::BucketId bid; + addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t"); + + CPPUNIT_ASSERT_EQUAL(std::string("Remove"), + testOp(new api::RemoveCommand( + bid, + document::DocumentId("userdoc:m:1:foo"), + api::Timestamp(1234)))); + + api::CreateVisitorCommand* cmd = new api::CreateVisitorCommand("foo", "bar", ""); + cmd->addBucketToBeVisited(document::BucketId(16, 1)); + cmd->addBucketToBeVisited(document::BucketId()); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create"), testOp(cmd)); +} + +void +Distributor_Test::testOperationsGeneratedAndStartedWithoutDuplicates() +{ + setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); + + for (uint32_t i = 0; i < 6; ++i) { + addNodesToBucketDB(document::BucketId(16, i), "0=1"); + } + + tickDistributorNTimes(20); + + CPPUNIT_ASSERT(!tick()); + + CPPUNIT_ASSERT_EQUAL(6, (int)_sender.commands.size()); +} + +void +Distributor_Test::testRecoveryModeOnClusterStateChange() +{ + setupDistributor(Redundancy(1), NodeCount(2), + "storage:1 .0.s:d distributor:1"); + _distributor->enableClusterState( + lib::ClusterState("storage:1 distributor:1")); + + CPPUNIT_ASSERT(_distributor->isInRecoveryMode()); + for (uint32_t i = 0; i < 3; ++i) { + addNodesToBucketDB(document::BucketId(16, i), "0=1"); + } + for (int i = 0; i < 3; ++i) { + tick(); + CPPUNIT_ASSERT(_distributor->isInRecoveryMode()); + } + tick(); + CPPUNIT_ASSERT(!_distributor->isInRecoveryMode()); + + _distributor->enableClusterState(lib::ClusterState("storage:2 distributor:1")); + CPPUNIT_ASSERT(_distributor->isInRecoveryMode()); +} + +void +Distributor_Test::testOperationsAreThrottled() +{ + setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); + getConfig().setMinPendingMaintenanceOps(1); + getConfig().setMaxPendingMaintenanceOps(1); + + for (uint32_t i = 0; i < 6; ++i) { + addNodesToBucketDB(document::BucketId(16, i), "0=1"); + } + tickDistributorNTimes(20); + CPPUNIT_ASSERT_EQUAL(1, (int)_sender.commands.size()); +} + +void +Distributor_Test::testRecoveryModeEntryResetsScanner() +{ + CPPUNIT_FAIL("TODO: refactor so this can be mocked and tested easily"); +} + +void +Distributor_Test::testReprioritizeBucketOnMaintenanceReply() +{ + CPPUNIT_FAIL("TODO: refactor so this can be mocked and tested easily"); +} + +void +Distributor_Test::testHandleUnknownMaintenanceReply() +{ + setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); + + { + api::SplitBucketCommand::SP cmd( + new api::SplitBucketCommand(document::BucketId(16, 1234))); + api::SplitBucketReply::SP reply(new api::SplitBucketReply(*cmd)); + + CPPUNIT_ASSERT(_distributor->handleReply(reply)); + } + + { + // RemoveLocationReply must be treated as a maintenance reply since + // it's what GC is currently built around. + auto cmd = std::make_shared<api::RemoveLocationCommand>( + "false", document::BucketId(30, 1234)); + auto reply = std::shared_ptr<api::StorageReply>(cmd->makeReply()); + CPPUNIT_ASSERT(_distributor->handleReply(reply)); + } +} + +void +Distributor_Test::testContainsTimeStatement() +{ + setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); + + CPPUNIT_ASSERT_EQUAL(false, getConfig().containsTimeStatement("")); + CPPUNIT_ASSERT_EQUAL(false, getConfig().containsTimeStatement("testdoctype1")); + CPPUNIT_ASSERT_EQUAL(false, getConfig().containsTimeStatement("testdoctype1.headerfield > 42")); + CPPUNIT_ASSERT_EQUAL(true, getConfig().containsTimeStatement("testdoctype1.headerfield > now()")); + CPPUNIT_ASSERT_EQUAL(true, getConfig().containsTimeStatement("testdoctype1.headerfield > now() - 3600")); + CPPUNIT_ASSERT_EQUAL(true, getConfig().containsTimeStatement("testdoctype1.headerfield == now() - 3600")); +} + +void +Distributor_Test::testUpdateBucketDatabase() +{ + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:3")); + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001) : " + "node(idx=0,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false), " + "node(idx=1,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false)" + ), + updateBucketDB("0:456,1:456,2:789", "2:r")); + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001) : " + "node(idx=0,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false), " + "node(idx=2,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false), " + "node(idx=1,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false)" + ), + updateBucketDB("0:456,1:456", "2:456")); + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001) : " + "node(idx=0,crc=0x315,docs=394/394,bytes=197/197,trusted=false,active=false), " + "node(idx=2,crc=0x14d,docs=166/166,bytes=83/83,trusted=false,active=false), " + "node(idx=1,crc=0x34a,docs=421/421,bytes=210/210,trusted=false,active=false)" + ), + updateBucketDB("0:456:t,1:456:t,2:123", "0:789,1:842,2:333")); + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001) : " + "node(idx=0,crc=0x315,docs=394/394,bytes=197/197,trusted=true,active=false), " + "node(idx=2,crc=0x14d,docs=166/166,bytes=83/83,trusted=false,active=false), " + "node(idx=1,crc=0x315,docs=394/394,bytes=197/197,trusted=true,active=false)" + ), + updateBucketDB("0:456:t,1:456:t,2:123", "0:789,1:789,2:333")); + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001) : " + "node(idx=2,crc=0x14d,docs=166/166,bytes=83/83,trusted=true,active=false)"), + updateBucketDB("0:456:t,1:456:t", "0:r,1:r,2:333")); + + // Copies are in sync so should still be trusted even if explicitly reset. + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001) : " + "node(idx=0,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false), " + "node(idx=2,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false), " + "node(idx=1,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false)" + ), + updateBucketDB("0:456,1:456", "2:456", ResetTrusted(true))); + + // When resetting, first inserted copy should not end up as implicitly trusted. + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001) : " + "node(idx=0,crc=0x1c8,docs=228/228,bytes=114/114,trusted=false,active=false), " + "node(idx=2,crc=0x14d,docs=166/166,bytes=83/83,trusted=false,active=false)" + ), + updateBucketDB("0:456", + "2:333", + ResetTrusted(true))); +} + +namespace { + +using namespace framework::defaultimplementation; + +class StatusRequestThread : public framework::Runnable +{ + StatusReporterDelegate& _reporter; + std::string _result; +public: + StatusRequestThread(StatusReporterDelegate& reporter) + : _reporter(reporter) + {} + void run(framework::ThreadHandle&) { + framework::HttpUrlPath path("/distributor?page=buckets"); + std::ostringstream stream; + _reporter.reportStatus(stream, path); + _result = stream.str(); + } + + std::string getResult() const { + return _result; + } +}; + +} + +void +Distributor_Test::testTickProcessesStatusRequests() +{ + setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); + + addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t"); + + // Must go via delegate since reportStatus is now just a rendering + // function and not a request enqueuer (see Distributor::handleStatusRequest). + StatusRequestThread thread(_distributor->_distributorStatusDelegate); + FakeClock clock; + ThreadPoolImpl pool(clock); + + uint64_t tickWaitMs = 5; + uint64_t tickMaxProcessTime = 5000; + int ticksBeforeWait = 1; + framework::Thread::UP tp(pool.startThread( + thread, "statustest", tickWaitMs, tickMaxProcessTime, ticksBeforeWait)); + + while (true) { + FastOS_Thread::Sleep(1); + framework::TickingLockGuard guard( + _distributor->_threadPool.freezeCriticalTicks()); + if (!_distributor->_statusToDo.empty()) break; + + } + CPPUNIT_ASSERT(tick()); + + tp->interruptAndJoin(0); + + CPPUNIT_ASSERT_CONTAIN("BucketId(0x4000000000000001)", thread.getResult()); +} + +void +Distributor_Test::testMetricUpdateHookUpdatesPendingMaintenanceMetrics() +{ + setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); + // To ensure we count all operations, not just those fitting within the + // pending window. + getConfig().setMinPendingMaintenanceOps(1); + getConfig().setMaxPendingMaintenanceOps(1); + + // 1 bucket must be merged, 1 must be split, 1 should be activated. + addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t/a,1=2/2/2"); + addNodesToBucketDB(document::BucketId(16, 2), + "0=100/10000000/200000/t/a,1=100/10000000/200000/t"); + addNodesToBucketDB(document::BucketId(16, 3), + "0=200/300/400/t,1=200/300/400/t"); + + // Go many full scanner rounds to check that metrics are set, not + // added to existing. + tickDistributorNTimes(50); + + // By this point, no hook has been called so the metrics have not been + // set. + typedef MaintenanceOperation MO; + { + const IdealStateMetricSet& metrics(getIdealStateManager().getMetrics()); + CPPUNIT_ASSERT_EQUAL(int64_t(0), + metrics.operations[MO::MERGE_BUCKET] + ->pending.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(0), metrics.operations[MO::SPLIT_BUCKET] + ->pending.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(0), + metrics.operations[MO::SET_BUCKET_STATE] + ->pending.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(0), metrics.operations[MO::DELETE_BUCKET] + ->pending.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(0), metrics.operations[MO::JOIN_BUCKET] + ->pending.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(0), + metrics.operations[MO::GARBAGE_COLLECTION] + ->pending.getLast()); + } + + // Force trigger update hook + vespalib::Monitor l; + _distributor->_metricUpdateHook.updateMetrics(metrics::MetricLockGuard(l)); + // Metrics should now be updated to the last complete working state + { + const IdealStateMetricSet& metrics(getIdealStateManager().getMetrics()); + CPPUNIT_ASSERT_EQUAL(int64_t(1), + metrics.operations[MO::MERGE_BUCKET] + ->pending.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(1), metrics.operations[MO::SPLIT_BUCKET] + ->pending.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(1), + metrics.operations[MO::SET_BUCKET_STATE] + ->pending.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(0), metrics.operations[MO::DELETE_BUCKET] + ->pending.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(0), metrics.operations[MO::JOIN_BUCKET] + ->pending.getLast()); + CPPUNIT_ASSERT_EQUAL(int64_t(0), + metrics.operations[MO::GARBAGE_COLLECTION] + ->pending.getLast()); + } +} + +void +Distributor_Test::testPriorityConfigIsPropagatedToDistributorConfiguration() +{ + using namespace vespa::config::content::core; + using ConfigBuilder = StorDistributormanagerConfigBuilder; + + setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); + + ConfigBuilder builder; + builder.priorityMergeMoveToIdealNode = 1; + builder.priorityMergeOutOfSyncCopies = 2; + builder.priorityMergeTooFewCopies = 3; + builder.priorityActivateNoExistingActive = 4; + builder.priorityActivateWithExistingActive = 5; + builder.priorityDeleteBucketCopy = 6; + builder.priorityJoinBuckets = 7; + builder.prioritySplitDistributionBits = 8; + builder.prioritySplitLargeBucket = 9; + builder.prioritySplitInconsistentBucket = 10; + builder.priorityGarbageCollection = 11; + + getConfig().configure(builder); + + const DistributorConfiguration::MaintenancePriorities& mp( + getConfig().getMaintenancePriorities()); + CPPUNIT_ASSERT_EQUAL(1, static_cast<int>(mp.mergeMoveToIdealNode)); + CPPUNIT_ASSERT_EQUAL(2, static_cast<int>(mp.mergeOutOfSyncCopies)); + CPPUNIT_ASSERT_EQUAL(3, static_cast<int>(mp.mergeTooFewCopies)); + CPPUNIT_ASSERT_EQUAL(4, static_cast<int>(mp.activateNoExistingActive)); + CPPUNIT_ASSERT_EQUAL(5, static_cast<int>(mp.activateWithExistingActive)); + CPPUNIT_ASSERT_EQUAL(6, static_cast<int>(mp.deleteBucketCopy)); + CPPUNIT_ASSERT_EQUAL(7, static_cast<int>(mp.joinBuckets)); + CPPUNIT_ASSERT_EQUAL(8, static_cast<int>(mp.splitDistributionBits)); + CPPUNIT_ASSERT_EQUAL(9, static_cast<int>(mp.splitLargeBucket)); + CPPUNIT_ASSERT_EQUAL(10, static_cast<int>(mp.splitInconsistentBucket)); + CPPUNIT_ASSERT_EQUAL(11, static_cast<int>(mp.garbageCollection)); +} + +void +Distributor_Test::testNoDbResurrectionForBucketNotOwnedInPendingState() +{ + setupDistributor(Redundancy(1), NodeCount(10), "storage:2 distributor:2"); + lib::ClusterState newState("storage:10 distributor:10"); + auto stateCmd = std::make_shared<api::SetSystemStateCommand>(newState); + // Force newState into being the pending state. According to the initial + // state we own the bucket, but according to the pending state, we do + // not. This must be handled correctly by the database update code. + getBucketDBUpdater().onSetSystemState(stateCmd); + + document::BucketId nonOwnedBucket(16, 3); + CPPUNIT_ASSERT(!getBucketDBUpdater() + .checkOwnershipInPendingState(nonOwnedBucket).isOwned()); + CPPUNIT_ASSERT(!getBucketDBUpdater().getDistributorComponent() + .checkOwnershipInPendingAndCurrentState(nonOwnedBucket) + .isOwned()); + + std::vector<BucketCopy> copies; + copies.emplace_back(1234, 0, api::BucketInfo(0x567, 1, 2)); + getExternalOperationHandler().updateBucketDatabase(nonOwnedBucket, copies, + DatabaseUpdate::CREATE_IF_NONEXISTING); + + CPPUNIT_ASSERT_EQUAL(std::string("NONEXISTING"), + dumpBucket(nonOwnedBucket)); +} + +void +Distributor_Test::testAddedDbBucketsWithoutGcTimestampImplicitlyGetCurrentTime() +{ + setupDistributor(Redundancy(1), NodeCount(10), "storage:2 distributor:2"); + getClock().setAbsoluteTimeInSeconds(101234); + document::BucketId bucket(16, 7654); + + std::vector<BucketCopy> copies; + copies.emplace_back(1234, 0, api::BucketInfo(0x567, 1, 2)); + getExternalOperationHandler().updateBucketDatabase(bucket, copies, + DatabaseUpdate::CREATE_IF_NONEXISTING); + BucketDatabase::Entry e(getBucket(bucket)); + CPPUNIT_ASSERT_EQUAL(uint32_t(101234), e->getLastGarbageCollectionTime()); +} + + +void +Distributor_Test::mergeStatsAreAccumulatedDuringDatabaseIteration() +{ + setupDistributor(Redundancy(2), NodeCount(3), "storage:3 distributor:1"); + // Copies out of sync. Not possible for distributor to _reliably_ tell + // which direction(s) data will flow, so for simplicity assume that we + // must sync both copies. + // Note that we mark certain copies as active to prevent the bucketstate + // checker from pre-empting the merges. + // -> syncing[0] += 1, syncing[2] += 1 + addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t/a,2=2/2/2"); + // Must add missing node 2 for bucket + // -> copyingOut[0] += 1, copyingIn[2] += 1 + addNodesToBucketDB(document::BucketId(16, 2), "0=1/1/1/t/a"); + // Moving from non-ideal node 1 to ideal node 2. Both nodes 0 and 1 will + // be involved in this merge, but only node 1 will be tagged as source only + // (i.e. to be deleted after the merge is completed). + // -> copyingOut[0] += 1, movingOut[1] += 1, copyingIn[2] += 1 + addNodesToBucketDB(document::BucketId(16, 3), "0=2/2/2/t/a,1=2/2/2/t"); + + // Go many full scanner rounds to check that stats are set, not + // added to existing. + tickDistributorNTimes(50); + + const auto& stats(_distributor->_maintenanceStats); + { + NodeMaintenanceStats wanted; + wanted.syncing = 1; + wanted.copyingOut = 2; + CPPUNIT_ASSERT_EQUAL(wanted, stats.perNodeStats.forNode(0)); + } + { + NodeMaintenanceStats wanted; + wanted.movingOut = 1; + CPPUNIT_ASSERT_EQUAL(wanted, stats.perNodeStats.forNode(1)); + } + { + NodeMaintenanceStats wanted; + wanted.syncing = 1; + wanted.copyingIn = 2; + CPPUNIT_ASSERT_EQUAL(wanted, stats.perNodeStats.forNode(2)); + } +} + +/** + * Since maintenance operations are prioritized differently, activation + * pre-empts merging and other ops. If this also implies pre-empting running + * their state checkers at all, we won't get any statistics from any other + * operations for the bucket. + */ +void +Distributor_Test::statsGeneratedForPreemptedOperations() +{ + setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); + // For this test it suffices to have a single bucket with multiple aspects + // wrong about it. In this case, let a bucket be both out of sync _and_ + // missing an active copy. This _should_ give a statistic with both nodes 0 + // and 1 requiring a sync. If instead merge stats generation is preempted + // by activation, we'll see no merge stats at all. + addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1,1=2/2/2"); + tickDistributorNTimes(50); + const auto& stats(_distributor->_maintenanceStats); + { + NodeMaintenanceStats wanted; + wanted.syncing = 1; + CPPUNIT_ASSERT_EQUAL(wanted, stats.perNodeStats.forNode(0)); + } + { + NodeMaintenanceStats wanted; + wanted.syncing = 1; + CPPUNIT_ASSERT_EQUAL(wanted, stats.perNodeStats.forNode(1)); + } +} + +void +Distributor_Test::hostInfoReporterConfigIsPropagatedToReporter() +{ + setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); + + // Default is enabled=true. + CPPUNIT_ASSERT(_distributor->_hostInfoReporter.isReportingEnabled()); + + ConfigBuilder builder; + builder.enableHostInfoReporting = false; + configureDistributor(builder); + + CPPUNIT_ASSERT(!_distributor->_hostInfoReporter.isReportingEnabled()); +} + +void +Distributor_Test::replicaCountingModeIsConfiguredToTrustedByDefault() +{ + setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); + CPPUNIT_ASSERT_EQUAL(ConfigBuilder::TRUSTED, currentReplicaCountingMode()); +} + +void +Distributor_Test::replicaCountingModeConfigIsPropagatedToMetricUpdater() +{ + setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); + ConfigBuilder builder; + builder.minimumReplicaCountingMode = ConfigBuilder::ANY; + configureDistributor(builder); + CPPUNIT_ASSERT_EQUAL(ConfigBuilder::ANY, currentReplicaCountingMode()); +} + +void +Distributor_Test::bucketActivationIsEnabledByDefault() +{ + setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); + CPPUNIT_ASSERT(getConfig().isBucketActivationDisabled() == false); +} + +void +Distributor_Test::bucketActivationConfigIsPropagatedToDistributorConfiguration() +{ + using namespace vespa::config::content::core; + using ConfigBuilder = StorDistributormanagerConfigBuilder; + + setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); + + ConfigBuilder builder; + builder.disableBucketActivation = true; + getConfig().configure(builder); + + CPPUNIT_ASSERT(getConfig().isBucketActivationDisabled()); +} + +} + +} diff --git a/storage/src/tests/distributor/distributortestutil.cpp b/storage/src/tests/distributor/distributortestutil.cpp new file mode 100644 index 00000000000..c2d878a253d --- /dev/null +++ b/storage/src/tests/distributor/distributortestutil.cpp @@ -0,0 +1,298 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <ctype.h> +#include <vespa/document/base/testdocman.h> +#include <vespa/storageframework/defaultimplementation/memory/nomemorymanager.h> +#include <tests/distributor/distributortestutil.h> +#include <vespa/vespalib/text/stringtokenizer.h> + +namespace storage { + +namespace distributor { + +void +DistributorTestUtil::createLinks() +{ + _node.reset(new TestDistributorApp(_config.getConfigId())); + _threadPool = framework::TickingThreadPool::createDefault("distributor"); + _distributor.reset(new Distributor( + _node->getComponentRegister(), + *_threadPool, + *this, + true, + _hostInfo, + &_messageSender)); + _component.reset(new storage::DistributorComponent(_node->getComponentRegister(), "distrtestutil")); +}; + +void +DistributorTestUtil::setupDistributor(int redundancy, + int nodeCount, + const std::string& systemState, + uint32_t earlyReturn, + bool requirePrimaryToBeWritten) +{ + lib::Distribution::DistributionConfig config( + lib::Distribution::getDefaultDistributionConfig( + redundancy, nodeCount)); + config.redundancy = redundancy; + config.initialRedundancy = earlyReturn; + config.ensurePrimaryPersisted = requirePrimaryToBeWritten; + lib::Distribution* distribution = new lib::Distribution(config); + _node->getComponentRegister().setDistribution( + lib::Distribution::SP(distribution)); + _distributor->enableClusterState(lib::ClusterState(systemState)); +} + +void +DistributorTestUtil::setRedundancy(uint32_t redundancy) +{ + _node->getComponentRegister().setDistribution(lib::Distribution::SP( + new lib::Distribution( + lib::Distribution::getDefaultDistributionConfig( + redundancy, 100)))); + _distributor->storageDistributionChanged(); +} + +void +DistributorTestUtil::setTypeRepo(const document::DocumentTypeRepo::SP &repo) +{ + _node->getComponentRegister().setDocumentTypeRepo(repo); +} + +void +DistributorTestUtil::close() +{ + _component.reset(0); + if (_distributor.get()) { + _distributor->onClose(); + } + _sender.clear(); + _node.reset(0); + _config = getStandardConfig(false); +} + +namespace { + std::string dumpVector(const std::vector<uint16_t>& vec) { + std::ostringstream ost; + for (uint32_t i = 0; i < vec.size(); ++i) { + if (i != 0) { + ost << ","; + } + ost << vec[i]; + } + return ost.str(); + } +} + +std::string +DistributorTestUtil::getNodes(document::BucketId id) +{ + BucketDatabase::Entry entry = getBucket(id); + + if (!entry.valid()) { + return id.toString(); + } else { + std::vector<uint16_t> nodes = entry->getNodes(); + std::sort(nodes.begin(), nodes.end()); + + std::ostringstream ost; + ost << id << ": " << dumpVector(nodes); + return ost.str(); + } +} + +std::string +DistributorTestUtil::getIdealStr(document::BucketId id, const lib::ClusterState& state) +{ + if (!getExternalOperationHandler().ownsBucketInState(state, id)) { + return id.toString(); + } + + std::vector<uint16_t> nodes; + _component->getDistribution()->getIdealNodes(lib::NodeType::STORAGE, + state, + id, + nodes); + std::sort(nodes.begin(), nodes.end()); + std::ostringstream ost; + ost << id << ": " << dumpVector(nodes); + return ost.str(); +} + +void +DistributorTestUtil::addIdealNodes(const lib::ClusterState& state, + const document::BucketId& id) +{ + BucketDatabase::Entry entry = getBucket(id); + + if (!entry.valid()) { + entry = BucketDatabase::Entry(id); + } + + std::vector<uint16_t> res; + assert(_component.get()); + _component->getDistribution()->getIdealNodes(lib::NodeType::STORAGE, + state, + id, + res); + + for (uint32_t i = 0; i < res.size(); ++i) { + if (state.getNodeState(lib::Node(lib::NodeType::STORAGE, res[i])).getState() != + lib::State::MAINTENANCE) + { + entry->addNode(BucketCopy(0, res[i], api::BucketInfo(1,1,1)), + toVector<uint16_t>(0)); + } + } + + getBucketDatabase().update(entry); +} + +void +DistributorTestUtil::addNodesToBucketDB(const document::BucketId& id, + const std::string& nodeStr) +{ + BucketDatabase::Entry entry = getBucket(id); + + if (!entry.valid()) { + entry = BucketDatabase::Entry(id); + } + + entry->clear(); + + vespalib::StringTokenizer tokenizer(nodeStr, ","); + for (uint32_t i = 0; i < tokenizer.size(); ++i) { + vespalib::StringTokenizer tok2(tokenizer[i], "="); + vespalib::StringTokenizer tok3(tok2[1], "/"); + + api::BucketInfo info(atoi(tok3[0].c_str()), + atoi(tok3.size() > 1 ? tok3[1].c_str() : tok3[0].c_str()), + atoi(tok3.size() > 2 ? tok3[2].c_str() : tok3[0].c_str())); + + size_t flagsIdx = 3; + + // Meta info override? For simplicity, require both meta count and size + if (tok3.size() > 4 && (!tok3[3].empty() && isdigit(tok3[3][0]))) { + info.setMetaCount(atoi(tok3[3].c_str())); + info.setUsedFileSize(atoi(tok3[4].c_str())); + flagsIdx = 5; + } + + if ((tok3.size() > flagsIdx + 1) && tok3[flagsIdx + 1] == "a") { + info.setActive(); + } else { + info.setActive(false); + } + if ((tok3.size() > flagsIdx + 2) && tok3[flagsIdx + 2] == "r") { + info.setReady(); + } else { + info.setReady(false); + } + + uint16_t idx = atoi(tok2[0].c_str()); + BucketCopy node( + 0, + idx, + info); + + // Allow user to manually override trusted and active. + if (tok3.size() > flagsIdx && tok3[flagsIdx] == "t") { + node.setTrusted(); + } + + entry->addNodeManual(node); + } + + getBucketDatabase().update(entry); +} + +void +DistributorTestUtil::removeFromBucketDB(const document::BucketId& id) +{ + getBucketDatabase().remove(id); +} + +void +DistributorTestUtil::addIdealNodes(const document::BucketId& id) +{ + addIdealNodes(getExternalOperationHandler().getClusterState(), id); +} + +void +DistributorTestUtil::insertBucketInfo(document::BucketId id, + uint16_t node, + uint32_t checksum, + uint32_t count, + uint32_t size, + bool trusted, + bool active) +{ + api::BucketInfo info(checksum, count, size); + insertBucketInfo(id, node, info, trusted, active); +} + +void +DistributorTestUtil::insertBucketInfo(document::BucketId id, + uint16_t node, + const api::BucketInfo& info, + bool trusted, + bool active) +{ + BucketDatabase::Entry entry = getBucketDatabase().get(id); + if (!entry.valid()) { + entry = BucketDatabase::Entry(id, BucketInfo()); + } + + api::BucketInfo info2(info); + if (active) { + info2.setActive(); + } + BucketCopy copy(getExternalOperationHandler().getUniqueTimestamp(), node, info2); + + entry->addNode(copy.setTrusted(trusted), toVector<uint16_t>(0)); + + getBucketDatabase().update(entry); +} + +std::string +DistributorTestUtil::dumpBucket(const document::BucketId& bid) +{ + return getBucketDatabase().get(bid).toString(); +} + +void +DistributorTestUtil::sendReply(Operation& op, + int idx, + api::ReturnCode::Result result) +{ + if (idx == -1) { + idx = _sender.commands.size() - 1; + } + assert(idx >= 0 && idx < static_cast<int>(_sender.commands.size())); + + std::shared_ptr<api::StorageCommand> cmd = _sender.commands[idx]; + api::StorageReply::SP reply(cmd->makeReply().release()); + reply->setResult(result); + op.receive(_sender, reply); +} + +BucketDatabase::Entry +DistributorTestUtil::getBucket(const document::BucketId& bId) const +{ + return _distributor->getBucketDatabase().get(bId); +} + +void +DistributorTestUtil::disableBucketActivationInConfig(bool disable) +{ + vespa::config::content::core::StorDistributormanagerConfigBuilder config; + config.disableBucketActivation = disable; + getConfig().configure(config); +} + +} + +} + + diff --git a/storage/src/tests/distributor/distributortestutil.h b/storage/src/tests/distributor/distributortestutil.h new file mode 100644 index 00000000000..43b56859d0d --- /dev/null +++ b/storage/src/tests/distributor/distributortestutil.h @@ -0,0 +1,200 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <tests/common/dummystoragelink.h> +#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> +#include <vespa/storage/common/hostreporter/hostinfo.h> +#include <vespa/storage/distributor/distributor.h> +#include <vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h> +#include <vespa/storage/storageutil/utils.h> +#include <tests/common/teststorageapp.h> +#include <tests/distributor/messagesenderstub.h> +#include <vespa/storageapi/message/state.h> +#include <tests/common/testhelper.h> + +namespace storage { + +namespace distributor { + +class DistributorTestUtil : private DoneInitializeHandler +{ +public: + DistributorTestUtil() + : _messageSender(_sender, _senderDown) + { + _config = getStandardConfig(false); + } + virtual ~DistributorTestUtil() {}; + + /** + * Sets up the storage link chain. + */ + void createLinks(); + void setTypeRepo(const document::DocumentTypeRepo::SP &repo); + + void close(); + + /** + * Returns a string with the nodes currently stored in the bucket + * database for the given bucket. + */ + std::string getNodes(document::BucketId id); + + /** + * Returns a string with the ideal state nodes for the given bucket. + */ + std::string getIdealStr(document::BucketId id, const lib::ClusterState& state); + + /** + * Adds the ideal nodes for the given bucket and the given cluster state + * to the bucket database. + */ + void addIdealNodes(const lib::ClusterState& state, const document::BucketId& id); + + /** + * Adds all the ideal nodes for the given bucket to the bucket database. + */ + void addIdealNodes(const document::BucketId& id); + + /** + * Parses the given string to a set of node => bucket info data, + * and inserts them as nodes in the given bucket. + * Format: + * "node1=checksum/docs/size,node2=checksum/docs/size" + */ + void addNodesToBucketDB(const document::BucketId& id, const std::string& nodeStr); + + /** + * Removes the given bucket from the bucket database. + */ + void removeFromBucketDB(const document::BucketId& id); + + /** + * Inserts the given bucket information for the given bucket and node in + * the bucket database. + */ + void insertBucketInfo(document::BucketId id, + uint16_t node, + uint32_t checksum, + uint32_t count, + uint32_t size, + bool trusted = false, + bool active = false); + + /** + * Inserts the given bucket information for the given bucket and node in + * the bucket database. + */ + void insertBucketInfo(document::BucketId id, + uint16_t node, + const api::BucketInfo& info, + bool trusted = false, + bool active = false); + + std::string dumpBucket(const document::BucketId& bucket); + + /** + * Replies to message idx sent upwards with the given result code. + * If idx = -1, replies to the last command received upwards. + */ + void sendReply(Operation& op, + int idx = -1, + api::ReturnCode::Result result = api::ReturnCode::OK); + + BucketDBUpdater& getBucketDBUpdater() { + return _distributor->_bucketDBUpdater; + } + IdealStateManager& getIdealStateManager() { + return _distributor->_idealStateManager; + } + ExternalOperationHandler& getExternalOperationHandler() { + return _distributor->_externalOperationHandler; + } + + Distributor& getDistributor() { + return *_distributor; + } + + bool tick() { + framework::ThreadWaitInfo res( + framework::ThreadWaitInfo::NO_MORE_CRITICAL_WORK_KNOWN); + { + framework::TickingLockGuard lock( + _distributor->_threadPool.freezeCriticalTicks()); + res.merge(_distributor->doCriticalTick(0)); + } + res.merge(_distributor->doNonCriticalTick(0)); + return !res.waitWanted(); + } + + DistributorConfiguration& getConfig() { + return const_cast<DistributorConfiguration&>(_distributor->getConfig()); + } + + vdstestlib::DirConfig& getDirConfig() { + return _config; + } + + BucketDatabase& getBucketDatabase() { return _distributor->getBucketDatabase(); } + + framework::defaultimplementation::FakeClock& getClock() { return _node->getClock(); } + DistributorComponentRegister& getComponentRegister() { return _node->getComponentRegister(); } + DistributorComponentRegisterImpl& getComponentRegisterImpl() { return _node->getComponentRegister(); } + + StorageComponent& getComponent() { + if (_component.get() == 0) { + _component.reset(new storage::DistributorComponent( + _node->getComponentRegister(), "distributor_test_utils")); + } + return *_component; + } + + void setupDistributor(int redundancy, + int nodeCount, + const std::string& systemState, + uint32_t earlyReturn = false, + bool requirePrimaryToBeWritten = true); + + void setRedundancy(uint32_t redundancy); + + virtual void notifyDoneInitializing() {} + + // Must implement this for storage server interface for now + virtual api::Timestamp getUniqueTimestamp() { + return _component->getUniqueTimestamp(); + } + + void disableBucketActivationInConfig(bool disable); + + BucketDatabase::Entry getBucket(const document::BucketId& bId) const; + +protected: + vdstestlib::DirConfig _config; + std::unique_ptr<TestDistributorApp> _node; + framework::TickingThreadPool::UP _threadPool; + std::unique_ptr<Distributor> _distributor; + std::unique_ptr<storage::DistributorComponent> _component; + MessageSenderStub _sender; + MessageSenderStub _senderDown; + HostInfo _hostInfo; + + struct MessageSenderImpl : public ChainedMessageSender { + MessageSenderStub& _sender; + MessageSenderStub& _senderDown; + MessageSenderImpl(MessageSenderStub& up, MessageSenderStub& down) + : _sender(up), _senderDown(down) {} + + void sendUp(const std::shared_ptr<api::StorageMessage>& msg) { + _sender.send(msg); + } + void sendDown(const std::shared_ptr<api::StorageMessage>& msg) { + _senderDown.send(msg); + } + }; + MessageSenderImpl _messageSender; +}; + +} + +} + diff --git a/storage/src/tests/distributor/externaloperationhandlertest.cpp b/storage/src/tests/distributor/externaloperationhandlertest.cpp new file mode 100644 index 00000000000..ce8149b4bac --- /dev/null +++ b/storage/src/tests/distributor/externaloperationhandlertest.cpp @@ -0,0 +1,176 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> + +#include <tests/distributor/distributortestutil.h> +#include <vespa/storage/distributor/externaloperationhandler.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/vdstestlib/cppunit/macros.h> + +namespace storage { +namespace distributor { + +class ExternalOperationHandlerTest : public CppUnit::TestFixture, + public DistributorTestUtil +{ + CPPUNIT_TEST_SUITE(ExternalOperationHandlerTest); + CPPUNIT_TEST(testBucketSplitMask); + CPPUNIT_TEST(testOperationRejectedOnWrongDistribution); + CPPUNIT_TEST(testOperationRejectedOnPendingWrongDistribution); + CPPUNIT_TEST_SUITE_END(); + + document::BucketId findNonOwnedUserBucketInState(vespalib::stringref state); + document::BucketId findOwned1stNotOwned2ndInStates( + vespalib::stringref state1, + vespalib::stringref state2); + + std::shared_ptr<api::StorageMessage> makeGetCommandForUser(uint64_t id); + +protected: + void testBucketSplitMask(); + void testOperationRejectedOnWrongDistribution(); + void testOperationRejectedOnPendingWrongDistribution(); + +public: + void tearDown() { + close(); + } + +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(ExternalOperationHandlerTest); + +void +ExternalOperationHandlerTest::testBucketSplitMask() +{ + { + createLinks(); + getDirConfig().getConfig("stor-distributormanager").set("minsplitcount", "16"); + + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0xffff), + getExternalOperationHandler().getBucketId(document::DocumentId( + vespalib::make_string("userdoc:ns:%d::", 0xffff)) + ).stripUnused()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0), + getExternalOperationHandler().getBucketId(document::DocumentId( + vespalib::make_string("userdoc:ns:%d::", 0x10000)) + ).stripUnused()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0xffff), + getExternalOperationHandler().getBucketId(document::DocumentId( + vespalib::make_string("userdoc:ns:%d::", 0xffff)) + ).stripUnused()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x100), + getExternalOperationHandler().getBucketId(document::DocumentId( + vespalib::make_string("userdoc:ns:%d::", 0x100)) + ).stripUnused()); + close(); + } + { + getDirConfig().getConfig("stor-distributormanager").set("minsplitcount", "20"); + createLinks(); + CPPUNIT_ASSERT_EQUAL(document::BucketId(20, 0x11111), + getExternalOperationHandler().getBucketId(document::DocumentId( + vespalib::make_string("userdoc:ns:%d::", 0x111111)) + ).stripUnused()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(20, 0x22222), + getExternalOperationHandler().getBucketId(document::DocumentId( + vespalib::make_string("userdoc:ns:%d::", 0x222222)) + ).stripUnused()); + } +} + +document::BucketId +ExternalOperationHandlerTest::findNonOwnedUserBucketInState( + vespalib::stringref statestr) +{ + lib::ClusterState state(statestr); + for (uint64_t i = 1; i < 1000; ++i) { + document::BucketId bucket(32, i); + if (!getExternalOperationHandler().ownsBucketInState(state, bucket)) { + return bucket; + } + } + throw std::runtime_error("no appropriate bucket found"); +} + +document::BucketId +ExternalOperationHandlerTest::findOwned1stNotOwned2ndInStates( + vespalib::stringref statestr1, + vespalib::stringref statestr2) +{ + lib::ClusterState state1(statestr1); + lib::ClusterState state2(statestr2); + for (uint64_t i = 1; i < 1000; ++i) { + document::BucketId bucket(32, i); + if (getExternalOperationHandler().ownsBucketInState(state1, bucket) + && !getExternalOperationHandler().ownsBucketInState(state2, bucket)) + { + return bucket; + } + } + throw std::runtime_error("no appropriate bucket found"); +} + +std::shared_ptr<api::StorageMessage> +ExternalOperationHandlerTest::makeGetCommandForUser(uint64_t id) +{ + document::DocumentId docId(document::UserDocIdString("userdoc:foo:" + vespalib::make_string("%lu", id) + ":bar")); + std::shared_ptr<api::StorageMessage> cmd( + new api::GetCommand(document::BucketId(0), docId, "[all]")); + return cmd; +} + +void +ExternalOperationHandlerTest::testOperationRejectedOnWrongDistribution() +{ + createLinks(); + std::string state("distributor:2 storage:2"); + setupDistributor(1, 2, state); + + document::BucketId bucket(findNonOwnedUserBucketInState(state)); + auto cmd = makeGetCommandForUser(bucket.withoutCountBits()); + + Operation::SP genOp; + CPPUNIT_ASSERT(getExternalOperationHandler().handleMessage(cmd, genOp)); + CPPUNIT_ASSERT(!genOp.get()); + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.replies.size()); + CPPUNIT_ASSERT_EQUAL( + std::string("ReturnCode(WRONG_DISTRIBUTION, " + "distributor:2 storage:2)"), + _sender.replies[0]->getResult().toString()); +} + +void +ExternalOperationHandlerTest::testOperationRejectedOnPendingWrongDistribution() +{ + createLinks(); + std::string current("distributor:2 storage:2"); + std::string pending("distributor:3 storage:3"); + setupDistributor(1, 3, current); + + document::BucketId b(findOwned1stNotOwned2ndInStates(current, pending)); + + // Trigger pending cluster state + auto stateCmd = std::make_shared<api::SetSystemStateCommand>( + lib::ClusterState(pending)); + getBucketDBUpdater().onSetSystemState(stateCmd); + + auto cmd = makeGetCommandForUser(b.withoutCountBits()); + + Operation::SP genOp; + CPPUNIT_ASSERT(getExternalOperationHandler().handleMessage(cmd, genOp)); + CPPUNIT_ASSERT(!genOp.get()); + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.replies.size()); + // Fail back with _pending_ cluster state so client can start trying + // correct distributor immediately. If that distributor has not yet + // completed processing its pending cluster state, it'll return the + // old (current) cluster state, causing the client to bounce between + // the two until the pending states have been resolved. This is pretty + // much inevitable with the current design. + CPPUNIT_ASSERT_EQUAL( + std::string("ReturnCode(WRONG_DISTRIBUTION, " + "distributor:3 storage:3)"), + _sender.replies[0]->getResult().toString()); +} +} // distributor +} // storage diff --git a/storage/src/tests/distributor/garbagecollectiontest.cpp b/storage/src/tests/distributor/garbagecollectiontest.cpp new file mode 100644 index 00000000000..399222f0e34 --- /dev/null +++ b/storage/src/tests/distributor/garbagecollectiontest.cpp @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <vespa/storageapi/message/removelocation.h> +#include <vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h> +#include <vespa/storage/distributor/idealstatemanager.h> +#include <tests/distributor/distributortestutil.h> + +namespace storage { +namespace distributor { + +class GarbageCollectionOperationTest : public CppUnit::TestFixture, public DistributorTestUtil +{ + CPPUNIT_TEST_SUITE(GarbageCollectionOperationTest); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST_SUITE_END(); + +protected: + void testSimple(); + +public: + void setUp() { + createLinks(); + }; + + void tearDown() { + close(); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(GarbageCollectionOperationTest); + +void +GarbageCollectionOperationTest::testSimple() +{ + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:2")); + addNodesToBucketDB(document::BucketId(16, 1), "0=250/50/300,1=250/50/300"); + getConfig().setGarbageCollection("music.date < 34", 3600); + + GarbageCollectionOperation op("storage", + BucketAndNodes(document::BucketId(16, 1), + toVector<uint16_t>(0, 1))); + + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL((size_t)2, _sender.commands.size()); + + getClock().setAbsoluteTimeInSeconds(34); + + for (uint32_t i = 0; i < 2; ++i) { + std::shared_ptr<api::StorageCommand> msg = _sender.commands[i]; + CPPUNIT_ASSERT(msg->getType() == api::MessageType::REMOVELOCATION); + + api::RemoveLocationCommand* tmp = (api::RemoveLocationCommand*)msg.get(); + CPPUNIT_ASSERT_EQUAL(vespalib::string("music.date < 34"), + tmp->getDocumentSelection()); + + std::shared_ptr<api::StorageReply> reply(tmp->makeReply().release()); + api::RemoveLocationReply* sreply = (api::RemoveLocationReply*)reply.get(); + sreply->setBucketInfo(api::BucketInfo(666, 90, 500)); + + op.receive(_sender, reply); + } + + BucketDatabase::Entry entry = getBucket(document::BucketId(16, 1)); + CPPUNIT_ASSERT(entry.valid()); + CPPUNIT_ASSERT_EQUAL(2, (int)entry->getNodeCount()); + CPPUNIT_ASSERT_EQUAL(34, (int)entry->getLastGarbageCollectionTime()); + CPPUNIT_ASSERT_EQUAL(api::BucketInfo(666, 90, 500), + entry->getNodeRef(0).getBucketInfo()); + CPPUNIT_ASSERT_EQUAL(api::BucketInfo(666, 90, 500), + entry->getNodeRef(1).getBucketInfo()); +} + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/getoperationtest.cpp b/storage/src/tests/distributor/getoperationtest.cpp new file mode 100644 index 00000000000..12853be2e42 --- /dev/null +++ b/storage/src/tests/distributor/getoperationtest.cpp @@ -0,0 +1,567 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> + +#include <vespa/config/helper/configgetter.h> +#include <vespa/document/config/config-documenttypes.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/storage/distributor/externaloperationhandler.h> +#include <vespa/storage/distributor/distributormetricsset.h> +#include <vespa/storageapi/message/persistence.h> +#include <tests/distributor/distributortestutil.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <iomanip> +#include <iostream> +#include <memory> +#include <vespa/storage/distributor/operations/external/getoperation.h> + +using std::shared_ptr; +using config::ConfigGetter; +using document::DocumenttypesConfig; +using config::FileSpec; + +namespace storage { +namespace distributor { + +class GetOperationTest : public CppUnit::TestFixture, public DistributorTestUtil { + CPPUNIT_TEST_SUITE(GetOperationTest); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST(testNotFound); + CPPUNIT_TEST(testResendOnStorageFailure); + CPPUNIT_TEST(testResendOnStorageFailureAllFail); + CPPUNIT_TEST(testSendToIdealCopyIfBucketInSync); + CPPUNIT_TEST(testReturnNotFoundWhenBucketNotInDb); + CPPUNIT_TEST(testAskAllNodesIfBucketIsInconsistent); + CPPUNIT_TEST(testSendToAllInvalidNodesWhenInconsistent); + CPPUNIT_TEST(testAskTrustedNodeIfBucketIsInconsistent); + CPPUNIT_TEST(testInconsistentSplit); // Test that we ask all nodes if a bucket is inconsistent. + CPPUNIT_TEST(testSendToAllInvalidCopies); + CPPUNIT_TEST(testMultiInconsistentBucket); + CPPUNIT_TEST(testMultiInconsistentBucketFail); + CPPUNIT_TEST(testMultiInconsistentBucketNotFound); + CPPUNIT_TEST(testMultiInconsistentBucketNotFoundDeleted); + CPPUNIT_TEST(testMultipleCopiesWithFailureOnLocalNode); + CPPUNIT_TEST(canGetDocumentsWhenAllReplicaNodesRetired); + CPPUNIT_TEST_SUITE_END(); + + document::DocumentTypeRepo::SP _repo; + +public: + document::DocumentId docId; + document::BucketId bucketId; + std::unique_ptr<Operation> op; + + void setUp() { + _repo.reset( + new document::DocumentTypeRepo(*ConfigGetter<DocumenttypesConfig>:: + getConfig("config-doctypes", FileSpec("config-doctypes.cfg")))); + createLinks(); + + docId = document::DocumentId(document::DocIdString("test", "uri")); + bucketId = getExternalOperationHandler().getBucketId(docId); + }; + + void tearDown() { + close(); + op.reset(); + } + + void sendGet() { + std::shared_ptr<api::GetCommand> msg( + new api::GetCommand(document::BucketId(0), docId, "[all]")); + + op.reset(new GetOperation(getExternalOperationHandler(), + msg, + getDistributor().getMetrics(). + gets[msg->getLoadType()])); + op->start(_sender, framework::MilliSecTime(0)); + } + + void sendReply(uint32_t idx, + api::ReturnCode::Result result, + std::string authorVal, uint32_t timestamp) + { + if (idx == (uint32_t)-1) { + idx = _sender.commands.size() - 1; + } + + std::shared_ptr<api::StorageCommand> msg2 = _sender.commands[idx]; + CPPUNIT_ASSERT_EQUAL(api::MessageType::GET, msg2->getType()); + + api::GetCommand* tmp = static_cast<api::GetCommand*>(msg2.get()); + document::Document::SP doc; + + if (authorVal.length()) { + const document::DocumentType* type(_repo->getDocumentType("text/html")); + doc = document::Document::SP( + new document::Document(*type, docId)); + + doc->setValue(doc->getField("author"), + document::StringFieldValue(authorVal)); + } + + api::GetReply* reply = new api::GetReply(*tmp, doc, timestamp); + reply->setResult(result); + + op->receive(_sender, std::shared_ptr<api::StorageReply>(reply)); + } + + void replyWithFailure() { + sendReply(-1, api::ReturnCode::IO_FAILURE, "", 0); + } + + void replyWithNotFound() { + sendReply(-1, api::ReturnCode::OK, "", 0); + } + + void replyWithDocument() { + sendReply(-1, api::ReturnCode::OK, "foo", 100); + } + + std::string getLastReplyAuthor() { + api::StorageMessage& msg = *_sender.replies[_sender.replies.size() - 1]; + + if (msg.getType() == api::MessageType::GET_REPLY) { + document::Document::SP doc( + dynamic_cast<api::GetReply&>(msg).getDocument()); + + return doc->getValue(doc->getField("author"))->toString(); + } else { + std::ostringstream ost; + ost << "Last reply was not a GET reply, but " << msg; + return ost.str(); + } + } + + void setClusterState(const std::string& clusterState) { + _distributor->enableClusterState(lib::ClusterState(clusterState)); + } + + void testSimple(); + void testReturnNotFoundWhenBucketNotInDb(); + void testNotFound(); + void testResendOnStorageFailure(); + void testResendOnStorageFailureAllFail(); + void testSendToIdealCopyIfBucketInSync(); + void testAskAllNodesIfBucketIsInconsistent(); + void testSendToAllInvalidNodesWhenInconsistent(); + void testAskTrustedNodeIfBucketIsInconsistent(); + void testInconsistentSplit(); + void testMultiInconsistentBucket(); + void testMultiInconsistentBucketFail(); + void testMultiInconsistentBucketNotFound(); + void testMultiInconsistentBucketNotFoundDeleted(); + void testSendToAllInvalidCopies(); + void testMultipleCopiesWithFailureOnLocalNode(); + void canGetDocumentsWhenAllReplicaNodesRetired(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(GetOperationTest); + +void +GetOperationTest::testSimple() +{ + setClusterState("distributor:1 storage:2"); + + addNodesToBucketDB(bucketId, "0=4,1=4"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get => 0"), + _sender.getCommands(true)); + + replyWithDocument(); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 100) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +GetOperationTest::testAskTrustedNodeIfBucketIsInconsistent() +{ + setClusterState("distributor:1 storage:4"); + + addNodesToBucketDB(bucketId, "0=100/3/10,1=200/4/12/t"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL(std::string("Get => 1"), + _sender.getCommands(true)); + + replyWithDocument(); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 100) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +GetOperationTest::testAskAllNodesIfBucketIsInconsistent() +{ + setClusterState("distributor:1 storage:4"); + + addNodesToBucketDB(bucketId, "0=100/3/10,1=200/4/12"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get => 0,Get => 1"), + _sender.getCommands(true)); + + sendReply(0, api::ReturnCode::OK, "newauthor", 2); + sendReply(1, api::ReturnCode::OK, "oldauthor", 1); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 2) ReturnCode(NONE)"), + _sender.getLastReply()); + + CPPUNIT_ASSERT_EQUAL(std::string("newauthor"), getLastReplyAuthor()); +} + + +void +GetOperationTest::testSendToAllInvalidCopies() +{ + setClusterState("distributor:1 storage:4"); + + addNodesToBucketDB(bucketId, "2=0/0/1,3=0/0/1"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get => 2,Get => 3"), + _sender.getCommands(true)); + + sendReply(0, api::ReturnCode::OK, "newauthor", 2); + sendReply(1, api::ReturnCode::OK, "oldauthor", 1); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 2) ReturnCode(NONE)"), + _sender.getLastReply()); + + CPPUNIT_ASSERT_EQUAL(std::string("newauthor"), getLastReplyAuthor()); +} + +void +GetOperationTest::testSendToAllInvalidNodesWhenInconsistent() +{ + setClusterState("distributor:1 storage:4"); + + addNodesToBucketDB(bucketId, "0=100,1=200,2=0/0/1,3=0/0/1"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get => 2,Get => 3,Get => 0,Get => 1"), + _sender.getCommands(true)); + + sendReply(0, api::ReturnCode::OK, "newauthor", 2); + sendReply(1, api::ReturnCode::OK, "oldauthor", 1); + sendReply(2, api::ReturnCode::OK, "oldauthor", 1); + sendReply(3, api::ReturnCode::OK, "oldauthor", 1); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 2) ReturnCode(NONE)"), + _sender.getLastReply()); + + CPPUNIT_ASSERT_EQUAL(std::string("newauthor"), getLastReplyAuthor()); +} + +void +GetOperationTest::testInconsistentSplit() +{ + setClusterState("distributor:1 storage:4"); + + addNodesToBucketDB(document::BucketId(16, 0x2a52), "0=100"); + addNodesToBucketDB(document::BucketId(17, 0x2a52), "1=200"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get => 0,Get => 1"), + _sender.getCommands(true)); + + sendReply(0, api::ReturnCode::OK, "newauthor", 2); + sendReply(1, api::ReturnCode::OK, "oldauthor", 1); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 2) ReturnCode(NONE)"), + _sender.getLastReply()); + + CPPUNIT_ASSERT_EQUAL(std::string("newauthor"), getLastReplyAuthor()); +} + + +void +GetOperationTest::testMultiInconsistentBucketNotFound() +{ + setClusterState("distributor:1 storage:4"); + + addNodesToBucketDB(bucketId, "0=100,2=100,1=200,3=200"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get => 0,Get => 1"), + _sender.getCommands(true)); + + sendReply(0, api::ReturnCode::OK, "newauthor", 2); + sendReply(1, api::ReturnCode::OK, "", 0); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 2) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +GetOperationTest::testMultiInconsistentBucketNotFoundDeleted() +{ + setClusterState("distributor:1 storage:4"); + + addNodesToBucketDB(bucketId, "0=100,2=100,1=200,3=200"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get => 0,Get => 1"), + _sender.getCommands(true)); + + sendReply(0, api::ReturnCode::OK, "newauthor", 2); + // This signifies that the latest change was that the document was deleted + // at timestamp 3. + sendReply(1, api::ReturnCode::OK, "", 3); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 3) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +GetOperationTest::testMultiInconsistentBucket() +{ + setClusterState("distributor:1 storage:4"); + + addNodesToBucketDB(bucketId, "0=100,2=100,1=200,3=200"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get => 0,Get => 1"), + _sender.getCommands(true)); + + sendReply(0, api::ReturnCode::OK, "newauthor", 2); + sendReply(1, api::ReturnCode::OK, "oldauthor", 1); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 2) ReturnCode(NONE)"), + _sender.getLastReply()); + + CPPUNIT_ASSERT_EQUAL(std::string("newauthor"), getLastReplyAuthor()); +} + +void +GetOperationTest::testMultiInconsistentBucketFail() +{ + setClusterState("distributor:1 storage:4"); + + addNodesToBucketDB(bucketId, "0=100,2=100,1=200,3=200"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get => 0,Get => 1"), + _sender.getCommands(true)); + + sendReply(0, api::ReturnCode::OK, "newauthor", 1); + sendReply(1, api::ReturnCode::DISK_FAILURE, "", 0); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get(BucketId(0x4000000000002a52), doc:test:uri) => 3"), + _sender.getLastCommand()); + + replyWithDocument(); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 100) ReturnCode(NONE)"), + _sender.getLastReply()); +} + + +void +GetOperationTest::testReturnNotFoundWhenBucketNotInDb() +{ + setClusterState("distributor:1 storage:1"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 0) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +GetOperationTest::testNotFound() +{ + setClusterState("distributor:1 storage:1"); + + addNodesToBucketDB(bucketId, "0=100"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get(BucketId(0x4000000000002a52), doc:test:uri) => 0"), + _sender.getLastCommand()); + + replyWithNotFound(); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 0) ReturnCode(NONE)"), + _sender.getLastReply()); + + CPPUNIT_ASSERT_EQUAL(1, (int)(getDistributor(). + getMetrics().gets[documentapi::LoadType::DEFAULT]. + failures.notfound.getValue())); +} + +void +GetOperationTest::testResendOnStorageFailure() +{ + setClusterState("distributor:1 storage:3"); + + // Add two nodes that are not trusted. GET should retry each one of them + // if one fails. + addNodesToBucketDB(bucketId, "1=100,2=100"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get(BucketId(0x4000000000002a52), doc:test:uri) => 1"), + _sender.getLastCommand()); + + replyWithFailure(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get(BucketId(0x4000000000002a52), doc:test:uri) => 2"), + _sender.getLastCommand()); + + replyWithDocument(); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 100) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +GetOperationTest::testResendOnStorageFailureAllFail() +{ + setClusterState("distributor:1 storage:3"); + + // Add two nodes that are not trusted. GET should retry each one of them + // if one fails. + addNodesToBucketDB(bucketId, "1=100,2=100"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get(BucketId(0x4000000000002a52), doc:test:uri) => 1"), + _sender.getLastCommand()); + + replyWithFailure(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get(BucketId(0x4000000000002a52), doc:test:uri) => 2"), + _sender.getLastCommand()); + + replyWithFailure(); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 0) ReturnCode(IO_FAILURE)"), + _sender.getLastReply()); +} + +void +GetOperationTest::testSendToIdealCopyIfBucketInSync() +{ + setClusterState("distributor:1 storage:4"); + + addNodesToBucketDB(bucketId, "1=100,2=100,3=100"); + + sendGet(); + + // Should always send to node 1 (follow bucket db order) + CPPUNIT_ASSERT_EQUAL( + std::string("Get(BucketId(0x4000000000002a52), doc:test:uri) => 1"), + _sender.getLastCommand()); + + replyWithDocument(); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 100) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +GetOperationTest::testMultipleCopiesWithFailureOnLocalNode() +{ + setClusterState("distributor:1 storage:4"); + + // Node 0 is local copy to distributor 0 and will be preferred when + // sending initially. + addNodesToBucketDB(document::BucketId(16, 0x2a52), "2=100,0=100"); + + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get => 0"), + _sender.getCommands(true)); + + // Fail local node; no reply must be sent yet since we've got more nodes + // to try. + sendReply(0, api::ReturnCode::TIMEOUT, "", 0); + + // Retry with remaining copy on node 2. + CPPUNIT_ASSERT_EQUAL( + std::string("Get => 0,Get => 2"), + _sender.getCommands(true)); + + sendReply(1, api::ReturnCode::OK, "newestauthor", 3); + + CPPUNIT_ASSERT_EQUAL( + std::string("GetReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 3) ReturnCode(NONE)"), + _sender.getLastReply()); + + CPPUNIT_ASSERT_EQUAL(std::string("newestauthor"), getLastReplyAuthor()); +} + +void +GetOperationTest::canGetDocumentsWhenAllReplicaNodesRetired() +{ + setClusterState("distributor:1 storage:2 .0.s:r .1.s:r"); + addNodesToBucketDB(bucketId, "0=4,1=4"); + sendGet(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get => 0"), + _sender.getCommands(true)); +} + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/idealstatemanagertest.cpp b/storage/src/tests/distributor/idealstatemanagertest.cpp new file mode 100644 index 00000000000..9c97a2ba967 --- /dev/null +++ b/storage/src/tests/distributor/idealstatemanagertest.cpp @@ -0,0 +1,268 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storage/distributor/bucketdbupdater.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/storage/common/bucketmessages.h> +#include <vespa/storage/distributor/idealstatemanager.h> +#include <vespa/storage/distributor/operations/idealstate/mergeoperation.h> +#include <vespa/storage/distributor/operations/idealstate/removebucketoperation.h> +#include <vespa/storage/distributor/operations/idealstate/setbucketstateoperation.h> +#include <vespa/storage/distributor/operations/idealstate/splitoperation.h> +#include <vespa/storageapi/message/stat.h> +#include <vespa/storageapi/message/visitor.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storage/storageutil/utils.h> +#include <tests/distributor/distributortestutil.h> +#include <vespa/storage/distributor/statecheckers.h> +#include <vespa/storageapi/message/state.h> + +namespace storage { +namespace distributor { + +class IdealStateManagerTest : public CppUnit::TestFixture, + public DistributorTestUtil +{ +public: + IdealStateManagerTest() {} + void setUp() { + createLinks(); + }; + + void tearDown() { + close(); + } + + void testSibling(); + void testClearActiveOnNodeDown(); + void testRecheckWhenActive(); + void testRecheckWhenPending(); + void testOpsGenerationBusy(); + void testStatusPage(); + void testDisabledStateChecker(); + void testBlockIdealStateOpsOnFullRequestBucketInfo(); + void testBlockCheckForAllOperationsToSpecificBucket(); + + void setSystemState(const lib::ClusterState& systemState) { + _distributor->enableClusterState(systemState); + } + + CPPUNIT_TEST_SUITE(IdealStateManagerTest); + CPPUNIT_TEST(testSibling); + CPPUNIT_TEST(testClearActiveOnNodeDown); + CPPUNIT_TEST(testRecheckWhenActive); + CPPUNIT_TEST(testStatusPage); + CPPUNIT_TEST(testDisabledStateChecker); + CPPUNIT_TEST(testBlockIdealStateOpsOnFullRequestBucketInfo); + CPPUNIT_TEST(testBlockCheckForAllOperationsToSpecificBucket); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(IdealStateManagerTest); + +void +IdealStateManagerTest::testSibling() +{ + CPPUNIT_ASSERT_EQUAL(document::BucketId(1,1), + getIdealStateManager().getDistributorComponent() + .getSibling(document::BucketId(1, 0))); + CPPUNIT_ASSERT_EQUAL(document::BucketId(1,0), + getIdealStateManager().getDistributorComponent() + .getSibling(document::BucketId(1, 1))); + CPPUNIT_ASSERT_EQUAL(document::BucketId(2,3), + getIdealStateManager().getDistributorComponent() + .getSibling(document::BucketId(2, 1))); + CPPUNIT_ASSERT_EQUAL(document::BucketId(2,1), + getIdealStateManager().getDistributorComponent() + .getSibling(document::BucketId(2, 3))); +} + +void +IdealStateManagerTest::testStatusPage() { + close(); + getDirConfig().getConfig("stor-distributormanager").set("splitsize", "100"); + getDirConfig().getConfig("stor-distributormanager").set("splitcount", "1000000"); + getDirConfig().getConfig("stor-distributormanager").set("joinsize", "0"); + getDirConfig().getConfig("stor-distributormanager").set("joincount", "0"); + createLinks(); + setupDistributor(1, 1, "distributor:1 storage:1"); + + insertBucketInfo(document::BucketId(16, 5), 0, 0xff, 100, 200, true, true); + insertBucketInfo(document::BucketId(16, 2), 0, 0xff, 10, 10, true, true); + + std::ostringstream ost; + getIdealStateManager().getBucketStatus(ost); + + CPPUNIT_ASSERT_EQUAL(std::string("BucketId(0x4000000000000002) : [node(idx=0,crc=0xff,docs=10/10,bytes=10/10,trusted=true,active=true)]<br>\n" + "<b>BucketId(0x4000000000000005):</b> <i> : split: [Splitting bucket because its maximum size (200 b, 100 docs, 100 meta, 200 b total) is " + "higher than the configured limit of (100, 1000000)]</i> [node(idx=0,crc=0xff,docs=100/100,bytes=200/200,trusted=true," + "active=true)]<br>\n"), + ost.str()); +} + +void +IdealStateManagerTest::testDisabledStateChecker() { + setupDistributor(1, 1, "distributor:1 storage:1"); + + getConfig().setSplitSize(100); + getConfig().setSplitCount(1000000); + getConfig().disableStateChecker("SplitBucket"); + + insertBucketInfo(document::BucketId(16, 5), 0, 0xff, 100, 200, true, true); + insertBucketInfo(document::BucketId(16, 2), 0, 0xff, 10, 10, true, true); + + std::ostringstream ost; + getIdealStateManager().getBucketStatus(ost); + + CPPUNIT_ASSERT_EQUAL(std::string( + "BucketId(0x4000000000000002) : [node(idx=0,crc=0xff,docs=10/10,bytes=10/10,trusted=true,active=true)]<br>\n" + "<b>BucketId(0x4000000000000005):</b> <i> : split: [Splitting bucket because its maximum size (200 b, 100 docs, 100 meta, 200 b total) is " + "higher than the configured limit of (100, 1000000)]</i> [node(idx=0,crc=0xff,docs=100/100,bytes=200/200,trusted=true," + "active=true)]<br>\n"), + ost.str()); + + tick(); + CPPUNIT_ASSERT_EQUAL(std::string(""), + _distributor->getActiveIdealStateOperations()); + +} + +void +IdealStateManagerTest::testClearActiveOnNodeDown() +{ + setSystemState(lib::ClusterState("distributor:1 storage:3")); + for (int i = 1; i < 4; i++) { + insertBucketInfo(document::BucketId(16, i), 0, 0xff, 100, 200); + insertBucketInfo(document::BucketId(16, i), 1, 0xffe, 1020, 2300); + insertBucketInfo(document::BucketId(16, i), 2, 0xfff, 1030, 2400); + } + + tick(); + + // Start all three operations. + for (uint32_t i = 0; i < 3; ++i) { + tick(); + } + + CPPUNIT_ASSERT_EQUAL( + std::string("setbucketstate to [0] BucketId(0x4000000000000001) (pri 100)\n" + "setbucketstate to [0] BucketId(0x4000000000000002) (pri 100)\n" + "setbucketstate to [0] BucketId(0x4000000000000003) (pri 100)\n"), + _distributor->getActiveIdealStateOperations()); + + setSystemState(lib::ClusterState("distributor:1 storage:3 .0.s:d")); + + CPPUNIT_ASSERT_EQUAL(std::string(""), + _distributor->getActiveIdealStateOperations()); + CPPUNIT_ASSERT_EQUAL(uint32_t(0), + _distributor->getPendingMessageTracker() + .getNodeInfo().getPendingCount(0)); +} + +void +IdealStateManagerTest::testRecheckWhenActive() +{ + for (uint32_t j = 0; j < 3; j++) { + insertBucketInfo(document::BucketId(16, 1), j, 0xff - j, 100, 200); + } + + setSystemState(lib::ClusterState("distributor:1 storage:3")); + + tick(); + + CPPUNIT_ASSERT_EQUAL( + std::string("setbucketstate to [0] BucketId(0x4000000000000001) (pri 100)\n"), + _distributor->getActiveIdealStateOperations()); + + tick(); + + CPPUNIT_ASSERT_EQUAL( + std::string("setbucketstate to [0] BucketId(0x4000000000000001) (pri 100)\n"), + _distributor->getActiveIdealStateOperations()); + + tick(); + + CPPUNIT_ASSERT_EQUAL( + std::string("setbucketstate to [0] BucketId(0x4000000000000001) (pri 100)\n"), + _distributor->getActiveIdealStateOperations()); +} + +void +IdealStateManagerTest::testBlockIdealStateOpsOnFullRequestBucketInfo() +{ + setupDistributor(2, 10, "distributor:1 storage:2"); + + framework::defaultimplementation::FakeClock clock; + PendingMessageTracker tracker(_node->getComponentRegister()); + + document::BucketId bid(16, 1234); + std::vector<document::BucketId> buckets; + + // RequestBucketInfoCommand does not have a specific bucketid since it's + // sent to the entire node. It will then use a null bucketid. + { + std::shared_ptr<api::RequestBucketInfoCommand> msg( + new api::RequestBucketInfoCommand(buckets)); + msg->setAddress( + api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 4)); + tracker.insert(msg); + } + + { + RemoveBucketOperation op("storage", + BucketAndNodes(bid, toVector<uint16_t>(3, 4))); + CPPUNIT_ASSERT(op.isBlocked(tracker)); + } + + { + // Don't trigger on requests to other nodes. + RemoveBucketOperation op("storage", + BucketAndNodes(bid, toVector<uint16_t>(3, 5))); + CPPUNIT_ASSERT(!op.isBlocked(tracker)); + } + + // Don't block on null-bucket messages that aren't RequestBucketInfo. + { + std::shared_ptr<api::CreateVisitorCommand> msg( + new api::CreateVisitorCommand("foo", "bar", "baz")); + msg->setAddress( + api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 7)); + tracker.insert(msg); + } + + { + RemoveBucketOperation op("storage", + BucketAndNodes(bid, toVector<uint16_t>(7))); + CPPUNIT_ASSERT(!op.isBlocked(tracker)); + } +} + +void +IdealStateManagerTest::testBlockCheckForAllOperationsToSpecificBucket() +{ + setupDistributor(2, 10, "distributor:1 storage:2"); + framework::defaultimplementation::FakeClock clock; + PendingMessageTracker tracker(_node->getComponentRegister()); + document::BucketId bid(16, 1234); + + { + auto msg = std::make_shared<api::JoinBucketsCommand>(bid); + msg->setAddress( + api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 4)); + tracker.insert(msg); + } + { + RemoveBucketOperation op("storage", + BucketAndNodes(bid, toVector<uint16_t>(7))); + // Not blocked for exact node match. + CPPUNIT_ASSERT(!op.checkBlock(bid, tracker)); + // But blocked for bucket match! + CPPUNIT_ASSERT(op.checkBlockForAllNodes(bid, tracker)); + } +} + +} // distributor +} // storage + diff --git a/storage/src/tests/distributor/joinbuckettest.cpp b/storage/src/tests/distributor/joinbuckettest.cpp new file mode 100644 index 00000000000..ec7e3aaac32 --- /dev/null +++ b/storage/src/tests/distributor/joinbuckettest.cpp @@ -0,0 +1,127 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storage/distributor/operations/idealstate/joinoperation.h> +#include <vespa/storage/distributor/idealstatemanager.h> +#include <tests/distributor/distributortestutil.h> + +namespace storage { +namespace distributor { + +class JoinOperationTest : public CppUnit::TestFixture, public DistributorTestUtil +{ + CPPUNIT_TEST_SUITE(JoinOperationTest); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST(sendSparseJoinsToNodesWithoutBothSourceBuckets); + CPPUNIT_TEST_SUITE_END(); + + void checkSourceBucketsAndSendReply( + JoinOperation& op, + size_t msgIndex, + const std::vector<document::BucketId>& wantedIds); + +protected: + void testSimple(); + void sendSparseJoinsToNodesWithoutBothSourceBuckets(); + +public: + void setUp() { + createLinks(); + }; + + void tearDown() { + close(); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(JoinOperationTest); + +void +JoinOperationTest::testSimple() +{ + getConfig().setJoinCount(100); + getConfig().setJoinSize(1000); + + addNodesToBucketDB(document::BucketId(33, 1), "0=250/50/300"); + addNodesToBucketDB(document::BucketId(33, 0x100000001), "0=300/40/200"); + + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:1")); + + JoinOperation op("storage", + BucketAndNodes(document::BucketId(32, 0), + toVector<uint16_t>(0)), + toVector(document::BucketId(33, 1), + document::BucketId(33, 0x100000001))); + + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + checkSourceBucketsAndSendReply(op, 0, {{33, 1}, {33, 0x100000001}}); + + CPPUNIT_ASSERT(!getBucket(document::BucketId(33, 0x100000001)).valid()); + CPPUNIT_ASSERT(!getBucket(document::BucketId(33, 1)).valid()); + + BucketDatabase::Entry entry = getBucket(document::BucketId(32, 0)); + CPPUNIT_ASSERT(entry.valid()); + CPPUNIT_ASSERT_EQUAL((uint16_t)0, entry->getNodeRef(0).getNode()); + CPPUNIT_ASSERT_EQUAL(api::BucketInfo(666, 90, 500), + entry->getNodeRef(0).getBucketInfo()); +} + +void +JoinOperationTest::checkSourceBucketsAndSendReply( + JoinOperation& op, + size_t msgIndex, + const std::vector<document::BucketId>& wantedIds) +{ + CPPUNIT_ASSERT(_sender.commands.size() > msgIndex); + + std::shared_ptr<api::StorageCommand> msg(_sender.commands[msgIndex]); + CPPUNIT_ASSERT_EQUAL(api::MessageType::JOINBUCKETS, msg->getType()); + + api::JoinBucketsCommand& joinCmd( + dynamic_cast<api::JoinBucketsCommand&>(*msg)); + CPPUNIT_ASSERT_EQUAL(wantedIds, joinCmd.getSourceBuckets()); + + std::shared_ptr<api::StorageReply> reply(joinCmd.makeReply()); + api::JoinBucketsReply& sreply( + dynamic_cast<api::JoinBucketsReply&>(*reply)); + sreply.setBucketInfo(api::BucketInfo(666, 90, 500)); + + op.receive(_sender, reply); +} + +/** + * If the set of buckets kept on nodes is disjoint, send sparse joins (same + * bucket id used as both source buckets) for those nodes having only one of + * the buckets. + */ +void +JoinOperationTest::sendSparseJoinsToNodesWithoutBothSourceBuckets() +{ + getConfig().setJoinCount(100); + getConfig().setJoinSize(1000); + + addNodesToBucketDB(document::BucketId(33, 1), "0=250/50/300,1=250/50/300"); + addNodesToBucketDB(document::BucketId(33, 0x100000001), "0=300/40/200"); + + _distributor->enableClusterState( + lib::ClusterState("distributor:1 storage:2")); + + JoinOperation op("storage", + BucketAndNodes(document::BucketId(32, 0), + toVector<uint16_t>(0, 1)), + toVector(document::BucketId(33, 1), + document::BucketId(33, 0x100000001))); + + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + checkSourceBucketsAndSendReply(op, 0, {{33, 1}, {33, 0x100000001}}); + checkSourceBucketsAndSendReply(op, 1, {{33, 1}, {33, 1}}); +} + +} + +} diff --git a/storage/src/tests/distributor/maintenancemocks.h b/storage/src/tests/distributor/maintenancemocks.h new file mode 100644 index 00000000000..923f7edec2b --- /dev/null +++ b/storage/src/tests/distributor/maintenancemocks.h @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <sstream> +#include <vespa/storage/distributor/maintenance/maintenanceprioritygenerator.h> +#include <vespa/storage/distributor/maintenance/maintenanceoperationgenerator.h> +#include <vespa/storage/distributor/operationstarter.h> +#include <vespa/storage/distributor/operations/operation.h> +#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> + +namespace storage { +namespace distributor { + +class MockMaintenancePriorityGenerator + : public MaintenancePriorityGenerator +{ + MaintenancePriorityAndType prioritize( + const document::BucketId&, + NodeMaintenanceStatsTracker& stats) const + { + stats.incMovingOut(1); + stats.incCopyingIn(2); + return MaintenancePriorityAndType( + MaintenancePriority(MaintenancePriority::VERY_HIGH), + MaintenanceOperation::MERGE_BUCKET); + } +}; + + +class MockOperation : public MaintenanceOperation +{ + document::BucketId _bucketId; + std::string _reason; + bool _shouldBlock; +public: + MockOperation(const document::BucketId& bucketId) + : _bucketId(bucketId), + _shouldBlock(false) + {} + + std::string toString() const { + return _bucketId.toString(); + } + + void onClose(DistributorMessageSender&) { + } + const char* getName() const { + return "MockOperation"; + } + virtual const std::string& getDetailedReason() const { + return _reason; + } + void onStart(DistributorMessageSender&) { + } + void onReceive(DistributorMessageSender&, const std::shared_ptr<api::StorageReply>&) { + } + bool isBlocked(const PendingMessageTracker&) const { + return _shouldBlock; + } + void setShouldBlock(bool shouldBlock) { + _shouldBlock = shouldBlock; + } +}; + +class MockMaintenanceOperationGenerator + : public MaintenanceOperationGenerator +{ +public: + MaintenanceOperation::SP generate(const document::BucketId& id) const { + return MaintenanceOperation::SP(new MockOperation(id)); + } + + std::vector<MaintenanceOperation::SP> generateAll( + const document::BucketId& id, + NodeMaintenanceStatsTracker& tracker) const + { + (void) tracker; + std::vector<MaintenanceOperation::SP> ret; + ret.push_back(MaintenanceOperation::SP(new MockOperation(id))); + return ret; + } + +}; + +class MockOperationStarter + : public OperationStarter +{ + std::ostringstream _started; + std::vector<Operation::SP> _operations; + bool _shouldStart; +public: + MockOperationStarter() + : _shouldStart(true) + {} + + bool start(const std::shared_ptr<Operation>& operation, + Priority priority) + { + if (_shouldStart) { + _started << operation->toString() + << ", pri " << static_cast<int>(priority) + << "\n"; + _operations.push_back(operation); + } + return _shouldStart; + } + + void setShouldStartOperations(bool shouldStart) { + _shouldStart = shouldStart; + } + + std::vector<Operation::SP>& getOperations() { + return _operations; + } + + std::string toString() const { + return _started.str(); + } +}; + +} +} + diff --git a/storage/src/tests/distributor/maintenanceschedulertest.cpp b/storage/src/tests/distributor/maintenanceschedulertest.cpp new file mode 100644 index 00000000000..4316bfd137c --- /dev/null +++ b/storage/src/tests/distributor/maintenanceschedulertest.cpp @@ -0,0 +1,108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <string> +#include <sstream> +#include <memory> +#include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> +#include <vespa/storage/distributor/maintenance/maintenancescheduler.h> +#include <vespa/storage/distributor/bucketdb/mapbucketdatabase.h> +#include <tests/distributor/maintenancemocks.h> + +namespace storage { + +namespace distributor { + +using document::BucketId; +typedef MaintenancePriority Priority; +typedef MaintenanceScheduler::WaitTimeMs WaitTimeMs; + +class MaintenanceSchedulerTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(MaintenanceSchedulerTest); + CPPUNIT_TEST(testPriorityClearedAfterScheduled); + CPPUNIT_TEST(testOperationIsScheduled); + CPPUNIT_TEST(testNoOperationsToSchedule); + CPPUNIT_TEST(testSuppressLowPrioritiesInEmergencyMode); + CPPUNIT_TEST(testPriorityNotClearedIfOperationNotStarted); + CPPUNIT_TEST_SUITE_END(); + + std::unique_ptr<SimpleBucketPriorityDatabase> _priorityDb; + std::unique_ptr<MockMaintenanceOperationGenerator> _operationGenerator; + std::unique_ptr<MockOperationStarter> _operationStarter; + std::unique_ptr<MaintenanceScheduler> _scheduler; + + void addBucketToDb(int bucketNum); +public: + void testPriorityClearedAfterScheduled(); + void testOperationIsScheduled(); + void testNoOperationsToSchedule(); + void testSuppressLowPrioritiesInEmergencyMode(); + void testPriorityNotClearedIfOperationNotStarted(); + + void setUp(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(MaintenanceSchedulerTest); + +void +MaintenanceSchedulerTest::setUp() +{ + _priorityDb.reset(new SimpleBucketPriorityDatabase()); + _operationGenerator.reset(new MockMaintenanceOperationGenerator()); + _operationStarter.reset(new MockOperationStarter()); + _scheduler.reset(new MaintenanceScheduler(*_operationGenerator, + *_priorityDb, + *_operationStarter)); +} + +void +MaintenanceSchedulerTest::testPriorityClearedAfterScheduled() +{ + _priorityDb->setPriority(PrioritizedBucket(BucketId(16, 1), Priority::VERY_HIGH)); + _scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE); + CPPUNIT_ASSERT_EQUAL(std::string(), _priorityDb->toString()); +} + +void +MaintenanceSchedulerTest::testOperationIsScheduled() +{ + _priorityDb->setPriority(PrioritizedBucket(BucketId(16, 1), Priority::MEDIUM)); + _scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE); + CPPUNIT_ASSERT_EQUAL(std::string("BucketId(0x4000000000000001), pri 100\n"), + _operationStarter->toString()); +} + +void +MaintenanceSchedulerTest::testNoOperationsToSchedule() +{ + WaitTimeMs waitMs(_scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE)); + CPPUNIT_ASSERT_EQUAL(WaitTimeMs(1), waitMs); + CPPUNIT_ASSERT_EQUAL(std::string(), _operationStarter->toString()); +} + +void +MaintenanceSchedulerTest::testSuppressLowPrioritiesInEmergencyMode() +{ + _priorityDb->setPriority(PrioritizedBucket(BucketId(16, 1), Priority::HIGH)); + _priorityDb->setPriority(PrioritizedBucket(BucketId(16, 2), Priority::VERY_HIGH)); + CPPUNIT_ASSERT_EQUAL(WaitTimeMs(0), _scheduler->tick(MaintenanceScheduler::RECOVERY_SCHEDULING_MODE)); + CPPUNIT_ASSERT_EQUAL(WaitTimeMs(1), _scheduler->tick(MaintenanceScheduler::RECOVERY_SCHEDULING_MODE)); + CPPUNIT_ASSERT_EQUAL(std::string("BucketId(0x4000000000000002), pri 0\n"), + _operationStarter->toString()); + CPPUNIT_ASSERT_EQUAL(std::string("PrioritizedBucket(BucketId(0x4000000000000001), pri HIGH)\n"), + _priorityDb->toString()); +} + +void +MaintenanceSchedulerTest::testPriorityNotClearedIfOperationNotStarted() +{ + _priorityDb->setPriority(PrioritizedBucket(BucketId(16, 1), Priority::HIGH)); + _operationStarter->setShouldStartOperations(false); + WaitTimeMs waitMs(_scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE)); + CPPUNIT_ASSERT_EQUAL(WaitTimeMs(1), waitMs); + CPPUNIT_ASSERT_EQUAL(std::string("PrioritizedBucket(BucketId(0x4000000000000001), pri HIGH)\n"), + _priorityDb->toString()); +} + +} +} diff --git a/storage/src/tests/distributor/mapbucketdatabasetest.cpp b/storage/src/tests/distributor/mapbucketdatabasetest.cpp new file mode 100644 index 00000000000..ab8e5add65f --- /dev/null +++ b/storage/src/tests/distributor/mapbucketdatabasetest.cpp @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/document_runnable.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <cppunit/extensions/HelperMacros.h> +#include <vespa/storage/distributor/bucketdb/mapbucketdatabase.h> +#include <vespa/storage/storageutil/utils.h> +#include <tests/distributor/bucketdatabasetest.h> + +namespace storage { +namespace distributor { + +struct MapBucketDatabaseTest : public BucketDatabaseTest { + MapBucketDatabase _db; + + virtual BucketDatabase& db() { return _db; }; + + CPPUNIT_TEST_SUITE(MapBucketDatabaseTest); + SETUP_DATABASE_TESTS(); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(MapBucketDatabaseTest); + +} +} diff --git a/storage/src/tests/distributor/mergelimitertest.cpp b/storage/src/tests/distributor/mergelimitertest.cpp new file mode 100644 index 00000000000..fd86e071579 --- /dev/null +++ b/storage/src/tests/distributor/mergelimitertest.cpp @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/linkedptr.h> +#include <vespa/storage/distributor/operations/idealstate/mergelimiter.h> +#include <vespa/vdstestlib/cppunit/macros.h> + +namespace storage { +namespace distributor { + +struct MergeLimiterTest : public CppUnit::TestFixture +{ + void testKeepsAllBelowLimit(); + void testLessThanMaxUntrusted(); + void testMoreThanMaxUntrusted(); + void testAllUntrustedLessThanMaxVariants(); + void testAllUntrustedMoreThanMaxVariants(); + void testSourceOnlyLast(); + + CPPUNIT_TEST_SUITE(MergeLimiterTest); + CPPUNIT_TEST(testKeepsAllBelowLimit); + CPPUNIT_TEST(testLessThanMaxUntrusted); + CPPUNIT_TEST(testMoreThanMaxUntrusted); + CPPUNIT_TEST(testAllUntrustedLessThanMaxVariants); + CPPUNIT_TEST(testAllUntrustedMoreThanMaxVariants); + CPPUNIT_TEST(testSourceOnlyLast); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(MergeLimiterTest); + +namespace { + typedef vespalib::LinkedPtr<BucketCopy> BucketCopyPtr; + std::vector<BucketCopyPtr> _bucketDatabase; + + struct NodeFactory { + std::vector<MergeMetaData> _nodes; + + NodeFactory& add(int index, int crc) { + _bucketDatabase.push_back(BucketCopyPtr( + new BucketCopy(0, index, api::BucketInfo(crc, 5, 10)))); + _nodes.push_back(MergeMetaData(index, *_bucketDatabase.back())); + return *this; + } + NodeFactory& addTrusted(int index, int crc) { + add(index, crc); + _bucketDatabase.back()->setTrusted(true); + return *this; + } + NodeFactory& setSourceOnly() { + _nodes.back()._sourceOnly = true; + return *this; + } + + operator const MergeLimiter::NodeArray&() const { return _nodes; } + }; + + #define ASSERT_LIMIT(maxNodes, nodes, result) \ + { \ + MergeLimiter limiter(maxNodes); \ + limiter.limitMergeToMaxNodes(nodes); \ + std::ostringstream actual; \ + for (uint32_t i=0; i<nodes.size(); ++i) { \ + if (i != 0) actual << ","; \ + actual << nodes[i]._nodeIndex; \ + if (nodes[i]._sourceOnly) actual << 's'; \ + } \ + CPPUNIT_ASSERT_EQUAL(std::string(result), actual.str()); \ + } +} + +// If there is <= max nodes, then none should be removed. +void +MergeLimiterTest::testKeepsAllBelowLimit() +{ + MergeLimiter::NodeArray nodes(NodeFactory() + .addTrusted(3, 0x4) + .addTrusted(5, 0x4) + .add(9, 0x6) + .add(2, 0x6) + .add(4, 0x5)); + + ASSERT_LIMIT(8, nodes, "3,5,9,2,4"); +} +// If less than max nodes is untrusted, merge all untrusted copies with a +// trusted one. (Optionally with extra trusted copies if there is space) +void +MergeLimiterTest::testLessThanMaxUntrusted() +{ + MergeLimiter::NodeArray nodes(NodeFactory() + .addTrusted(3, 0x4) + .addTrusted(5, 0x4) + .add(9, 0x6) + .add(2, 0x6) + .add(4, 0x5)); + ASSERT_LIMIT(4, nodes, "2,4,9,5"); +} +// With more than max untrusted, just merge one trusted with as many untrusted +// that fits. +void +MergeLimiterTest::testMoreThanMaxUntrusted() +{ + MergeLimiter::NodeArray nodes(NodeFactory() + .addTrusted(3, 0x4) + .addTrusted(5, 0x4) + .add(9, 0x6) + .add(2, 0x6) + .add(13, 0x9) + .add(1, 0x7) + .add(4, 0x5)); + ASSERT_LIMIT(4, nodes, "2,13,1,5"); +} +// With nothing trusted. If there is <= max different variants (checksums), +// merge one of each variant. After this merge, all these nodes can be set +// trusted. (Except for any source only ones) +void +MergeLimiterTest::testAllUntrustedLessThanMaxVariants() +{ + MergeLimiter::NodeArray nodes(NodeFactory() + .add(3, 0x4) + .add(5, 0x4) + .add(9, 0x6) + .add(2, 0x6) + .add(13, 0x3) + .add(1, 0x3) + .add(4, 0x3)); + ASSERT_LIMIT(4, nodes, "5,2,4,3"); +} +// With nothing trusted and more than max variants, we just have to merge one +// of each variant until we end up with less than max variants. +void +MergeLimiterTest::testAllUntrustedMoreThanMaxVariants() +{ + MergeLimiter::NodeArray nodes(NodeFactory() + .add(3, 0x4) + .add(5, 0x5) + .add(9, 0x6) + .add(2, 0x6) + .add(13, 0x3) + .add(1, 0x9) + .add(4, 0x8)); + ASSERT_LIMIT(4, nodes, "3,5,2,13"); +} + +// With more than max untrusted, just merge one trusted with as many untrusted +// that fits. +void +MergeLimiterTest::testSourceOnlyLast() +{ + MergeLimiter::NodeArray nodes(NodeFactory() + .addTrusted(3, 0x4) + .addTrusted(5, 0x4).setSourceOnly() + .add(9, 0x6) + .add(2, 0x6).setSourceOnly() + .add(13, 0x9) + .add(1, 0x7) + .add(4, 0x5)); + ASSERT_LIMIT(4, nodes, "13,1,2s,5s"); +} + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/mergeoperationtest.cpp b/storage/src/tests/distributor/mergeoperationtest.cpp new file mode 100644 index 00000000000..a2373731bc3 --- /dev/null +++ b/storage/src/tests/distributor/mergeoperationtest.cpp @@ -0,0 +1,430 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <boost/lexical_cast.hpp> +#include <cppunit/extensions/HelperMacros.h> +#include <iomanip> +#include <iostream> +#include <memory> +#include <tests/common/dummystoragelink.h> +#include <vespa/storage/distributor/idealstatemanager.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storage/distributor/operations/idealstate/mergeoperation.h> +#include <vespa/storage/distributor/pendingmessagetracker.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storage/distributor/bucketdbupdater.h> +#include <tests/distributor/distributortestutil.h> +#include <vespa/vespalib/text/stringtokenizer.h> + +using std::shared_ptr; + +namespace storage { +namespace distributor { + +class MergeOperationTest : public CppUnit::TestFixture, + public DistributorTestUtil +{ + CPPUNIT_TEST_SUITE(MergeOperationTest); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST(testFailIfSourceOnlyCopiesChanged); + CPPUNIT_TEST(testGenerateNodeList); + CPPUNIT_TEST(doNotRemoveCopiesWithPendingMessages); + CPPUNIT_TEST(testDoNotRemoveActiveSourceOnlyCopies); + CPPUNIT_TEST(testMarkRedundantTrustedCopiesAsSourceOnly); + CPPUNIT_TEST(onlyMarkRedundantRetiredReplicasAsSourceOnly); + CPPUNIT_TEST_SUITE_END(); + + std::unique_ptr<PendingMessageTracker> _pendingTracker; + +protected: + void testSimple(); + void testFailIfSourceOnlyCopiesChanged(); + void testGenerateNodeList(); + void doNotRemoveCopiesWithPendingMessages(); + void testDoNotRemoveActiveSourceOnlyCopies(); + void testMarkRedundantTrustedCopiesAsSourceOnly(); + void onlyMarkRedundantRetiredReplicasAsSourceOnly(); + +public: + void setUp() { + createLinks(); + _pendingTracker.reset(new PendingMessageTracker(getComponentRegister())); + _sender.setPendingMessageTracker(*_pendingTracker); + } + + void tearDown() { + close(); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(MergeOperationTest); + +void +MergeOperationTest::testSimple() +{ + getClock().setAbsoluteTimeInSeconds(10); + + addNodesToBucketDB(document::BucketId(16, 1), + "0=10/1/1/t," + "1=20/1/1," + "2=10/1/1/t"); + + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:3")); + + MergeOperation op(BucketAndNodes(document::BucketId(16, 1), + toVector<uint16_t>(0, 1, 2))); + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string( + "MergeBucketCommand(BucketId(0x4000000000000001), to time 10000000, " + "cluster state version: 0, nodes: [0, 2, 1 (source only)], chain: [], " + "reasons to start: ) => 0"), + _sender.getLastCommand(true)); + + sendReply(op); + + CPPUNIT_ASSERT_EQUAL( + std::string("DeleteBucketCommand(BucketId(0x4000000000000001)) " + "Reasons to start: => 1"), + _sender.getLastCommand(true)); + +} + +void +MergeOperationTest::testFailIfSourceOnlyCopiesChanged() +{ + getClock().setAbsoluteTimeInSeconds(10); + + addNodesToBucketDB(document::BucketId(16, 1), + "0=10/1/1/t," + "1=20/1/1," + "2=10/1/1/t"); + + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:3")); + + MergeOperation op(BucketAndNodes(document::BucketId(16, 1), + toVector<uint16_t>(0, 1, 2))); + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + std::string merge("MergeBucketCommand(BucketId(0x4000000000000001), to time 10000000, " + "cluster state version: 0, nodes: [0, 2, 1 (source only)], chain: [], " + "reasons to start: ) => 0"); + + CPPUNIT_ASSERT_EQUAL(merge, _sender.getLastCommand(true)); + { + const api::MergeBucketCommand& cmd( + dynamic_cast<api::MergeBucketCommand&>(*_sender.commands[0])); + CPPUNIT_ASSERT_EQUAL(uint16_t(0), cmd.getSourceIndex()); + } + + // Source-only copy changed during merge + addNodesToBucketDB(document::BucketId(16, 1), + "0=10/1/1/t," + "1=40/1/1," + "2=10/1/1/t"); + sendReply(op); + // Should not be a remove here! + CPPUNIT_ASSERT_EQUAL(merge, _sender.getLastCommand(true)); + CPPUNIT_ASSERT(!op.ok()); +} + +namespace { +std::string getNodeList(std::string state, uint32_t redundancy, std::string existing) { + lib::Distribution distribution( + lib::Distribution::getDefaultDistributionConfig(redundancy)); + lib::ClusterState clusterState(state); + vespalib::StringTokenizer st(existing, ","); + std::vector<BucketCopy> bucketDB(st.size()); + for (uint32_t i = 0; i < st.size(); i++) { + std::string num = st[i]; + size_t pos = num.find('t'); + bool trusted = false; + + if (pos != std::string::npos) { + num.erase(pos); + trusted = true; + } + bucketDB[i] = BucketCopy(0, atoi(num.c_str()), + api::BucketInfo(1, 2, 3)); + bucketDB[i].setTrusted(trusted); + } + std::vector<MergeMetaData> nodes(st.size()); + for (uint32_t i = 0; i < st.size(); i++) { + nodes[i] = MergeMetaData(bucketDB[i].getNode(), bucketDB[i]); + } + MergeLimiter limiter(16); + MergeOperation::generateSortedNodeList(distribution, clusterState, + document::BucketId(32, 1), + limiter, nodes); + std::ostringstream actual; + for (uint32_t i = 0; i < nodes.size(); i++) { + if (i != 0) { + actual << ","; + } + actual << nodes[i]._nodeIndex; + if (nodes[i]._sourceOnly) { + actual << "s"; + } + } + return actual.str(); +} +} + +void +MergeOperationTest::testGenerateNodeList() +{ + // If this fails, the distribution has changed and the rest of the test will + // likely fail + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0,9,2,1,4"), + getNodeList("storage:10", 10, "0,1,2,3,4,5,6,7,8,9")); + + // Nodes that are initializing should be treated as up + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7s,6s"), + getNodeList("storage:10 .3.s:i .5.s:i", 2, "7,6,3,5")); // Ideal: 3,5 + + // Order is given by ideal state algorithm, not order of storagenodes in bucket db + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7"), + getNodeList("storage:10", 3, "3,7,5")); + + // Node not in ideal state will be used if not enough nodes in ideal state + CPPUNIT_ASSERT_EQUAL( + std::string("3,7,6"), + getNodeList("storage:10", 3, "3,7,6")); + + // Nodes not in ideal state will be included as source only after redundancy + // is reached + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,8s"), + getNodeList("storage:10", 3, "3,5,7,8")); + + // Need at least redundancy copies that are not source only + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,8,9s"), + getNodeList("storage:10", 3, "3,5,8,9")); + + // Order is given by storagenodes in bucket db + // when no nodes are in ideal state + CPPUNIT_ASSERT_EQUAL( + std::string("4,1,2"), + getNodeList("storage:10", 3, "4,1,2")); + + CPPUNIT_ASSERT_EQUAL( + std::string("3,0s,1s,2s,4s,5s,6s,7s,8s,9s"), + getNodeList("storage:10", 1, "0,1,2,3,4,5,6,7,8,9")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,0s,1s,2s,4s,6s,7s,8s,9s"), + getNodeList("storage:10", 2, "0,1,2,3,4,5,6,7,8,9")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,0s,1s,2s,4s,6s,8s,9s"), + getNodeList("storage:10", 3, "0,1,2,3,4,5,6,7,8,9")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,0s,1s,2s,4s,8s,9s"), + getNodeList("storage:10", 4, "0,1,2,3,4,5,6,7,8,9")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0s,1s,2s,4s,9s"), + getNodeList("storage:10", 5, "0,1,2,3,4,5,6,7,8,9")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0,1s,2s,4s,9s"), + getNodeList("storage:10", 6, "0,1,2,3,4,5,6,7,8,9")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0,9,1s,2s,4s"), + getNodeList("storage:10", 7, "0,1,2,3,4,5,6,7,8,9")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0,9,2,1s,4s"), + getNodeList("storage:10", 8, "0,1,2,3,4,5,6,7,8,9")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0,9,2,1,4s"), + getNodeList("storage:10", 9, "0,1,2,3,4,5,6,7,8,9")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0,9,2,1,4"), + getNodeList("storage:10", 10, "0,1,2,3,4,5,6,7,8,9")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,9s,8s,7s,6s,5s,4s,2s,1s,0s"), + getNodeList("storage:10", 1, "9,8,7,6,5,4,3,2,1,0")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,9s,8s,7s,6s,4s,2s,1s,0s"), + getNodeList("storage:10", 2, "9,8,7,6,5,4,3,2,1,0")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,9s,8s,6s,4s,2s,1s,0s"), + getNodeList("storage:10", 3, "9,8,7,6,5,4,3,2,1,0")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,9s,8s,4s,2s,1s,0s"), + getNodeList("storage:10", 4, "9,8,7,6,5,4,3,2,1,0")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,9s,4s,2s,1s,0s"), + getNodeList("storage:10", 5, "9,8,7,6,5,4,3,2,1,0")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0,9s,4s,2s,1s"), + getNodeList("storage:10", 6, "9,8,7,6,5,4,3,2,1,0")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0,9,4s,2s,1s"), + getNodeList("storage:10", 7, "9,8,7,6,5,4,3,2,1,0")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0,9,2,4s,1s"), + getNodeList("storage:10", 8, "9,8,7,6,5,4,3,2,1,0")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0,9,2,1,4s"), + getNodeList("storage:10", 9, "9,8,7,6,5,4,3,2,1,0")); + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0,9,2,1,4"), + getNodeList("storage:10", 10, "9,8,7,6,5,4,3,2,1,0")); + + // Trusted copies should not be source only. + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0,9,1,2,4s"), + getNodeList("storage:10", 7, "0,1t,2t,3,4,5,6,7,8,9")); + + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0,9,2,1s,4s"), + getNodeList("storage:10", 7, "0,1,2t,3,4,5,6,7,8,9")); + + // Retired nodes are not in ideal state + // Ideal: 5,7 + CPPUNIT_ASSERT_EQUAL( + std::string("0,2,3s"), + getNodeList("storage:10 .3.s:r", 2, "0,2,3")); + // Ideal: 5,7,6 + CPPUNIT_ASSERT_EQUAL( + std::string("0,2,3"), + getNodeList("storage:10 .3.s:r", 3, "0,2,3")); +} + +void +MergeOperationTest::doNotRemoveCopiesWithPendingMessages() +{ + document::BucketId bucket(16, 1); + + getClock().setAbsoluteTimeInSeconds(10); + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:3")); + addNodesToBucketDB(bucket, + "0=10/1/1/t," + "1=20/1/1," + "2=10/1/1/t"); + + MergeOperation op(BucketAndNodes(bucket, + toVector<uint16_t>(0, 1, 2))); + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + std::string merge("MergeBucketCommand(BucketId(0x4000000000000001), to time 10000000, " + "cluster state version: 0, nodes: [0, 2, 1 (source only)], chain: [], " + "reasons to start: ) => 0"); + + CPPUNIT_ASSERT_EQUAL(merge, _sender.getLastCommand(true)); + + // Suddenly a wild operation appears to the source only copy! + // Removes are blocked by all and any operation types, so can just choose + // at will. + api::StorageMessage::SP msg( + new api::SetBucketStateCommand(bucket, api::SetBucketStateCommand::ACTIVE)); + msg->setAddress(api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 1)); + _pendingTracker->insert(msg); + + + sendReply(op); + // Should not be a remove here! + CPPUNIT_ASSERT_EQUAL(merge, _sender.getLastCommand(true)); + CPPUNIT_ASSERT(!op.ok()); +} + +void +MergeOperationTest::testDoNotRemoveActiveSourceOnlyCopies() +{ + getClock().setAbsoluteTimeInSeconds(10); + + addNodesToBucketDB(document::BucketId(16, 1), + "0=10/1/1/t," + "1=20/1/1/u/a," + "2=10/1/1/t"); + + _distributor->enableClusterState( + lib::ClusterState("distributor:1 storage:3")); + MergeOperation op(BucketAndNodes(document::BucketId(16, 1), + toVector<uint16_t>(0, 1, 2))); + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + std::string merge( + "MergeBucketCommand(BucketId(0x4000000000000001), to time " + "10000000, cluster state version: 0, nodes: [0, 2, 1 " + "(source only)], chain: [], reasons to start: ) => 0"); + CPPUNIT_ASSERT_EQUAL(merge, _sender.getLastCommand(true)); + + sendReply(op); + // No DeleteBucket shall have been sent + CPPUNIT_ASSERT_EQUAL(merge, _sender.getLastCommand(true)); +} + +void +MergeOperationTest::testMarkRedundantTrustedCopiesAsSourceOnly() +{ + // This test uses the same distribution as testGenerateNodeList(), i.e. + // an ideal state sequence of [3, 5, 7, 6, 8, 0, 9, 2, 1, 4] + + // 3 redundancy, 5 trusted -> 2 trusted source only. + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6s,8s"), + getNodeList("storage:10", 3, "3t,5t,7t,6t,8t")); + + // 3 redundancy, 4 trusted -> 1 trusted source only. + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6s,8s"), + getNodeList("storage:10", 3, "3t,5t,7t,6t,8")); + + // 3 redundancy, 3 trusted -> 0 trusted source only, 2 non-trusted sources. + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6s,8s"), + getNodeList("storage:10", 3, "3t,5t,7t,6,8")); + + // 3 redundancy, 4 trusted -> 1 source only trusted. + // We allow marking a trusted, non-ideal copy as source even when we don't + // have #redundancy trusted _ideal_ copies, as long as we're left with >= + // #redundancy trusted copies in total. + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8s"), + getNodeList("storage:10", 3, "3t,5t,7,6t,8t")); + + // Not sufficient number of trusted copies to mark any as source only. + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8"), + getNodeList("storage:10", 3, "3t,5,7,6t,8t")); + + // Same as above, with all trusted copies being non-ideal. + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8"), + getNodeList("storage:10", 3, "3,5,7,6t,8t")); + + // #redundancy of trusted, but none are ideal. Non-ideal trusted should + // not be marked as source only (though we can mark non-trusted non-ideal + // node as source only). + // Note the node reordering since trusted are added before the rest. + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,8,0,9,6s"), + getNodeList("storage:10", 3, "3,5,7,6,8t,0t,9t")); + + // But allow for removing excess trusted, non-ideal copies. + CPPUNIT_ASSERT_EQUAL( + std::string("3,5,7,6,8,0,9s"), + getNodeList("storage:10", 3, "3,5,7,6t,8t,0t,9t")); +} + +void +MergeOperationTest::onlyMarkRedundantRetiredReplicasAsSourceOnly() +{ + // No nodes in ideal state and all nodes are retired. With redundancy of 2 + // we can only mark the last replica in the DB as source-only. Retired + // nodes are meant as source-only due to being migrated away from, but + // source-only nodes will have their replica removed after a successful + // merge, which we cannot allow to happen here. + CPPUNIT_ASSERT_EQUAL( + std::string("0,1,2s"), + getNodeList("storage:3 .0.s.:r .1.s:r .2.s:r", 2, "1,0,2")); +} + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/messagesenderstub.cpp b/storage/src/tests/distributor/messagesenderstub.cpp new file mode 100644 index 00000000000..88210a94848 --- /dev/null +++ b/storage/src/tests/distributor/messagesenderstub.cpp @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <tests/distributor/messagesenderstub.h> +#include <tests/distributor/distributortestutil.h> + +namespace storage { + +std::string +MessageSenderStub::getLastCommand(bool verbose) const +{ + if (commands.empty()) { + throw std::logic_error("Expected command where there was none"); + } + return dumpMessage(*commands[commands.size() - 1], + true, + verbose); + +} + +std::string +MessageSenderStub::dumpMessage(const api::StorageMessage& msg, + bool includeAddress, + bool verbose) const +{ + std::ostringstream ost; + + if (verbose) { + ost << msg; + } else { + ost << msg.getType().getName(); + } + + if (includeAddress && msg.getAddress()) { + ost << " => " << msg.getAddress()->getIndex(); + } + if (verbose && msg.getType().isReply()) { + ost << " " << dynamic_cast<const api::StorageReply&>(msg).getResult(); + } + + return ost.str(); +} + +std::string +MessageSenderStub::getCommands(bool includeAddress, bool verbose, uint32_t fromIdx) const +{ + std::ostringstream ost; + + for (uint32_t i = fromIdx; i < commands.size(); i++) { + if (i != fromIdx) { + ost << ","; + } + + ost << dumpMessage(*commands[i], includeAddress, verbose); + } + + return ost.str(); +} + +std::string +MessageSenderStub::getLastReply(bool verbose) const +{ + if (replies.empty()) { + throw std::logic_error("Expected reply where there was none"); + } + + return dumpMessage(*replies.back(), + true, + verbose); + +} + +std::string +MessageSenderStub::getReplies(bool includeAddress, bool verbose) const +{ + std::ostringstream ost; + for (uint32_t i = 0; i < replies.size(); i++) { + if (i != 0) { + ost << ","; + } + + ost << dumpMessage(*replies[i], includeAddress, verbose); + } + + return ost.str(); +} + +} + diff --git a/storage/src/tests/distributor/messagesenderstub.h b/storage/src/tests/distributor/messagesenderstub.h new file mode 100644 index 00000000000..d70c5355868 --- /dev/null +++ b/storage/src/tests/distributor/messagesenderstub.h @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/storage/distributor/distributor.h> +#include <vespa/storage/distributor/distributormessagesender.h> + +namespace storage { + +struct MessageSenderStub : distributor::DistributorMessageSender +{ + std::vector<std::shared_ptr<api::StorageCommand> > commands; + std::vector<std::shared_ptr<api::StorageReply> > replies; + + MessageSenderStub() + : _clusterName("storage"), + _pendingMessageTracker(0) + {} + + void clear() { + commands.clear(); + replies.clear(); + } + + virtual void sendCommand(const std::shared_ptr<api::StorageCommand>& cmd) + { + commands.push_back(cmd); + } + + virtual void sendReply(const std::shared_ptr<api::StorageReply>& reply) + { + replies.push_back(reply); + } + + std::string getLastCommand(bool verbose = true) const; + + std::string getCommands(bool includeAddress = false, + bool verbose = false, + uint32_t fromIndex = 0) const; + + std::string getLastReply(bool verbose = true) const; + + std::string getReplies(bool includeAddress = false, + bool verbose = false) const; + + std::string dumpMessage(const api::StorageMessage& msg, + bool includeAddress, + bool verbose) const; + + virtual int getDistributorIndex() const { + return 0; + } + + virtual const std::string& getClusterName() const { + return _clusterName; + } + + virtual const distributor::PendingMessageTracker& getPendingMessageTracker() const { + assert(_pendingMessageTracker); + return *_pendingMessageTracker; + } + + void setPendingMessageTracker(distributor::PendingMessageTracker& tracker) { + _pendingMessageTracker = &tracker; + } +private: + std::string _clusterName; + distributor::PendingMessageTracker* _pendingMessageTracker; +}; + +} + diff --git a/storage/src/tests/distributor/nodeinfotest.cpp b/storage/src/tests/distributor/nodeinfotest.cpp new file mode 100644 index 00000000000..883e6ba7668 --- /dev/null +++ b/storage/src/tests/distributor/nodeinfotest.cpp @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <iomanip> +#include <iostream> +#include <memory> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> +#include <vespa/storage/distributor/bucketdbupdater.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/vdslib/state/random.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storage/distributor/pendingclusterstate.h> +#include <vespa/vespalib/text/stringtokenizer.h> +#include <vespa/storage/distributor/nodeinfo.h> + +#include <iostream> +#include <fstream> +#include <string> + +namespace storage { +namespace distributor { + +class NodeInfoTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(NodeInfoTest); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST_SUITE_END(); +public: + void testSimple(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(NodeInfoTest); + +void +NodeInfoTest::testSimple() +{ + framework::defaultimplementation::FakeClock clock; + NodeInfo info(clock); + + CPPUNIT_ASSERT_EQUAL(0, (int)info.getPendingCount(3)); + CPPUNIT_ASSERT_EQUAL(0, (int)info.getPendingCount(9)); + + info.incPending(3); + info.incPending(3); + info.incPending(3); + info.incPending(3); + info.decPending(3); + info.decPending(4); + info.incPending(7); + info.incPending(4); + info.decPending(3); + + CPPUNIT_ASSERT_EQUAL(2, (int)info.getPendingCount(3)); + CPPUNIT_ASSERT_EQUAL(1, (int)info.getPendingCount(4)); + CPPUNIT_ASSERT_EQUAL(1, (int)info.getPendingCount(7)); + CPPUNIT_ASSERT_EQUAL(0, (int)info.getPendingCount(5)); + + info.setBusy(5); + clock.addSecondsToTime(10); + info.setBusy(1); + clock.addSecondsToTime(20); + info.setBusy(42); + + CPPUNIT_ASSERT_EQUAL(true, info.isBusy(5)); + CPPUNIT_ASSERT_EQUAL(true, info.isBusy(1)); + CPPUNIT_ASSERT_EQUAL(true, info.isBusy(42)); + CPPUNIT_ASSERT_EQUAL(false, info.isBusy(7)); + + clock.addSecondsToTime(42); + + CPPUNIT_ASSERT_EQUAL(false, info.isBusy(5)); + CPPUNIT_ASSERT_EQUAL(false, info.isBusy(1)); + CPPUNIT_ASSERT_EQUAL(true, info.isBusy(42)); + CPPUNIT_ASSERT_EQUAL(false, info.isBusy(7)); + +} + +} + +} diff --git a/storage/src/tests/distributor/nodemaintenancestatstrackertest.cpp b/storage/src/tests/distributor/nodemaintenancestatstrackertest.cpp new file mode 100644 index 00000000000..f1c177e7939 --- /dev/null +++ b/storage/src/tests/distributor/nodemaintenancestatstrackertest.cpp @@ -0,0 +1,102 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> + +#include <vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h> + +namespace storage { +namespace distributor { + +class NodeMaintenanceStatsTrackerTest : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE(NodeMaintenanceStatsTrackerTest); + CPPUNIT_TEST(emptyStatsInstancesAreEqual); + CPPUNIT_TEST(statsFieldsAffectEqualityComparison); + CPPUNIT_TEST(requestingNonExistingNodeGivesEmptyStats); + CPPUNIT_TEST(statsAreTrackedPerNode); + CPPUNIT_TEST_SUITE_END(); + + void emptyStatsInstancesAreEqual(); + void statsFieldsAffectEqualityComparison(); + void requestingNonExistingNodeGivesEmptyStats(); + void statsAreTrackedPerNode(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(NodeMaintenanceStatsTrackerTest); + +void +NodeMaintenanceStatsTrackerTest::emptyStatsInstancesAreEqual() +{ + NodeMaintenanceStats a; + NodeMaintenanceStats b; + CPPUNIT_ASSERT_EQUAL(a, b); +} + +void +NodeMaintenanceStatsTrackerTest::statsFieldsAffectEqualityComparison() +{ + NodeMaintenanceStats a; + NodeMaintenanceStats b; + + a.movingOut = 1; + CPPUNIT_ASSERT(!(a == b)); + b.movingOut = 1; + CPPUNIT_ASSERT(a == b); + + a.syncing = 1; + CPPUNIT_ASSERT(!(a == b)); + b.syncing = 1; + CPPUNIT_ASSERT(a == b); + + a.copyingIn = 1; + CPPUNIT_ASSERT(!(a == b)); + b.copyingIn = 1; + CPPUNIT_ASSERT(a == b); + + a.copyingOut = 1; + CPPUNIT_ASSERT(!(a == b)); + b.copyingOut = 1; + CPPUNIT_ASSERT(a == b); +} + +void +NodeMaintenanceStatsTrackerTest::requestingNonExistingNodeGivesEmptyStats() +{ + NodeMaintenanceStatsTracker tracker; + NodeMaintenanceStats wanted; + CPPUNIT_ASSERT_EQUAL(wanted, tracker.forNode(0)); +} + +void +NodeMaintenanceStatsTrackerTest::statsAreTrackedPerNode() +{ + NodeMaintenanceStatsTracker tracker; + NodeMaintenanceStats wanted; + + tracker.incMovingOut(0); + wanted.movingOut = 1; + CPPUNIT_ASSERT_EQUAL(wanted, tracker.forNode(0)); + wanted.movingOut = 0; + CPPUNIT_ASSERT_EQUAL(wanted, tracker.forNode(1)); + + tracker.incMovingOut(0); + wanted.movingOut = 2; + CPPUNIT_ASSERT_EQUAL(wanted, tracker.forNode(0)); + + tracker.incMovingOut(1); + wanted.movingOut = 1; + CPPUNIT_ASSERT_EQUAL(wanted, tracker.forNode(1)); + + tracker.incSyncing(1); + tracker.incCopyingIn(1); + tracker.incCopyingOut(1); + wanted.syncing = 1; + wanted.copyingIn = 1; + wanted.copyingOut = 1; + CPPUNIT_ASSERT_EQUAL(wanted, tracker.forNode(1)); +} + +} // distributor +} // storage + diff --git a/storage/src/tests/distributor/operationtargetresolvertest.cpp b/storage/src/tests/distributor/operationtargetresolvertest.cpp new file mode 100644 index 00000000000..5b23d3a7a9e --- /dev/null +++ b/storage/src/tests/distributor/operationtargetresolvertest.cpp @@ -0,0 +1,316 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> + +#include <iomanip> +#include <iostream> +#include <memory> +#include <vespa/config/helper/configgetter.h> +#include <vespa/document/config/config-documenttypes.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/state.h> +#include <tests/distributor/distributortestutil.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/vdslib/distribution/idealnodecalculatorimpl.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/distributor/operationtargetresolverimpl.h> + +using document::BucketId; + +namespace storage { +namespace distributor { + +struct OperationTargetResolverTest : public CppUnit::TestFixture, + public DistributorTestUtil +{ + + document::DocumentTypeRepo::SP _repo; + const document::DocumentType* _html_type; + std::unique_ptr<Operation> op; + + void testSimple(); + void testMultipleNodes(); + void testChooseIdealStateWhenManyCopies(); + void testChooseHighestSplitBucket(); + void testChooseHighestSplitBucketPerNode(); + void testChooseHighestSplitBucketWithTrusted(); + void testInconsistentBucketsAreNotExplicitlyCreated(); + void testNoTrustedOrIdealStateCopyAvailable(); + void testCreateMissingCopies(); + void testNoExistingCopies(); + void testCountMaintenanceNodesAsDown(); + void testResolvingDoesNotMutateDatabase(); + void testTrustedOverIdealState(); + + BucketInstanceList getInstances(const BucketId& bid, + bool stripToRedundancy); + + void setUp() { + _repo.reset(new document::DocumentTypeRepo( + *config::ConfigGetter<document::DocumenttypesConfig>::getConfig( + "config-doctypes", config::FileSpec("config-doctypes.cfg")))); + _html_type = _repo->getDocumentType("text/html"); + createLinks(); + }; + + void tearDown() { + close(); + } + + CPPUNIT_TEST_SUITE(OperationTargetResolverTest); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST(testMultipleNodes); + CPPUNIT_TEST(testChooseIdealStateWhenManyCopies); + CPPUNIT_TEST(testChooseHighestSplitBucket); + CPPUNIT_TEST(testChooseHighestSplitBucketPerNode); + CPPUNIT_TEST(testChooseHighestSplitBucketWithTrusted); + CPPUNIT_TEST(testNoTrustedOrIdealStateCopyAvailable); + CPPUNIT_TEST(testInconsistentBucketsAreNotExplicitlyCreated); + CPPUNIT_TEST(testCreateMissingCopies); + CPPUNIT_TEST(testNoExistingCopies); + CPPUNIT_TEST(testCountMaintenanceNodesAsDown); + CPPUNIT_TEST(testResolvingDoesNotMutateDatabase); + CPPUNIT_TEST(testTrustedOverIdealState); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(OperationTargetResolverTest); + +namespace { + + // Create assertion that makes it easy to write tests, and report correct + // line for problem at command line +#define ASSERT_THAT(id) \ + { \ + struct MyAsserter : public Asserter { \ + void assertEqualMsg(std::string t1, OperationTargetList t2, \ + OperationTargetList t3) { \ + CPPUNIT_ASSERT_EQUAL_MSG(t1, t2, t3); \ + } \ + }; \ + _asserters.push_back(new MyAsserter); \ + } \ + TestTargets::createTest(id, *this, *_asserters.back()) + + struct Asserter { + virtual ~Asserter() {} + virtual void assertEqualMsg(std::string t1, + OperationTargetList t2, + OperationTargetList t3) = 0; + }; + std::vector<Asserter*> _asserters; + struct TestTargets { + const BucketId& _id; + OperationTargetList _expected; + OperationTargetResolverTest& _test; + Asserter& _asserter; + + TestTargets(const BucketId& id, + OperationTargetResolverTest& test, + Asserter& asserter) + : _id(id), _test(test), _asserter(asserter) {} + + ~TestTargets() { + BucketInstanceList result(_test.getInstances(_id, true)); + BucketInstanceList all(_test.getInstances(_id, false)); + _asserter.assertEqualMsg( + all.toString(), _expected, result.createTargets()); + delete _asserters.back(); + _asserters.pop_back(); + } + + TestTargets& sendsTo(const BucketId& id, uint16_t node) { + _expected.push_back(OperationTarget( + id, lib::Node(lib::NodeType::STORAGE, node), false)); + return *this; + } + TestTargets& createsAt(const BucketId& id, uint16_t node) { + _expected.push_back(OperationTarget( + id, lib::Node(lib::NodeType::STORAGE, node), true)); + return *this; + } + + static TestTargets createTest(const BucketId& id, + OperationTargetResolverTest& test, + Asserter& asserter) + { + return TestTargets(id, test, asserter); + } + }; + + +} // anonymous + +BucketInstanceList +OperationTargetResolverTest::getInstances(const BucketId& id, + bool stripToRedundancy) +{ + lib::IdealNodeCalculatorImpl idealNodeCalc; + idealNodeCalc.setDistribution(getExternalOperationHandler().getDistribution()); + idealNodeCalc.setClusterState(getExternalOperationHandler().getClusterState()); + OperationTargetResolverImpl resolver( + getExternalOperationHandler().getBucketDatabase(), idealNodeCalc, 16, + getExternalOperationHandler().getDistribution().getRedundancy()); + if (stripToRedundancy) { + return resolver.getInstances(OperationTargetResolver::PUT, id); + } else { + return resolver.getAllInstances(OperationTargetResolver::PUT, id); + } +} + +/* + * Test basic case with no inconsistencies + */ +void +OperationTargetResolverTest::testSimple() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + addNodesToBucketDB(BucketId(16, 0), "0=0,1=0"); + + ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(16, 0), 1) + .sendsTo(BucketId(16, 0), 0); +} + +void +OperationTargetResolverTest::testMultipleNodes() +{ + setupDistributor(1, 2, "storage:2 distributor:1"); + + for (int i = 0; i < 100; ++i) { + addNodesToBucketDB(BucketId(16, i), "0=0,1=0"); + + lib::IdealNodeCalculatorImpl idealNodeCalc; + idealNodeCalc.setDistribution(getExternalOperationHandler().getDistribution()); + idealNodeCalc.setClusterState(getExternalOperationHandler().getClusterState()); + lib::IdealNodeList idealNodes( + idealNodeCalc.getIdealStorageNodes(BucketId(16, i))); + uint16_t expectedNode = idealNodes[0].getIndex(); + ASSERT_THAT(BucketId(32, i)).sendsTo(BucketId(16, i), expectedNode); + } +} + +void +OperationTargetResolverTest::testChooseIdealStateWhenManyCopies() +{ + setupDistributor(2, 4, "storage:4 distributor:1"); + addNodesToBucketDB(BucketId(16, 0), "0=0,1=0,2=0,3=0"); // ideal nodes: 1, 3 + ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(16, 0), 1) + .sendsTo(BucketId(16, 0), 3); +} + +void +OperationTargetResolverTest::testTrustedOverIdealState() +{ + setupDistributor(2, 4, "storage:4 distributor:1"); + addNodesToBucketDB(BucketId(16, 0), "0=0/0/0/t,1=0,2=0/0/0/t,3=0"); + // ideal nodes: 1, 3 + ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(16, 0), 0) + .sendsTo(BucketId(16, 0), 2); +} + +void +OperationTargetResolverTest::testChooseHighestSplitBucket() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + // 0, 1 are both in ideal state for both buckets. + addNodesToBucketDB(BucketId(16, 0), "0=0,1=0"); + addNodesToBucketDB(BucketId(17, 0), "0=0,1=0"); + ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(17, 0), 1) + .sendsTo(BucketId(17, 0), 0); +} + +void +OperationTargetResolverTest::testChooseHighestSplitBucketPerNode() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + addNodesToBucketDB(BucketId(16, 0), "1=0"); + addNodesToBucketDB(BucketId(17, 0), "0=0"); + ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(17, 0), 0) + .sendsTo(BucketId(16, 0), 1); +} + +void +OperationTargetResolverTest::testChooseHighestSplitBucketWithTrusted() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + // Unfinished split scenario: split done on 0, not on 1. + // Copy on 1 is only remaining for (16, 0), so always trusted. + addNodesToBucketDB(BucketId(16, 0), "1=1/2/3/t"); + addNodesToBucketDB(BucketId(17, 0), "0=2/3/4/t"); + addNodesToBucketDB(BucketId(17, 1ULL << 16), "0=3/4/5/t"); + ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(17, 0), 0) + .sendsTo(BucketId(16, 0), 1); +} + +void +OperationTargetResolverTest::testInconsistentBucketsAreNotExplicitlyCreated() +{ + setupDistributor(2, 2, "bits:8 storage:2 distributor:1"); + addNodesToBucketDB(BucketId(15, 0), "1=9/9/9/t"); + addNodesToBucketDB(BucketId(16, 1 << 15), "0=9/9/9/t"); + // (32, 0) belongs in (16, 0) subtree, but it does not exist. We cannot + // create a bucket on (15, 0) node 0 since that will explicitly introduce + // an inconsistent bucket in its local state. Note that we still _send_ to + // the inconsistent (15, 0) bucket since it already exists and will be + // split out very soon anyway. This is predominantly to avoid making things + // even worse than they are and to avoid the edge case in bug 7296087. + ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(15, 0), 1) + .createsAt(BucketId(16, 0), 0); +} + +void +OperationTargetResolverTest::testNoTrustedOrIdealStateCopyAvailable() +{ + setupDistributor(2, 4, "storage:4 distributor:1"); + addNodesToBucketDB(BucketId(16, 0), "0=0,2=0"); + addNodesToBucketDB(BucketId(18, 0), "0=0"); // ideal nodes: 1, 3 + ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(18, 0), 0) + .sendsTo(BucketId(16, 0), 2); +} + +void +OperationTargetResolverTest::testCreateMissingCopies() +{ + setupDistributor(4, 10, "storage:10 distributor:1"); + addNodesToBucketDB(BucketId(16, 0), "6=0"); + addNodesToBucketDB(BucketId(18, 0), "4=0"); // ideal nodes: 6, 8, 7, 1 + + ASSERT_THAT(BucketId(32, 0)).sendsTo(BucketId(18, 0), 4) + .sendsTo(BucketId(16, 0), 6) + .createsAt(BucketId(18, 0), 8) + .createsAt(BucketId(18, 0), 7); +} + +void +OperationTargetResolverTest::testNoExistingCopies() +{ + setupDistributor(2, 5, "storage:5 distributor:1"); + + ASSERT_THAT(BucketId(32, 0)).createsAt(BucketId(16, 0), 1) + .createsAt(BucketId(16, 0), 3); +} + +void +OperationTargetResolverTest::testCountMaintenanceNodesAsDown() +{ + setupDistributor(2, 5, "storage:5 .1.s:m distributor:1"); + + ASSERT_THAT(BucketId(32, 0)).createsAt(BucketId(16, 0), 3) + .createsAt(BucketId(16, 0), 2); +} + +void +OperationTargetResolverTest::testResolvingDoesNotMutateDatabase() +{ + setupDistributor(2, 5, "storage:5 distributor:1"); + + ASSERT_THAT(BucketId(32, 0)).createsAt(BucketId(16, 0), 1) + .createsAt(BucketId(16, 0), 3); + + CPPUNIT_ASSERT_EQUAL(std::string("NONEXISTING"), + dumpBucket(BucketId(0x4000000000000000))); +} + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/pendingmessagetrackertest.cpp b/storage/src/tests/distributor/pendingmessagetrackertest.cpp new file mode 100644 index 00000000000..f69525836be --- /dev/null +++ b/storage/src/tests/distributor/pendingmessagetrackertest.cpp @@ -0,0 +1,674 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> + +#include <vespa/document/base/testdocman.h> +#include <vespa/storage/distributor/pendingmessagetracker.h> +#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/vdslib/state/random.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <fstream> +#include <sstream> +#include <iomanip> +#include <iostream> +#include <memory> +#include <string> +#include <iterator> + +namespace storage { +namespace distributor { + +// Workaround typedef for not (yet) running with --std=c++14 which supports +// user defined literals. Once we do, replace ms(123) with 123ms. +using ms = std::chrono::milliseconds; + +class PendingMessageTrackerCallback_Test : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(PendingMessageTrackerCallback_Test); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST(testMultipleMessages); + CPPUNIT_TEST(testStartPage); + CPPUNIT_TEST(testGetPendingMessageTypes); + CPPUNIT_TEST(testHasPendingMessage); + CPPUNIT_TEST(testGetAllMessagesForSingleBucket); + CPPUNIT_TEST(nodeStatsCanBeOutputStreamed); + CPPUNIT_TEST(totalPutLatencyIsInitiallyZero); + CPPUNIT_TEST(statsNotAlteredBeforeReplyReceived); + CPPUNIT_TEST(totalPutLatencyIsTrackedForSingleRequest); + CPPUNIT_TEST(statsAreTrackedSeparatelyPerNode); + CPPUNIT_TEST(onlyPutMessagesAreTracked); + CPPUNIT_TEST(totalPutLatencyIsAggregatedAcrossRequests); + CPPUNIT_TEST(clearingMessagesDoesNotAffectStats); + CPPUNIT_TEST(timeTravellingClockLatenciesNotRegistered); + CPPUNIT_TEST(statsSnapshotIncludesAllNodes); + CPPUNIT_TEST(latencyProviderForwardsToImplementation); + CPPUNIT_TEST_SUITE_END(); + +public: + void testSimple(); + void testMultipleMessages(); + void testStartPage(); + void testGetPendingMessageTypes(); + void testHasPendingMessage(); + void testGetAllMessagesForSingleBucket(); + void nodeStatsCanBeOutputStreamed(); + void totalPutLatencyIsInitiallyZero(); + void statsNotAlteredBeforeReplyReceived(); + void totalPutLatencyIsTrackedForSingleRequest(); + void statsAreTrackedSeparatelyPerNode(); + void onlyPutMessagesAreTracked(); + void totalPutLatencyIsAggregatedAcrossRequests(); + void clearingMessagesDoesNotAffectStats(); + void timeTravellingClockLatenciesNotRegistered(); + void statsSnapshotIncludesAllNodes(); + void latencyProviderForwardsToImplementation(); + +private: + void insertMessages(PendingMessageTracker& tracker); + + OperationStats makeOpStats(std::chrono::milliseconds totalLatency, + uint64_t numRequests) const + { + OperationStats stats; + stats.totalLatency = totalLatency; + stats.numRequests = numRequests; + return stats; + } +}; + +bool +operator==(const OperationStats& a, const OperationStats& b) +{ + return (a.totalLatency == b.totalLatency + && a.numRequests == b.numRequests); +} + +namespace { + +class RequestBuilder { + uint16_t _toNode; + std::chrono::milliseconds _atTime; +public: + RequestBuilder() + : _toNode(0), + _atTime() + { + } + + RequestBuilder& atTime(std::chrono::milliseconds t) { + _atTime = t; + return *this; + } + + RequestBuilder& toNode(uint16_t node) { + _toNode = node; + return *this; + } + + uint16_t toNode() const { return _toNode; } + std::chrono::milliseconds atTime() const { return _atTime; } +}; + +class Fixture +{ + StorageComponentRegisterImpl _compReg; + framework::defaultimplementation::FakeClock _clock; + std::unique_ptr<PendingMessageTracker> _tracker; + document::TestDocMan _testDocMan; +public: + + Fixture() + : _compReg(), + _clock(), + _tracker(), + _testDocMan() + { + _compReg.setClock(_clock); + _clock.setAbsoluteTimeInSeconds(1); + // Have to set clock in compReg before constructing tracker, or it'll + // flip out and die on an explicit nullptr check. + _tracker = std::unique_ptr<PendingMessageTracker>( + new PendingMessageTracker(_compReg)); + } + + std::shared_ptr<api::PutCommand> sendPut(const RequestBuilder& builder) { + assignMockedTime(builder.atTime()); + auto put = createPutToNode(builder.toNode()); + _tracker->insert(put); + return put; + } + + void sendPutReply(api::PutCommand& putCmd, + const RequestBuilder& builder) + { + assignMockedTime(builder.atTime()); + auto putReply = putCmd.makeReply(); + _tracker->reply(*putReply); + } + + std::shared_ptr<api::RemoveCommand> sendRemove( + const RequestBuilder& builder) + { + assignMockedTime(builder.atTime()); + auto remove = createRemoveToNode(builder.toNode()); + _tracker->insert(remove); + return remove; + } + + void sendRemoveReply(api::RemoveCommand& removeCmd, + const RequestBuilder& builder) + { + assignMockedTime(builder.atTime()); + auto removeReply = removeCmd.makeReply(); + _tracker->reply(*removeReply); + } + + void sendPutAndReplyWithLatency(uint16_t node, + std::chrono::milliseconds latency) + { + auto put = sendPut(RequestBuilder().atTime(ms(1000)).toNode(node)); + sendPutReply(*put, RequestBuilder().atTime(ms(1000) + latency)); + } + + OperationStats getNodePutOperationStats(uint16_t node) { + return _tracker->getNodeStats(node).puts; + } + + PendingMessageTracker& tracker() { return *_tracker; } + +private: + std::string createDummyIdString(const document::BucketId& bucket) const { + std::ostringstream id; + id << "id:foo:testdoctype1:n=" << bucket.getId() << ":foo"; + return id.str(); + } + + document::Document::SP createDummyDocumentForBucket( + const document::BucketId& bucket) const + { + return _testDocMan.createDocument("foobar", + createDummyIdString(bucket)); + } + + api::StorageMessageAddress makeStorageAddress(uint16_t node) const { + return {"storage", lib::NodeType::STORAGE, node}; + } + + std::shared_ptr<api::PutCommand> createPutToNode(uint16_t node) const { + document::BucketId bucket(16, 1234); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bucket, + createDummyDocumentForBucket(bucket), + api::Timestamp(123456))); + cmd->setAddress(makeStorageAddress(node)); + return cmd; + } + + std::shared_ptr<api::RemoveCommand> createRemoveToNode( + uint16_t node) const + { + document::BucketId bucket(16, 1234); + std::shared_ptr<api::RemoveCommand> cmd( + new api::RemoveCommand(bucket, + document::DocumentId( + createDummyIdString(bucket)), + api::Timestamp(123456))); + cmd->setAddress(makeStorageAddress(node)); + return cmd; + } + + void assignMockedTime(std::chrono::milliseconds time) { + _clock.setAbsoluteTimeInMicroSeconds(time.count() * 1000); + } +}; + + +} + +CPPUNIT_TEST_SUITE_REGISTRATION(PendingMessageTrackerCallback_Test); + +void +PendingMessageTrackerCallback_Test::testSimple() +{ + StorageComponentRegisterImpl compReg; + framework::defaultimplementation::FakeClock clock; + compReg.setClock(clock); + clock.setAbsoluteTimeInSeconds(1); + PendingMessageTracker tracker(compReg); + + std::shared_ptr<api::RemoveCommand> remove( + new api::RemoveCommand( + document::BucketId(16, 1234), + document::DocumentId("userdoc:footype:1234:foo"), 1001)); + remove->setAddress( + api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 0)); + tracker.insert(remove); + + { + std::ostringstream ost; + tracker.reportStatus(ost, framework::HttpUrlPath("/pendingmessages?order=bucket")); + + CPPUNIT_ASSERT_CONTAIN( + std::string( + "<b>BucketId(0x40000000000004d2)</b>\n" + "<ul>\n" + "<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> " + "Remove(BucketId(0x40000000000004d2), " + "userdoc:footype:1234:foo, timestamp 1001)</li>\n" + "</ul>\n"), + ost.str()); + } + + api::RemoveReply reply(*remove); + tracker.reply(reply); + + { + std::ostringstream ost; + tracker.reportStatus(ost, framework::HttpUrlPath("/pendingmessages?order=bucket")); + + CPPUNIT_ASSERT_MSG(ost.str(), + ost.str().find("doc:") == std::string::npos); + } +} + +void +PendingMessageTrackerCallback_Test::insertMessages(PendingMessageTracker& tracker) +{ + for (uint32_t i = 0; i < 4; i++) { + std::ostringstream ost; + ost << "userdoc:footype:1234:" << i; + std::shared_ptr<api::RemoveCommand> remove( + new api::RemoveCommand( + document::BucketId(16, 1234), + document::DocumentId(ost.str()), 1000 + i)); + remove->setAddress( + api::StorageMessageAddress("storage", + lib::NodeType::STORAGE, i % 2)); + tracker.insert(remove); + } + + for (uint32_t i = 0; i < 4; i++) { + std::ostringstream ost; + ost << "userdoc:footype:4567:" << i; + std::shared_ptr<api::RemoveCommand> remove(new api::RemoveCommand(document::BucketId(16, 4567), document::DocumentId(ost.str()), 2000 + i)); + remove->setAddress(api::StorageMessageAddress("storage", lib::NodeType::STORAGE, i % 2)); + tracker.insert(remove); + } +} + +void +PendingMessageTrackerCallback_Test::testStartPage() +{ + StorageComponentRegisterImpl compReg; + framework::defaultimplementation::FakeClock clock; + compReg.setClock(clock); + PendingMessageTracker tracker(compReg); + + { + std::ostringstream ost; + tracker.reportStatus(ost, framework::HttpUrlPath("/pendingmessages")); + + CPPUNIT_ASSERT_CONTAIN( + std::string( + "<h1>Pending messages to storage nodes</h1>\n" + "View:\n" + "<ul>\n" + "<li><a href=\"?order=bucket\">Group by bucket</a></li>" + "<li><a href=\"?order=node\">Group by node</a></li>"), + ost.str()); + + } +} + +void +PendingMessageTrackerCallback_Test::testMultipleMessages() +{ + StorageComponentRegisterImpl compReg; + framework::defaultimplementation::FakeClock clock; + compReg.setClock(clock); + clock.setAbsoluteTimeInSeconds(1); + PendingMessageTracker tracker(compReg); + + insertMessages(tracker); + + { + std::ostringstream ost; + tracker.reportStatus(ost, framework::HttpUrlPath("/pendingmessages?order=bucket")); + + CPPUNIT_ASSERT_CONTAIN( + std::string( + "<b>BucketId(0x40000000000004d2)</b>\n" + "<ul>\n" + "<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:0, timestamp 1000)</li>\n" + "<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:2, timestamp 1002)</li>\n" + "<li><i>Node 1</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:1, timestamp 1001)</li>\n" + "<li><i>Node 1</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:3, timestamp 1003)</li>\n" + "</ul>\n" + "<b>BucketId(0x40000000000011d7)</b>\n" + "<ul>\n" + "<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000011d7), userdoc:footype:4567:0, timestamp 2000)</li>\n" + "<li><i>Node 0</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000011d7), userdoc:footype:4567:2, timestamp 2002)</li>\n" + "<li><i>Node 1</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000011d7), userdoc:footype:4567:1, timestamp 2001)</li>\n" + "<li><i>Node 1</i>: <b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000011d7), userdoc:footype:4567:3, timestamp 2003)</li>\n" + "</ul>\n"), + ost.str()); + } + + { + std::ostringstream ost; + tracker.reportStatus(ost, framework::HttpUrlPath("/pendingmessages?order=node")); + + CPPUNIT_ASSERT_CONTAIN(std::string( + "<b>Node 0 (pending count: 4)</b>\n" + "<ul>\n" + "<li><b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:0, timestamp 1000)</li>\n" + "<li><b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:2, timestamp 1002)</li>\n" + "<li><b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000011d7), userdoc:footype:4567:0, timestamp 2000)</li>\n" + "<li><b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000011d7), userdoc:footype:4567:2, timestamp 2002)</li>\n" + "</ul>\n" + "<b>Node 1 (pending count: 4)</b>\n" + "<ul>\n" + "<li><b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:1, timestamp 1001)</li>\n" + "<li><b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000004d2), userdoc:footype:1234:3, timestamp 1003)</li>\n" + "<li><b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000011d7), userdoc:footype:4567:1, timestamp 2001)</li>\n" + "<li><b>1970-01-01 00:00:01</b> Remove(BucketId(0x40000000000011d7), userdoc:footype:4567:3, timestamp 2003)</li>\n" + "</ul>\n" + ), ost.str()); + } +} + +namespace { + +template <typename T> +std::string setToString(const std::set<T>& s) +{ + std::ostringstream ost; + ost << '{'; + for (typename std::set<T>::const_iterator i(s.begin()), e(s.end()); + i != e; ++i) + { + if (i != s.begin()) { + ost << ','; + } + ost << *i; + } + ost << '}'; + return ost.str(); +} + +} + +namespace { + +class TestChecker : public PendingMessageTracker::Checker +{ +public: + uint8_t pri; + + TestChecker() : pri(UINT8_MAX) {} + + bool check(uint32_t msgType, uint16_t node, uint8_t p) { + (void) node; + if (msgType == api::MessageType::REMOVE_ID) { + pri = p; + return false; + } + + return true; + } +}; + + +} + +void +PendingMessageTrackerCallback_Test::testGetPendingMessageTypes() +{ + StorageComponentRegisterImpl compReg; + framework::defaultimplementation::FakeClock clock; + compReg.setClock(clock); + clock.setAbsoluteTimeInSeconds(1); + PendingMessageTracker tracker(compReg); + document::BucketId bid(16, 1234); + + std::shared_ptr<api::RemoveCommand> remove( + new api::RemoveCommand( + bid, + document::DocumentId("userdoc:footype:1234:foo"), 1001)); + remove->setAddress( + api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 0)); + tracker.insert(remove); + + { + TestChecker checker; + tracker.checkPendingMessages(0, bid, checker); + CPPUNIT_ASSERT_EQUAL(127, (int)checker.pri); + } + + { + TestChecker checker; + tracker.checkPendingMessages(0, document::BucketId(16, 1235), checker); + CPPUNIT_ASSERT_EQUAL(255, (int)checker.pri); + } + + { + TestChecker checker; + tracker.checkPendingMessages(1, bid, checker); + CPPUNIT_ASSERT_EQUAL(255, (int)checker.pri); + } +} + +void +PendingMessageTrackerCallback_Test::testHasPendingMessage() +{ + StorageComponentRegisterImpl compReg; + framework::defaultimplementation::FakeClock clock; + compReg.setClock(clock); + clock.setAbsoluteTimeInSeconds(1); + PendingMessageTracker tracker(compReg); + document::BucketId bid(16, 1234); + + CPPUNIT_ASSERT(!tracker.hasPendingMessage(1, bid, api::MessageType::REMOVE_ID)); + + { + std::shared_ptr<api::RemoveCommand> remove( + new api::RemoveCommand( + bid, + document::DocumentId("userdoc:footype:1234:foo"), 1001)); + remove->setAddress( + api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 1)); + tracker.insert(remove); + } + + CPPUNIT_ASSERT(tracker.hasPendingMessage(1, bid, api::MessageType::REMOVE_ID)); + CPPUNIT_ASSERT(!tracker.hasPendingMessage(0, bid, api::MessageType::REMOVE_ID)); + CPPUNIT_ASSERT(!tracker.hasPendingMessage(2, bid, api::MessageType::REMOVE_ID)); + CPPUNIT_ASSERT(!tracker.hasPendingMessage(1, + document::BucketId(16, 1233), + api::MessageType::REMOVE_ID)); + CPPUNIT_ASSERT(!tracker.hasPendingMessage(1, bid, api::MessageType::DELETEBUCKET_ID)); +} + +namespace { + +class OperationEnumerator : public PendingMessageTracker::Checker +{ + std::ostringstream ss; +public: + bool check(uint32_t msgType, uint16_t node, uint8_t p) override { + (void) p; + ss << api::MessageType::get(static_cast<api::MessageType::Id>(msgType)) + .getName() + << " -> " << node + << "\n"; + + return true; + } + + std::string str() const { return ss.str(); } +}; + +} // anon ns + +void +PendingMessageTrackerCallback_Test::testGetAllMessagesForSingleBucket() +{ + StorageComponentRegisterImpl compReg; + framework::defaultimplementation::FakeClock clock; + compReg.setClock(clock); + clock.setAbsoluteTimeInSeconds(1); + PendingMessageTracker tracker(compReg); + + insertMessages(tracker); + + { + OperationEnumerator enumerator; + tracker.checkPendingMessages(document::BucketId(16, 1234), enumerator); + CPPUNIT_ASSERT_EQUAL(std::string("Remove -> 0\n" + "Remove -> 0\n" + "Remove -> 1\n" + "Remove -> 1\n"), + enumerator.str()); + } + { + OperationEnumerator enumerator; + tracker.checkPendingMessages(document::BucketId(16, 9876), enumerator); + CPPUNIT_ASSERT_EQUAL(std::string(""), enumerator.str()); + } +} + +void +PendingMessageTrackerCallback_Test::nodeStatsCanBeOutputStreamed() +{ + NodeStats stats; + stats.puts = makeOpStats(ms(56789), 10); + + std::ostringstream os; + os << stats; + std::string expected( + "NodeStats(puts=OperationStats(" + "totalLatency=56789ms, " + "numRequests=10))"); + CPPUNIT_ASSERT_EQUAL(expected, os.str()); +} + +void +PendingMessageTrackerCallback_Test::totalPutLatencyIsInitiallyZero() +{ + Fixture fixture; + CPPUNIT_ASSERT_EQUAL(makeOpStats(ms(0), 0), + fixture.getNodePutOperationStats(0)); +} + +void +PendingMessageTrackerCallback_Test::statsNotAlteredBeforeReplyReceived() +{ + Fixture fixture; + fixture.sendPut(RequestBuilder().atTime(ms(1000)).toNode(0)); + CPPUNIT_ASSERT_EQUAL(makeOpStats(ms(0), 0), + fixture.getNodePutOperationStats(0)); +} + +void +PendingMessageTrackerCallback_Test::totalPutLatencyIsTrackedForSingleRequest() +{ + Fixture fixture; + fixture.sendPutAndReplyWithLatency(0, ms(500)); + + CPPUNIT_ASSERT_EQUAL(makeOpStats(ms(500), 1), + fixture.getNodePutOperationStats(0)); +} + +void +PendingMessageTrackerCallback_Test::statsAreTrackedSeparatelyPerNode() +{ + Fixture fixture; + fixture.sendPutAndReplyWithLatency(0, ms(500)); + fixture.sendPutAndReplyWithLatency(1, ms(600)); + + CPPUNIT_ASSERT_EQUAL(makeOpStats(ms(500), 1), + fixture.getNodePutOperationStats(0)); + CPPUNIT_ASSERT_EQUAL(makeOpStats(ms(600), 1), + fixture.getNodePutOperationStats(1)); +} + +// Necessarily, this test will have to be altered when we add tracking of +// other message types as well. +void +PendingMessageTrackerCallback_Test::onlyPutMessagesAreTracked() +{ + Fixture fixture; + auto remove = fixture.sendRemove( + RequestBuilder().atTime(ms(1000)).toNode(0)); + fixture.sendRemoveReply(*remove, RequestBuilder().atTime(ms(2000))); + CPPUNIT_ASSERT_EQUAL(makeOpStats(ms(0), 0), + fixture.getNodePutOperationStats(0)); +} + +void +PendingMessageTrackerCallback_Test::totalPutLatencyIsAggregatedAcrossRequests() +{ + Fixture fixture; + // Model 2 concurrent puts to node 0. + fixture.sendPutAndReplyWithLatency(0, ms(500)); + fixture.sendPutAndReplyWithLatency(0, ms(600)); + CPPUNIT_ASSERT_EQUAL(makeOpStats(ms(1100), 2), + fixture.getNodePutOperationStats(0)); +} + +void +PendingMessageTrackerCallback_Test::clearingMessagesDoesNotAffectStats() +{ + Fixture fixture; + fixture.sendPutAndReplyWithLatency(2, ms(2000)); + fixture.tracker().clearMessagesForNode(2); + CPPUNIT_ASSERT_EQUAL(makeOpStats(ms(2000), 1), + fixture.getNodePutOperationStats(2)); +} + +void +PendingMessageTrackerCallback_Test::timeTravellingClockLatenciesNotRegistered() +{ + Fixture fixture; + auto put = fixture.sendPut(RequestBuilder().atTime(ms(1000)).toNode(0)); + fixture.sendPutReply(*put, RequestBuilder().atTime(ms(999))); + // Latency increase of zero, but we do count the request itself. + CPPUNIT_ASSERT_EQUAL(makeOpStats(ms(0), 1), + fixture.getNodePutOperationStats(0)); +} + +void +PendingMessageTrackerCallback_Test::statsSnapshotIncludesAllNodes() +{ + Fixture fixture; + fixture.sendPutAndReplyWithLatency(0, ms(500)); + fixture.sendPutAndReplyWithLatency(1, ms(600)); + + NodeStatsSnapshot snapshot = fixture.tracker().getLatencyStatistics(); + + CPPUNIT_ASSERT_EQUAL(size_t(2), snapshot.nodeToStats.size()); + CPPUNIT_ASSERT_EQUAL(makeOpStats(ms(500), 1), + snapshot.nodeToStats[0].puts); + CPPUNIT_ASSERT_EQUAL(makeOpStats(ms(600), 1), + snapshot.nodeToStats[1].puts); +} + +void +PendingMessageTrackerCallback_Test::latencyProviderForwardsToImplementation() +{ + Fixture fixture; + fixture.sendPutAndReplyWithLatency(0, ms(500)); + + LatencyStatisticsProvider& provider( + fixture.tracker().getLatencyStatisticsProvider()); + NodeStatsSnapshot snapshot = provider.getLatencyStatistics(); + + CPPUNIT_ASSERT_EQUAL(size_t(1), snapshot.nodeToStats.size()); + CPPUNIT_ASSERT_EQUAL(makeOpStats(ms(500), 1), + snapshot.nodeToStats[0].puts); +} + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/putoperationtest.cpp b/storage/src/tests/distributor/putoperationtest.cpp new file mode 100644 index 00000000000..011b34cd1e3 --- /dev/null +++ b/storage/src/tests/distributor/putoperationtest.cpp @@ -0,0 +1,704 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> + +#include <vespa/config/helper/configgetter.h> +#include <vespa/document/config/config-documenttypes.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/storage/distributor/operations/external/putoperation.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/state.h> +#include <tests/distributor/distributortestutil.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/text/stringtokenizer.h> +#include <iomanip> +#include <iostream> +#include <memory> + +using std::shared_ptr; +using config::ConfigGetter; +using document::DocumenttypesConfig; +using config::FileSpec; +using vespalib::string; +using namespace document; +using namespace storage; +using namespace storage::api; +using namespace storage::lib; +using namespace std::literals::string_literals; + +namespace storage { + +namespace distributor { + +class PutOperationTest : public CppUnit::TestFixture, + public DistributorTestUtil { + CPPUNIT_TEST_SUITE(PutOperationTest); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST(testBucketDatabaseGetsSpecialEntryWhenCreateBucketSent); + CPPUNIT_TEST(testSendInlineSplitBeforePutIfBucketTooLarge); + CPPUNIT_TEST(testDoNotSendInlineSplitIfNotConfigured); + CPPUNIT_TEST(testNodeRemovedOnReply); + CPPUNIT_TEST(testDoNotSendCreateBucketIfAlreadyPending); + CPPUNIT_TEST(testMultipleCopies); + CPPUNIT_TEST(testMultipleCopiesEarlyReturnPrimaryNotRequired); + CPPUNIT_TEST(testMultipleCopiesEarlyReturnPrimaryRequired); + CPPUNIT_TEST(testMultipleCopiesEarlyReturnPrimaryRequiredNotDone); + CPPUNIT_TEST_IGNORED(testDoNotRevertOnFailureAfterEarlyReturn); + CPPUNIT_TEST(testStorageFailed); + CPPUNIT_TEST(testRevertSuccessfulCopiesWhenOneFails); + CPPUNIT_TEST(testNoRevertIfRevertDisabled); + CPPUNIT_TEST(testNoStorageNodes); + CPPUNIT_TEST(testUpdateCorrectBucketOnRemappedPut); + CPPUNIT_TEST(testTargetNodes); + CPPUNIT_TEST(testDoNotResurrectDownedNodesInBucketDB); + CPPUNIT_TEST(sendToRetiredNodesIfNoUpNodesAvailable); + CPPUNIT_TEST(replicaImplicitlyActivatedWhenActivationIsNotDisabled); + CPPUNIT_TEST(replicaNotImplicitlyActivatedWhenActivationIsDisabled); + CPPUNIT_TEST_SUITE_END(); + + DocumentTypeRepo::SP _repo; + const DocumentType* _html_type; + std::unique_ptr<Operation> op; + +protected: + void testSimple(); + void testBucketDatabaseGetsSpecialEntryWhenCreateBucketSent(); + void testSendInlineSplitBeforePutIfBucketTooLarge(); + void testDoNotSendInlineSplitIfNotConfigured(); + void testNodeRemovedOnReply(); + void testDoNotSendCreateBucketIfAlreadyPending(); + void testStorageFailed(); + void testNoReply(); + void testMultipleCopies(); + void testRevertSuccessfulCopiesWhenOneFails(); + void testNoRevertIfRevertDisabled(); + void testInconsistentChecksum(); + void testNoStorageNodes(); + void testMultipleCopiesEarlyReturnPrimaryNotRequired(); + void testMultipleCopiesEarlyReturnPrimaryRequired(); + void testMultipleCopiesEarlyReturnPrimaryRequiredNotDone(); + void testDoNotRevertOnFailureAfterEarlyReturn(); + void testUpdateCorrectBucketOnRemappedPut(); + void testBucketNotFound(); + void testTargetNodes(); + void testDoNotResurrectDownedNodesInBucketDB(); + void sendToRetiredNodesIfNoUpNodesAvailable(); + void replicaImplicitlyActivatedWhenActivationIsNotDisabled(); + void replicaNotImplicitlyActivatedWhenActivationIsDisabled(); + + void doTestCreationWithBucketActivationDisabled(bool disabled); + +public: + void setUp() { + _repo.reset( + new DocumentTypeRepo(*ConfigGetter<DocumenttypesConfig> + ::getConfig("config-doctypes", FileSpec("config-doctypes.cfg")))); + _html_type = _repo->getDocumentType("text/html"); + createLinks(); + }; + + void tearDown() { + close(); + } + + document::BucketId createAndSendSampleDocument(uint32_t timeout); + std::string getNodes(const std::string& infoString); + + void sendReply(int idx = -1, + api::ReturnCode::Result result + = api::ReturnCode::OK, + api::BucketInfo info = api::BucketInfo(1,2,3,4,5)) + { + CPPUNIT_ASSERT(!_sender.commands.empty()); + if (idx == -1) { + idx = _sender.commands.size() - 1; + } else if (static_cast<size_t>(idx) >= _sender.commands.size()) { + throw std::logic_error("Specified message index is greater " + "than number of received messages"); + } + + std::shared_ptr<api::StorageCommand> msg = _sender.commands[idx]; + api::StorageReply::SP reply(msg->makeReply().release()); + dynamic_cast<api::BucketInfoReply*>(reply.get())->setBucketInfo(info); + reply->setResult(result); + + op->receive(_sender, reply); + } + + void sendPut(std::shared_ptr<api::PutCommand> msg) { + op.reset(new PutOperation(getExternalOperationHandler(), + msg, + getDistributor().getMetrics(). + puts[msg->getLoadType()])); + op->start(_sender, framework::MilliSecTime(0)); + } + + Document::SP createDummyDocument(const char* ns, + const char* id) const + { + return Document::SP( + new Document(*_html_type, + DocumentId(DocIdString(ns, id)))); + + } + + std::shared_ptr<api::PutCommand> createPut( + const Document::SP doc) const + { + return std::shared_ptr<api::PutCommand>( + new api::PutCommand(document::BucketId(0), doc, 100)); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(PutOperationTest); + +document::BucketId +PutOperationTest::createAndSendSampleDocument(uint32_t timeout) { + Document::SP + doc(new Document(*_html_type, + DocumentId(DocIdString("test", "test")))); + + document::BucketId id = getExternalOperationHandler().getBucketId(doc->getId()); + addIdealNodes(id); + + std::shared_ptr<api::PutCommand> msg( + new api::PutCommand(document::BucketId(0), + doc, + 0)); + msg->setTimestamp(100); + msg->setPriority(128); + msg->setTimeout(timeout); + sendPut(msg); + return id; +} + +namespace { + +typedef int Redundancy; +typedef int NodeCount; +typedef uint32_t ReturnAfter; +typedef bool RequirePrimaryWritten; + +} + +void +PutOperationTest::testSimple() +{ + setupDistributor(1, 1, "storage:1 distributor:1"); + createAndSendSampleDocument(180); + + CPPUNIT_ASSERT_EQUAL(std::string("Put(BucketId(0x4000000000008b13), " + "doc:test:test, timestamp 100, size 33) => 0"), + _sender.getCommands(true, true)); + + sendReply(); + + CPPUNIT_ASSERT_EQUAL(std::string("PutReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 100) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +PutOperationTest::testBucketDatabaseGetsSpecialEntryWhenCreateBucketSent() +{ + setupDistributor(2, 1, "storage:1 distributor:1"); + + Document::SP doc(createDummyDocument("test", "test")); + document::BucketId bucketId(getExternalOperationHandler().getBucketId(doc->getId())); + sendPut(createPut(doc)); + + // Database updated before CreateBucket is sent + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000008b13) : " + "node(idx=0,crc=0x1,docs=0/0,bytes=0/0,trusted=true,active=true)"), + dumpBucket(getExternalOperationHandler().getBucketId(doc->getId()))); + + CPPUNIT_ASSERT_EQUAL(std::string("Create bucket => 0,Put => 0"), + _sender.getCommands(true)); +} + +void +PutOperationTest::testSendInlineSplitBeforePutIfBucketTooLarge() +{ + setupDistributor(1, 1, "storage:1 distributor:1"); + getConfig().setSplitCount(1024); + getConfig().setSplitSize(1000000); + + addNodesToBucketDB(document::BucketId(0x4000000000002a52), "0=10000/10000/10000/t"); + + sendPut(createPut(createDummyDocument("test", "uri"))); + + CPPUNIT_ASSERT_EQUAL( + std::string("SplitBucketCommand(BucketId(0x4000000000002a52)Max doc count: " + "1024, Max total doc size: 1000000) Reasons to start: " + "[Splitting bucket because its maximum size (10000 b, 10000 docs, 10000 meta, 10000 b total) is " + "higher than the configured limit of (1000000, 1024)] => 0," + "Put(BucketId(0x4000000000002a52), doc:test:uri, timestamp 100, " + "size 32) => 0"), + _sender.getCommands(true, true)); +} + +void +PutOperationTest::testDoNotSendInlineSplitIfNotConfigured() +{ + setupDistributor(1, 1, "storage:1 distributor:1"); + getConfig().setSplitCount(1024); + getConfig().setDoInlineSplit(false); + + addNodesToBucketDB(document::BucketId(0x4000000000002a52), "0=10000/10000/10000/t"); + + sendPut(createPut(createDummyDocument("test", "uri"))); + + CPPUNIT_ASSERT_EQUAL( + std::string( + "Put(BucketId(0x4000000000002a52), doc:test:uri, timestamp 100, " + "size 32) => 0"), + _sender.getCommands(true, true)); +} + +void +PutOperationTest::testNodeRemovedOnReply() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + createAndSendSampleDocument(180); + + CPPUNIT_ASSERT_EQUAL( + std::string("Put(BucketId(0x4000000000008b13), " + "doc:test:test, timestamp 100, size 33) => 1," + "Put(BucketId(0x4000000000008b13), " + "doc:test:test, timestamp 100, size 33) => 0"), + _sender.getCommands(true, true)); + + getExternalOperationHandler().removeNodeFromDB(document::BucketId(16, 0x8b13), 0); + + sendReply(0); + sendReply(1); + + CPPUNIT_ASSERT_EQUAL(std::string( + "PutReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 100) ReturnCode(BUCKET_DELETED, " + "BucketId(0x4000000000008b13) was deleted from nodes [0] " + "after message was sent but before it was done. " + "Sent to [1,0])"), + _sender.getLastReply()); +} + +void +PutOperationTest::testStorageFailed() +{ + setupDistributor(2, 1, "storage:1 distributor:1"); + + createAndSendSampleDocument(180); + + sendReply(-1, api::ReturnCode::INTERNAL_FAILURE); + + CPPUNIT_ASSERT_EQUAL(std::string("PutReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 100) ReturnCode(INTERNAL_FAILURE)"), + _sender.getLastReply(true)); +} + +void +PutOperationTest::testMultipleCopies() +{ + setupDistributor(3, 4, "storage:4 distributor:1"); + + Document::SP doc(createDummyDocument("test", "test")); + sendPut(createPut(doc)); + + CPPUNIT_ASSERT_EQUAL(std::string("Create bucket => 3,Create bucket => 1," + "Create bucket => 0,Put => 3,Put => 1,Put => 0"), + _sender.getCommands(true)); + + for (uint32_t i = 0; i < 6; i++) { + sendReply(i); + } + + CPPUNIT_ASSERT_EQUAL( + std::string("PutReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 100) ReturnCode(NONE)"), + _sender.getLastReply(true)); + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000008b13) : " + "node(idx=3,crc=0x1,docs=2/4,bytes=3/5,trusted=true,active=false), " + "node(idx=1,crc=0x1,docs=2/4,bytes=3/5,trusted=true,active=false), " + "node(idx=0,crc=0x1,docs=2/4,bytes=3/5,trusted=true,active=false)"), + dumpBucket(getExternalOperationHandler().getBucketId(doc->getId()))); +} + + +void +PutOperationTest::testMultipleCopiesEarlyReturnPrimaryRequired() +{ + setupDistributor(3, 4, "storage:4 distributor:1", 2, true); + + sendPut(createPut(createDummyDocument("test", "test"))); + + CPPUNIT_ASSERT_EQUAL(std::string("Create bucket => 3,Create bucket => 1," + "Create bucket => 0,Put => 3,Put => 1,Put => 0"), + _sender.getCommands(true)); + + // Reply to 2 CreateBucket, including primary + for (uint32_t i = 0; i < 2; i++) { + sendReply(i); + } + // Reply to 2 puts, including primary + for (uint32_t i = 0; i < 2; i++) { + sendReply(3 + i); + } + + CPPUNIT_ASSERT_EQUAL( + std::string( + "PutReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 100) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +PutOperationTest::testMultipleCopiesEarlyReturnPrimaryNotRequired() +{ + setupDistributor(3, 4, "storage:4 distributor:1", 2, false); + + sendPut(createPut(createDummyDocument("test", "test"))); + + CPPUNIT_ASSERT_EQUAL(std::string("Create bucket => 3,Create bucket => 1," + "Create bucket => 0,Put => 3,Put => 1,Put => 0"), + _sender.getCommands(true)); + + // Reply only to 2 nodes (but not the primary) + for (uint32_t i = 1; i < 3; i++) { + sendReply(i); // CreateBucket + } + for (uint32_t i = 1; i < 3; i++) { + sendReply(3 + i); // Put + } + + CPPUNIT_ASSERT_EQUAL( + std::string("PutReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 100) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +PutOperationTest::testMultipleCopiesEarlyReturnPrimaryRequiredNotDone() +{ + setupDistributor(3, 4, "storage:4 distributor:1", 2, true); + + sendPut(createPut(createDummyDocument("test", "test"))); + + CPPUNIT_ASSERT_EQUAL(std::string("Create bucket => 3,Create bucket => 1," + "Create bucket => 0,Put => 3,Put => 1,Put => 0"), + _sender.getCommands(true)); + + // Reply only to 2 nodes (but not the primary) + sendReply(1); + sendReply(2); + sendReply(4); + sendReply(5); + + CPPUNIT_ASSERT_EQUAL(0, (int)_sender.replies.size()); +} + +void +PutOperationTest::testDoNotRevertOnFailureAfterEarlyReturn() +{ + setupDistributor(Redundancy(3),NodeCount(4), "storage:4 distributor:1", + ReturnAfter(2), RequirePrimaryWritten(false)); + + sendPut(createPut(createDummyDocument("test", "test"))); + + CPPUNIT_ASSERT_EQUAL(std::string("Create bucket => 3,Create bucket => 1," + "Create bucket => 0,Put => 3,Put => 1,Put => 0"), + _sender.getCommands(true)); + + for (uint32_t i = 0; i < 3; i++) { + sendReply(i); // CreateBucket + } + for (uint32_t i = 0; i < 2; i++) { + sendReply(3 + i); // Put + } + + CPPUNIT_ASSERT_EQUAL( + std::string( + "PutReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 100) ReturnCode(NONE)"), + _sender.getLastReply()); + + sendReply(5, api::ReturnCode::INTERNAL_FAILURE); + // Should not be any revert commands sent + CPPUNIT_ASSERT_EQUAL(std::string("Create bucket => 3,Create bucket => 1," + "Create bucket => 0,Put => 3,Put => 1,Put => 0"), + _sender.getCommands(true)); +} + +void +PutOperationTest::testRevertSuccessfulCopiesWhenOneFails() +{ + setupDistributor(3, 4, "storage:4 distributor:1"); + + createAndSendSampleDocument(180); + + CPPUNIT_ASSERT_EQUAL(std::string("Put => 3,Put => 1,Put => 0"), + _sender.getCommands(true)); + + for (uint32_t i = 0; i < 2; i++) { + sendReply(i); + } + + sendReply(2, api::ReturnCode::INTERNAL_FAILURE); + + CPPUNIT_ASSERT_EQUAL(std::string("PutReply(doc:test:test, " + "BucketId(0x0000000000000000), timestamp 100) " + "ReturnCode(INTERNAL_FAILURE)"), + _sender.getLastReply(true)); + + CPPUNIT_ASSERT_EQUAL(std::string("Revert => 3,Revert => 1"), + _sender.getCommands(true, false, 3)); +} + +void +PutOperationTest::testNoRevertIfRevertDisabled() +{ + close(); + getDirConfig().getConfig("stor-distributormanager") + .set("enable_revert", "false"); + setUp(); + setupDistributor(3, 4, "storage:4 distributor:1"); + + createAndSendSampleDocument(180); + + CPPUNIT_ASSERT_EQUAL(std::string("Put => 3,Put => 1,Put => 0"), + _sender.getCommands(true)); + + for (uint32_t i = 0; i < 2; i++) { + sendReply(i); + } + + sendReply(2, api::ReturnCode::INTERNAL_FAILURE); + + CPPUNIT_ASSERT_EQUAL(std::string("PutReply(doc:test:test, " + "BucketId(0x0000000000000000), timestamp 100) " + "ReturnCode(INTERNAL_FAILURE)"), + _sender.getLastReply(true)); + + CPPUNIT_ASSERT_EQUAL(std::string(""), + _sender.getCommands(true, false, 3)); +} + +void +PutOperationTest::testDoNotSendCreateBucketIfAlreadyPending() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + + Document::SP doc(createDummyDocument("test", "uri")); + sendPut(createPut(doc)); + + CPPUNIT_ASSERT_EQUAL(std::string("Create bucket => 1,Create bucket => 0," + "Put => 1,Put => 0"), + _sender.getCommands(true)); + + // Manually shove sent messages into pending message tracker, since + // this isn't done automatically. + for (size_t i = 0; i < _sender.commands.size(); ++i) { + getExternalOperationHandler().getDistributor().getPendingMessageTracker() + .insert(_sender.commands[i]); + } + + sendPut(createPut(doc)); + + CPPUNIT_ASSERT_EQUAL(std::string("Create bucket => 1,Create bucket => 0," + "Put => 1,Put => 0," + "Put => 1,Put => 0"), + _sender.getCommands(true)); +} + +void +PutOperationTest::testNoStorageNodes() +{ + setupDistributor(2, 1, "storage:0 distributor:1"); + createAndSendSampleDocument(180); + CPPUNIT_ASSERT_EQUAL(std::string("PutReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 100) ReturnCode(NOT_CONNECTED, " + "Can't store document: No storage nodes available)"), + _sender.getLastReply(true)); +} + +void +PutOperationTest::testUpdateCorrectBucketOnRemappedPut() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + + Document::SP doc(new Document(*_html_type, DocumentId( + UserDocIdString("userdoc:test:13:uri")))); + + addNodesToBucketDB(document::BucketId(16,13), "0=0,1=0"); + + sendPut(createPut(doc)); + + CPPUNIT_ASSERT_EQUAL(std::string("Put => 0,Put => 1"), + _sender.getCommands(true)); + + { + std::shared_ptr<api::StorageCommand> msg2 = _sender.commands[0]; + std::shared_ptr<api::StorageReply> reply(msg2->makeReply().release()); + PutReply* sreply = (PutReply*)reply.get(); + sreply->remapBucketId(document::BucketId(17, 13)); + sreply->setBucketInfo(api::BucketInfo(1,2,3,4,5)); + op->receive(_sender, reply); + } + + sendReply(1); + + CPPUNIT_ASSERT_EQUAL(std::string("PutReply(userdoc:test:13:uri, " + "BucketId(0x0000000000000000), " + "timestamp 100) ReturnCode(NONE)"), + _sender.getLastReply()); + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x440000000000000d) : " + "node(idx=0,crc=0x1,docs=2/4,bytes=3/5,trusted=true,active=false)"), + dumpBucket(document::BucketId(17, 13))); +} + +BucketInfo +parseBucketInfoString(const std::string& nodeList) { + vespalib::StringTokenizer tokenizer(nodeList, ","); + + BucketInfo entry; + for (uint32_t i = 0; i < tokenizer.size(); i++) { + vespalib::StringTokenizer tokenizer2(tokenizer[i], "-"); + int node = atoi(tokenizer2[0].c_str()); + int size = atoi(tokenizer2[1].c_str()); + bool trusted = (tokenizer2[2] == "true"); + + entry.addNode(BucketCopy(0, + node, + api::BucketInfo(size, size * 1000, size * 2000)) + .setTrusted(trusted), + toVector<uint16_t>(0)); + } + + return entry; +} + +std::string +PutOperationTest::getNodes(const std::string& infoString) { + Document::SP doc(createDummyDocument("test", "uri")); + document::BucketId bid(getExternalOperationHandler().getBucketId(doc->getId())); + + BucketInfo entry = parseBucketInfoString(infoString); + + std::ostringstream ost; + + std::vector<uint16_t> targetNodes; + std::vector<uint16_t> createNodes; + PutOperation::getTargetNodes(getExternalOperationHandler().getIdealNodes(bid), + targetNodes, createNodes, entry, 2); + + ost << "target( "; + for (uint32_t i = 0; i < targetNodes.size(); i++) { + ost << targetNodes[i] << " "; + } + ost << ") create( "; + for (uint32_t i = 0; i < createNodes.size(); i++) { + ost << createNodes[i] << " "; + } + ost << ")"; + + return ost.str(); +} + +void +PutOperationTest::testTargetNodes() +{ + setupDistributor(2, 6, "storage:6 distributor:1"); + + // Ideal state of bucket is 1,3. + CPPUNIT_ASSERT_EQUAL(std::string("target( 1 3 ) create( 1 3 )"), getNodes("")); + CPPUNIT_ASSERT_EQUAL(std::string("target( 1 3 ) create( 3 )"), getNodes("1-1-true")); + CPPUNIT_ASSERT_EQUAL(std::string("target( 1 3 ) create( 3 )"), getNodes("1-1-false")); + CPPUNIT_ASSERT_EQUAL(std::string("target( 3 4 5 ) create( )"), getNodes("3-1-true,4-1-true,5-1-true")); + CPPUNIT_ASSERT_EQUAL(std::string("target( 3 4 ) create( )"), getNodes("3-2-true,4-2-true,5-1-false")); + CPPUNIT_ASSERT_EQUAL(std::string("target( 1 3 4 ) create( )"), getNodes("3-2-true,4-2-true,1-1-false")); + CPPUNIT_ASSERT_EQUAL(std::string("target( 4 5 ) create( )"), getNodes("4-2-false,5-1-false")); + CPPUNIT_ASSERT_EQUAL(std::string("target( 1 4 ) create( 1 )"), getNodes("4-1-true")); +} + +void +PutOperationTest::testDoNotResurrectDownedNodesInBucketDB() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + + Document::SP doc(createDummyDocument("test", "uri")); + document::BucketId bId = getExternalOperationHandler().getBucketId(doc->getId()); + + addNodesToBucketDB(bId, "0=1/2/3/t,1=1/2/3/t"); + + sendPut(createPut(doc)); + + CPPUNIT_ASSERT_EQUAL(std::string("Put => 1,Put => 0"), + _sender.getCommands(true)); + + _distributor->enableClusterState(ClusterState("distributor:1 storage:2 .1.s:d")); + addNodesToBucketDB(bId, "0=1/2/3/t"); // This will actually remove node #1. + + sendReply(0, api::ReturnCode::OK, api::BucketInfo(9,9,9)); + sendReply(1, api::ReturnCode::OK, api::BucketInfo(5,6,7)); + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000002a52) : " + "node(idx=0,crc=0x5,docs=6/6,bytes=7/7,trusted=true,active=false)"), + dumpBucket(getExternalOperationHandler().getBucketId(doc->getId()))); +} + +void +PutOperationTest::sendToRetiredNodesIfNoUpNodesAvailable() +{ + setupDistributor(Redundancy(2), NodeCount(2), + "distributor:1 storage:2 .0.s:r .1.s:r"); + Document::SP doc(createDummyDocument("test", "uri")); + document::BucketId bucket( + getExternalOperationHandler().getBucketId(doc->getId())); + addNodesToBucketDB(bucket, "0=1/2/3/t,1=1/2/3/t"); + + sendPut(createPut(doc)); + + CPPUNIT_ASSERT_EQUAL("Put => 0,Put => 1"s, + _sender.getCommands(true)); +} + +void +PutOperationTest::doTestCreationWithBucketActivationDisabled(bool disabled) +{ + setupDistributor(Redundancy(2), NodeCount(2), "distributor:1 storage:1"); + disableBucketActivationInConfig(disabled); + + Document::SP doc(createDummyDocument("test", "uri")); + sendPut(createPut(doc)); + + CPPUNIT_ASSERT_EQUAL(std::string("Create bucket => 0,Put => 0"), + _sender.getCommands(true)); + auto cmd = _sender.commands[0]; + auto createCmd = std::dynamic_pointer_cast<api::CreateBucketCommand>(cmd); + CPPUNIT_ASSERT(createCmd.get() != nullptr); + // There's only 1 content node, so if activation were not disabled, it + // should always be activated. + CPPUNIT_ASSERT_EQUAL(!disabled, createCmd->getActive()); +} + +void +PutOperationTest::replicaImplicitlyActivatedWhenActivationIsNotDisabled() +{ + doTestCreationWithBucketActivationDisabled(false); +} + +void +PutOperationTest::replicaNotImplicitlyActivatedWhenActivationIsDisabled() +{ + doTestCreationWithBucketActivationDisabled(true); +} + +} + +} diff --git a/storage/src/tests/distributor/removebucketoperationtest.cpp b/storage/src/tests/distributor/removebucketoperationtest.cpp new file mode 100644 index 00000000000..aeceefa15a0 --- /dev/null +++ b/storage/src/tests/distributor/removebucketoperationtest.cpp @@ -0,0 +1,150 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storage/distributor/operations/idealstate/removebucketoperation.h> +#include <vespa/storage/distributor/idealstatemanager.h> +#include <vespa/storage/distributor/pendingmessagetracker.h> +#include <vespa/storage/storageutil/utils.h> +#include <tests/distributor/distributortestutil.h> + +namespace storage { +namespace distributor { + +class RemoveBucketOperationTest : public CppUnit::TestFixture, + public DistributorTestUtil +{ + CPPUNIT_TEST_SUITE(RemoveBucketOperationTest); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST(testBucketInfoMismatchFailure); + CPPUNIT_TEST(testFailWithInvalidBucketInfo); + CPPUNIT_TEST_SUITE_END(); + +protected: + void testSimple(); + void testBucketInfoMismatchFailure(); + void testFailWithInvalidBucketInfo(); + +public: + void setUp() { + createLinks(); + }; + + void tearDown() { + close(); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(RemoveBucketOperationTest); + +void +RemoveBucketOperationTest::testSimple() +{ + addNodesToBucketDB(document::BucketId(16, 1), + "0=10/100/1/t," + "1=10/100/1/t," + "2=10/100/1/t"); + setRedundancy(1); + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:3")); + + RemoveBucketOperation op("storage", + BucketAndNodes(document::BucketId(16, 1), + toVector<uint16_t>(1,2))); + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + + CPPUNIT_ASSERT_EQUAL(std::string("Delete bucket => 1," + "Delete bucket => 2"), + _sender.getCommands(true)); + + sendReply(op, 0); + sendReply(op, 1); + + CPPUNIT_ASSERT_EQUAL( + std::string( + "BucketId(0x4000000000000001) : " + "node(idx=0,crc=0xa,docs=100/100,bytes=1/1,trusted=true,active=false)"), + dumpBucket(document::BucketId(16, 1))); +} + +/** + * Test that receiving a DeleteBucket failure from a storage node that sends + * back actual bucket info reinserts that bucket info into the distributor + * bucket database. + */ +void +RemoveBucketOperationTest::testBucketInfoMismatchFailure() +{ + addNodesToBucketDB(document::BucketId(16, 1), "1=0/0/0/t"); + + getComponentRegisterImpl().setDistribution(std::shared_ptr<lib::Distribution>( + new lib::Distribution( + lib::Distribution::getDefaultDistributionConfig(1, 10)))); + + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:2")); + + RemoveBucketOperation op("storage", + BucketAndNodes(document::BucketId(16, 1), + toVector<uint16_t>(1))); + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Delete bucket => 1"), + _sender.getCommands(true)); + + CPPUNIT_ASSERT_EQUAL((size_t) 1, _sender.commands.size()); + std::shared_ptr<api::StorageCommand> msg2 = _sender.commands[0]; + std::shared_ptr<api::StorageReply> reply(msg2->makeReply().release()); + dynamic_cast<api::DeleteBucketReply&>(*reply).setBucketInfo( + api::BucketInfo(10, 100, 1)); + reply->setResult(api::ReturnCode::REJECTED); + op.receive(_sender, reply); + + // RemoveBucketOperation should reinsert bucketinfo into database + CPPUNIT_ASSERT_EQUAL( + std::string( + "BucketId(0x4000000000000001) : " + "node(idx=1,crc=0xa,docs=100/100,bytes=1/1,trusted=true,active=false)"), + dumpBucket(document::BucketId(16, 1))); +} + +/** + * Test that receiving a DeleteBucket failure from a storage node that does + * not include valid BucketInfo in its reply does not reinsert the bucket + * into the distributor. + */ +void +RemoveBucketOperationTest::testFailWithInvalidBucketInfo() +{ + addNodesToBucketDB(document::BucketId(16, 1), "1=0/0/0/t"); + + getComponentRegisterImpl().setDistribution(std::shared_ptr<lib::Distribution>( + new lib::Distribution( + lib::Distribution::getDefaultDistributionConfig(1, 10)))); + + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:2")); + + RemoveBucketOperation op("storage", + BucketAndNodes(document::BucketId(16, 1), + toVector<uint16_t>(1))); + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Delete bucket => 1"), + _sender.getCommands(true)); + + CPPUNIT_ASSERT_EQUAL((size_t) 1, _sender.commands.size()); + std::shared_ptr<api::StorageCommand> msg2 = _sender.commands[0]; + std::shared_ptr<api::StorageReply> reply(msg2->makeReply().release()); + reply->setResult(api::ReturnCode::ABORTED); + op.receive(_sender, reply); + + CPPUNIT_ASSERT_EQUAL(std::string("NONEXISTING"), + dumpBucket(document::BucketId(16, 1))); +} + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/removelocationtest.cpp b/storage/src/tests/distributor/removelocationtest.cpp new file mode 100644 index 00000000000..7a1bba86303 --- /dev/null +++ b/storage/src/tests/distributor/removelocationtest.cpp @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <iomanip> +#include <iostream> +#include <memory> +#include <tests/common/dummystoragelink.h> +#include <vespa/storageapi/message/removelocation.h> +#include <vespa/storage/distributor/operations/external/removelocationoperation.h> +#include <tests/distributor/distributortestutil.h> + +namespace storage { +namespace distributor { + +class RemoveLocationOperationTest : public CppUnit::TestFixture, + public DistributorTestUtil +{ + CPPUNIT_TEST_SUITE(RemoveLocationOperationTest); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST_SUITE_END(); + +protected: + void testSimple(); + +public: + std::unique_ptr<RemoveLocationOperation> op; + + void setUp() { + createLinks(); + }; + + void tearDown() { + close(); + } + + void sendRemoveLocation(const std::string& selection) { + std::shared_ptr<api::RemoveLocationCommand> msg( + new api::RemoveLocationCommand(selection, document::BucketId(0))); + + op.reset(new RemoveLocationOperation(getExternalOperationHandler(), + msg, + getDistributor().getMetrics(). + removelocations[msg->getLoadType()])); + + op->start(_sender, framework::MilliSecTime(0)); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(RemoveLocationOperationTest); + +void +RemoveLocationOperationTest::testSimple() +{ + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:3")); + + addNodesToBucketDB(document::BucketId(34, 0x000001234), "0=1,1=1"); + addNodesToBucketDB(document::BucketId(34, 0x100001234), "0=1,2=1"); + addNodesToBucketDB(document::BucketId(34, 0x200001234), "0=1,2=1"); + addNodesToBucketDB(document::BucketId(34, 0x300001234), "1=1,2=1"); + + sendRemoveLocation("id.user=4660"); + + CPPUNIT_ASSERT_EQUAL( + std::string("Remove selection(id.user=4660): BucketInfoCommand() => 0," + "Remove selection(id.user=4660): BucketInfoCommand() => 1," + "Remove selection(id.user=4660): BucketInfoCommand() => 0," + "Remove selection(id.user=4660): BucketInfoCommand() => 2," + "Remove selection(id.user=4660): BucketInfoCommand() => 0," + "Remove selection(id.user=4660): BucketInfoCommand() => 2," + "Remove selection(id.user=4660): BucketInfoCommand() => 1," + "Remove selection(id.user=4660): BucketInfoCommand() => 2"), + _sender.getCommands(true, true)); + + for (uint32_t i = 0; i < 8; ++i) { + sendReply(*op, i); + } + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketInfoReply(BucketInfo(invalid)) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/removeoperationtest.cpp b/storage/src/tests/distributor/removeoperationtest.cpp new file mode 100644 index 00000000000..7907541a7c7 --- /dev/null +++ b/storage/src/tests/distributor/removeoperationtest.cpp @@ -0,0 +1,203 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <iomanip> +#include <iostream> +#include <memory> +#include <tests/common/dummystoragelink.h> +#include <vespa/storage/distributor/externaloperationhandler.h> +#include <vespa/storageapi/message/persistence.h> +#include <tests/distributor/distributortestutil.h> +#include <vespa/storage/distributor/operations/external/removeoperation.h> + +namespace storage { +namespace distributor { + +class RemoveOperationTest : public CppUnit::TestFixture, + public DistributorTestUtil +{ + CPPUNIT_TEST_SUITE(RemoveOperationTest); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST(testNotFound); + CPPUNIT_TEST(testStorageFailure); + CPPUNIT_TEST(testNotInDB); + CPPUNIT_TEST(testMultipleCopies); + CPPUNIT_TEST(canSendRemoveWhenAllReplicaNodesRetired); + CPPUNIT_TEST_SUITE_END(); + +protected: + void testSimple(); + void testNotFound(); + void testStorageFailure(); + void testNoReply(); + void testNotInDB(); + void testMultipleCopies(); + void testRevert(); + void canSendRemoveWhenAllReplicaNodesRetired(); + +public: + document::DocumentId docId; + document::BucketId bucketId; + std::unique_ptr<RemoveOperation> op; + + void setUp() { + createLinks(); + + docId = document::DocumentId(document::DocIdString("test", "uri")); + bucketId = getExternalOperationHandler().getBucketId(docId); + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:4")); + }; + + void tearDown() { + close(); + } + + void sendRemove(document::DocumentId dId) { + std::shared_ptr<api::RemoveCommand> msg( + new api::RemoveCommand(document::BucketId(0), dId, 100)); + + op.reset(new RemoveOperation(getExternalOperationHandler(), + msg, + getDistributor().getMetrics(). + removes[msg->getLoadType()])); + + op->start(_sender, framework::MilliSecTime(0)); + } + + void replyToMessage(RemoveOperation& callback, + uint32_t index, + uint64_t oldTimestamp) + { + if (index == (uint32_t)-1) { + index = _sender.commands.size() - 1; + } + + std::shared_ptr<api::StorageMessage> msg2 = _sender.commands[index]; + api::RemoveCommand* removec = dynamic_cast<api::RemoveCommand*>(msg2.get()); + std::unique_ptr<api::StorageReply> reply(removec->makeReply()); + api::RemoveReply* removeR = static_cast<api::RemoveReply*>(reply.get()); + removeR->setOldTimestamp(oldTimestamp); + callback.onReceive(_sender, + std::shared_ptr<api::StorageReply>(reply.release())); + } + + void sendRemove() { + sendRemove(docId); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(RemoveOperationTest); + +void +RemoveOperationTest::testSimple() +{ + addNodesToBucketDB(bucketId, "1=0"); + + sendRemove(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Remove(BucketId(0x4000000000002a52), doc:test:uri, " + "timestamp 100) => 1"), + _sender.getLastCommand()); + + replyToMessage(*op, -1, 34); + + CPPUNIT_ASSERT_EQUAL( + std::string("RemoveReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 100, removed doc from 34) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +RemoveOperationTest::testNotFound() +{ + addNodesToBucketDB(bucketId, "1=0"); + + sendRemove(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Remove(BucketId(0x4000000000002a52), doc:test:uri, " + "timestamp 100) => 1"), + _sender.getLastCommand()); + + replyToMessage(*op, -1, 0); + + CPPUNIT_ASSERT_EQUAL( + std::string("RemoveReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 100, not found) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +RemoveOperationTest::testStorageFailure() +{ + addNodesToBucketDB(bucketId, "1=0"); + + sendRemove(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Remove(BucketId(0x4000000000002a52), doc:test:uri, " + "timestamp 100) => 1"), + _sender.getLastCommand()); + + sendReply(*op, -1, api::ReturnCode::INTERNAL_FAILURE); + + CPPUNIT_ASSERT_EQUAL( + std::string("RemoveReply(BucketId(0x0000000000000000), doc:test:uri, " + "timestamp 100, not found) ReturnCode(INTERNAL_FAILURE)"), + _sender.getLastReply()); +} + +void +RemoveOperationTest::testNotInDB() +{ + sendRemove(); + + CPPUNIT_ASSERT_EQUAL(std::string("RemoveReply(BucketId(0x0000000000000000), " + "doc:test:uri, timestamp 100, not found) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +RemoveOperationTest::testMultipleCopies() +{ + addNodesToBucketDB(bucketId, "1=0, 2=0, 3=0"); + + sendRemove(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Remove(BucketId(0x4000000000002a52), doc:test:uri, " + "timestamp 100) => 1," + "Remove(BucketId(0x4000000000002a52), doc:test:uri, " + "timestamp 100) => 2," + "Remove(BucketId(0x4000000000002a52), doc:test:uri, " + "timestamp 100) => 3"), + _sender.getCommands(true, true)); + + replyToMessage(*op, 0, 34); + replyToMessage(*op, 1, 34); + replyToMessage(*op, 2, 75); + + CPPUNIT_ASSERT_EQUAL( + std::string("RemoveReply(BucketId(0x0000000000000000), " + "doc:test:uri, timestamp 100, removed doc from 75) ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +RemoveOperationTest::canSendRemoveWhenAllReplicaNodesRetired() +{ + _distributor->enableClusterState( + lib::ClusterState("distributor:1 storage:1 .0.s:r")); + addNodesToBucketDB(bucketId, "0=123"); + sendRemove(); + + CPPUNIT_ASSERT_EQUAL( + std::string("Remove(BucketId(0x4000000000002a52), doc:test:uri, " + "timestamp 100) => 0"), + _sender.getLastCommand()); +} + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/simplebucketprioritydatabasetest.cpp b/storage/src/tests/distributor/simplebucketprioritydatabasetest.cpp new file mode 100644 index 00000000000..a066649477c --- /dev/null +++ b/storage/src/tests/distributor/simplebucketprioritydatabasetest.cpp @@ -0,0 +1,143 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <string> +#include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> + +namespace storage { + +namespace distributor { + +using document::BucketId; +typedef MaintenancePriority Priority; + +class SimpleBucketPriorityDatabaseTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(SimpleBucketPriorityDatabaseTest); + CPPUNIT_TEST(testIteratorRangeIsEqualOnEmptyDatabase); + CPPUNIT_TEST(testCanGetPrioritizedBucket); + CPPUNIT_TEST(testIterateOverMultiplePriorities); + CPPUNIT_TEST(testMultipleSetPriorityForOneBucket); + CPPUNIT_TEST(testIterateOverMultipleBucketsWithMultiplePriorities); + CPPUNIT_TEST(testNoMaintenanceNeededClearsBucketFromDatabase); + CPPUNIT_TEST_SUITE_END(); + + typedef SimpleBucketPriorityDatabase::const_iterator const_iterator; + +public: + void testIteratorRangeIsEqualOnEmptyDatabase(); + void testCanGetPrioritizedBucket(); + void testIterateOverMultiplePriorities(); + void testMultipleSetPriorityForOneBucket(); + void testIterateOverMultipleBucketsWithMultiplePriorities(); + void testNoMaintenanceNeededClearsBucketFromDatabase(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(SimpleBucketPriorityDatabaseTest); + +void +SimpleBucketPriorityDatabaseTest::testIteratorRangeIsEqualOnEmptyDatabase() +{ + SimpleBucketPriorityDatabase queue; + const_iterator begin(queue.begin()); + const_iterator end(queue.end()); + + CPPUNIT_ASSERT(begin == end); + CPPUNIT_ASSERT(begin == begin); + CPPUNIT_ASSERT(end == end); +} + +void +SimpleBucketPriorityDatabaseTest::testCanGetPrioritizedBucket() +{ + SimpleBucketPriorityDatabase queue; + + PrioritizedBucket lowPriBucket(BucketId(16, 1234), Priority::VERY_LOW); + queue.setPriority(lowPriBucket); + + PrioritizedBucket highest(*queue.begin()); + CPPUNIT_ASSERT_EQUAL(lowPriBucket, highest); +} + +void +SimpleBucketPriorityDatabaseTest::testIterateOverMultiplePriorities() +{ + SimpleBucketPriorityDatabase queue; + + PrioritizedBucket lowPriBucket(BucketId(16, 1234), Priority::LOW); + PrioritizedBucket highPriBucket(BucketId(16, 4321), Priority::HIGH); + queue.setPriority(lowPriBucket); + queue.setPriority(highPriBucket); + + const_iterator iter(queue.begin()); + CPPUNIT_ASSERT_EQUAL(highPriBucket, *iter); + ++iter; + CPPUNIT_ASSERT(iter != queue.end()); + CPPUNIT_ASSERT_EQUAL(lowPriBucket, *iter); + ++iter; + CPPUNIT_ASSERT(iter == queue.end()); +} + +void +SimpleBucketPriorityDatabaseTest::testMultipleSetPriorityForOneBucket() +{ + SimpleBucketPriorityDatabase queue; + + PrioritizedBucket lowPriBucket(BucketId(16, 1234), Priority::LOW); + PrioritizedBucket highPriBucket(BucketId(16, 1234), Priority::HIGH); + + queue.setPriority(lowPriBucket); + queue.setPriority(highPriBucket); + + const_iterator iter(queue.begin()); + CPPUNIT_ASSERT_EQUAL(highPriBucket, *iter); + ++iter; + CPPUNIT_ASSERT(iter == queue.end()); +} + +void +SimpleBucketPriorityDatabaseTest::testNoMaintenanceNeededClearsBucketFromDatabase() +{ + SimpleBucketPriorityDatabase queue; + + PrioritizedBucket highPriBucket(BucketId(16, 1234), Priority::HIGH); + PrioritizedBucket noPriBucket(BucketId(16, 1234), + Priority::NO_MAINTENANCE_NEEDED); + queue.setPriority(highPriBucket); + queue.setPriority(noPriBucket); + + const_iterator iter(queue.begin()); + CPPUNIT_ASSERT(iter == queue.end()); +} + +void +SimpleBucketPriorityDatabaseTest::testIterateOverMultipleBucketsWithMultiplePriorities() +{ + SimpleBucketPriorityDatabase queue; + + PrioritizedBucket lowPriBucket1(BucketId(16, 1), Priority::LOW); + PrioritizedBucket lowPriBucket2(BucketId(16, 2), Priority::LOW); + PrioritizedBucket mediumPriBucket(BucketId(16, 3), Priority::MEDIUM); + PrioritizedBucket highPriBucket1(BucketId(16, 4), Priority::HIGH); + PrioritizedBucket highPriBucket2(BucketId(16, 5), Priority::HIGH); + + queue.setPriority(highPriBucket1); + queue.setPriority(lowPriBucket2); + queue.setPriority(mediumPriBucket); + queue.setPriority(highPriBucket2); + queue.setPriority(lowPriBucket1); + + const_iterator iter(queue.begin()); + PrioritizedBucket lastBucket(BucketId(), Priority::PRIORITY_LIMIT); + for (int i = 0; i < 5; ++i) { + CPPUNIT_ASSERT(iter != queue.end()); + CPPUNIT_ASSERT(!iter->moreImportantThan(lastBucket)); + lastBucket = *iter; + ++iter; + } + CPPUNIT_ASSERT(iter == queue.end()); +} + +} +} + diff --git a/storage/src/tests/distributor/simplemaintenancescannertest.cpp b/storage/src/tests/distributor/simplemaintenancescannertest.cpp new file mode 100644 index 00000000000..512a10bbd9a --- /dev/null +++ b/storage/src/tests/distributor/simplemaintenancescannertest.cpp @@ -0,0 +1,220 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/distributor/maintenance/simplemaintenancescanner.h> +#include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> +#include <vespa/storage/distributor/bucketdb/mapbucketdatabase.h> +#include <tests/distributor/maintenancemocks.h> + +#include <string> +#include <sstream> +#include <memory> +#include <algorithm> +#include <iterator> + +namespace storage { + +namespace distributor { + +using document::BucketId; +typedef MaintenancePriority Priority; + +class SimpleMaintenanceScannerTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(SimpleMaintenanceScannerTest); + CPPUNIT_TEST(testPrioritizeSingleBucket); + CPPUNIT_TEST(testPrioritizeMultipleBuckets); + CPPUNIT_TEST(testPendingMaintenanceOperationStatistics); + CPPUNIT_TEST(perNodeMaintenanceStatsAreTracked); + CPPUNIT_TEST(testReset); + CPPUNIT_TEST_SUITE_END(); + + using PendingStats = SimpleMaintenanceScanner::PendingMaintenanceStats; + + std::string dumpPriorityDbToString(const BucketPriorityDatabase&) const; + + std::unique_ptr<MockMaintenancePriorityGenerator> _priorityGenerator; + std::unique_ptr<MapBucketDatabase> _bucketDb; + std::unique_ptr<SimpleBucketPriorityDatabase> _priorityDb; + std::unique_ptr<SimpleMaintenanceScanner> _scanner; + + void addBucketToDb(int bucketNum); + + bool scanEntireDatabase(int expected); + + std::string stringifyGlobalPendingStats(const PendingStats&) const; + +public: + void testPrioritizeSingleBucket(); + void testPrioritizeMultipleBuckets(); + void testPendingMaintenanceOperationStatistics(); + void perNodeMaintenanceStatsAreTracked(); + void testReset(); + + void setUp(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(SimpleMaintenanceScannerTest); + +void +SimpleMaintenanceScannerTest::setUp() +{ + _priorityGenerator.reset(new MockMaintenancePriorityGenerator()); + _bucketDb.reset(new MapBucketDatabase()); + _priorityDb.reset(new SimpleBucketPriorityDatabase()); + _scanner.reset(new SimpleMaintenanceScanner(*_priorityDb, *_priorityGenerator, *_bucketDb)); +} + +void +SimpleMaintenanceScannerTest::addBucketToDb(int bucketNum) +{ + BucketDatabase::Entry entry(BucketId(16, bucketNum), BucketInfo()); + _bucketDb->update(entry); +} + +std::string +SimpleMaintenanceScannerTest::stringifyGlobalPendingStats( + const PendingStats& stats) const +{ + std::ostringstream ss; + ss << stats.global; + return ss.str(); +} + +void +SimpleMaintenanceScannerTest::testPrioritizeSingleBucket() +{ + addBucketToDb(1); + std::string expected("PrioritizedBucket(BucketId(0x4000000000000001), pri VERY_HIGH)\n"); + + CPPUNIT_ASSERT(!_scanner->scanNext().isDone()); + CPPUNIT_ASSERT_EQUAL(expected, _priorityDb->toString()); + + CPPUNIT_ASSERT(_scanner->scanNext().isDone()); + CPPUNIT_ASSERT_EQUAL(expected, _priorityDb->toString()); +} + +namespace { + std::string sortLines(const std::string& source) { + vespalib::StringTokenizer st(source,"\n",""); + std::vector<std::string> lines; + std::copy(st.begin(), st.end(), std::back_inserter(lines)); + std::sort(lines.begin(), lines.end()); + std::ostringstream ost; + for (auto& line : lines) { + ost << line << "\n"; + } + return ost.str(); + } +} + +void +SimpleMaintenanceScannerTest::testPrioritizeMultipleBuckets() +{ + addBucketToDb(1); + addBucketToDb(2); + addBucketToDb(3); + std::string expected("PrioritizedBucket(BucketId(0x4000000000000001), pri VERY_HIGH)\n" + "PrioritizedBucket(BucketId(0x4000000000000002), pri VERY_HIGH)\n" + "PrioritizedBucket(BucketId(0x4000000000000003), pri VERY_HIGH)\n"); + + CPPUNIT_ASSERT(scanEntireDatabase(3)); + CPPUNIT_ASSERT_EQUAL(sortLines(expected), + sortLines(_priorityDb->toString())); +} + +bool +SimpleMaintenanceScannerTest::scanEntireDatabase(int expected) +{ + for (int i = 0; i < expected; ++i) { + if (_scanner->scanNext().isDone()) { + return false; + } + } + return _scanner->scanNext().isDone(); +} + +void +SimpleMaintenanceScannerTest::testReset() +{ + addBucketToDb(1); + addBucketToDb(3); + + CPPUNIT_ASSERT(scanEntireDatabase(2)); + std::string expected("PrioritizedBucket(BucketId(0x4000000000000001), pri VERY_HIGH)\n" + "PrioritizedBucket(BucketId(0x4000000000000003), pri VERY_HIGH)\n"); + CPPUNIT_ASSERT_EQUAL(expected, _priorityDb->toString()); + + addBucketToDb(2); + CPPUNIT_ASSERT(scanEntireDatabase(0)); + CPPUNIT_ASSERT_EQUAL(expected, _priorityDb->toString()); + + _scanner->reset(); + CPPUNIT_ASSERT(scanEntireDatabase(3)); + + expected = "PrioritizedBucket(BucketId(0x4000000000000001), pri VERY_HIGH)\n" + "PrioritizedBucket(BucketId(0x4000000000000002), pri VERY_HIGH)\n" + "PrioritizedBucket(BucketId(0x4000000000000003), pri VERY_HIGH)\n"; + CPPUNIT_ASSERT_EQUAL(sortLines(expected), sortLines(_priorityDb->toString())); +} + +void +SimpleMaintenanceScannerTest::testPendingMaintenanceOperationStatistics() +{ + addBucketToDb(1); + addBucketToDb(3); + + std::string expectedEmpty("delete bucket: 0, merge bucket: 0, " + "split bucket: 0, join bucket: 0, " + "set bucket state: 0, garbage collection: 0"); + { + auto stats(_scanner->getPendingMaintenanceStats()); + CPPUNIT_ASSERT_EQUAL(expectedEmpty, stringifyGlobalPendingStats(stats)); + } + + CPPUNIT_ASSERT(scanEntireDatabase(2)); + + // All mock operations generated have the merge type. + { + auto stats(_scanner->getPendingMaintenanceStats()); + std::string expected("delete bucket: 0, merge bucket: 2, " + "split bucket: 0, join bucket: 0, " + "set bucket state: 0, garbage collection: 0"); + CPPUNIT_ASSERT_EQUAL(expected, stringifyGlobalPendingStats(stats)); + } + + _scanner->reset(); + { + auto stats(_scanner->getPendingMaintenanceStats()); + CPPUNIT_ASSERT_EQUAL(expectedEmpty, stringifyGlobalPendingStats(stats)); + } +} + +void +SimpleMaintenanceScannerTest::perNodeMaintenanceStatsAreTracked() +{ + addBucketToDb(1); + addBucketToDb(3); + { + auto stats(_scanner->getPendingMaintenanceStats()); + NodeMaintenanceStats emptyStats; + CPPUNIT_ASSERT_EQUAL(emptyStats, stats.perNodeStats.forNode(0)); + } + CPPUNIT_ASSERT(scanEntireDatabase(2)); + // Mock is currently hardwired to increment movingOut for node 1 and + // copyingIn for node 2 per bucket iterated (we've got 2). + auto stats(_scanner->getPendingMaintenanceStats()); + { + NodeMaintenanceStats wantedNode1Stats; + wantedNode1Stats.movingOut = 2; + CPPUNIT_ASSERT_EQUAL(wantedNode1Stats, stats.perNodeStats.forNode(1)); + } + { + NodeMaintenanceStats wantedNode2Stats; + wantedNode2Stats.copyingIn = 2; + CPPUNIT_ASSERT_EQUAL(wantedNode2Stats, stats.perNodeStats.forNode(2)); + } +} + +} +} diff --git a/storage/src/tests/distributor/splitbuckettest.cpp b/storage/src/tests/distributor/splitbuckettest.cpp new file mode 100644 index 00000000000..d0fa69d600e --- /dev/null +++ b/storage/src/tests/distributor/splitbuckettest.cpp @@ -0,0 +1,353 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <iomanip> +#include <iostream> +#include <memory> +#include <tests/common/dummystoragelink.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storage/distributor/operations/idealstate/splitoperation.h> +#include <vespa/storage/common/bucketmessages.h> +#include <vespa/document/base/documentid.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storage/distributor/idealstatemanager.h> +#include <vespa/storageapi/message/multioperation.h> +#include <tests/distributor/distributortestutil.h> + +using std::shared_ptr; +using namespace document; + +namespace storage { + +namespace distributor { + +class SplitOperationTest : public CppUnit::TestFixture, + public DistributorTestUtil +{ + CPPUNIT_TEST_SUITE(SplitOperationTest); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST(testMultiNodeFailure); + CPPUNIT_TEST(testCopyTrustedStatusNotCarriedOverAfterSplit); + CPPUNIT_TEST(testOperationBlockedByPendingJoin); + CPPUNIT_TEST_SUITE_END(); + + uint32_t splitByteSize; + uint32_t tooLargeBucketSize; + uint32_t splitCount; + uint32_t maxSplitBits; + +protected: + void testSimple(); + void testMultiNodeFailure(); + void testCopyTrustedStatusNotCarriedOverAfterSplit(); + void testOperationBlockedByPendingJoin(); + +public: + SplitOperationTest(); + + void setUp() { + createLinks(); + getConfig().setSplitCount(splitCount); + getConfig().setSplitSize(splitByteSize); + + } + + void tearDown() { + close(); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(SplitOperationTest); + +SplitOperationTest::SplitOperationTest() + : splitByteSize(10*1024*1024), + tooLargeBucketSize(splitByteSize * 1.1), + splitCount(UINT32_MAX), + maxSplitBits(58) +{ +} + +void +SplitOperationTest::testSimple() +{ + _distributor->enableClusterState( + lib::ClusterState("distributor:1 storage:1")); + + insertBucketInfo(document::BucketId(16, 1), 0, 0xabc, 1000, + tooLargeBucketSize, 250); + + SplitOperation op("storage", + BucketAndNodes(document::BucketId(16, 1), + toVector<uint16_t>(0)), + maxSplitBits, + splitCount, + splitByteSize); + + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + { + CPPUNIT_ASSERT_EQUAL(size_t(1), _sender.commands.size()); + + std::shared_ptr<api::StorageCommand> msg = _sender.commands[0]; + CPPUNIT_ASSERT(msg->getType() == api::MessageType::SPLITBUCKET); + CPPUNIT_ASSERT_EQUAL( + api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 0) + .toString(), + msg->getAddress()->toString()); + + std::shared_ptr<api::StorageReply> reply(msg->makeReply().release()); + api::SplitBucketReply* sreply( + static_cast<api::SplitBucketReply*>(reply.get())); + + sreply->getSplitInfo().push_back(api::SplitBucketReply::Entry( + document::BucketId(17, 1), + api::BucketInfo(100, 600, 5000000))); + + sreply->getSplitInfo().push_back(api::SplitBucketReply::Entry( + document::BucketId(17, 0x10001), + api::BucketInfo(110, 400, 6000000))); + + op.receive(_sender, reply); + } + + CPPUNIT_ASSERT(!getBucket(document::BucketId(16, 1)).valid()); + + { + BucketDatabase::Entry entry = getBucket(document::BucketId(17, 1)); + + CPPUNIT_ASSERT(entry.valid()); + CPPUNIT_ASSERT_EQUAL((uint16_t)0, entry->getNodeRef(0).getNode()); + CPPUNIT_ASSERT_EQUAL((uint32_t)100, entry->getNodeRef(0).getChecksum()); + CPPUNIT_ASSERT_EQUAL((uint32_t)5000000, + entry->getNodeRef(0).getTotalDocumentSize()); + CPPUNIT_ASSERT_EQUAL((uint32_t)600, + entry->getNodeRef(0).getDocumentCount()); + } + + { + BucketDatabase::Entry entry(getBucket(document::BucketId(17, 0x10001))); + + CPPUNIT_ASSERT(entry.valid()); + CPPUNIT_ASSERT_EQUAL((uint16_t)0, entry->getNodeRef(0).getNode()); + CPPUNIT_ASSERT_EQUAL((uint32_t)110, entry->getNodeRef(0).getChecksum()); + CPPUNIT_ASSERT_EQUAL((uint32_t)6000000, + entry->getNodeRef(0).getTotalDocumentSize()); + CPPUNIT_ASSERT_EQUAL((uint32_t)400, + entry->getNodeRef(0).getDocumentCount()); + } +} + +void +SplitOperationTest::testMultiNodeFailure() +{ + { + BucketDatabase::Entry entry(document::BucketId(16, 1)); + + BucketCopy copy(0, 0, api::BucketInfo(250, 1000, tooLargeBucketSize)); + entry->addNode(copy, toVector<uint16_t>(0)); + + entry->addNode(BucketCopy(0, 1, copy.getBucketInfo()), + toVector<uint16_t>(0)); + getBucketDatabase().update(entry); + } + + _distributor->enableClusterState( + lib::ClusterState("distributor:1 storage:2")); + + + SplitOperation op("storage", + BucketAndNodes(document::BucketId(16, 1), + toVector<uint16_t>(0,1)), + maxSplitBits, + splitCount, + splitByteSize); + + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + { + CPPUNIT_ASSERT_EQUAL((size_t)2, _sender.commands.size()); + + { + std::shared_ptr<api::StorageCommand> msg = _sender.commands[0]; + CPPUNIT_ASSERT(msg->getType() == api::MessageType::SPLITBUCKET); + CPPUNIT_ASSERT_EQUAL( + api::StorageMessageAddress("storage", + lib::NodeType::STORAGE, 0).toString(), + msg->getAddress()->toString()); + + api::SplitBucketReply* sreply( + static_cast<api::SplitBucketReply*>( + msg->makeReply().release())); + sreply->setResult(api::ReturnCode::OK); + + sreply->getSplitInfo().push_back(api::SplitBucketReply::Entry( + document::BucketId(17, 1), + api::BucketInfo(100, 600, 5000000))); + + sreply->getSplitInfo().push_back(api::SplitBucketReply::Entry( + document::BucketId(17, 0x10001), + api::BucketInfo(110, 400, 6000000))); + + op.receive(_sender, std::shared_ptr<api::StorageReply>(sreply)); + } + + sendReply(op, 1, api::ReturnCode::NOT_CONNECTED); + } + + { + BucketDatabase::Entry entry = getBucket(document::BucketId(16, 1)); + + CPPUNIT_ASSERT(entry.valid()); + CPPUNIT_ASSERT_EQUAL((uint32_t)1, entry->getNodeCount()); + + CPPUNIT_ASSERT_EQUAL((uint16_t)1, entry->getNodeRef(0).getNode()); + CPPUNIT_ASSERT_EQUAL((uint32_t)250, entry->getNodeRef(0).getChecksum()); + CPPUNIT_ASSERT_EQUAL(tooLargeBucketSize, + entry->getNodeRef(0).getTotalDocumentSize()); + CPPUNIT_ASSERT_EQUAL((uint32_t)1000, + entry->getNodeRef(0).getDocumentCount()); + } + + { + BucketDatabase::Entry entry = getBucket(document::BucketId(17, 1)); + + CPPUNIT_ASSERT(entry.valid()); + CPPUNIT_ASSERT_EQUAL((uint32_t)1, entry->getNodeCount()); + + CPPUNIT_ASSERT_EQUAL((uint16_t)0, entry->getNodeRef(0).getNode()); + CPPUNIT_ASSERT_EQUAL((uint32_t)100, entry->getNodeRef(0).getChecksum()); + CPPUNIT_ASSERT_EQUAL((uint32_t)5000000, + entry->getNodeRef(0).getTotalDocumentSize()); + CPPUNIT_ASSERT_EQUAL((uint32_t)600, + entry->getNodeRef(0).getDocumentCount()); + } + + { + BucketDatabase::Entry entry(getBucket(document::BucketId(17, 0x10001))); + + CPPUNIT_ASSERT(entry.valid()); + CPPUNIT_ASSERT_EQUAL((uint32_t)1, entry->getNodeCount()); + + CPPUNIT_ASSERT_EQUAL((uint16_t)0, entry->getNodeRef(0).getNode()); + CPPUNIT_ASSERT_EQUAL((uint32_t)110, entry->getNodeRef(0).getChecksum()); + CPPUNIT_ASSERT_EQUAL((uint32_t)6000000, + entry->getNodeRef(0).getTotalDocumentSize()); + CPPUNIT_ASSERT_EQUAL((uint32_t)400, + entry->getNodeRef(0).getDocumentCount()); + } +} + +void +SplitOperationTest::testCopyTrustedStatusNotCarriedOverAfterSplit() +{ + _distributor->enableClusterState( + lib::ClusterState("distributor:1 storage:2")); + + document::BucketId sourceBucket(16, 1); + /* + * Need 3 nodes to reproduce bug 6418516. Otherwise, the source bucket is + * left with only 1 copy which implicitly becomes trusted. When this copy + * is then split, the distributor db will automatically un-trust all buckets + * since it sees that multiple copies are trusted that are not consistent + * with each other. This prevents the bug from being visible. + */ + addNodesToBucketDB(sourceBucket, "0=150/20/30000000/t,1=450/50/60000/u," + "2=550/60/70000"); + + SplitOperation op("storage", + BucketAndNodes(sourceBucket, toVector<uint16_t>(0, 1)), + maxSplitBits, + splitCount, + splitByteSize); + + op.setIdealStateManager(&getIdealStateManager()); + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(size_t(3), _sender.commands.size()); + + std::vector<document::BucketId> childBuckets; + childBuckets.push_back(document::BucketId(17, 1)); + childBuckets.push_back(document::BucketId(17, 0x10001)); + + // Note: only 2 out of 3 requests replied to! + for (int i = 0; i < 2; ++i) { + std::shared_ptr<api::StorageCommand> msg = _sender.commands[i]; + CPPUNIT_ASSERT(msg->getType() == api::MessageType::SPLITBUCKET); + std::shared_ptr<api::StorageReply> reply(msg->makeReply().release()); + api::SplitBucketReply* sreply( + static_cast<api::SplitBucketReply*>(reply.get())); + + // Make sure copies differ so they cannot become implicitly trusted. + sreply->getSplitInfo().push_back(api::SplitBucketReply::Entry( + childBuckets[0], + api::BucketInfo(100 + i, 600, 5000000))); + sreply->getSplitInfo().push_back(api::SplitBucketReply::Entry( + childBuckets[1], + api::BucketInfo(110 + i, 400, 6000000))); + + op.receive(_sender, reply); + } + + CPPUNIT_ASSERT(getBucket(sourceBucket).valid()); // Still alive + + for (uint32_t i = 0; i < 2; ++i) { + BucketDatabase::Entry entry(getBucket(childBuckets[i])); + + CPPUNIT_ASSERT(entry.valid()); + CPPUNIT_ASSERT_EQUAL(size_t(2), entry->getNodes().size()); + + for (uint16_t j = 0; j < 2; ++j) { + CPPUNIT_ASSERT(!entry->getNodeRef(i).trusted()); + } + } +} + +void +SplitOperationTest::testOperationBlockedByPendingJoin() +{ + StorageComponentRegisterImpl compReg; + framework::defaultimplementation::FakeClock clock; + compReg.setClock(clock); + clock.setAbsoluteTimeInSeconds(1); + PendingMessageTracker tracker(compReg); + + _distributor->enableClusterState( + lib::ClusterState("distributor:1 storage:2")); + + document::BucketId joinTarget(2, 1); + std::vector<document::BucketId> joinSources = { + document::BucketId(3, 1), document::BucketId(3, 5) + }; + auto joinCmd = std::make_shared<api::JoinBucketsCommand>(joinTarget); + joinCmd->getSourceBuckets() = joinSources; + joinCmd->setAddress( + api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 0)); + + tracker.insert(joinCmd); + + insertBucketInfo(joinTarget, 0, 0xabc, 1000, 1234, 250); + + SplitOperation op("storage", + BucketAndNodes(joinTarget, toVector<uint16_t>(0)), + maxSplitBits, + splitCount, + splitByteSize); + + CPPUNIT_ASSERT(op.isBlocked(tracker)); + + // Now, pretend there's a join for another node in the same bucket. This + // will happen when a join is partially completed. + tracker.clearMessagesForNode(0); + CPPUNIT_ASSERT(!op.isBlocked(tracker)); + + joinCmd->setAddress( + api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 1)); + tracker.insert(joinCmd); + + CPPUNIT_ASSERT(op.isBlocked(tracker)); +} + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/statecheckerstest.cpp b/storage/src/tests/distributor/statecheckerstest.cpp new file mode 100644 index 00000000000..da444b9d22a --- /dev/null +++ b/storage/src/tests/distributor/statecheckerstest.cpp @@ -0,0 +1,1838 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storage/distributor/bucketdbupdater.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/storage/common/bucketmessages.h> +#include <vespa/storage/config/config-stor-distributormanager.h> +#include <vespa/storage/distributor/idealstatemanager.h> +#include <vespa/storage/distributor/operations/idealstate/mergeoperation.h> +#include <vespa/storage/distributor/operations/idealstate/removebucketoperation.h> +#include <vespa/storage/distributor/operations/idealstate/setbucketstateoperation.h> +#include <vespa/storage/distributor/operations/idealstate/splitoperation.h> +#include <vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h> +#include <vespa/storageapi/message/stat.h> +#include <vespa/storage/storageutil/utils.h> +#include <tests/distributor/distributortestutil.h> +#include <vespa/storage/distributor/statecheckers.h> +#include <vespa/storageapi/message/state.h> + +using namespace std::literals::string_literals; + +namespace storage { +namespace distributor { + +struct StateCheckersTest : public CppUnit::TestFixture, + public DistributorTestUtil +{ + StateCheckersTest() {} + + void setUp() { + createLinks(); + } + + void tearDown() { + close(); + } + + struct PendingMessage + { + uint32_t _msgType; + uint8_t _pri; + + PendingMessage() : _msgType(UINT32_MAX), _pri(0) {} + + PendingMessage(uint32_t msgType, uint8_t pri) + : _msgType(msgType), _pri(pri) {} + + bool shouldCheck() const { return _msgType != UINT32_MAX; } + }; + + void testSplit(); + void testInconsistentSplit(); + void splitCanBeScheduledWhenReplicasOnRetiredNodes(); + void testSynchronizeAndMove(); + void testDoNotMergeInconsistentlySplitBuckets(); + void doNotMoveReplicasWithinRetiredNodes(); + void testDeleteExtraCopies(); + void testDoNotDeleteActiveExtraCopies(); + void testConsistentCopiesOnRetiredNodesMayBeDeleted(); + void redundantCopyDeletedEvenWhenAllNodesRetired(); + void testJoin(); + void testDoNotJoinBelowClusterStateBitCount(); + void testAllowInconsistentJoinInDifferingSiblingIdealState(); + void testDoNotAllowInconsistentJoinWhenNotInIdealState(); + void testDoNotAllowInconsistentJoinWhenConfigDisabled(); + void testNoJoinWhenInvalidCopyExists(); + void testNoJoinOnDifferentNodes(); + void testNoJoinWhenCopyCountAboveRedundancyLevelsForLeftSibling(); + void testNoJoinWhenCopyCountAboveRedundancyLevelsForRightSibling(); + void testNoJoinWhenCopyCountAboveRedundancyLevelsForBothSiblings(); + void joinCanBeScheduledWhenReplicasOnRetiredNodes(); + void testBucketState(); + void testDoNotActivateNonReadyCopiesWhenIdealNodeInMaintenance(); + void testDoNotChangeActiveStateForInconsistentlySplitBuckets(); + void testNoActiveChangeForNonIdealCopiesWhenOtherwiseIdentical(); + void testBucketStatePerGroup(); + void allowActivationOfRetiredNodes(); + void inhibitBucketActivationIfDisabledInConfig(); + void inhibitBucketDeactivationIfDisabledInConfig(); + void retiredNodesOutOfSyncAreMerged(); + void testGarbageCollection(); + void gcInhibitedWhenIdealNodeInMaintenance(); + void testNoRemoveWhenIdealNodeInMaintenance(); + void testStepwiseJoinForSmallBucketsWithoutSiblings(); + void testNoStepwiseJoinWhenDisabledThroughConfig(); + void testNoStepwiseJoinWhenSingleSiblingTooLarge(); + void testStepwiseJoinMaySkipMultipleBitsWhenConsistent(); + void testStepwiseJoinDoesNotSkipBeyondLevelWithSibling(); + void contextPopulatesIdealStateContainers(); + void statsUpdatedWhenMergingDueToMove(); + void statsUpdatedWhenMergingDueToMissingCopy(); + void statsUpdatedWhenMergingDueToOutOfSyncCopies(); + + void enableClusterState(const lib::ClusterState& systemState) { + _distributor->enableClusterState(systemState); + } + + void insertJoinableBuckets(); + + void assertCurrentIdealState(const document::BucketId& bucket, + const std::vector<uint16_t> expected) + { + std::vector<uint16_t> idealNodes( + getIdealStateManager().getDistributorComponent() + .getDistribution().getIdealStorageNodes( + getIdealStateManager().getDistributorComponent() + .getClusterState(), + bucket, + "ui")); + CPPUNIT_ASSERT_EQUAL(expected, idealNodes); + } + + void enableInconsistentJoinInConfig(bool enabled); + + std::string testStateChecker( + StateChecker& checker, + StateChecker::Context& c, + bool includeBucketId = false, + const PendingMessage& blocker = PendingMessage(), + bool includeMessagePriority = false, + bool includeSchedulingPriority = false) + { + std::ostringstream ost; + + c.siblingBucket = getIdealStateManager().getDistributorComponent() + .getSibling(c.bucketId); + + std::vector<BucketDatabase::Entry> entries; + getBucketDatabase().getAll(c.bucketId, entries); + c.siblingEntry = getBucketDatabase().get(c.siblingBucket); + + c.entries = entries; + for (uint32_t j = 0; j < entries.size(); ++j) { + // Run checking only on this bucketid, but include all buckets + // owned by it or owners of it, so we can detect inconsistent split. + if (entries[j].getBucketId() == c.bucketId) { + c.entry = entries[j]; + + StateChecker::Result result(checker.check(c)); + IdealStateOperation::UP op(result.createOperation()); + if (op.get()) { + if (blocker.shouldCheck() + && op->shouldBlockThisOperation(blocker._msgType, + blocker._pri)) + { + return "BLOCKED"; + } + + if (!ost.str().empty()) { + ost << ","; + } + if (includeBucketId) { + ost << op->getBucketId() << ": "; + } + + ost << op->getDetailedReason(); + if (includeMessagePriority) { + ost << " (pri " + << static_cast<int>(op->getPriority()) + << ')'; + } + if (includeSchedulingPriority) { + ost << " (scheduling pri " + << MaintenancePriority::toString( + result.getPriority().getPriority()) + << ")"; + } + } + } + } + + if (ost.str().empty()) { + ost << "NO OPERATIONS GENERATED"; + } + + getBucketDatabase().clear(); + + return ost.str(); + } + + std::string testGarbageCollection(uint32_t prevTimestamp, + uint32_t nowTimestamp, + uint32_t checkInterval, + uint32_t lastChangeTime = 0, + bool includePriority = false); + + std::string testSplit(uint32_t splitCount, + uint32_t splitSize, + uint32_t minSplitBits, + const std::string& bucketInfo, + const PendingMessage& blocker = PendingMessage(), + bool includePriority = false); + + std::string testInconsistentSplit(const document::BucketId& bid, + bool includePriority = false); + + std::string testJoin(uint32_t joinCount, + uint32_t joinSize, + uint32_t minSplitBits, + const document::BucketId& bid, + const PendingMessage& blocker = PendingMessage(), + bool includePriority = false); + + struct CheckerParams { + std::string _bucketInfo; + std::string _clusterState {"distributor:1 storage:2"}; + std::string _expect; + static const PendingMessage NO_OP_BLOCKER; + const PendingMessage* _blockerMessage {&NO_OP_BLOCKER}; + uint32_t _redundancy {2}; + uint32_t _splitCount {0}; + uint32_t _splitSize {0}; + uint32_t _minSplitBits {0}; + bool _includeMessagePriority {false}; + bool _includeSchedulingPriority {false}; + + CheckerParams& expect(const std::string& e) { + _expect = e; + return *this; + } + CheckerParams& bucketInfo(const std::string& info) { + _bucketInfo = info; + return *this; + } + CheckerParams& clusterState(const std::string& state) { + _clusterState = state; + return *this; + } + CheckerParams& blockerMessage(const PendingMessage& blocker) { + _blockerMessage = &blocker; + return *this; + } + CheckerParams& redundancy(uint32_t r) { + _redundancy = r; + return *this; + } + CheckerParams& includeMessagePriority(bool includePri) { + _includeMessagePriority = includePri; + return *this; + } + CheckerParams& includeSchedulingPriority(bool includePri) { + _includeSchedulingPriority = includePri; + return *this; + } + }; + + template <typename CheckerImpl> + void runAndVerify(const CheckerParams& params) { + CheckerImpl checker; + + document::BucketId bid(17, 0); + addNodesToBucketDB(bid, params._bucketInfo); + setRedundancy(params._redundancy); + _distributor->enableClusterState( + lib::ClusterState(params._clusterState)); + NodeMaintenanceStatsTracker statsTracker; + StateChecker::Context c( + getExternalOperationHandler(), statsTracker, bid); + std::string result = testStateChecker( + checker, c, false, *params._blockerMessage, + params._includeMessagePriority, + params._includeSchedulingPriority); + CPPUNIT_ASSERT_EQUAL(params._expect, result); + } + + std::string testSynchronizeAndMove( + const std::string& bucketInfo, + const std::string& clusterState = "distributor:1 storage:2", + uint32_t redundancy = 2, + const PendingMessage& blocker = PendingMessage(), + bool includePriority = false); + + std::string testDeleteExtraCopies( + const std::string& bucketInfo, + uint32_t redundancy = 2, + const PendingMessage& blocker = PendingMessage(), + const std::string& clusterState = "", + bool includePriority = false); + + std::string testBucketState(const std::string& bucketInfo, + uint32_t redundancy = 2, + bool includePriority = false); + std::string testBucketStatePerGroup(const std::string& bucketInfo, + bool includePriority = false); + + CPPUNIT_TEST_SUITE(StateCheckersTest); + CPPUNIT_TEST(testSplit); + CPPUNIT_TEST(testInconsistentSplit); + CPPUNIT_TEST(splitCanBeScheduledWhenReplicasOnRetiredNodes); + CPPUNIT_TEST(testSynchronizeAndMove); + CPPUNIT_TEST(testDoNotMergeInconsistentlySplitBuckets); + CPPUNIT_TEST(doNotMoveReplicasWithinRetiredNodes); + CPPUNIT_TEST(retiredNodesOutOfSyncAreMerged); + CPPUNIT_TEST(testDoNotChangeActiveStateForInconsistentlySplitBuckets); + CPPUNIT_TEST(testDeleteExtraCopies); + CPPUNIT_TEST(testDoNotDeleteActiveExtraCopies); + CPPUNIT_TEST(testConsistentCopiesOnRetiredNodesMayBeDeleted); + CPPUNIT_TEST(redundantCopyDeletedEvenWhenAllNodesRetired); + CPPUNIT_TEST(testJoin); + CPPUNIT_TEST(testDoNotJoinBelowClusterStateBitCount); + CPPUNIT_TEST(testAllowInconsistentJoinInDifferingSiblingIdealState); + CPPUNIT_TEST(testDoNotAllowInconsistentJoinWhenNotInIdealState); + CPPUNIT_TEST(testDoNotAllowInconsistentJoinWhenConfigDisabled); + CPPUNIT_TEST(testNoJoinWhenInvalidCopyExists); + CPPUNIT_TEST(testNoJoinOnDifferentNodes); + CPPUNIT_TEST(testNoJoinWhenCopyCountAboveRedundancyLevelsForLeftSibling); + CPPUNIT_TEST(testNoJoinWhenCopyCountAboveRedundancyLevelsForRightSibling); + CPPUNIT_TEST(testNoJoinWhenCopyCountAboveRedundancyLevelsForBothSiblings); + CPPUNIT_TEST(joinCanBeScheduledWhenReplicasOnRetiredNodes); + CPPUNIT_TEST(testBucketState); + CPPUNIT_TEST(testDoNotActivateNonReadyCopiesWhenIdealNodeInMaintenance); + CPPUNIT_TEST(testNoActiveChangeForNonIdealCopiesWhenOtherwiseIdentical); + CPPUNIT_TEST(testBucketStatePerGroup); + CPPUNIT_TEST(allowActivationOfRetiredNodes); + CPPUNIT_TEST(inhibitBucketActivationIfDisabledInConfig); + CPPUNIT_TEST(inhibitBucketDeactivationIfDisabledInConfig); + CPPUNIT_TEST(testGarbageCollection); + CPPUNIT_TEST(gcInhibitedWhenIdealNodeInMaintenance); + CPPUNIT_TEST(testNoRemoveWhenIdealNodeInMaintenance); + CPPUNIT_TEST(testStepwiseJoinForSmallBucketsWithoutSiblings); + CPPUNIT_TEST(testNoStepwiseJoinWhenDisabledThroughConfig); + CPPUNIT_TEST(testNoStepwiseJoinWhenSingleSiblingTooLarge); + CPPUNIT_TEST(testStepwiseJoinMaySkipMultipleBitsWhenConsistent); + CPPUNIT_TEST(testStepwiseJoinDoesNotSkipBeyondLevelWithSibling); + CPPUNIT_TEST(contextPopulatesIdealStateContainers); + CPPUNIT_TEST(statsUpdatedWhenMergingDueToMove); + CPPUNIT_TEST(statsUpdatedWhenMergingDueToMissingCopy); + CPPUNIT_TEST(statsUpdatedWhenMergingDueToOutOfSyncCopies); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(StateCheckersTest); + +const StateCheckersTest::PendingMessage +StateCheckersTest::CheckerParams::NO_OP_BLOCKER; + +std::string StateCheckersTest::testSplit(uint32_t splitCount, + uint32_t splitSize, + uint32_t minSplitBits, + const std::string& bucketInfo, + const PendingMessage& blocker, + bool includePriority) +{ + document::BucketId bid(17, 0); + + addNodesToBucketDB(bid, bucketInfo); + + SplitBucketStateChecker checker; + NodeMaintenanceStatsTracker statsTracker; + StateChecker::Context c(getExternalOperationHandler(), statsTracker, bid); + getConfig().setSplitSize(splitSize); + getConfig().setSplitCount(splitCount); + getConfig().setMinimalBucketSplit(minSplitBits); + return testStateChecker(checker, c, false, blocker, includePriority); +} + + + +void +StateCheckersTest::testSplit() +{ + setupDistributor(3, 10, "distributor:1 storage:2"); + + CPPUNIT_ASSERT_EQUAL( + std::string("[Splitting bucket because its maximum size (2000 b, 10 docs, 10 meta, 2000 b total) " + "is higher than the configured limit of (1000, 4294967295)]"), + testSplit((uint32_t)-1, 1000, 16, "0=100/10/2000")); + + CPPUNIT_ASSERT_EQUAL( + std::string("[Splitting bucket because its maximum size (1000 b, " + "200 docs, 200 meta, 1000 b total) " + "is higher than the configured limit of (10000, 100)] " + "(pri 175)"), + testSplit(100, 10000, 16, "0=100/200/1000", PendingMessage(), true)); + + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testSplit(1000, 1000, 16, "0=100/200/200")); + + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testSplit(1000, 1000, 16, "0=100/200/200/2000/2000")); + + CPPUNIT_ASSERT_EQUAL( + std::string("[Splitting bucket because the current system size requires " + "a higher minimum split bit]"), + testSplit((uint32_t)-1, (uint32_t)-1, 24, "0=100/200/1000")); + + CPPUNIT_ASSERT_EQUAL( + std::string("[Splitting bucket because its maximum size (1000 b, 1000 docs, 1000 meta, 1000 b total) " + "is higher than the configured limit of (10000, 100)]"), + testSplit(100, 10000, 16, "0=100/10/10,1=100/1000/1000")); + + CPPUNIT_ASSERT_EQUAL( + std::string("[Splitting bucket because its maximum size (1000 b, 1000 docs, 1000 meta, 1000 b total) " + "is higher than the configured limit of (10000, 100)]"), + testSplit(100, 10000, 16, "0=1/0/0,1=100/1000/1000")); + + CPPUNIT_ASSERT_EQUAL( + std::string("[Splitting bucket because its maximum size (1000 b, 1000 docs, 1000 meta, 1000 b total) " + "is higher than the configured limit of (10000, 100)]"), + testSplit(100, 10000, 16, "0=0/0/1,1=100/1000/1000")); + + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testSplit(1000, 1000, 16, "0=100/1/200000")); + + CPPUNIT_ASSERT_EQUAL( + std::string("BLOCKED"), + testSplit(100, 10000, 16, "0=0/0/1,1=100/1000/1000", + PendingMessage(api::MessageType::SPLITBUCKET_ID, 0))); + + // Split on too high meta + CPPUNIT_ASSERT_EQUAL( + std::string("[Splitting bucket because its maximum size (1000 b, 100 docs, 2100 meta, 15000000 b total) " + "is higher than the configured limit of (10000000, 1000)]"), + testSplit(1000, 10000000, 16, "0=14/100/1000/2100/15000000")); + // Split on too high file size + CPPUNIT_ASSERT_EQUAL( + std::string("[Splitting bucket because its maximum size (1000 b, 100 docs, 1500 meta, 21000000 b total) " + "is higher than the configured limit of (10000000, 1000)]"), + testSplit(1000, 10000000, 16, "0=14/100/1000/1500/21000000")); + + // Don't block higher priority splits than what's already pending. + CPPUNIT_ASSERT_EQUAL( + std::string("[Splitting bucket because its maximum size (1000 b, 1000 docs, 1000 meta, 1000 b total) " + "is higher than the configured limit of (10000, 100)]"), + testSplit(100, 10000, 16, "0=100/10/10,1=100/1000/1000", + PendingMessage(api::MessageType::SPLITBUCKET_ID, 255))); + + // But must block equal priority splits that are already pending, or + // we'll end up spamming the nodes with splits! + // NOTE: assuming split priority of 175. + CPPUNIT_ASSERT_EQUAL( + std::string("BLOCKED"), + testSplit(100, 10000, 16, "0=0/0/1,1=100/1000/1000", + PendingMessage(api::MessageType::SPLITBUCKET_ID, 175))); + + // Don't split if we're already joining, since there's a window of time + // where the bucket will appear to be inconsistently split when the join + // is not finished on all the nodes. + CPPUNIT_ASSERT_EQUAL( + std::string("BLOCKED"), + testSplit(100, 10000, 16, "0=0/0/1,1=100/1000/1000", + PendingMessage(api::MessageType::JOINBUCKETS_ID, 175))); +} + +std::string +StateCheckersTest::testInconsistentSplit(const document::BucketId& bid, + bool includePriority) +{ + SplitInconsistentStateChecker checker; + NodeMaintenanceStatsTracker statsTracker; + StateChecker::Context c(getExternalOperationHandler(), statsTracker, bid); + return testStateChecker(checker, c, true, + PendingMessage(), includePriority); +} + +void +StateCheckersTest::testInconsistentSplit() +{ + setupDistributor(3, 10, "distributor:1 storage:2"); + + insertBucketInfo(document::BucketId(16, 1), 1, 0x1, 1, 1); + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testInconsistentSplit(document::BucketId(16, 1))); + + insertBucketInfo(document::BucketId(17, 1), 1, 0x1, 1, 1); + insertBucketInfo(document::BucketId(16, 1), 1, 0x1, 1, 1); + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001): [Bucket is inconsistently " + "split (list includes 0x4000000000000001, 0x4400000000000001) " + "Splitting it to improve the problem (max used bits 17)]"), + testInconsistentSplit(document::BucketId(16, 1))); + + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testInconsistentSplit(document::BucketId(17, 1))); + + insertBucketInfo(document::BucketId(17, 1), 0, 0x0, 0, 0); + insertBucketInfo(document::BucketId(16, 1), 1, 0x1, 1, 1); + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000001): [Bucket is inconsistently " + "split (list includes 0x4000000000000001, 0x4400000000000001) " + "Splitting it to improve the problem (max used bits " + "17)] (pri 110)"), + testInconsistentSplit(document::BucketId(16, 1), true)); + + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testInconsistentSplit(document::BucketId(17, 1))); +} + +void +StateCheckersTest::splitCanBeScheduledWhenReplicasOnRetiredNodes() +{ + setupDistributor(Redundancy(2), NodeCount(2), + "distributor:1 storage:2, .0.s:r .1.s:r"); + CPPUNIT_ASSERT_EQUAL( + "[Splitting bucket because its maximum size (2000 b, 10 docs, " + "10 meta, 2000 b total) is higher than the configured limit of " + "(1000, 4294967295)]"s, + testSplit(UINT32_MAX, 1000, 16, "0=100/10/2000")); +} + +std::string +StateCheckersTest::testJoin(uint32_t joinCount, + uint32_t joinSize, + uint32_t minSplitBits, + const document::BucketId& bid, + const PendingMessage& blocker, + bool includePriority) +{ + JoinBucketsStateChecker checker; + getConfig().setJoinSize(joinSize); + getConfig().setJoinCount(joinCount); + getConfig().setMinimalBucketSplit(minSplitBits); + + NodeMaintenanceStatsTracker statsTracker; + StateChecker::Context c(getExternalOperationHandler(), statsTracker, bid); + return testStateChecker(checker, c, true, blocker, includePriority); +} + +void +StateCheckersTest::insertJoinableBuckets() +{ + insertBucketInfo(document::BucketId(33, 1), 1, 0x1, 1, 1); + insertBucketInfo(document::BucketId(33, 0x100000001), 1, 0x1, 1, 1); +} + +void +StateCheckersTest::testJoin() +{ + setupDistributor(3, 10, "distributor:1 storage:2"); + + insertJoinableBuckets(); + CPPUNIT_ASSERT_EQUAL(std::string( + "BucketId(0x8000000000000001): " + "[Joining buckets BucketId(0x8400000000000001) and " + "BucketId(0x8400000100000001) because their size " + "(2 bytes, 2 docs) is less than the configured limit " + "of (100, 10)"), + testJoin(10, 100, 16, document::BucketId(33, 1))); + + insertJoinableBuckets(); + // Join size is 0, so only look at document count + CPPUNIT_ASSERT_EQUAL(std::string( + "BucketId(0x8000000000000001): " + "[Joining buckets BucketId(0x8400000000000001) and " + "BucketId(0x8400000100000001) because their size " + "(2 bytes, 2 docs) is less than the configured limit " + "of (0, 3) (pri 155)"), + testJoin(3, 0, 16, document::BucketId(33, 1), PendingMessage(), true)); + + insertJoinableBuckets(); + // Should not generate joins for both pairs, just the primary + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testJoin(10, 100, 16, document::BucketId(33, 0x100000001))); + + insertJoinableBuckets(); + // Should not generate join if min split bits is higher + CPPUNIT_ASSERT_EQUAL(std::string("NO OPERATIONS GENERATED"), + testJoin(10, 100, 33, document::BucketId(33, 1))); + + insertJoinableBuckets(); + // Meta data too big, no join + insertBucketInfo(document::BucketId(33, 1), 1, + api::BucketInfo(0x1, 1, 1, 1000, 1000)); + + CPPUNIT_ASSERT_EQUAL(std::string("NO OPERATIONS GENERATED"), + testJoin(10, 100, 16, document::BucketId(33, 1))); + + insertJoinableBuckets(); + // Bucket recently created + insertBucketInfo(document::BucketId(33, 1), 1, + api::BucketInfo(0x1, 0, 0, 0, 0)); + CPPUNIT_ASSERT_EQUAL(std::string("NO OPERATIONS GENERATED"), + testJoin(10, 100, 16, document::BucketId(33, 1))); + +} + +/** + * If distributor config says minsplitcount is 8, but cluster state says that + * distribution bit count is 16, we should not allow the join to take place. + * We don't properly handle the "reduce distribution bits" case in general, so + * the safest is to never violate this and to effectively make distribution + * bit increases a one-way street. + */ +void +StateCheckersTest::testDoNotJoinBelowClusterStateBitCount() +{ + setupDistributor(2, 2, "bits:16 distributor:1 storage:2"); + // Insert sibling buckets at 16 bits that are small enough to be joined + // unless there is special logic for dealing with distribution bits. + insertBucketInfo(document::BucketId(16, 1), 1, 0x1, 1, 1); + insertBucketInfo(document::BucketId(16, (1 << 15) | 1), 1, 0x1, 1, 1); + using ConfiguredMinSplitBits = uint32_t; + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testJoin(100, 100, ConfiguredMinSplitBits(8), + document::BucketId(16, 1))); +} + +void +StateCheckersTest::enableInconsistentJoinInConfig(bool enabled) +{ + vespa::config::content::core::StorDistributormanagerConfigBuilder config; + config.enableInconsistentJoin = enabled; + getConfig().configure(config); +} + +void +StateCheckersTest::testAllowInconsistentJoinInDifferingSiblingIdealState() +{ + // Normally, bucket siblings have an ideal state on the same node in order + // to enable joining these back together. However, the ideal disks assigned + // may differ and it's sufficient for a sibling bucket's ideal disk to be + // down on the node of its other sibling for it to be assigned a different + // node. In this case, there's no other way to get buckets joined back + // together than if we allow bucket replicas to get temporarily out of sync + // by _forcing_ a join across all replicas no matter their placement. + // This will trigger a merge to reconcile and move the new bucket copies to + // their ideal location. + setupDistributor(2, 3, "distributor:1 storage:3 .0.d:20 .0.d.14.s:d .2.d:20"); + document::BucketId sibling1(33, 0x000000001); // ideal disk 14 on node 0 + document::BucketId sibling2(33, 0x100000001); // ideal disk 1 on node 0 + + // Full node sequence sorted by score for sibling(1|2) is [0, 2, 1]. + // Node 0 cannot be used, so use 1 instead. + assertCurrentIdealState(sibling1, {2, 1}); + assertCurrentIdealState(sibling2, {0, 2}); + + insertBucketInfo(sibling1, 2, 0x1, 2, 3); + insertBucketInfo(sibling1, 1, 0x1, 2, 3); + insertBucketInfo(sibling2, 0, 0x1, 2, 3); + insertBucketInfo(sibling2, 2, 0x1, 2, 3); + + enableInconsistentJoinInConfig(true); + + CPPUNIT_ASSERT_EQUAL(std::string( + "BucketId(0x8000000000000001): " + "[Joining buckets BucketId(0x8400000000000001) and " + "BucketId(0x8400000100000001) because their size " + "(6 bytes, 4 docs) is less than the configured limit " + "of (100, 10)"), + testJoin(10, 100, 16, sibling1)); +} + +void +StateCheckersTest::testDoNotAllowInconsistentJoinWhenNotInIdealState() +{ + setupDistributor(2, 4, "distributor:1 storage:4 .0.d:20 .0.d.14.s:d .2.d:20 .3.d:20"); + document::BucketId sibling1(33, 0x000000001); + document::BucketId sibling2(33, 0x100000001); + + assertCurrentIdealState(sibling1, {3, 2}); + assertCurrentIdealState(sibling2, {3, 0}); + + insertBucketInfo(sibling1, 3, 0x1, 2, 3); + insertBucketInfo(sibling1, 2, 0x1, 2, 3); + insertBucketInfo(sibling2, 3, 0x1, 2, 3); + insertBucketInfo(sibling2, 1, 0x1, 2, 3); // not in ideal state + + enableInconsistentJoinInConfig(true); + + CPPUNIT_ASSERT_EQUAL(std::string("NO OPERATIONS GENERATED"), + testJoin(10, 100, 16, sibling1)); +} + +void +StateCheckersTest::testDoNotAllowInconsistentJoinWhenConfigDisabled() +{ + setupDistributor(2, 3, "distributor:1 storage:3 .0.d:20 .0.d.14.s:d .2.d:20"); + document::BucketId sibling1(33, 0x000000001); // ideal disk 14 on node 0 + document::BucketId sibling2(33, 0x100000001); // ideal disk 1 on node 0 + + // Full node sequence sorted by score for sibling(1|2) is [0, 2, 1]. + // Node 0 cannot be used, so use 1 instead. + assertCurrentIdealState(sibling1, {2, 1}); + assertCurrentIdealState(sibling2, {0, 2}); + + insertBucketInfo(sibling1, 2, 0x1, 2, 3); + insertBucketInfo(sibling1, 1, 0x1, 2, 3); + insertBucketInfo(sibling2, 0, 0x1, 2, 3); + insertBucketInfo(sibling2, 2, 0x1, 2, 3); + + enableInconsistentJoinInConfig(false); + + CPPUNIT_ASSERT_EQUAL(std::string("NO OPERATIONS GENERATED"), + testJoin(10, 100, 16, sibling1)); +} + +void +StateCheckersTest::testNoJoinWhenInvalidCopyExists() +{ + setupDistributor(3, 10, "distributor:1 storage:3"); + + insertBucketInfo(document::BucketId(33, 0x100000001), 1, 0x1, 1, 1); + // No join when there exists an invalid copy + insertBucketInfo(document::BucketId(33, 1), 1, api::BucketInfo()); + + CPPUNIT_ASSERT_EQUAL(std::string("NO OPERATIONS GENERATED"), + testJoin(10, 100, 16, document::BucketId(33, 1))); +} + +void +StateCheckersTest::testNoJoinOnDifferentNodes() +{ + setupDistributor(3, 10, "distributor:1 storage:2"); + + insertBucketInfo(document::BucketId(33, 0x000000001), 0, 0x1, 1, 1); + insertBucketInfo(document::BucketId(33, 0x100000001), 1, 0x1, 1, 1); + + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testJoin(10, 100, 16, document::BucketId(33, 0x1))); +} + +void +StateCheckersTest::testNoJoinWhenCopyCountAboveRedundancyLevelsForLeftSibling() +{ + setupDistributor(3, 10, "distributor:1 storage:2"); + setRedundancy(1); + insertBucketInfo(document::BucketId(33, 0x000000001), 0, 0x1, 1, 1); + insertBucketInfo(document::BucketId(33, 0x000000001), 1, 0x1, 1, 1); + insertBucketInfo(document::BucketId(33, 0x100000001), 0, 0x1, 1, 1); + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testJoin(10, 100, 16, document::BucketId(33, 0x1))); +} + +void +StateCheckersTest::testNoJoinWhenCopyCountAboveRedundancyLevelsForRightSibling() +{ + setupDistributor(3, 10, "distributor:1 storage:2"); + setRedundancy(1); + insertBucketInfo(document::BucketId(33, 0x000000001), 1, 0x1, 1, 1); + insertBucketInfo(document::BucketId(33, 0x100000001), 0, 0x1, 1, 1); + insertBucketInfo(document::BucketId(33, 0x100000001), 1, 0x1, 1, 1); + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testJoin(10, 100, 16, document::BucketId(33, 0x1))); +} + +void +StateCheckersTest::testNoJoinWhenCopyCountAboveRedundancyLevelsForBothSiblings() +{ + setupDistributor(3, 10, "distributor:1 storage:2"); + setRedundancy(1); + insertBucketInfo(document::BucketId(33, 0x000000001), 0, 0x1, 1, 1); + insertBucketInfo(document::BucketId(33, 0x000000001), 1, 0x1, 1, 1); + insertBucketInfo(document::BucketId(33, 0x100000001), 0, 0x1, 1, 1); + insertBucketInfo(document::BucketId(33, 0x100000001), 1, 0x1, 1, 1); + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testJoin(10, 100, 16, document::BucketId(33, 0x1))); +} + +std::string +StateCheckersTest::testSynchronizeAndMove(const std::string& bucketInfo, + const std::string& clusterState, + uint32_t redundancy, + const PendingMessage& blocker, + bool includePriority) +{ + document::BucketId bid(17, 0); + + addNodesToBucketDB(bid, bucketInfo); + + SynchronizeAndMoveStateChecker checker; + setRedundancy(redundancy); + + _distributor->enableClusterState(lib::ClusterState(clusterState)); + NodeMaintenanceStatsTracker statsTracker; + StateChecker::Context c(getExternalOperationHandler(), statsTracker, bid); + return testStateChecker(checker, c, false, blocker, includePriority); +} + +void +StateCheckersTest::testSynchronizeAndMove() +{ + // Plus if it was more obvious which nodes were in ideal state for various + // cluster states. (One possibility to override ideal state function for + // test) + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams().expect( + "[Synchronizing buckets with different checksums " + "node(idx=0,crc=0x1,docs=1/1,bytes=1/1,trusted=false," + "active=false), " + "node(idx=1,crc=0x2,docs=2/2,bytes=2/2,trusted=false," + "active=false)] " + "(scheduling pri MEDIUM)") + .bucketInfo("0=1,1=2") + .includeSchedulingPriority(true)); + + // If 1+ nodes in ideal state is in maintenance, do nothing + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("NO OPERATIONS GENERATED") + .bucketInfo("0=1,2=2") + .clusterState("distributor:1 storage:3 .1.s:m")); + + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("[Moving bucket to ideal node 3] " + "(scheduling pri VERY_LOW)") + .bucketInfo("0=1,1=1,2=1") + .clusterState("distributor:1 storage:4") + .includeSchedulingPriority(true)); + + // Not doing anything in ideal state + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("NO OPERATIONS GENERATED") + .bucketInfo("0=1,1=1,3=1") + .clusterState("distributor:1 storage:4")); + + // Both copies out of ideal state + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("[Moving bucket to ideal node 1]" + "[Moving bucket to ideal node 3] (pri 165) " + "(scheduling pri VERY_LOW)") + .clusterState("distributor:1 storage:5") + .bucketInfo("0=1,4=1,5=1") + .includeMessagePriority(true) + .includeSchedulingPriority(true)); + + // Too little redundancy and out of ideal state. Note that in this case, + // the non-ideal node is reported as a missing node and not with a "Moving + // bucket to ideal node" reason. + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("[Adding missing node 1]" + "[Adding missing node 3] (pri 120) " + "(scheduling pri MEDIUM)") + .bucketInfo("0=1") + .clusterState("distributor:1 storage:5") + .includeMessagePriority(true) + .includeSchedulingPriority(true)); + + // Synchronizing even when ideal state is in sync + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("[Synchronizing buckets with different checksums " + "node(idx=0,crc=0x3,docs=3/3,bytes=3/3,trusted=false," + "active=false), " + "node(idx=1,crc=0x3,docs=3/3,bytes=3/3,trusted=false," + "active=false), " + "node(idx=2,crc=0x0,docs=0/0,bytes=0/0,trusted=false," + "active=false)]") + .bucketInfo("0=3,1=3,2=0") + .clusterState("distributor:1 storage:3")); + + // Synchronize even when we have >= redundancy trusted copies and ideal + // nodes are in sync. + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("[Synchronizing buckets with different checksums " + "node(idx=0,crc=0x2,docs=3/3,bytes=4/4,trusted=false," + "active=false), " + "node(idx=1,crc=0x1,docs=2/2,bytes=3/3,trusted=true," + "active=false), " + "node(idx=2,crc=0x1,docs=2/2,bytes=3/3,trusted=true," + "active=false), " + "node(idx=3,crc=0x1,docs=2/2,bytes=3/3,trusted=true," + "active=false)] " + "(pri 120) (scheduling pri MEDIUM)") + .bucketInfo("0=2/3/4,1=1/2/3/t,2=1/2/3/t,3=1/2/3/t") + .clusterState("distributor:1 storage:5") + .includeMessagePriority(true) + .includeSchedulingPriority(true)); + + // Not doing anything if one of the buckets in ideal state is invalid + // but we have redundancy coverage otherwise + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("NO OPERATIONS GENERATED") + .bucketInfo("1=0/0/1,3=1") + .clusterState("distributor:1 storage:4")); + + // Not doing anything if all copies we have are invalid + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("NO OPERATIONS GENERATED") + .bucketInfo("1=0/0/1,3=0/0/1") + .clusterState("distributor:1 storage:4")); + + // Not doing anything if we have < redundancy copies but all existing + // copies are invalid. + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("NO OPERATIONS GENERATED") + .bucketInfo("1=0/0/1") + .clusterState("distributor:1 storage:4")); +} + +void +StateCheckersTest::testDoNotMergeInconsistentlySplitBuckets() +{ + // No merge generated if buckets are inconsistently split. + // This matches the case where a bucket has been split into 2 on one + // node and is not yet split on another; we should never try to merge + // either two of the split leaf buckets back onto the first node! + // Running state checker on a leaf: + addNodesToBucketDB(document::BucketId(16, 0), "0=2"); + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testSynchronizeAndMove("1=1", // 17 bits + "distributor:1 storage:4")); + // Running state checker on an inner node bucket: + addNodesToBucketDB(document::BucketId(18, 0), "0=2"); + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testSynchronizeAndMove("0=1", // 17 bits + "distributor:1 storage:4")); +} + +void +StateCheckersTest::doNotMoveReplicasWithinRetiredNodes() +{ + // Nodes 1 and 3 would be in ideal state if the nodes were not retired. + // Here, all nodes are retired and we should thus not do any sort of + // moving. + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("NO OPERATIONS GENERATED") + .bucketInfo("0=2,1=2") + .clusterState("distributor:1 storage:4 " + ".0.s:r .1.s:r .2.s:r .3.s:r")); +} + +void +StateCheckersTest::retiredNodesOutOfSyncAreMerged() +{ + // Normally, we'd do a merge that'd move the bucket to new nodes, leaving + // the out of sync retired nodes as source-only replicas. But here we + // don't have that choice and thus try to do the most useful thing we can + // with what we have available to us (which is to try to get things in + // sync). + runAndVerify<SynchronizeAndMoveStateChecker>( + CheckerParams() + .expect("[Synchronizing buckets with different checksums " + "node(idx=0,crc=0x1,docs=1/1,bytes=1/1,trusted=false," + "active=false), " + "node(idx=1,crc=0x2,docs=2/2,bytes=2/2,trusted=false," + "active=false)]") + .bucketInfo("0=1,1=2") + .clusterState("distributor:1 storage:4 " + ".0.s:r .1.s:r .2.s:r .3.s:r")); +} + +std::string +StateCheckersTest::testDeleteExtraCopies( + const std::string& bucketInfo, uint32_t redundancy, + const PendingMessage& blocker, + const std::string& clusterState, + bool includePriority) +{ + document::BucketId bid(17, 0); + + addNodesToBucketDB(bid, bucketInfo); + setRedundancy(redundancy); + + if (!clusterState.empty()) { + _distributor->enableClusterState(lib::ClusterState(clusterState)); + } + DeleteExtraCopiesStateChecker checker; + NodeMaintenanceStatsTracker statsTracker; + StateChecker::Context c(getExternalOperationHandler(), statsTracker, bid); + return testStateChecker(checker, c, false, blocker, includePriority); +} + + +void +StateCheckersTest::testDeleteExtraCopies() +{ + setupDistributor(2, 100, "distributor:1 storage:4"); + + { + std::vector<uint16_t> idealNodes( + getIdealStateManager().getDistributorComponent() + .getDistribution().getIdealStorageNodes( + getIdealStateManager().getDistributorComponent().getClusterState(), + document::BucketId(17, 0), + "ui")); + std::vector<uint16_t> wanted; + wanted.push_back(1); + wanted.push_back(3); + CPPUNIT_ASSERT_EQUAL(wanted, idealNodes); + } + + CPPUNIT_ASSERT_EQUAL_MSG( + "Remove empty buckets", + std::string("[Removing all copies since bucket is empty:node(idx=0,crc=0x0," + "docs=0/0,bytes=0/0,trusted=false,active=false)]" + " (pri 100)"), + testDeleteExtraCopies("0=0", 2, PendingMessage(), "", true)); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Remove extra trusted copy", + std::string("[Removing redundant in-sync copy from node 2]"), + testDeleteExtraCopies("3=3/3/3/t,1=3/3/3/t,2=3/3/3/t")); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Redundant copies in sync can be removed without trusted being a " + "factor of consideration. Ideal state copy not removed.", + std::string("[Removing redundant in-sync copy from node 2]"), + testDeleteExtraCopies("3=3/3/3,1=3/3/3/t,2=3/3/3/t")); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Need redundancy number of copies", + std::string("NO OPERATIONS GENERATED"), + testDeleteExtraCopies("0=3,1=3")); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Do not remove extra copies without enough trusted copies", + std::string("NO OPERATIONS GENERATED"), + testDeleteExtraCopies("0=0/0/1,1=3,2=3")); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Do not remove buckets that have meta entries", + std::string("NO OPERATIONS GENERATED"), + testDeleteExtraCopies("0=0/0/1,1=0/0/1")); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Do not remove any recently created copies", + std::string("NO OPERATIONS GENERATED"), + testDeleteExtraCopies("0=1/0/0/t,1=1/0/0/t,2=1/0/0/t")); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Do not remove untrusted copy that is out of sync", + std::string("NO OPERATIONS GENERATED"), + testDeleteExtraCopies("0=2/3/4,1=1/2/3/t,2=1/2/3/t")); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Do not remove out of sync copies, even if we have more than #" + "redundancy trusted copies", + std::string("NO OPERATIONS GENERATED"), + testDeleteExtraCopies("0=2/3/4,1=1/2/3/t,2=1/2/3/t,3=1/2/3/t")); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Don't remove unless we have enough trusted " + "copies to satisfy redundancy", + std::string("NO OPERATIONS GENERATED"), + testDeleteExtraCopies("0=2/3/4,1=1/2/3,2=2/3/4,3=1/2/3")); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Only remove empty copies unless all other copies are in sync", + std::string("[Removing empty copy from node 4]"), + testDeleteExtraCopies("0=2/3/4,1=1/2/3,2=2/3/4,3=1/2/3,4=0/0/0")); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Remove redundant empty copy", + std::string("[Removing empty copy from node 0]"), + testDeleteExtraCopies("1=2/3,3=1/2/3,0=0/0/0")); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Remove empty bucket with multiple copies", + std::string( + "[Removing all copies since bucket is empty:" + "node(idx=0,crc=0x0,docs=0/0,bytes=0/0,trusted=false,active=false), " + "node(idx=1,crc=0x0,docs=0/0,bytes=0/0,trusted=false,active=false), " + "node(idx=2,crc=0x0,docs=0/0,bytes=0/0,trusted=false,active=false)]"), + testDeleteExtraCopies("0=0/0/0,1=0/0/0,2=0/0/0")); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Pending persistence operation blocks delete", + std::string("BLOCKED"), + testDeleteExtraCopies("0=0/0/0,1=1/2/3/t,2=1/2/3/t", + 2, + PendingMessage(api::MessageType::PUT_ID, 255))); +} + +void +StateCheckersTest::testDoNotDeleteActiveExtraCopies() +{ + setupDistributor(2, 100, "distributor:1 storage:4"); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Do not delete redundant copy if it is marked active", + std::string("NO OPERATIONS GENERATED"), + testDeleteExtraCopies("3=3/3/3/t,1=3/3/3/t,2=3/3/3/t/a")); +} + +void +StateCheckersTest::testConsistentCopiesOnRetiredNodesMayBeDeleted() +{ + setupDistributor(2, 100, "distributor:1 storage:4 .1.s:r"); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Remove in-sync copy on node that is retired", + std::string("[Removing redundant in-sync copy from node 1]"), + testDeleteExtraCopies("3=3/3/3/t,1=3/3/3/t,2=3/3/3/t")); +} + +void +StateCheckersTest::redundantCopyDeletedEvenWhenAllNodesRetired() +{ + setupDistributor(2, 100, "distributor:1 storage:4 " + ".0.s:r .1.s:r .2.s:r .3.s:r"); + + CPPUNIT_ASSERT_EQUAL_MSG( + "Remove in-sync copy on node that is retired", + "[Removing redundant in-sync copy from node 2]"s, + testDeleteExtraCopies("3=3/3/3/t,1=3/3/3/t,2=3/3/3/t")); +} + +std::string StateCheckersTest::testBucketState( + const std::string& bucketInfo, uint32_t redundancy, + bool includePriority) +{ + document::BucketId bid(17, 0); + setRedundancy(redundancy); + addNodesToBucketDB(bid, bucketInfo); + + BucketStateStateChecker checker; + NodeMaintenanceStatsTracker statsTracker; + StateChecker::Context c(getExternalOperationHandler(), statsTracker, bid); + return testStateChecker(checker, c, false, PendingMessage(), + includePriority); +} + +void +StateCheckersTest::testBucketState() +{ + setupDistributor(2, 100, "distributor:1 storage:4"); + + { + // Set config explicitly so we can compare priorities for differing + // cases. + DistributorConfiguration::MaintenancePriorities mp; + mp.activateNoExistingActive = 90; + mp.activateWithExistingActive = 120; + getConfig().setMaintenancePriorities(mp); + } + + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testBucketState("")); + + // Node 1 is in ideal state + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 1 as active:" + " copy is ideal state priority 0] (pri 90)"), + testBucketState("1=2/3/4", 2, true)); + + // Node 3 is in ideal state + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 3 as active:" + " copy is ideal state priority 1]"), + testBucketState("3=2/3/4")); + + // No trusted nodes, but node 1 is first in ideal state. + // Also check bad case where more than 1 node is set as active just + // to ensure we can get out of that situation if it should ever happen. + // Nothing done with node 3 since is't not active and shouldn't be. + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 1 as active:" + " copy is ideal state priority 0]" + "[Setting node 0 as inactive]" + "[Setting node 2 as inactive] (pri 120)"), + testBucketState("0=3/4/5/u/a,1=3,2=4/5/6/u/a,3=3", 2, true)); + + // Test setting active when only node available is not contained + // within the resolved ideal state. + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 0 as active: first available copy]"), + testBucketState("0=2/3/4")); + + // A trusted ideal state copy should be set active rather than a non-trusted + // ideal state copy + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 3 as active:" + " copy is trusted and ideal state priority 1]" + "[Setting node 1 as inactive]"), + testBucketState("1=2/3/4/u/a,3=5/6/7/t")); + + // None of the ideal state copies are trusted but a non-ideal copy is. + // The trusted copy should be active. + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 2 as active: copy is trusted]"), + testBucketState("1=2/3/4,3=5/6/7/,2=8/9/10/t")); + + // Make sure bucket db ordering does not matter + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 2 as active: copy is trusted]"), + testBucketState("2=8/9/10/t,1=2/3/4,3=5/6/7")); + + // If copy is already active, we shouldn't generate operations + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testBucketState("1=2/3/4/t/a")); + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testBucketState("1=2/3/4,3=5/6/7/t/a")); + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testBucketState("2=8/9/10/t/a,1=2/3/4,3=5/6/7")); + + // If multiple buckets are active, deactive all but one + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 2 as inactive]" + "[Setting node 3 as inactive]"), + testBucketState("1=1/2/3/t/a,2=1/2/3/t/a,3=1/2/3/t/a")); + + // Invalid buckets should not be included + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testBucketState("1=0/0/1,3=0/0/1")); + + // Ready preferred over trusted & ideal state + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testBucketState("2=8/9/10/t/i/u,1=2/3/4/u/a/r,3=5/6/7")); + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 2 as active: copy is ready]" + "[Setting node 1 as inactive]"), + testBucketState("2=8/9/10/u/i/r,1=2/3/4/u/a/u,3=5/6/7/u/i/u")); + + // Prefer in ideal state if multiple copies ready + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 3 as active: copy is ready]" + "[Setting node 1 as inactive]"), + testBucketState("2=8/9/10/u/i/r,1=2/3/4/u/a/u,3=5/6/7/u/i/r")); + + // Prefer ideal state if all ready but no trusted + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 1 as active: copy is ready]"), + testBucketState("2=8/9/10/u/i/r,1=2/3/4/u/i/r,3=5/6/7/u/i/r")); + + // Prefer trusted over ideal state + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 2 as active: copy is ready and trusted]" + "[Setting node 1 as inactive]"), + testBucketState("2=8/9/10/t/i/r,1=2/3/4/u/a/r,3=5/6/7")); +} + +/** + * Users assume that setting nodes into maintenance will not cause extra load + * on the cluster, but activating non-ready copies because the active copy went + * into maintenance violates that assumption. See bug 6833209 for context and + * details. + */ +void +StateCheckersTest::testDoNotActivateNonReadyCopiesWhenIdealNodeInMaintenance() +{ + setupDistributor(2, 100, "distributor:1 storage:4 .1.s:m"); + // Ideal node 1 is in maintenance and no ready copy available. + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testBucketState("2=8/9/10/t/i/u,3=5/6/7")); + // But we should activate another copy iff there's another ready copy. + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 2 as active: copy is ready]"), + testBucketState("2=8/9/10/u/i/r,3=5/6/7/u/i/u")); +} + +/** + * We really do not want to activate buckets when they are inconsistent. + * See bug 6395693 for a set of reasons why. + */ +void +StateCheckersTest::testDoNotChangeActiveStateForInconsistentlySplitBuckets() +{ + setupDistributor(2, 100, "distributor:1 storage:4"); + // Running state checker on a leaf: + addNodesToBucketDB(document::BucketId(16, 0), "0=2"); + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testBucketState("1=1")); // 17 bits + // Running state checker on an inner node bucket: + addNodesToBucketDB(document::BucketId(18, 0), "0=2"); + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testSynchronizeAndMove("0=1")); // 17 bits +} + +/** + * If all existing copies are outside the ideal state, e.g. if the set of nodes + * in the cluster has changed significantly, we do not want to change the active + * state of copies needlessly iff the copies are otherwise equally scored in + * terms of activation eligibility. If we do not prioritize existing active + * copies higher in this case, it's possible that their ideal order has been + * permutated, causing another copy to rank higher in the ideal state node + * sequence. This would in turn activate the newly higher ranked copy and + * deactivate the previously active copy, causing transient search duplicates + * and uneeded work in the cluster; new copies will be created and indexed + * soon anyway. + * + * See bug 7278932. + */ +void +StateCheckersTest::testNoActiveChangeForNonIdealCopiesWhenOtherwiseIdentical() +{ + setupDistributor(2, 100, "distributor:1 storage:50"); + // 1 is more ideal than 3 in this state, but since they're both not part + // of the #redundancy ideal set, activation should not change hands. + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testBucketState("1=2/3/4/t/i/r,3=2/3/4/t/a/r")); + // Same applies if the copies aren't ready, since if a copy has been marked + // as active it will already have started background indexing. No need in + // undoing that if we don't have any better candidates going anyway. + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testBucketState("1=2/3/4/t,3=2/3/4/t/a")); +} + +std::string StateCheckersTest::testBucketStatePerGroup( + const std::string& bucketInfo, bool includePriority) +{ + document::BucketId bid(17, 0); + addNodesToBucketDB(bid, bucketInfo); + + BucketStateStateChecker checker; + NodeMaintenanceStatsTracker statsTracker; + StateChecker::Context c(getExternalOperationHandler(), statsTracker, bid); + return testStateChecker(checker, c, false, PendingMessage(), + includePriority); +} + +void +StateCheckersTest::testBucketStatePerGroup() +{ + setupDistributor(6, 20, "distributor:1 storage:12 .2.s:d .4.s:d .7.s:d"); + vespa::config::content::StorDistributionConfigBuilder config; + config.activePerLeafGroup = true; + config.redundancy = 6; + config.group.resize(4); + config.group[0].index = "invalid"; + config.group[0].name = "invalid"; + config.group[0].partitions = "2|2|*"; + config.group[1].index = "0"; + config.group[1].name = "left"; + config.group[1].nodes.resize(3); + config.group[1].nodes[0].index = 0; + config.group[1].nodes[1].index = 1; + config.group[1].nodes[2].index = 3; + config.group[2].index = "1"; + config.group[2].name = "right"; + config.group[2].nodes.resize(3); + config.group[2].nodes[0].index = 5; + config.group[2].nodes[1].index = 6; + config.group[2].nodes[2].index = 8; + config.group[3].index = "2"; + config.group[3].name = "middle"; + config.group[3].nodes.resize(3); + config.group[3].nodes[0].index = 9; + config.group[3].nodes[1].index = 10; + config.group[3].nodes[2].index = 11; + lib::Distribution::SP distr(new lib::Distribution(config)); + _node->getComponentRegister().setDistribution(distr); + + { + DistributorConfiguration::MaintenancePriorities mp; + mp.activateNoExistingActive = 90; + mp.activateWithExistingActive = 120; + getConfig().setMaintenancePriorities(mp); + } + + // Node 1 and 8 is is ideal state + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 1 as active: " + "copy is trusted and ideal state priority 4]" + "[Setting node 6 as active: " + "copy is trusted and ideal state priority 0] (pri 90)"), + testBucketStatePerGroup("0=2/3/4/t, 1=2/3/4/t, 3=2/3/4/t, " + "5=2/3/4/t, 6=2/3/4/t, 8=2/3/4/t", true)); + + // Data differ between groups + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 1 as active: " + "copy is trusted and ideal state priority 4]" + "[Setting node 6 as active: " + "copy is ideal state priority 0] (pri 90)"), + testBucketStatePerGroup("0=2/3/4/t, 1=2/3/4/t, 3=2/3/4/t, " + "5=5/6/7, 6=5/6/7, 8=5/6/7", true)); + + // Disable too + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 0 as inactive]" + "[Setting node 3 as inactive]" + "[Setting node 5 as inactive]" + "[Setting node 8 as inactive] (pri 90)"), + testBucketStatePerGroup("0=2/3/4/t/a, 1=2/3/4/t/a, 3=2/3/4/t/a, " + "5=2/3/4/t/a, 6=2/3/4/t/a, 8=2/3/4/t/a", + true)); + + // Node 1 and 8 is is ideal state + CPPUNIT_ASSERT_EQUAL( + std::string("[Setting node 1 as active: " + "copy is trusted and ideal state priority 4]" + "[Setting node 6 as active: " + "copy is trusted and ideal state priority 0]" + "[Setting node 9 as active: " + "copy is trusted and ideal state priority 2] (pri 90)"), + testBucketStatePerGroup("0=2/3/4/t, 1=2/3/4/t, 3=2/3/4/t, " + "5=2/3/4/t, 6=2/3/4/t, 8=2/3/4/t, " + "9=2/3/4/t, 10=2/3/4/t, 11=2/3/4/t", + true)); +} + +void +StateCheckersTest::allowActivationOfRetiredNodes() +{ + // All nodes in retired state implies that the ideal state is empty. But + // we still want to be able to shuffle bucket activations around in order + // to preserve coverage. + setupDistributor(2, 2, "distributor:1 storage:2 .0.s:r .1.s:r"); + CPPUNIT_ASSERT_EQUAL( + "[Setting node 1 as active: copy is trusted]" + "[Setting node 0 as inactive]"s, + testBucketState("0=2/3/4/u/a,1=5/6/7/t")); +} + +void +StateCheckersTest::inhibitBucketActivationIfDisabledInConfig() +{ + setupDistributor(2, 4, "distributor:1 storage:4"); + disableBucketActivationInConfig(true); + + // Node 1 is in ideal state and only replica and should be activated in + // an indexed cluster context (but not here). + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testBucketState("1=2/3/4", 2, true)); +} + +void +StateCheckersTest::inhibitBucketDeactivationIfDisabledInConfig() +{ + setupDistributor(2, 4, "distributor:1 storage:4"); + disableBucketActivationInConfig(true); + + // Multiple replicas which would have been deactivated. This test is mostly + // for the sake of completion; a scenario where buckets are active while + // having no indexed documents configured should not happen. + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testBucketState("1=1/2/3/t/a,2=1/2/3/t/a,3=1/2/3/t/a")); +} + +std::string StateCheckersTest::testGarbageCollection( + uint32_t prevTimestamp, uint32_t nowTimestamp, + uint32_t checkInterval, uint32_t lastChangeTime, + bool includePriority) +{ + BucketDatabase::Entry e(document::BucketId(17, 0)); + e.getBucketInfo().addNode(BucketCopy(prevTimestamp, 0, + api::BucketInfo(3,3,3)), + toVector((uint16_t)0)); + e.getBucketInfo().setLastGarbageCollectionTime(prevTimestamp); + getBucketDatabase().update(e); + + GarbageCollectionStateChecker checker; + getConfig().setGarbageCollection("music", checkInterval); + getConfig().setLastGarbageCollectionChangeTime(lastChangeTime); + NodeMaintenanceStatsTracker statsTracker; + StateChecker::Context c(getExternalOperationHandler(), statsTracker, + e.getBucketId()); + getClock().setAbsoluteTimeInSeconds(nowTimestamp); + return testStateChecker(checker, c, false, PendingMessage(), + includePriority); +} + +void +StateCheckersTest::testGarbageCollection() +{ + // BucketId(17, 0) has id (and thus 'hash') 0x4400000000000000. With a + // check interval modulo of 3600, this implies a start point of 848. + + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testGarbageCollection(900, 3600 + 847, 3600)); + + CPPUNIT_ASSERT_EQUAL( + std::string("[Needs garbage collection: Last check at 900, current time 4448, " + "configured interval 3600]"), + testGarbageCollection(900, 3600 + 848, 3600)); + + CPPUNIT_ASSERT_EQUAL( + std::string("[Needs garbage collection: Last check at 3, current time 4000, " + "configured interval 3600]"), + testGarbageCollection(3, 4000, 3600)); + + // GC start point 3648. + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testGarbageCollection(3, 3647, 8000)); + + CPPUNIT_ASSERT_EQUAL( + std::string("[Needs garbage collection: Last check at 3, current time 4000, " + "configured interval 3600]"), + testGarbageCollection(3, 4000, 3600)); + + // GC explicitly disabled. + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testGarbageCollection(3, 4000, 0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testGarbageCollection(3, 3, 1)); + + CPPUNIT_ASSERT_EQUAL( + std::string("[Needs garbage collection: Last check at 3, current time 4000, " + "configured interval 300] (pri 200)"), + testGarbageCollection(3, 4000, 300, 1, true)); + + CPPUNIT_ASSERT_EQUAL( + std::string("NO OPERATIONS GENERATED"), + testGarbageCollection(3850, 4000, 300, 1)); +} + +/** + * When a node is in maintenance, we want to do our best to avoid any unneeded + * changes to the bucket replicas' states, as this will require re-syncing of + * the replicas when the node out of maintenance. Consequently we should not + * trigger GC for buckets when this is the case. + */ +void +StateCheckersTest::gcInhibitedWhenIdealNodeInMaintenance() +{ + // Redundancy is 3, so with only 3 nodes, node 1 is guaranteed to be part of + // the ideal state of any bucket in the system. + setupDistributor(3, 3, "distributor:1 storage:3 .1.s:m"); + document::BucketId bucket(17, 0); + addNodesToBucketDB(bucket, "0=10/100/1/true," + "1=10/100/1/true," + "2=10/100/1/true"); + BucketDatabase::Entry e(getBucketDatabase().get(bucket)); + e.getBucketInfo().setLastGarbageCollectionTime(3); + getBucketDatabase().update(e); + + GarbageCollectionStateChecker checker; + getConfig().setGarbageCollection("music", 3600); + getConfig().setLastGarbageCollectionChangeTime(0); + NodeMaintenanceStatsTracker statsTracker; + StateChecker::Context c(getExternalOperationHandler(), statsTracker, + bucket); + getClock().setAbsoluteTimeInSeconds(4000); + // Would normally (in a non-maintenance case) trigger GC due to having + // overshot the GC check cycle. + auto result = testStateChecker(checker, c, false, PendingMessage(), false); + + CPPUNIT_ASSERT_EQUAL(std::string("NO OPERATIONS GENERATED"), result); +} + +/* + * Bug 6656726, comment #25. Merge state checker does not execute if an ideal + * node is in maintenance, so for symmetry we need to do the same for deletes + * (it's bad mojo to potentially delete something that would've been merged + * had it not been for a node being in maintenance). + */ +void +StateCheckersTest::testNoRemoveWhenIdealNodeInMaintenance() +{ + CPPUNIT_ASSERT_EQUAL_MSG( + "Do not remove when ideal node is in maintenance mode", + std::string("NO OPERATIONS GENERATED"), + testDeleteExtraCopies("0=10/100/1/true," + "1=10/100/1/true," + "2=10/100/1/true", + 2, PendingMessage(), + "distributor:1 storage:3 .1.s:m")); +} + +/* + * Just joining buckets where both children are present is not enough to + * ensure any system can compact its bucket tree. We must therefore + * gradually hoist buckets higher into the tree when possible in order + * to converge in a state where as many buckets as possible have siblings + * on the same level. + * + * See bug 6768991 for context. + */ +void +StateCheckersTest::testStepwiseJoinForSmallBucketsWithoutSiblings() +{ + setupDistributor(3, 10, "distributor:1 storage:2 bits:1"); + vespa::config::content::core::StorDistributormanagerConfigBuilder config; + config.enableJoinForSiblingLessBuckets = true; + getConfig().configure(config); + // Buckets without siblings but that should be step-wise joined back + // into bucket (2, 1). + insertBucketInfo(document::BucketId(3, 1), 1, 0x1, 1, 1); + insertBucketInfo(document::BucketId(3, 0x3), 1, 0x1, 1, 1); + CPPUNIT_ASSERT_EQUAL(std::string( + "BucketId(0x0800000000000001): " + "[Joining buckets BucketId(0x0c00000000000001) and " + "BucketId(0x0c00000000000001) because their size " + "(1 bytes, 1 docs) is less than the configured limit " + "of (100, 10)"), + testJoin(10, 100, 2, document::BucketId(3, 1))); + + // Other bucket should be joined as well. Together the two join targets + // will transform into a mighty sibling pair that can rule the galaxy + // (and also be joined together afterwards)! + insertBucketInfo(document::BucketId(3, 1), 1, 0x1, 1, 1); + insertBucketInfo(document::BucketId(3, 0x3), 1, 0x1, 1, 1); + CPPUNIT_ASSERT_EQUAL(std::string( + "BucketId(0x0800000000000003): " + "[Joining buckets BucketId(0x0c00000000000003) and " + "BucketId(0x0c00000000000003) because their size " + "(1 bytes, 1 docs) is less than the configured limit " + "of (100, 10)"), + testJoin(10, 100, 2, document::BucketId(3, 0x3))); +} + +void +StateCheckersTest::testNoStepwiseJoinWhenDisabledThroughConfig() +{ + setupDistributor(3, 10, "distributor:1 storage:2 bits:1"); + vespa::config::content::core::StorDistributormanagerConfigBuilder config; + config.enableJoinForSiblingLessBuckets = false; + getConfig().configure(config); + + // Buckets without siblings but that would have been step-wise joined back + // into bucket 1 if it had been config-enabled. + insertBucketInfo(document::BucketId(3, 1), 1, 0x1, 1, 1); + insertBucketInfo(document::BucketId(3, 0x3), 1, 0x1, 1, 1); + CPPUNIT_ASSERT_EQUAL(std::string("NO OPERATIONS GENERATED"), + testJoin(10, 100, 1, document::BucketId(3, 1))); +} + +void +StateCheckersTest::testNoStepwiseJoinWhenSingleSiblingTooLarge() +{ + setupDistributor(3, 10, "distributor:1 storage:2 bits:1"); + vespa::config::content::core::StorDistributormanagerConfigBuilder config; + config.enableJoinForSiblingLessBuckets = true; + getConfig().configure(config); + + // Bucket is exactly at the boundary where it's too big. + insertBucketInfo(document::BucketId(3, 1), 1, 0x1, 10, 100); + insertBucketInfo(document::BucketId(3, 0x3), 1, 0x1, 1, 1); + CPPUNIT_ASSERT_EQUAL(std::string("NO OPERATIONS GENERATED"), + testJoin(10, 100, 1, document::BucketId(3, 1))); +} + +void +StateCheckersTest::testStepwiseJoinMaySkipMultipleBitsWhenConsistent() +{ + setupDistributor(2, 10, "distributor:1 storage:2 bits:8"); + vespa::config::content::core::StorDistributormanagerConfigBuilder config; + config.enableJoinForSiblingLessBuckets = true; + getConfig().configure(config); + + insertBucketInfo(document::BucketId(16, 1), 1, 0x1, 1, 1); + // No buckets further up in the tree, can join up to the distribution bit + // limit at 8. + CPPUNIT_ASSERT_EQUAL(std::string( + "BucketId(0x2000000000000001): " + "[Joining buckets BucketId(0x4000000000000001) and " + "BucketId(0x4000000000000001) because their size " + "(1 bytes, 1 docs) is less than the configured limit " + "of (100, 10)"), + testJoin(10, 100, 8, document::BucketId(16, 1))); +} + +void +StateCheckersTest::testStepwiseJoinDoesNotSkipBeyondLevelWithSibling() +{ + setupDistributor(2, 10, "distributor:1 storage:2 bits:8"); + vespa::config::content::core::StorDistributormanagerConfigBuilder config; + config.enableJoinForSiblingLessBuckets = true; + getConfig().configure(config); + + // All 0-branch children + insertBucketInfo(document::BucketId(16, 0), 1, 0x1, 1, 1); + // 0-branches down to level 10, then 1-branch down to level 11. This means + // the (16, 0) bucket cannot be moved further up than level 11 as it has a + // sibling there (0x2c00000000000400 sibling of 0x2c00000000000000). + insertBucketInfo(document::BucketId(11, 1 << 10), 1, 0x1, 1, 1); + CPPUNIT_ASSERT_EQUAL(std::string( + "BucketId(0x2c00000000000000): " + "[Joining buckets BucketId(0x4000000000000000) and " + "BucketId(0x4000000000000000) because their size " + "(1 bytes, 1 docs) is less than the configured limit " + "of (100, 10)"), + testJoin(10, 100, 8, document::BucketId(16, 0))); +} + +void +StateCheckersTest::joinCanBeScheduledWhenReplicasOnRetiredNodes() +{ + setupDistributor(1, 1, "distributor:1 storage:1 .0.s.:r"); + insertJoinableBuckets(); + CPPUNIT_ASSERT_EQUAL( + "BucketId(0x8000000000000001): " + "[Joining buckets BucketId(0x8400000000000001) and " + "BucketId(0x8400000100000001) because their size " + "(2 bytes, 2 docs) is less than the configured limit " + "of (100, 10)"s, + testJoin(10, 100, 16, document::BucketId(33, 1))); +} + +void +StateCheckersTest::contextPopulatesIdealStateContainers() +{ + // 1 and 3 are ideal nodes for bucket {17, 0} + setupDistributor(2, 100, "distributor:1 storage:4"); + + NodeMaintenanceStatsTracker statsTracker; + StateChecker::Context c(getExternalOperationHandler(), statsTracker, {17, 0}); + + CPPUNIT_ASSERT_EQUAL((std::vector<uint16_t>{1, 3}), c.idealState); + CPPUNIT_ASSERT_EQUAL(size_t(2), c.unorderedIdealState.size()); + CPPUNIT_ASSERT(c.unorderedIdealState.find(1) + != c.unorderedIdealState.end()); + CPPUNIT_ASSERT(c.unorderedIdealState.find(3) + != c.unorderedIdealState.end()); +} + +namespace { + +template <typename Checker> +class StateCheckerRunner +{ + StateCheckersTest& _fixture; + NodeMaintenanceStatsTracker _statsTracker; + std::string _result; +public: + StateCheckerRunner(StateCheckersTest& fixture) + : _fixture(fixture) + { + } + + StateCheckerRunner& addToDb(const document::BucketId& bid, + const std::string& bucketInfo) + { + _fixture.addNodesToBucketDB(bid, bucketInfo); + return *this; + } + + StateCheckerRunner& redundancy(uint32_t red) { + _fixture.setRedundancy(red); + return *this; + } + + StateCheckerRunner& clusterState(const std::string& state) { + _fixture.enableClusterState(lib::ClusterState(state)); + return *this; + } + + // Run the templated state checker with the provided parameters, updating + // _result with the ideal state operations triggered. + // NOTE: resets the bucket database! + void runFor(const document::BucketId& bid) { + Checker checker; + StateChecker::Context c(_fixture.getExternalOperationHandler(), _statsTracker, bid); + _result = _fixture.testStateChecker( + checker, c, false, StateCheckersTest::PendingMessage(), false); + } + + const std::string& result() const { return _result; } + const NodeMaintenanceStatsTracker& stats() const { + return _statsTracker; + } +}; + +} // anon ns + +void +StateCheckersTest::statsUpdatedWhenMergingDueToMove() +{ + StateCheckerRunner<SynchronizeAndMoveStateChecker> runner(*this); + // Ideal state for bucket {17,0} in given cluster state is [1, 3] + runner.addToDb({17, 0}, "0=1,1=1,2=1") + .clusterState("distributor:1 storage:4") + .runFor({17, 0}); + // Node 1 treated as copy source, but not as move source. + { + NodeMaintenanceStats wanted; + wanted.copyingOut = 1; + CPPUNIT_ASSERT_EQUAL(wanted, runner.stats().forNode(1)); + } + // Moving 1 bucket from nodes {0, 2} into 3. + // Note that we do not at this point in time distinguish _which_ of these + // will do the actual data movement to node 3. + { + NodeMaintenanceStats wanted; + wanted.copyingIn = 1; + CPPUNIT_ASSERT_EQUAL(wanted, runner.stats().forNode(3)); + } + { + NodeMaintenanceStats wanted; + wanted.movingOut = 1; + CPPUNIT_ASSERT_EQUAL(wanted, runner.stats().forNode(0)); + CPPUNIT_ASSERT_EQUAL(wanted, runner.stats().forNode(2)); + } +} + +void +StateCheckersTest::statsUpdatedWhenMergingDueToMissingCopy() +{ + StateCheckerRunner<SynchronizeAndMoveStateChecker> runner(*this); + // Ideal state for bucket {17,0} in given cluster state is [1, 3] + runner.addToDb({17, 0}, "1=1") + .clusterState("distributor:1 storage:4") + .runFor({17, 0}); + + { + NodeMaintenanceStats wanted; + wanted.copyingIn = 1; + CPPUNIT_ASSERT_EQUAL(wanted, runner.stats().forNode(3)); + } + { + NodeMaintenanceStats wanted; + wanted.copyingOut = 1; + CPPUNIT_ASSERT_EQUAL(wanted, runner.stats().forNode(1)); + } +} + +void +StateCheckersTest::statsUpdatedWhenMergingDueToOutOfSyncCopies() +{ + StateCheckerRunner<SynchronizeAndMoveStateChecker> runner(*this); + runner.addToDb({17, 0}, "1=1,3=2") + .clusterState("distributor:1 storage:4") + .runFor({17, 0}); + { + NodeMaintenanceStats wanted; + wanted.syncing = 1; + CPPUNIT_ASSERT_EQUAL(wanted, runner.stats().forNode(1)); + CPPUNIT_ASSERT_EQUAL(wanted, runner.stats().forNode(3)); + } +} + +} // distributor +} // storage} diff --git a/storage/src/tests/distributor/statoperationtest.cpp b/storage/src/tests/distributor/statoperationtest.cpp new file mode 100644 index 00000000000..22fee6e44d7 --- /dev/null +++ b/storage/src/tests/distributor/statoperationtest.cpp @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/storageapi/message/stat.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <tests/distributor/distributortestutil.h> +#include <vespa/storage/distributor/operations/external/statbucketoperation.h> +#include <vespa/storage/distributor/operations/external/statbucketlistoperation.h> + +namespace storage { +namespace distributor { + +struct StatOperationTest : public CppUnit::TestFixture, + public DistributorTestUtil +{ + void setUp() { + createLinks(); + }; + + void tearDown() { + close(); + } + + void testBucketInfo(); + void testBucketList(); + + CPPUNIT_TEST_SUITE(StatOperationTest); + CPPUNIT_TEST(testBucketInfo); + CPPUNIT_TEST(testBucketList); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(StatOperationTest); + +void +StatOperationTest::testBucketInfo() +{ + _distributor->enableClusterState(lib::ClusterState("distributor:1 storage:2")); + + addNodesToBucketDB(document::BucketId(16, 5), + "0=4/2/100,1=4/2/100"); + + StatBucketOperation op( + getExternalOperationHandler(), + std::shared_ptr<api::StatBucketCommand>( + new api::StatBucketCommand(document::BucketId(16, 5), ""))); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Statbucket => 0,Statbucket => 1"), + _sender.getCommands(true)); + + { + api::StatBucketCommand* tmp( + static_cast<api::StatBucketCommand*>(_sender.commands[0].get())); + api::StatBucketReply* reply = new api::StatBucketReply(*tmp, "foo"); + op.receive(_sender, std::shared_ptr<api::StorageReply>(reply)); + } + + { + api::StatBucketCommand* tmp( + static_cast<api::StatBucketCommand*>(_sender.commands[1].get())); + api::StatBucketReply* reply = new api::StatBucketReply(*tmp, "bar"); + op.receive(_sender, std::shared_ptr<api::StorageReply>(reply)); + } + + api::StatBucketReply* replyback( + static_cast<api::StatBucketReply*>(_sender.replies.back().get())); + CPPUNIT_ASSERT_CONTAIN("foo", replyback->getResults()); + CPPUNIT_ASSERT_CONTAIN("bar", replyback->getResults()); +} + +void +StatOperationTest::testBucketList() { + setupDistributor(2, 2, "distributor:1 storage:2"); + + getConfig().setSplitCount(10); + getConfig().setSplitSize(100); + + for (uint32_t i = 0; i < 2; ++i) { + insertBucketInfo(document::BucketId(16, 5), i, + 0xff, 100, 200, true, (i == 1)); + } + + std::shared_ptr<api::GetBucketListCommand> msg( + new api::GetBucketListCommand(document::BucketId(16, 5))); + + StatBucketListOperation op( + getExternalOperationHandler().getBucketDatabase(), + getIdealStateManager(), + getExternalOperationHandler().getIndex(), + msg); + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(1, (int)_sender.replies.size()); + + api::GetBucketListReply* repl( + dynamic_cast<api::GetBucketListReply*>(_sender.replies[0].get())); + + CPPUNIT_ASSERT_EQUAL(1, (int)repl->getBuckets().size()); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 5), + repl->getBuckets()[0]._bucket); + CPPUNIT_ASSERT_EQUAL( + vespalib::string( + "[distributor:0] split: " + "[Splitting bucket because its maximum size (200 b, 100 docs, 100 meta, 200 b total) " + "is higher than the configured limit of (100, 10)] " + "[node(idx=0,crc=0xff,docs=100/100,bytes=200/200,trusted=true,active=false), " + "node(idx=1,crc=0xff,docs=100/100,bytes=200/200,trusted=true,active=true)]"), + repl->getBuckets()[0]._bucketInformation); +} + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/statusreporterdelegatetest.cpp b/storage/src/tests/distributor/statusreporterdelegatetest.cpp new file mode 100644 index 00000000000..f05eebed0ce --- /dev/null +++ b/storage/src/tests/distributor/statusreporterdelegatetest.cpp @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <tests/common/testhelper.h> +#include <tests/distributor/distributortestutil.h> + +#include <vespa/storage/distributor/statusreporterdelegate.h> + +namespace storage { +namespace distributor { + +class StatusReporterDelegateTest : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE(StatusReporterDelegateTest); + CPPUNIT_TEST(testDelegateInvokesDelegatorOnStatusRequest); + CPPUNIT_TEST_SUITE_END(); + + void testDelegateInvokesDelegatorOnStatusRequest(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(StatusReporterDelegateTest); + +namespace { + +// We really ought to get GoogleMock as part of our testing suite... +class MockDelegator : public StatusDelegator +{ + mutable std::ostringstream _calls; + bool handleStatusRequest(const DelegatedStatusRequest& request) const { + _calls << "Request(" << request.path << ")"; + return request.reporter.reportStatus(request.outputStream, request.path); + } +public: + std::string getCalls() const { + return _calls.str(); + } +}; + +class MockStatusReporter : public framework::StatusReporter +{ +public: + MockStatusReporter() + : framework::StatusReporter("foo", "Bar") + {} + vespalib::string getReportContentType( + const framework::HttpUrlPath&) const + { + return "foo/bar"; + } + + bool reportStatus(std::ostream& os, + const framework::HttpUrlPath& path) const + { + os << "reportStatus with " << path; + return true; + } +}; + +} + +void +StatusReporterDelegateTest::testDelegateInvokesDelegatorOnStatusRequest() +{ + vdstestlib::DirConfig config(getStandardConfig(false)); + TestDistributorApp app(config.getConfigId()); + + MockDelegator mockDelegator; + MockStatusReporter reporter; + + StatusReporterDelegate delegate(app.getComponentRegister(), + mockDelegator, + reporter); + framework::HttpUrlPath path("dummy"); + CPPUNIT_ASSERT_EQUAL(vespalib::string("foo/bar"), + delegate.getReportContentType(path)); + + std::ostringstream ss; + CPPUNIT_ASSERT(delegate.reportStatus(ss, path)); + + CPPUNIT_ASSERT_EQUAL(std::string("Request(dummy)"), + mockDelegator.getCalls()); + CPPUNIT_ASSERT_EQUAL(std::string("reportStatus with dummy"), + ss.str()); +} + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/throttlingoperationstartertest.cpp b/storage/src/tests/distributor/throttlingoperationstartertest.cpp new file mode 100644 index 00000000000..5c4ba99563c --- /dev/null +++ b/storage/src/tests/distributor/throttlingoperationstartertest.cpp @@ -0,0 +1,142 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <string> +#include <sstream> +#include <memory> +#include <vespa/storage/distributor/throttlingoperationstarter.h> +#include <tests/distributor/maintenancemocks.h> + +namespace storage { + +namespace distributor { + +using document::BucketId; + +class ThrottlingOperationStarterTest : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(ThrottlingOperationStarterTest); + CPPUNIT_TEST(testOperationNotThrottledWhenSlotAvailable); + CPPUNIT_TEST(testOperationStartingIsForwardedToImplementation); + CPPUNIT_TEST(testOperationThrottledWhenNoAvailableSlots); + CPPUNIT_TEST(testThrottlingWithMaxPendingRange); + CPPUNIT_TEST(testStartingOperationsFillsUpPendingWindow); + CPPUNIT_TEST(testFinishingOperationsAllowsMoreToStart); + CPPUNIT_TEST_SUITE_END(); + + std::shared_ptr<Operation> createMockOperation() { + return std::shared_ptr<Operation>(new MockOperation(BucketId(16, 1))); + } + + std::unique_ptr<MockOperationStarter> _starterImpl; + std::unique_ptr<ThrottlingOperationStarter> _operationStarter; + +public: + void testOperationNotThrottledWhenSlotAvailable(); + void testOperationStartingIsForwardedToImplementation(); + void testOperationThrottledWhenNoAvailableSlots(); + void testThrottlingWithMaxPendingRange(); + void testStartingOperationsFillsUpPendingWindow(); + void testFinishingOperationsAllowsMoreToStart(); + + void setUp(); + void tearDown(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(ThrottlingOperationStarterTest); + +void +ThrottlingOperationStarterTest::setUp() +{ + _starterImpl.reset(new MockOperationStarter()); + _operationStarter.reset(new ThrottlingOperationStarter(*_starterImpl)); +} + +void +ThrottlingOperationStarterTest::tearDown() +{ + // Must clear before _operationStarter goes out of scope, or operation + // destructors will try to call method on destroyed object. + _starterImpl->getOperations().clear(); +} + +void +ThrottlingOperationStarterTest::testOperationNotThrottledWhenSlotAvailable() +{ + CPPUNIT_ASSERT(_operationStarter->start(createMockOperation(), + OperationStarter::Priority(0))); +} + +void +ThrottlingOperationStarterTest::testOperationStartingIsForwardedToImplementation() +{ + CPPUNIT_ASSERT(_operationStarter->start(createMockOperation(), + OperationStarter::Priority(0))); + CPPUNIT_ASSERT_EQUAL(std::string("BucketId(0x4000000000000001), pri 0\n"), + _starterImpl->toString()); +} + +void +ThrottlingOperationStarterTest::testOperationThrottledWhenNoAvailableSlots() +{ + _operationStarter->setMaxPendingRange(0, 0); + CPPUNIT_ASSERT(!_operationStarter->start(createMockOperation(), + OperationStarter::Priority(0))); +} + +void +ThrottlingOperationStarterTest::testThrottlingWithMaxPendingRange() +{ + _operationStarter->setMaxPendingRange(0, 1); + CPPUNIT_ASSERT(!_operationStarter->canStart(0, OperationStarter::Priority(255))); + CPPUNIT_ASSERT(_operationStarter->canStart(0, OperationStarter::Priority(0))); + + _operationStarter->setMaxPendingRange(1, 1); + CPPUNIT_ASSERT(_operationStarter->canStart(0, OperationStarter::Priority(255))); + CPPUNIT_ASSERT(_operationStarter->canStart(0, OperationStarter::Priority(0))); + + _operationStarter->setMaxPendingRange(1, 3); + CPPUNIT_ASSERT(!_operationStarter->canStart(1, OperationStarter::Priority(255))); + CPPUNIT_ASSERT(_operationStarter->canStart(1, OperationStarter::Priority(100))); + CPPUNIT_ASSERT(_operationStarter->canStart(1, OperationStarter::Priority(0))); + CPPUNIT_ASSERT(_operationStarter->canStart(2, OperationStarter::Priority(0))); + CPPUNIT_ASSERT(!_operationStarter->canStart(3, OperationStarter::Priority(0))); + CPPUNIT_ASSERT(!_operationStarter->canStart(4, OperationStarter::Priority(0))); +} + +void +ThrottlingOperationStarterTest::testStartingOperationsFillsUpPendingWindow() +{ + _operationStarter->setMaxPendingRange(1, 3); + CPPUNIT_ASSERT(_operationStarter->start(createMockOperation(), + OperationStarter::Priority(255))); + CPPUNIT_ASSERT(!_operationStarter->start(createMockOperation(), + OperationStarter::Priority(255))); + CPPUNIT_ASSERT(_operationStarter->start(createMockOperation(), + OperationStarter::Priority(100))); + CPPUNIT_ASSERT(!_operationStarter->start(createMockOperation(), + OperationStarter::Priority(100))); + CPPUNIT_ASSERT(_operationStarter->start(createMockOperation(), + OperationStarter::Priority(0))); + CPPUNIT_ASSERT(!_operationStarter->start(createMockOperation(), + OperationStarter::Priority(0))); +} + +void +ThrottlingOperationStarterTest::testFinishingOperationsAllowsMoreToStart() +{ + _operationStarter->setMaxPendingRange(1, 1); + CPPUNIT_ASSERT(_operationStarter->start(createMockOperation(), + OperationStarter::Priority(255))); + CPPUNIT_ASSERT(!_operationStarter->start(createMockOperation(), + OperationStarter::Priority(255))); + CPPUNIT_ASSERT(!_starterImpl->getOperations().empty()); + + _starterImpl->getOperations().pop_back(); + + CPPUNIT_ASSERT(_operationStarter->start(createMockOperation(), + OperationStarter::Priority(255))); + CPPUNIT_ASSERT(!_starterImpl->getOperations().empty()); +} + +} +} diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp new file mode 100644 index 00000000000..f6346c9755f --- /dev/null +++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp @@ -0,0 +1,1194 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/config/helper/configgetter.h> +#include <cppunit/extensions/HelperMacros.h> +#include <vespa/document/config/config-documenttypes.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/base/testdocrepo.h> +#include <vespa/document/update/arithmeticvalueupdate.h> +#include <iomanip> +#include <iostream> +#include <memory> +#include <tests/common/dummystoragelink.h> +#include <vespa/storage/distributor/externaloperationhandler.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storage/distributor/operations/external/twophaseupdateoperation.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/storageapi/message/batch.h> +#include <vespa/storageapi/message/bucket.h> +#include <tests/distributor/distributortestutil.h> +#include <tests/distributor/messagesenderstub.h> + +namespace storage { +namespace distributor { + +using std::shared_ptr; +using config::ConfigGetter; +using document::DocumenttypesConfig; +using config::FileSpec; +using namespace document; +using namespace storage; +using namespace storage::distributor; +using namespace storage::api; +using namespace storage::lib; + +using namespace std::literals::string_literals; + +class TwoPhaseUpdateOperationTest : public CppUnit::TestFixture, + public DistributorTestUtil +{ + CPPUNIT_TEST_SUITE(TwoPhaseUpdateOperationTest); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST(testNonExisting); + CPPUNIT_TEST(testUpdateFailed); + CPPUNIT_TEST(testFastPathInconsistentTimestamps); + CPPUNIT_TEST(testFastPathInconsistentTimestampsNotFound); + CPPUNIT_TEST(testFastPathInconsistentTimestampsUpdateError); + CPPUNIT_TEST(testFastPathInconsistentTimestampsGetError); + CPPUNIT_TEST(testFastPathInconsistentTimestampsPutError); + CPPUNIT_TEST(testFastPathInconsistentTimestampsPutNotStarted); + CPPUNIT_TEST(testFastPathInconsistentTimestampsInconsistentSplit); + CPPUNIT_TEST(testFastPathPropagatesMessageSettingsToUpdate); + CPPUNIT_TEST(testNofM); + CPPUNIT_TEST(testSafePathUpdatesNewestReceivedDocument); + CPPUNIT_TEST(testCreateIfNonExistentCreatesDocumentIfAllEmptyGets); + CPPUNIT_TEST(testUpdateFailsIfSafePathHasFailedPut); + CPPUNIT_TEST(testUpdateFailsIfSafePathGetsFail); + CPPUNIT_TEST(testUpdateFailsIfApplyThrowsException); + CPPUNIT_TEST(testNonExistingWithAutoCreate); + CPPUNIT_TEST(testSafePathFailsUpdateWhenMismatchingTimestampConstraint); + CPPUNIT_TEST(testSafePathUpdatePropagatesMessageSettingsToGetsAndPuts); + CPPUNIT_TEST(testSafePathPropagatesMbusTracesFromReplies); + CPPUNIT_TEST(testUpdateFailsIfOwnershipChangesBetweenGetAndPut); + CPPUNIT_TEST(testSafePathConditionMismatchFailsWithTasError); + CPPUNIT_TEST(testSafePathConditionMatchSendsPutsWithUpdatedDoc); + CPPUNIT_TEST(testSafePathConditionParseFailureFailsWithIllegalParamsError); + CPPUNIT_TEST(testSafePathConditonUnknownDocTypeFailsWithIllegalParamsError); + CPPUNIT_TEST(testSafePathConditionWithMissingDocFailsWithTasError); + CPPUNIT_TEST(testFastPathCloseEdgeSendsCorrectReply); + CPPUNIT_TEST(testSafePathCloseEdgeSendsCorrectReply); + CPPUNIT_TEST_SUITE_END(); + + document::TestDocRepo _testRepo; + DocumentTypeRepo::SP _repo; + const DocumentType* _doc_type; + +protected: + void testSimple(); + void testNonExisting(); + void testUpdateFailed(); + void testFastPathInconsistentTimestamps(); + void testFastPathInconsistentTimestampsNotFound(); + void testFastPathInconsistentTimestampsUpdateError(); + void testFastPathInconsistentTimestampsGetError(); + void testFastPathInconsistentTimestampsPutError(); + void testFastPathInconsistentTimestampsPutNotStarted(); + void testFastPathInconsistentTimestampsInconsistentSplit(); + void testFastPathPropagatesMessageSettingsToUpdate(); + void testNofM(); + void testSafePathUpdatesNewestReceivedDocument(); + void testCreateIfNonExistentCreatesDocumentIfAllEmptyGets(); + void testUpdateFailsIfSafePathHasFailedPut(); + void testUpdateFailsIfSafePathGetsFail(); + void testUpdateFailsIfApplyThrowsException(); + void testNonExistingWithAutoCreate(); + void testSafePathFailsUpdateWhenMismatchingTimestampConstraint(); + void testSafePathUpdatePropagatesMessageSettingsToGetsAndPuts(); + void testSafePathPropagatesMbusTracesFromReplies(); + void testUpdateFailsIfOwnershipChangesBetweenGetAndPut(); + void testSafePathConditionMismatchFailsWithTasError(); + void testSafePathConditionMatchSendsPutsWithUpdatedDoc(); + void testSafePathConditionParseFailureFailsWithIllegalParamsError(); + void testSafePathConditonUnknownDocTypeFailsWithIllegalParamsError(); + void testSafePathConditionWithMissingDocFailsWithTasError(); + void testFastPathCloseEdgeSendsCorrectReply(); + void testSafePathCloseEdgeSendsCorrectReply(); + + void checkMessageSettingsPropagatedTo( + const api::StorageCommand::SP& msg) const; + + std::string getUpdatedValueFromLastPut(MessageSenderStub&); +public: + void setUp() { + _repo = _testRepo.getTypeRepoSp(); + _doc_type = _repo->getDocumentType("testdoctype1"); + createLinks(); + setTypeRepo(_repo); + getClock().setAbsoluteTimeInSeconds(200); + } + + void tearDown() { + close(); + } + + void replyToMessage(Operation& callback, + MessageSenderStub& sender, + uint32_t index, + uint64_t oldTimestamp, + api::ReturnCode::Result result = api::ReturnCode::OK); + + void replyToPut( + Operation& callback, + MessageSenderStub& sender, + uint32_t index, + api::ReturnCode::Result result = api::ReturnCode::OK, + const std::string& traceMsg = ""); + + void replyToCreateBucket( + Operation& callback, + MessageSenderStub& sender, + uint32_t index, + api::ReturnCode::Result result = api::ReturnCode::OK); + + void replyToGet( + Operation& callback, + MessageSenderStub& sender, + uint32_t index, + uint64_t oldTimestamp, + bool haveDocument = true, + api::ReturnCode::Result result = api::ReturnCode::OK, + const std::string& traceMsg = ""); + + struct UpdateOptions { + bool _makeInconsistentSplit; + bool _createIfNonExistent; + bool _withError; + api::Timestamp _timestampToUpdate; + documentapi::TestAndSetCondition _condition; + + UpdateOptions() + : _makeInconsistentSplit(false), + _createIfNonExistent(false), + _withError(false), + _timestampToUpdate(0), + _condition() + { + } + + UpdateOptions& makeInconsistentSplit(bool mis) { + _makeInconsistentSplit = mis; + return *this; + } + UpdateOptions& createIfNonExistent(bool cine) { + _createIfNonExistent = cine; + return *this; + } + UpdateOptions& withError(bool error = true) { + _withError = error; + return *this; + } + UpdateOptions& timestampToUpdate(api::Timestamp ts) { + _timestampToUpdate = ts; + return *this; + } + UpdateOptions& condition(vespalib::stringref cond) { + _condition = documentapi::TestAndSetCondition(cond); + return *this; + } + }; + + std::shared_ptr<TwoPhaseUpdateOperation> + sendUpdate(const std::string& bucketState, + const UpdateOptions& options = UpdateOptions()); + + void assertAbortedUpdateReplyWithContextPresent( + const MessageSenderStub& closeSender) const; + +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(TwoPhaseUpdateOperationTest); + +void +TwoPhaseUpdateOperationTest::replyToMessage( + Operation& callback, + MessageSenderStub& sender, + uint32_t index, + uint64_t oldTimestamp, + api::ReturnCode::Result result) +{ + std::shared_ptr<api::StorageMessage> msg2 = sender.commands.at(index); + UpdateCommand& updatec = dynamic_cast<UpdateCommand&>(*msg2); + std::unique_ptr<api::StorageReply> reply(updatec.makeReply()); + static_cast<api::UpdateReply*>(reply.get())->setOldTimestamp(oldTimestamp); + reply->setResult(api::ReturnCode(result, "")); + + callback.receive(sender, + std::shared_ptr<StorageReply>(reply.release())); +} + +void +TwoPhaseUpdateOperationTest::replyToPut( + Operation& callback, + MessageSenderStub& sender, + uint32_t index, + api::ReturnCode::Result result, + const std::string& traceMsg) +{ + std::shared_ptr<api::StorageMessage> msg2 = sender.commands.at(index); + PutCommand& putc = dynamic_cast<PutCommand&>(*msg2); + std::unique_ptr<api::StorageReply> reply(putc.makeReply()); + reply->setResult(api::ReturnCode(result, "")); + if (!traceMsg.empty()) { + MBUS_TRACE(reply->getTrace(), 1, traceMsg); + } + callback.receive(sender, + std::shared_ptr<StorageReply>(reply.release())); +} + +void +TwoPhaseUpdateOperationTest::replyToCreateBucket( + Operation& callback, + MessageSenderStub& sender, + uint32_t index, + api::ReturnCode::Result result) +{ + std::shared_ptr<api::StorageMessage> msg2 = sender.commands.at(index); + CreateBucketCommand& putc = dynamic_cast<CreateBucketCommand&>(*msg2); + std::unique_ptr<api::StorageReply> reply(putc.makeReply()); + reply->setResult(api::ReturnCode(result, "")); + callback.receive(sender, + std::shared_ptr<StorageReply>(reply.release())); +} + +void +TwoPhaseUpdateOperationTest::replyToGet( + Operation& callback, + MessageSenderStub& sender, + uint32_t index, + uint64_t oldTimestamp, + bool haveDocument, + api::ReturnCode::Result result, + const std::string& traceMsg) +{ + const api::GetCommand& get( + static_cast<const api::GetCommand&>(*sender.commands.at(index))); + std::shared_ptr<api::StorageReply> reply; + + if (haveDocument) { + auto doc(std::make_shared<Document>( + *_doc_type, DocumentId(DocIdString("test", "test")))); + doc->setValue("headerval", IntFieldValue(oldTimestamp)); + + reply = std::make_shared<api::GetReply>(get, doc, oldTimestamp); + } else { + reply = std::make_shared<api::GetReply>(get, Document::SP(), 0); + } + reply->setResult(api::ReturnCode(result, "")); + if (!traceMsg.empty()) { + MBUS_TRACE(reply->getTrace(), 1, traceMsg); + } + + callback.receive(sender, reply); +} + +namespace { + +struct DummyTransportContext : api::TransportContext { + // No methods to implement. +}; + +} + +std::shared_ptr<TwoPhaseUpdateOperation> +TwoPhaseUpdateOperationTest::sendUpdate(const std::string& bucketState, + const UpdateOptions& options) +{ + document::DocumentUpdate::SP update; + if (!options._withError) { + update = std::make_shared<document::DocumentUpdate>( + *_doc_type, + document::DocumentId(document::DocIdString("test", "test"))); + document::FieldUpdate fup(_doc_type->getField("headerval")); + fup.addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10)); + update->addUpdate(fup); + } else { + // Create an update to a different doctype than the one returned as + // part of the Get. Just a sneaky way to force an eval error. + auto* badDocType = _repo->getDocumentType("testdoctype2"); + update = std::make_shared<document::DocumentUpdate>( + *badDocType, + document::DocumentId(document::DocIdString("test", "test"))); + document::FieldUpdate fup(badDocType->getField("onlyinchild")); + fup.addUpdate(ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 10)); + update->addUpdate(fup); + } + update->setCreateIfNonExistent(options._createIfNonExistent); + + document::BucketId id = getExternalOperationHandler().getBucketId(update->getId()); + document::BucketId id2 = document::BucketId(id.getUsedBits() + 1, id.getRawId()); + + if (bucketState.length()) { + addNodesToBucketDB(id, bucketState); + } + + if (options._makeInconsistentSplit) { + addNodesToBucketDB(id2, bucketState); + } + + auto msg(std::make_shared<api::UpdateCommand>( + document::BucketId(0), update, api::Timestamp(0))); + // Misc settings for checking that propagation works. + msg->getTrace().setLevel(6); + msg->setTimeout(6789); + msg->setPriority(99); + if (options._timestampToUpdate) { + msg->setOldTimestamp(options._timestampToUpdate); + } + msg->setCondition(options._condition); + msg->setTransportContext(std::make_unique<DummyTransportContext>()); + + ExternalOperationHandler& handler = getExternalOperationHandler(); + return std::make_shared<TwoPhaseUpdateOperation>( + handler, msg, getDistributor().getMetrics()); +} + + +void +TwoPhaseUpdateOperationTest::testSimple() +{ + setupDistributor(1, 1, "storage:1 distributor:1"); + + std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0"), + sender.getCommands(true)); + + replyToMessage(*cb, sender, 0, 90); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 90) ReturnCode(NONE)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testNonExisting() +{ + setupDistributor(1, 1, "storage:1 distributor:1"); + + std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 0) ReturnCode(NONE)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testUpdateFailed() +{ + setupDistributor(1, 1, "storage:1 distributor:1"); + + std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0"), + sender.getCommands(true)); + + replyToMessage(*cb, sender, 0, 90, api::ReturnCode::INTERNAL_FAILURE); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 0) " + "ReturnCode(INTERNAL_FAILURE)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testFastPathInconsistentTimestamps() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + + std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0,Update => 1"), + sender.getCommands(true)); + + replyToMessage(*cb, sender, 0, 90); + replyToMessage(*cb, sender, 1, 110); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get(BucketId(0x4000000000008b13), doc:test:test) => 1"), + sender.getLastCommand(true)); + + replyToGet(*cb, sender, 2, 110); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0,Update => 1,Get => 1,Put => 1,Put => 0"), + sender.getCommands(true)); + + CPPUNIT_ASSERT(sender.replies.empty()); + + replyToPut(*cb, sender, 3); + replyToPut(*cb, sender, 4); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 110 Was inconsistent " + "(best node 1)) ReturnCode(NONE)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testFastPathInconsistentTimestampsNotFound() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + + std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0,Update => 1"), + sender.getCommands(true)); + + replyToMessage(*cb, sender, 0, 90); + replyToMessage(*cb, sender, 1, 110); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get(BucketId(0x4000000000008b13), doc:test:test) => 1"), + sender.getLastCommand(true)); + CPPUNIT_ASSERT(sender.replies.empty()); + + replyToGet(*cb, sender, 2, 110, false); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 110 Was inconsistent " + "(best node 1)) ReturnCode(INTERNAL_FAILURE)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testFastPathInconsistentTimestampsUpdateError() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + + std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0,Update => 1"), + sender.getCommands(true)); + + replyToMessage(*cb, sender, 0, 90); + CPPUNIT_ASSERT(sender.replies.empty()); + replyToMessage(*cb, sender, 1, 110, api::ReturnCode::IO_FAILURE); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 90) " + "ReturnCode(IO_FAILURE)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testFastPathInconsistentTimestampsGetError() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + + std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0,Update => 1"), + sender.getCommands(true)); + + replyToMessage(*cb, sender, 0, 90); + replyToMessage(*cb, sender, 1, 110); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get(BucketId(0x4000000000008b13), doc:test:test) => 1"), + sender.getLastCommand(true)); + + CPPUNIT_ASSERT(sender.replies.empty()); + replyToGet(*cb, sender, 2, 110, false, api::ReturnCode::IO_FAILURE); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 110 Was inconsistent " + "(best node 1)) ReturnCode(IO_FAILURE)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testFastPathInconsistentTimestampsPutError() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3")); + + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0,Update => 1"), + sender.getCommands(true)); + + replyToMessage(*cb, sender, 0, 90); + replyToMessage(*cb, sender, 1, 110); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get(BucketId(0x4000000000008b13), doc:test:test) => 1"), + sender.getLastCommand(true)); + + replyToGet(*cb, sender, 2, 110); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0,Update => 1,Get => 1,Put => 1,Put => 0"), + sender.getCommands(true)); + + replyToPut(*cb, sender, 3, api::ReturnCode::IO_FAILURE); + CPPUNIT_ASSERT(sender.replies.empty()); + replyToPut(*cb, sender, 4); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 110 Was inconsistent " + "(best node 1)) ReturnCode(IO_FAILURE)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testFastPathInconsistentTimestampsPutNotStarted() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3")); + + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0,Update => 1"), + sender.getCommands(true)); + + replyToMessage(*cb, sender, 0, 90); + replyToMessage(*cb, sender, 1, 110); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get(BucketId(0x4000000000008b13), doc:test:test) => 1"), + sender.getLastCommand(true)); + checkMessageSettingsPropagatedTo(sender.commands.back()); + + _distributor->enableClusterState(lib::ClusterState("storage:0 distributor:1")); + CPPUNIT_ASSERT(sender.replies.empty()); + replyToGet(*cb, sender, 2, 110); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 110 Was inconsistent " + "(best node 1)) ReturnCode(NOT_CONNECTED, " + "Can't store document: No storage nodes available)"), + sender.getLastReply(true)); +} + + +void +TwoPhaseUpdateOperationTest::testFastPathInconsistentTimestampsInconsistentSplit() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=1/2/3", + UpdateOptions().makeInconsistentSplit(true))); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + std::string wanted("Get(BucketId(0x4000000000008b13), doc:test:test) => 0," + "Get(BucketId(0x4400000000008b13), doc:test:test) => 0"); + + std::string text = sender.getCommands(true, true); + CPPUNIT_ASSERT_EQUAL(wanted, text); + + replyToGet(*cb, sender, 0, 90); + replyToGet(*cb, sender, 1, 120); + + CPPUNIT_ASSERT_EQUAL( + std::string( + "Put(BucketId(0x4400000000008b13), doc:test:test, " + "timestamp 200000000, size 52) => 1," + "Put(BucketId(0x4400000000008b13), doc:test:test, " + "timestamp 200000000, size 52) => 0"), + sender.getCommands(true, true, 2)); + + replyToPut(*cb, sender, 2); + CPPUNIT_ASSERT(sender.replies.empty()); + replyToPut(*cb, sender, 3); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 120) " + "ReturnCode(NONE)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::checkMessageSettingsPropagatedTo( + const api::StorageCommand::SP& msg) const +{ + // Settings set in sendUpdate(). + CPPUNIT_ASSERT_EQUAL(uint32_t(6), msg->getTrace().getLevel()); + CPPUNIT_ASSERT_EQUAL(uint32_t(6789), msg->getTimeout()); + CPPUNIT_ASSERT_EQUAL(uint8_t(99), msg->getPriority()); +} + +void +TwoPhaseUpdateOperationTest::testFastPathPropagatesMessageSettingsToUpdate() +{ + setupDistributor(1, 1, "storage:1 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Update => 0"), sender.getCommands(true)); + + StorageCommand::SP msg(sender.commands.back()); + checkMessageSettingsPropagatedTo(msg); +} + +void +TwoPhaseUpdateOperationTest::testNofM() +{ + setupDistributor(2, 2, "storage:2 distributor:1", 1); + + std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0,Update => 1"), + sender.getCommands(true)); + + CPPUNIT_ASSERT(sender.replies.empty()); + replyToMessage(*cb, sender, 0, 90); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 90) ReturnCode(NONE)"), + sender.getLastReply(true)); + + replyToMessage(*cb, sender, 1, 123); +} + +std::string +TwoPhaseUpdateOperationTest::getUpdatedValueFromLastPut( + MessageSenderStub& sender) +{ + Document::SP doc(dynamic_cast<api::PutCommand&>(*sender.commands.back()) + .getDocument()); + FieldValue::UP value(doc->getValue("headerval")); + return value->toString(); +} + +void +TwoPhaseUpdateOperationTest::testSafePathUpdatesNewestReceivedDocument() +{ + setupDistributor(3, 3, "storage:3 distributor:1"); + // 0,1 in sync. 2 out of sync. + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4")); + + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("Get(BucketId(0x4000000000008b13), doc:test:test) => 0," + "Get(BucketId(0x4000000000008b13), doc:test:test) => 2"), + sender.getCommands(true, true)); + replyToGet(*cb, sender, 0, 50); + replyToGet(*cb, sender, 1, 70); + + CPPUNIT_ASSERT_EQUAL( + std::string( + "Put(BucketId(0x4000000000008b13), doc:test:test, " + "timestamp 200000000, size 52) => 1," + "Put(BucketId(0x4000000000008b13), doc:test:test, " + "timestamp 200000000, size 52) => 0," + "Put(BucketId(0x4000000000008b13), doc:test:test, " + "timestamp 200000000, size 52) => 2"), + sender.getCommands(true, true, 2)); + // Make sure Put contains an updated document (+10 arith. update on field + // whose value equals gotten timestamp). In this case we want 70 -> 80. + CPPUNIT_ASSERT_EQUAL(std::string("80"), getUpdatedValueFromLastPut(sender)); + + replyToPut(*cb, sender, 2); + replyToPut(*cb, sender, 3); + CPPUNIT_ASSERT(sender.replies.empty()); + replyToPut(*cb, sender, 4); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 70) " + "ReturnCode(NONE)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testCreateIfNonExistentCreatesDocumentIfAllEmptyGets() +{ + setupDistributor(3, 3, "storage:3 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4", + UpdateOptions().createIfNonExistent(true))); + + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Get => 0,Get => 2"), + sender.getCommands(true)); + replyToGet(*cb, sender, 0, 0, false); + replyToGet(*cb, sender, 1, 0, false); + // Since create-if-non-existent is set, distributor should create doc from + // scratch. + CPPUNIT_ASSERT_EQUAL( + std::string( + "Put(BucketId(0x4000000000008b13), doc:test:test, " + "timestamp 200000000, size 52) => 1," + "Put(BucketId(0x4000000000008b13), doc:test:test, " + "timestamp 200000000, size 52) => 0," + "Put(BucketId(0x4000000000008b13), doc:test:test, " + "timestamp 200000000, size 52) => 2"), + sender.getCommands(true, true, 2)); + + CPPUNIT_ASSERT_EQUAL(std::string("10"), getUpdatedValueFromLastPut(sender)); + + replyToPut(*cb, sender, 2); + replyToPut(*cb, sender, 3); + CPPUNIT_ASSERT(sender.replies.empty()); + replyToPut(*cb, sender, 4); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 200000000) " + "ReturnCode(NONE)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testUpdateFailsIfSafePathHasFailedPut() +{ + setupDistributor(3, 3, "storage:3 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4", + UpdateOptions().createIfNonExistent(true))); + + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Get => 0,Get => 2"), + sender.getCommands(true)); + replyToGet(*cb, sender, 0, 0, false); + replyToGet(*cb, sender, 1, 0, false); + // Since create-if-non-existent is set, distributor should create doc from + // scratch. + CPPUNIT_ASSERT_EQUAL(std::string("Put => 1,Put => 0,Put => 2"), + sender.getCommands(true, false, 2)); + + replyToPut(*cb, sender, 2); + replyToPut(*cb, sender, 3); + CPPUNIT_ASSERT(sender.replies.empty()); + replyToPut(*cb, sender, 4, api::ReturnCode::IO_FAILURE); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 200000000) " + "ReturnCode(IO_FAILURE)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testUpdateFailsIfSafePathGetsFail() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=2/3/4", + UpdateOptions().createIfNonExistent(true))); + + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Get => 0,Get => 1"), + sender.getCommands(true)); + replyToGet(*cb, sender, 0, 0, false, api::ReturnCode::IO_FAILURE); + CPPUNIT_ASSERT(sender.replies.empty()); + replyToGet(*cb, sender, 1, 0, false, api::ReturnCode::IO_FAILURE); + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 0) " + "ReturnCode(IO_FAILURE)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testUpdateFailsIfApplyThrowsException() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + // Create update for wrong doctype which will fail the update. + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().withError())); + + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Get => 0,Get => 1"), + sender.getCommands(true)); + replyToGet(*cb, sender, 0, 50); + CPPUNIT_ASSERT(sender.replies.empty()); + replyToGet(*cb, sender, 1, 70); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 70) " + "ReturnCode(INTERNAL_FAILURE, Can not apply a " + "\"testdoctype2\" document update to a " + "\"testdoctype1\" document.)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testNonExistingWithAutoCreate() +{ + setupDistributor(1, 1, "storage:1 distributor:1"); + + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("", UpdateOptions().createIfNonExistent(true))); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string( + "CreateBucketCommand(BucketId(0x4000000000008b13), active) " + "Reasons to start: => 0," + "Put(BucketId(0x4000000000008b13), doc:test:test, " + "timestamp 200000000, size 52) => 0"), + sender.getCommands(true, true)); + + CPPUNIT_ASSERT_EQUAL(std::string("10"), getUpdatedValueFromLastPut(sender)); + + replyToCreateBucket(*cb, sender, 0); + CPPUNIT_ASSERT(sender.replies.empty()); + replyToPut(*cb, sender, 1); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 200000000) " + "ReturnCode(NONE)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testSafePathFailsUpdateWhenMismatchingTimestampConstraint() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=2/3/4", + UpdateOptions().timestampToUpdate(1234))); + + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Get => 0,Get => 1"), + sender.getCommands(true)); + replyToGet(*cb, sender, 0, 100); + CPPUNIT_ASSERT(sender.replies.empty()); + replyToGet(*cb, sender, 1, 110); + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 0) " + "ReturnCode(NONE, No document with requested " + "timestamp found)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testSafePathUpdatePropagatesMessageSettingsToGetsAndPuts() +{ + setupDistributor(3, 3, "storage:3 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Get => 0,Get => 2"), + sender.getCommands(true)); + checkMessageSettingsPropagatedTo(sender.commands.at(0)); + checkMessageSettingsPropagatedTo(sender.commands.at(1)); + replyToGet(*cb, sender, 0, 50); + replyToGet(*cb, sender, 1, 70); + CPPUNIT_ASSERT_EQUAL(std::string("Put => 1,Put => 0,Put => 2"), + sender.getCommands(true, false, 2)); + checkMessageSettingsPropagatedTo(sender.commands.at(2)); + checkMessageSettingsPropagatedTo(sender.commands.at(3)); + checkMessageSettingsPropagatedTo(sender.commands.at(4)); + replyToPut(*cb, sender, 2); + replyToPut(*cb, sender, 3); + replyToPut(*cb, sender, 4); +} + +void +TwoPhaseUpdateOperationTest::testSafePathPropagatesMbusTracesFromReplies() +{ + setupDistributor(3, 3, "storage:3 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Get => 0,Get => 2"), + sender.getCommands(true)); + replyToGet(*cb, sender, 0, 50, true, + api::ReturnCode::OK, "hello earthlings"); + replyToGet(*cb, sender, 1, 70); + CPPUNIT_ASSERT_EQUAL(std::string("Put => 1,Put => 0,Put => 2"), + sender.getCommands(true, false, 2)); + replyToPut(*cb, sender, 2, api::ReturnCode::OK, "fooo"); + replyToPut(*cb, sender, 3, api::ReturnCode::OK, "baaa"); + CPPUNIT_ASSERT(sender.replies.empty()); + replyToPut(*cb, sender, 4); + + CPPUNIT_ASSERT_EQUAL(std::string("Update Reply"), + sender.getLastReply(false)); + + std::string trace(sender.replies.back()->getTrace().toString()); + //std::cout << "\n\n" << trace << "\n\n"; + CPPUNIT_ASSERT(trace.find("hello earthlings") != std::string::npos); + CPPUNIT_ASSERT(trace.find("fooo") != std::string::npos); + CPPUNIT_ASSERT(trace.find("baaa") != std::string::npos); +} + +void +TwoPhaseUpdateOperationTest::testUpdateFailsIfOwnershipChangesBetweenGetAndPut() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + + // Update towards inconsistent bucket invokes safe path. + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=2/3/4")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Get => 0,Get => 1"), + sender.getCommands(true)); + + // Alter cluster state so that distributor is now down (technically the + // entire cluster is down in this state, but this should not matter). In + // this new state, the distributor no longer owns the bucket in question + // and the operation should thus be failed. We must not try to send Puts + // to a bucket we no longer own. + _distributor->enableClusterState( + lib::ClusterState("storage:2 distributor:1 .0.s:d")); + getBucketDatabase().clear(); + replyToGet(*cb, sender, 0, 70); + replyToGet(*cb, sender, 1, 70); + + // BUCKET_NOT_FOUND is a transient error code which should cause the client + // to re-send the operation, presumably to the correct distributor the next + // time. + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 70) " + "ReturnCode(BUCKET_NOT_FOUND, Distributor lost " + "ownership of bucket between executing the read " + "and write phases of a two-phase update operation)"), + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testSafePathConditionMismatchFailsWithTasError() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition( + "testdoctype1.headerval==120"))); + + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + // Newest doc has headerval==110, not 120. + replyToGet(*cb, sender, 0, 100); + replyToGet(*cb, sender, 1, 110); + CPPUNIT_ASSERT_EQUAL( + "UpdateReply(doc:test:test, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 0) " + "ReturnCode(TEST_AND_SET_CONDITION_FAILED, " + "Condition did not match document)"s, + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testSafePathConditionMatchSendsPutsWithUpdatedDoc() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition( + "testdoctype1.headerval==110"))); + + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + replyToGet(*cb, sender, 0, 100); + replyToGet(*cb, sender, 1, 110); + CPPUNIT_ASSERT_EQUAL("Put => 1,Put => 0"s, + sender.getCommands(true, false, 2)); +} + +void +TwoPhaseUpdateOperationTest::testSafePathConditionParseFailureFailsWithIllegalParamsError() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition( + "testdoctype1.san==fran...cisco"))); + + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + replyToGet(*cb, sender, 0, 100); + replyToGet(*cb, sender, 1, 110); + // NOTE: condition is currently not attempted parsed until Gets have been + // replied to. This may change in the future. + // XXX reliance on parser/exception error message is very fragile. + CPPUNIT_ASSERT_EQUAL( + "UpdateReply(doc:test:test, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 0) " + "ReturnCode(ILLEGAL_PARAMETERS, " + "Failed to parse test and set condition: " + "Unexpected token at position 16 " + "('==fran...c') in query 'testdoctype1." + "san==fran...cisco',)"s, + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testSafePathConditonUnknownDocTypeFailsWithIllegalParamsError() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition( + "langbein.headerval=1234"))); + + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + replyToGet(*cb, sender, 0, 100); + replyToGet(*cb, sender, 1, 110); + // NOTE: condition is currently not attempted parsed until Gets have been + // replied to. This may change in the future. + CPPUNIT_ASSERT_EQUAL( + "UpdateReply(doc:test:test, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 0) " + "ReturnCode(ILLEGAL_PARAMETERS, " + "Failed to parse test and set condition: " + "Document type langbein not found)"s, + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::testSafePathConditionWithMissingDocFailsWithTasError() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition( + "testdoctype1.headerval==120"))); + + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + // Both Gets return nothing at all, nothing at all. + replyToGet(*cb, sender, 0, 100, false); + replyToGet(*cb, sender, 1, 110, false); + CPPUNIT_ASSERT_EQUAL( + "UpdateReply(doc:test:test, " + "BucketId(0x0000000000000000), " + "timestamp 0, timestamp of updated doc: 0) " + "ReturnCode(TEST_AND_SET_CONDITION_FAILED, " + "Document did not exist)"s, + sender.getLastReply(true)); +} + +void +TwoPhaseUpdateOperationTest::assertAbortedUpdateReplyWithContextPresent( + const MessageSenderStub& closeSender) const +{ + CPPUNIT_ASSERT_EQUAL(size_t(1), closeSender.replies.size()); + StorageReply::SP reply(closeSender.replies.back()); + CPPUNIT_ASSERT_EQUAL(api::MessageType::UPDATE_REPLY, reply->getType()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ABORTED, + reply->getResult().getResult()); + auto context = reply->getTransportContext(); // Transfers ownership + CPPUNIT_ASSERT(context.get()); +} + +void +TwoPhaseUpdateOperationTest::testFastPathCloseEdgeSendsCorrectReply() +{ + setupDistributor(1, 1, "storage:1 distributor:1"); + // Only 1 replica; consistent with itself by definition. + std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL("Update => 0"s, sender.getCommands(true)); + // Close the operation. This should generate a single reply that is + // bound to the original command. We can identify rogue replies by these + // not having a transport context, as these are unique_ptrs that are + // moved to the reply upon the first reply construction. Any subsequent or + // erroneous replies will not have this context attached to themselves. + MessageSenderStub closeSender; + cb->onClose(closeSender); + + assertAbortedUpdateReplyWithContextPresent(closeSender); +} + +void +TwoPhaseUpdateOperationTest::testSafePathCloseEdgeSendsCorrectReply() +{ + setupDistributor(2, 2, "storage:2 distributor:1"); + + std::shared_ptr<TwoPhaseUpdateOperation> cb( + sendUpdate("0=1/2/3,1=2/3/4")); // Inconsistent replicas. + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Get => 0,Get => 1"), + sender.getCommands(true)); + // Closing the operation should now only return an ABORTED reply for + // the UpdateCommand, _not_ from the nested, pending Get operation (which + // will implicitly generate an ABORTED reply for the synthesized Get + // command passed to it). + MessageSenderStub closeSender; + cb->onClose(closeSender); + + assertAbortedUpdateReplyWithContextPresent(closeSender); +} + +// XXX currently differs in behavior from content nodes in that updates for +// document IDs without explicit doctypes will _not_ be auto-failed on the +// distributor. + +// XXX shouldn't be necessary to have any special handling of create-if... and +// test-and-set right? They appear fully mutually exclusive. + +// XXX: test case where update reply has been sent but callback still +// has pending messages (e.g. n-of-m case). + +} // distributor +} // storage diff --git a/storage/src/tests/distributor/updateoperationtest.cpp b/storage/src/tests/distributor/updateoperationtest.cpp new file mode 100644 index 00000000000..912d0235e42 --- /dev/null +++ b/storage/src/tests/distributor/updateoperationtest.cpp @@ -0,0 +1,210 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <iomanip> +#include <iostream> +#include <memory> +#include <tests/common/dummystoragelink.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/storageapi/message/bucket.h> +#include <tests/distributor/distributortestutil.h> +#include <vespa/config/helper/configgetter.h> +#include <vespa/document/config/config-documenttypes.h> +#include <tests/distributor/messagesenderstub.h> +#include <vespa/storage/distributor/operations/external/updateoperation.h> + +using std::shared_ptr; +using namespace document; +using namespace storage; +using namespace storage::distributor; +using namespace storage::api; +using namespace std; +using namespace storage::lib; +using config::ConfigGetter; +using config::FileSpec; +using vespalib::string; + +class UpdateOperation_Test : public CppUnit::TestFixture, + public DistributorTestUtil +{ + CPPUNIT_TEST_SUITE(UpdateOperation_Test); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST(testNotFound); + CPPUNIT_TEST(testMultiNode); + CPPUNIT_TEST(testMultiNodeInconsistentTimestamp); + CPPUNIT_TEST_SUITE_END(); + + DocumentTypeRepo::SP _repo; + const DocumentType *_html_type; + +protected: + void testSimple(); + void testNotFound(); + void testMultiNode(); + void testMultiNodeInconsistentTimestamp(); + +public: + void setUp() { + _repo.reset( + new DocumentTypeRepo(*ConfigGetter<DocumenttypesConfig>:: + getConfig("config-doctypes", FileSpec("config-doctypes.cfg")))); + _html_type = _repo->getDocumentType("text/html"); + createLinks(); + } + + void tearDown() { + close(); + } + + void replyToMessage( + UpdateOperation& callback, + MessageSenderStub& sender, + uint32_t index, + uint64_t oldTimestamp, + api::BucketInfo info = api::BucketInfo(2,4,6)); + + std::shared_ptr<UpdateOperation> + sendUpdate(const std::string& bucketState); + + document::BucketId _bId; +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(UpdateOperation_Test); + +std::shared_ptr<UpdateOperation> +UpdateOperation_Test::sendUpdate(const std::string& bucketState) +{ + document::DocumentUpdate::SP update( + new document::DocumentUpdate( + *_html_type, + document::DocumentId(document::DocIdString("test", "test")))); + + _bId = getExternalOperationHandler().getBucketId(update->getId()); + + addNodesToBucketDB(_bId, bucketState); + + std::shared_ptr<api::UpdateCommand> msg( + new api::UpdateCommand(document::BucketId(0), + update, + 100)); + + ExternalOperationHandler& handler = getExternalOperationHandler(); + return std::shared_ptr<UpdateOperation>( + new UpdateOperation(handler, + msg, + getDistributor().getMetrics().updates[msg->getLoadType()])); +} + + +void +UpdateOperation_Test::replyToMessage( + UpdateOperation& callback, + MessageSenderStub& sender, + uint32_t index, + uint64_t oldTimestamp, + api::BucketInfo info) +{ + std::shared_ptr<api::StorageMessage> msg2 = sender.commands[index]; + UpdateCommand* updatec = dynamic_cast<UpdateCommand*>(msg2.get()); + std::unique_ptr<api::StorageReply> reply(updatec->makeReply()); + UpdateReply* updateR = static_cast<api::UpdateReply*>(reply.get()); + updateR->setOldTimestamp(oldTimestamp); + updateR->setBucketInfo(info); + + callback.onReceive(sender, + std::shared_ptr<StorageReply>(reply.release())); +} + +void +UpdateOperation_Test::testSimple() +{ + setupDistributor(1, 1, "storage:1 distributor:1"); + + std::shared_ptr<UpdateOperation> cb(sendUpdate("0=1/2/3")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0"), + sender.getCommands(true)); + + replyToMessage(*cb, sender, 0, 90); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 100, timestamp of updated doc: 90) ReturnCode(NONE)"), + sender.getLastReply(true)); +} + +void +UpdateOperation_Test::testNotFound() +{ + setupDistributor(1, 1, "storage:1 distributor:1"); + + std::shared_ptr<UpdateOperation> cb(sendUpdate("0=1/2/3")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0"), + sender.getCommands(true)); + + replyToMessage(*cb, sender, 0, 0); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 100, timestamp of updated doc: 0) ReturnCode(NONE)"), + sender.getLastReply(true)); +} + +void +UpdateOperation_Test::testMultiNode() +{ + setupDistributor(2, 2, "distributor:1 storage:2"); + std::shared_ptr<UpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0,Update => 1"), + sender.getCommands(true)); + + replyToMessage(*cb, sender, 0, 120); + replyToMessage(*cb, sender, 1, 120); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 100, timestamp of updated doc: 120) ReturnCode(NONE)"), + sender.getLastReply(true)); + + CPPUNIT_ASSERT_EQUAL( + std::string( + _bId.toString() + " : " + "node(idx=1,crc=0x2,docs=4/4,bytes=6/6,trusted=true,active=false), " + "node(idx=0,crc=0x2,docs=4/4,bytes=6/6,trusted=true,active=false)"), + dumpBucket(_bId)); +} + +void +UpdateOperation_Test::testMultiNodeInconsistentTimestamp() +{ + setupDistributor(2, 2, "distributor:1 storage:2"); + std::shared_ptr<UpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3")); + MessageSenderStub sender; + cb->start(sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL( + std::string("Update => 0,Update => 1"), + sender.getCommands(true)); + + replyToMessage(*cb, sender, 0, 119); + replyToMessage(*cb, sender, 1, 120); + + CPPUNIT_ASSERT_EQUAL( + std::string("UpdateReply(doc:test:test, BucketId(0x0000000000000000), " + "timestamp 100, timestamp of updated doc: 120 Was inconsistent " + "(best node 1)) ReturnCode(NONE)"), + sender.getLastReply(true)); +} + diff --git a/storage/src/tests/distributor/visitoroperationtest.cpp b/storage/src/tests/distributor/visitoroperationtest.cpp new file mode 100644 index 00000000000..a8f28a73fb6 --- /dev/null +++ b/storage/src/tests/distributor/visitoroperationtest.cpp @@ -0,0 +1,1646 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <iomanip> +#include <iostream> +#include <memory> +#include <math.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storageapi/message/datagram.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/storage/distributor/operations/external/visitoroperation.h> +#include <vespa/storage/distributor/operations/external/visitororder.h> +#include <tests/distributor/distributortestutil.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/vdstestlib/cppunit/macros.h> + +using namespace document; +using namespace storage::api; +using namespace storage::lib; +using namespace std::string_literals; + +namespace storage { +namespace distributor { + +class VisitorOperationTest : public CppUnit::TestFixture, + public DistributorTestUtil { + CPPUNIT_TEST_SUITE(VisitorOperationTest); + CPPUNIT_TEST(testParameterForwarding); + CPPUNIT_TEST(testShutdown); + CPPUNIT_TEST(testNoBucket); + CPPUNIT_TEST(testOnlySuperBucketAndProgressAllowed); + CPPUNIT_TEST(testRetiredStorageNode); + CPPUNIT_TEST(testNoResendAfterTimeoutPassed); + CPPUNIT_TEST(testDistributorNotReady); + CPPUNIT_TEST(testInvalidOrderDocSelection); + CPPUNIT_TEST(testNonExistingBucket); + CPPUNIT_TEST(testUserSingleBucket); + CPPUNIT_TEST(testUserInconsistentlySplitBucket); + CPPUNIT_TEST(testBucketRemovedWhileVisitorPending); + CPPUNIT_TEST(testEmptyBucketsVisitedWhenVisitingRemoves); + CPPUNIT_TEST(testResendToOtherStorageNodeOnFailure); + CPPUNIT_TEST(testTimeoutOnlyAfterReplyFromAllStorageNodes); + CPPUNIT_TEST(testTimeoutDoesNotOverrideCriticalError); + CPPUNIT_TEST(testWrongDistribution); + CPPUNIT_TEST(testWrongDistributionInPendingState); + CPPUNIT_TEST(testVisitorAbortedIfNodeIsMarkedAsDown); + CPPUNIT_TEST(testBucketHighBitCount); + CPPUNIT_TEST(testBucketLowBitCount); + CPPUNIT_TEST(testParallelVisitorsToOneStorageNode); + CPPUNIT_TEST(testParallelVisitorsResendOnlyFailing); + CPPUNIT_TEST(testParallelVisitorsToOneStorageNodeOneSuperBucket); + CPPUNIT_TEST(testVisitWhenOneBucketCopyIsInvalid); + CPPUNIT_TEST(testVisitingWhenAllBucketsAreInvalid); + CPPUNIT_TEST(testInconsistencyHandling); + CPPUNIT_TEST(testVisitIdealNode); + CPPUNIT_TEST(testNoResendingOnCriticalFailure); + CPPUNIT_TEST(testFailureOnAllNodes); + CPPUNIT_TEST(testVisitOrder); + CPPUNIT_TEST(testVisitInChunks); + CPPUNIT_TEST(testVisitOrderSplitPastOrderBits); + CPPUNIT_TEST(testVisitOrderInconsistentlySplit); + CPPUNIT_TEST(testUserVisitorOrder); + CPPUNIT_TEST(testUserVisitorOrderSplitPastOrderBits); + CPPUNIT_TEST(testNoClientReplyBeforeAllStorageRepliesReceived); + CPPUNIT_TEST(testSkipFailedSubBucketsWhenVisitingInconsistent); + CPPUNIT_TEST(testQueueTimeoutIsFactorOfTotalTimeout); + CPPUNIT_TEST_SUITE_END(); + +protected: + void testParameterForwarding(); + void testShutdown(); + void testNoBucket(); + void testOnlySuperBucketAndProgressAllowed(); + void testRetiredStorageNode(); + void testNoResendAfterTimeoutPassed(); + void testDistributorNotReady(); + void testInvalidOrderDocSelection(); + void testNonExistingBucket(); + void testUserSingleBucket(); + void testUserInconsistentlySplitBucket(); + void testBucketRemovedWhileVisitorPending(); + void testEmptyBucketsVisitedWhenVisitingRemoves(); + void testResendToOtherStorageNodeOnFailure(); + void testTimeoutOnlyAfterReplyFromAllStorageNodes(); + void testTimeoutDoesNotOverrideCriticalError(); + void testAbortNonExisting(); + void testAbort(); + void testWrongDistribution(); + void testWrongDistributionInPendingState(); + void testVisitorAbortedIfNodeIsMarkedAsDown(); + void testBucketHighBitCount(); + void testBucketLowBitCount(); + void testParallelVisitorsToOneStorageNode(); + void testParallelVisitorsResendOnlyFailing(); + void testParallelVisitorsToOneStorageNodeOneSuperBucket(); + void testVisitWhenOneBucketCopyIsInvalid(); + void testVisitingWhenAllBucketsAreInvalid(); + void testInconsistencyHandling(); + void testVisitIdealNode(); + void testNoResendingOnCriticalFailure(); + void testFailureOnAllNodes(); + void testVisitOrder(); + void testVisitInChunks(); + void testVisitOrderSplitPastOrderBits(); + void testVisitOrderInconsistentlySplit(); + void testUserVisitorOrder(); + void testUserVisitorOrderSplitPastOrderBits(); + void testUserVisitorOrderInconsistentlySplit(); + void testNoClientReplyBeforeAllStorageRepliesReceived(); + void testSkipFailedSubBucketsWhenVisitingInconsistent(); + void testQueueTimeoutIsFactorOfTotalTimeout(); +public: + VisitorOperationTest() + : defaultConfig(framework::MilliSecTime(0), + 100, + 100) + {} + + void setUp() { + createLinks(); + nullId = document::BucketId(0, 0); + doneId = document::BucketId(INT_MAX); + }; + + void tearDown() { + close(); + } + + enum {MAX_PENDING = 2}; +private: + document::BucketId nullId; + document::BucketId doneId; + VisitorOperation::Config defaultConfig; + + api::CreateVisitorCommand::SP + createVisitorCommand(std::string instanceId, + document::BucketId superBucket, + document::BucketId lastBucket, + uint32_t maxBuckets = 8, + uint32_t timeoutMS = 500, + bool visitInconsistentBuckets = false, + bool visitRemoves = false, + std::string libraryName = "dumpvisitor", + document::OrderingSpecification::Order visitorOrdering = + document::OrderingSpecification::ASCENDING, + const std::string& docSelection = "") + { + api::CreateVisitorCommand::SP cmd( + new api::CreateVisitorCommand(libraryName, instanceId, docSelection)); + cmd->setControlDestination("controldestination"); + cmd->setDataDestination("datadestination"); + cmd->setFieldSet("[header]"); + if (visitRemoves) { + cmd->setVisitRemoves(); + } + cmd->setFromTime(10); + cmd->setToTime(100); + + cmd->addBucketToBeVisited(superBucket); + cmd->addBucketToBeVisited(lastBucket); + + cmd->setMaximumPendingReplyCount(VisitorOperationTest::MAX_PENDING); + cmd->setMaxBucketsPerVisitor(maxBuckets); + cmd->setTimeout(timeoutMS); + if (visitInconsistentBuckets) { + cmd->setVisitInconsistentBuckets(); + } + cmd->setVisitorOrdering(visitorOrdering); + return cmd; + } + + std::string + serializeVisitorCommand(int idx = -1) { + if (idx == -1) { + idx = _sender.commands.size() - 1; + } + + std::ostringstream ost; + + CreateVisitorCommand* cvc = dynamic_cast<CreateVisitorCommand*>( + _sender.commands[idx].get()); + + ost << *cvc << " Buckets: [ "; + for (uint32_t i = 0; i < cvc->getBuckets().size(); ++i) { + ost << cvc->getBuckets()[i] << " "; + } + ost << "]"; + return ost.str(); + } + + /** + Starts a visitor where we expect no createVisitorCommands to be sent + to storage, either due to error or due to no data actually stored. + */ + std::string runEmptyVisitor(api::CreateVisitorCommand::SP msg) { + VisitorOperation op(getExternalOperationHandler(), + msg, + defaultConfig); + op.start(_sender, framework::MilliSecTime(0)); + return _sender.getLastReply(); + } + + const std::vector<BucketId>& getBucketsFromLastCommand() { + const CreateVisitorCommand& cvc( + dynamic_cast<const CreateVisitorCommand&>( + *_sender.commands[_sender.commands.size() - 1])); + return cvc.getBuckets(); + } + + std::pair<std::string, std::string> + runVisitor(document::BucketId id, + document::BucketId lastId, + uint32_t maxBuckets); + + std::string doOrderedVisitor(document::BucketId startBucket); + + void doStandardVisitTest(const std::string& clusterState); + + std::unique_ptr<VisitorOperation> startOperationWith2StorageNodeVisitors( + bool inconsistent); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(VisitorOperationTest); + +void +VisitorOperationTest::testParameterForwarding() +{ + doStandardVisitTest("distributor:1 storage:1"); +} + +void +VisitorOperationTest::doStandardVisitTest(const std::string& clusterState) +{ + _distributor->enableClusterState(ClusterState(clusterState)); + + // Create bucket in bucketdb + document::BucketId id(uint64_t(0x400000000000007b)); + addNodesToBucketDB(id, "0=1/1/1/t"); + + // Send create visitor + vespalib::string instanceId("testParameterForwarding"); + vespalib::string libraryName("dumpvisitor"); + vespalib::string docSelection(""); + api::CreateVisitorCommand::SP msg( + new api::CreateVisitorCommand(libraryName, + instanceId, + docSelection)); + vespalib::string controlDestination("controldestination"); + msg->setControlDestination(controlDestination); + vespalib::string dataDestination("datadestination"); + msg->setDataDestination(dataDestination); + msg->setMaximumPendingReplyCount(VisitorOperationTest::MAX_PENDING); + msg->setMaxBucketsPerVisitor(8); + msg->setFromTime(10); + msg->setToTime(0); + msg->addBucketToBeVisited(id); + msg->addBucketToBeVisited(nullId); + msg->setFieldSet("[header]"); + msg->setVisitRemoves(); + msg->setTimeout(1234); + msg->getTrace().setLevel(7); + + VisitorOperation op(getExternalOperationHandler(), + msg, + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0"), + _sender.getCommands(true)); + + // Receive create visitor command for storage and simulate reply + api::StorageMessage::SP rep0 = _sender.commands[0]; + CreateVisitorCommand* cvc = dynamic_cast<CreateVisitorCommand*>(rep0.get()); + CPPUNIT_ASSERT(cvc); + CPPUNIT_ASSERT_EQUAL(libraryName, cvc->getLibraryName()); + CPPUNIT_ASSERT_EQUAL(instanceId, cvc->getInstanceId().substr(0, instanceId.length())); + CPPUNIT_ASSERT_EQUAL(docSelection, cvc->getDocumentSelection()); + CPPUNIT_ASSERT_EQUAL(controlDestination, cvc->getControlDestination()); + CPPUNIT_ASSERT_EQUAL(dataDestination, cvc->getDataDestination()); + CPPUNIT_ASSERT_EQUAL((unsigned int) VisitorOperationTest::MAX_PENDING, cvc->getMaximumPendingReplyCount()); + CPPUNIT_ASSERT_EQUAL((unsigned int) 8, cvc->getMaxBucketsPerVisitor()); + CPPUNIT_ASSERT_EQUAL((size_t) 1, cvc->getBuckets().size()); + CPPUNIT_ASSERT_EQUAL((api::Timestamp) 10, cvc->getFromTime()); + CPPUNIT_ASSERT(cvc->getToTime() > 0); + CPPUNIT_ASSERT_EQUAL(vespalib::string("[header]"), cvc->getFieldSet()); + CPPUNIT_ASSERT_EQUAL((bool) 1, cvc->visitRemoves()); + CPPUNIT_ASSERT_EQUAL(uint32_t(1234), cvc->getTimeout()); + CPPUNIT_ASSERT_EQUAL(uint32_t(7), cvc->getTrace().getLevel()); + + sendReply(op); + + CPPUNIT_ASSERT_EQUAL(std::string("CreateVisitorReply(" + "last=BucketId(0x000000007fffffff)) " + "ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +VisitorOperationTest::testShutdown() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:1")); + + // Create bucket in bucketdb + document::BucketId id(uint64_t(0x400000000000007b)); + addNodesToBucketDB(id, "0=1/1/1/t"); + + // Send create visitor + vespalib::string instanceId("testShutdown"); + vespalib::string libraryName("dumpvisitor"); + vespalib::string docSelection(""); + api::CreateVisitorCommand::SP msg( + new api::CreateVisitorCommand(libraryName, + instanceId, + docSelection)); + msg->addBucketToBeVisited(id); + msg->addBucketToBeVisited(nullId); + + VisitorOperation op(getExternalOperationHandler(), msg, defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0"), + _sender.getCommands(true)); + + op.onClose(_sender); // This will fail the visitor + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(ABORTED, Process is shutting down)"), + _sender.getLastReply()); +} + +void +VisitorOperationTest::testNoBucket() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:1")); + + // Send create visitor + api::CreateVisitorCommand::SP msg(new api::CreateVisitorCommand( + "dumpvisitor", "instance", "")); + + CPPUNIT_ASSERT_EQUAL(std::string( + "CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(ILLEGAL_PARAMETERS, No buckets in " + "CreateVisitorCommand for visitor 'instance')"), + runEmptyVisitor(msg)); +} + +void +VisitorOperationTest::testOnlySuperBucketAndProgressAllowed() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:1")); + + // Send create visitor + api::CreateVisitorCommand::SP msg(new api::CreateVisitorCommand( + "dumpvisitor", "instance", "")); + msg->addBucketToBeVisited(nullId); + msg->addBucketToBeVisited(nullId); + msg->addBucketToBeVisited(nullId); + + CPPUNIT_ASSERT_EQUAL(std::string( + "CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(ILLEGAL_PARAMETERS, CreateVisitorCommand " + "does not contain 2 buckets for visitor " + "'instance')"), + runEmptyVisitor(msg)); +} + +void +VisitorOperationTest::testRetiredStorageNode() +{ + doStandardVisitTest("distributor:1 storage:1 .0.s:r"); +} + +void +VisitorOperationTest::testNoResendAfterTimeoutPassed() +{ + document::BucketId id(uint64_t(0x400000000000007b)); + + _distributor->enableClusterState(ClusterState("distributor:1 storage:2")); + addNodesToBucketDB(id, "0=1/1/1/t,1=1/1/1/t"); + + VisitorOperation op( + getExternalOperationHandler(), + createVisitorCommand("lowtimeoutbusy", id, nullId, 8, 20), + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0"), + _sender.getCommands(true)); + + getClock().addMilliSecondsToTime(22); + + sendReply(op, -1, api::ReturnCode::BUSY); + + CPPUNIT_ASSERT_EQUAL( + std::string( + "CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(ABORTED, Timeout of 20 ms is running out)"), + _sender.getLastReply()); +} + +void +VisitorOperationTest::testDistributorNotReady() +{ + _distributor->enableClusterState(ClusterState("distributor:0 storage:0")); + document::BucketId id(uint64_t(0x400000000000007b)); + CPPUNIT_ASSERT_EQUAL( + std::string( + "CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(NODE_NOT_READY, No distributors available when " + "processing visitor 'notready')"), + runEmptyVisitor(createVisitorCommand("notready", id, nullId))); +} + +// Distributor only parses selection if in the order doc case (which is detected +// by first checking if string contains "order" which it must to refer to +// "id.order" !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +void +VisitorOperationTest::testInvalidOrderDocSelection() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:1")); + document::BucketId id(0x400000000000007b); + addNodesToBucketDB(id, "0=1/1/1/t"); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(ILLEGAL_PARAMETERS, Failed to parse document select " + "string 'id.order(10,3)=1 and dummy': Document type dummy not found)"), + runEmptyVisitor( + createVisitorCommand("invalidOrderDoc", + id, + nullId, + 8, + 500, + false, + false, + "dumpvisitor", + document::OrderingSpecification::ASCENDING, + "id.order(10,3)=1 and dummy"))); +} + +void +VisitorOperationTest::testNonExistingBucket() +{ + document::BucketId id(uint64_t(0x400000000000007b)); + _distributor->enableClusterState(ClusterState("distributor:1 storage:1")); + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x000000007fffffff)) " + "ReturnCode(NONE)"), + runEmptyVisitor( + createVisitorCommand("nonExistingBucket", + id, + nullId))); +} + +void +VisitorOperationTest::testUserSingleBucket() +{ + document::BucketId id(uint64_t(0x400000000000007b)); + document::BucketId userid(uint64_t(0x800000000000007b)); + _distributor->enableClusterState(ClusterState("distributor:1 storage:1")); + + addNodesToBucketDB(id, "0=1/1/1/t"); + + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("userSingleBucket", + userid, + nullId, + 8, + 500, + false, + false, + "dumpvisitor", + document::OrderingSpecification::ASCENDING, + "true"), + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL_MSG(_sender.getLastReply(), + std::string("Visitor Create => 0"), + _sender.getCommands(true)); + sendReply(op); + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x000000007fffffff)) " + "ReturnCode(NONE)"), + _sender.getLastReply()); +} + +std::pair<std::string, std::string> +VisitorOperationTest::runVisitor(document::BucketId id, + document::BucketId lastId, + uint32_t maxBuckets) +{ + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("inconsistentSplit", + id, + lastId, + maxBuckets, + 500, + false, + false, + "dumpvisitor", + document::OrderingSpecification::ASCENDING, + "true"), + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + + sendReply(op); + + std::pair<std::string, std::string> retVal = + std::make_pair(serializeVisitorCommand(), _sender.getLastReply()); + + _sender.clear(); + + return retVal; +} + +void +VisitorOperationTest::testUserInconsistentlySplitBucket() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:1")); + + // Not containing (19, 0x40001) + addNodesToBucketDB(document::BucketId(17, 0x0), "0=1/1/1/t"); + addNodesToBucketDB(document::BucketId(18, 0x20001), "0=1/1/1/t"); + addNodesToBucketDB(document::BucketId(19, 0x1), "0=1/1/1/t"); + + // Containing (19, 0x40001) + addNodesToBucketDB(document::BucketId(17, 0x1), "0=1/1/1/t"); + addNodesToBucketDB(document::BucketId(18, 0x1), "0=1/1/1/t"); + + // Equal to (19, 0x40001) + addNodesToBucketDB(document::BucketId(19, 0x40001), "0=1/1/1/t"); + + // Contained in (19, 0x40001) + addNodesToBucketDB(document::BucketId(20, 0x40001), "0=1/1/1/t"); + addNodesToBucketDB(document::BucketId(20, 0xc0001), "0=1/1/1/t"); + addNodesToBucketDB(document::BucketId(21, 0x40001), "0=1/1/1/t"); + addNodesToBucketDB(document::BucketId(21, 0x140001), "0=1/1/1/t"); + + document::BucketId id(19, 0x40001); + + { + std::pair<std::string, std::string> val( + runVisitor(id, nullId, 100)); + + CPPUNIT_ASSERT_EQUAL(std::string( + "CreateVisitorCommand(dumpvisitor, true, 7 buckets) " + "Buckets: [ BucketId(0x4400000000000001) " + "BucketId(0x4800000000000001) " + "BucketId(0x4c00000000040001) " + "BucketId(0x5000000000040001) " + "BucketId(0x5400000000040001) " + "BucketId(0x5400000000140001) " + "BucketId(0x50000000000c0001) ]"), + val.first); + + CPPUNIT_ASSERT_EQUAL(std::string( + "CreateVisitorReply(last=BucketId(0x000000007fffffff)) " + "ReturnCode(NONE)"), + val.second); + } +} + +void +VisitorOperationTest::testBucketRemovedWhileVisitorPending() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:1")); + + // Create bucket in bucketdb + document::BucketId id(uint64_t(0x400000000000007b)); + + addNodesToBucketDB(id, "0=1/1/1/t"); + + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("removefrombucketdb", + id, + nullId), + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0"), + _sender.getCommands(true)); + + removeFromBucketDB(id); + + sendReply(op, -1, api::ReturnCode::NOT_CONNECTED); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(BUCKET_NOT_FOUND)"), + _sender.getLastReply()); +} + +void +VisitorOperationTest::testEmptyBucketsVisitedWhenVisitingRemoves() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:1")); + document::BucketId id(uint64_t(0x400000000000007b)); + addNodesToBucketDB(id, "0=0/0/0/1/2/t"); + + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("emptybucket", + id, + nullId, + 8, + 500, + false, + true), + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + + // Since visitRemoves is true, the empty bucket will be visited + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0"), + _sender.getCommands(true)); +} + +void +VisitorOperationTest::testResendToOtherStorageNodeOnFailure() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:2")); + document::BucketId id(uint64_t(0x400000000000007b)); + + addNodesToBucketDB(id, "0=1/1/1/t,1=1/1/1/t"); + + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("emptyinconsistent", + id, + nullId), + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0"), + _sender.getCommands(true)); + + sendReply(op, -1, api::ReturnCode::NOT_CONNECTED); + CPPUNIT_ASSERT_EQUAL(""s, _sender.getReplies()); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0,Visitor Create => 1"), + _sender.getCommands(true)); +} + +// Since MessageBus handles timeouts for us implicitly, we make the assumption +// that we can safely wait for all replies to be received before sending a +// client reply and that this won't cause things to hang for indeterminate +// amounts of time. +void +VisitorOperationTest::testTimeoutOnlyAfterReplyFromAllStorageNodes() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:2")); + + // Contained in (16, 0x1) + addNodesToBucketDB(document::BucketId(17, 0x00001), "0=1/1/1/t"); + addNodesToBucketDB(document::BucketId(17, 0x10001), "1=1/1/1/t"); + + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("timeout2bucketson2nodes", + document::BucketId(16, 1), + nullId, + 8), + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL("Visitor Create => 0,Visitor Create => 1"s, + _sender.getCommands(true)); + + getClock().addMilliSecondsToTime(501); + + sendReply(op, 0); + CPPUNIT_ASSERT_EQUAL(""s, _sender.getReplies()); // No reply yet. + + sendReply(op, 1, api::ReturnCode::BUSY); + + CPPUNIT_ASSERT_EQUAL( + "CreateVisitorReply(last=BucketId(0x4400000000000001)) " + "ReturnCode(ABORTED, Timeout of 500 ms is running out)"s, + _sender.getLastReply()); + + // XXX This is sub-optimal in the case that we time out but all storage + // visitors return OK, as we'll then be failing an operation that + // technically went fine. However, this is assumed to happen sufficiently + // rarely (requires timing to be so that mbus timouts don't happen for + // neither client -> distributor nor distributor -> storage for the + // operation to possibly could have been considered successful) that we + // don't bother to add complexity for handling it as a special case. +} + +void +VisitorOperationTest::testTimeoutDoesNotOverrideCriticalError() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:2")); + addNodesToBucketDB(document::BucketId(17, 0x00001), "0=1/1/1/t"); + addNodesToBucketDB(document::BucketId(17, 0x10001), "1=1/1/1/t"); + + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("timeout2bucketson2nodes", + document::BucketId(16, 1), + nullId, + 8, + 500), // ms timeout + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + CPPUNIT_ASSERT_EQUAL("Visitor Create => 0,Visitor Create => 1"s, + _sender.getCommands(true)); + + getClock().addMilliSecondsToTime(501); + // Technically has timed out at this point, but should still report the + // critical failure. + sendReply(op, 0, api::ReturnCode::INTERNAL_FAILURE); + CPPUNIT_ASSERT_EQUAL(""s, _sender.getReplies()); + sendReply(op, 1, api::ReturnCode::BUSY); + + CPPUNIT_ASSERT_EQUAL( + "CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(INTERNAL_FAILURE, [from content node 0] )"s, + _sender.getLastReply()); +} + +void +VisitorOperationTest::testWrongDistribution() +{ + setupDistributor(1, 100, "distributor:100 storage:2"); + + document::BucketId id(uint64_t(0x400000000000127b)); + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(WRONG_DISTRIBUTION, distributor:100 storage:2)"), + runEmptyVisitor(createVisitorCommand("wrongdist", id, nullId))); +} + +void +VisitorOperationTest::testWrongDistributionInPendingState() +{ + // Force bucket to belong to this distributor in currently enabled state. + setupDistributor(1, 100, "distributor:1 storage:2"); + // Trigger pending cluster state. Note: increase in storage node count + // to force resending of bucket info requests. + auto stateCmd = std::make_shared<api::SetSystemStateCommand>( + lib::ClusterState("distributor:100 storage:3")); + getBucketDBUpdater().onSetSystemState(stateCmd); + + document::BucketId id(uint64_t(0x400000000000127b)); + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(WRONG_DISTRIBUTION, distributor:100 storage:3)"), + runEmptyVisitor(createVisitorCommand("wrongdistpending", id, nullId))); +} + +// If the current node state changes, this alters the node's cluster state +// internally without this change being part of a new version. As a result, +// we cannot answer with WRONG_DISTRIBUTION as the client expects to see a +// higher version number. +// See ticket 6353382 for details. +void +VisitorOperationTest::testVisitorAbortedIfNodeIsMarkedAsDown() +{ + setupDistributor(1, 10, "distributor:10 .0.s:s storage:10"); + + document::BucketId id(uint64_t(0x400000000000127b)); + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(ABORTED, Distributor is shutting down)"), + runEmptyVisitor(createVisitorCommand("wrongdist", id, nullId))); +} + +void +VisitorOperationTest::testBucketHighBitCount() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:1 bits:16")); + + document::BucketId id(18, 0x0); + addNodesToBucketDB(id, "0=1/1/1/t"); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(WRONG_DISTRIBUTION, distributor:1 storage:1)"), + runEmptyVisitor(createVisitorCommand("buckethigbit", id, nullId))); + + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("buckethighbitcount", + id, + nullId, + 8, + 500, + false, + false, + "dumpvisitor", + document::OrderingSpecification::ASCENDING, + "true"), + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0"), + _sender.getCommands(true)); +} + +void +VisitorOperationTest::testBucketLowBitCount() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:1 bits:16")); + + document::BucketId id(1, 0x0); + addNodesToBucketDB(id, "0=1/1/1/t"); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(WRONG_DISTRIBUTION, distributor:1 storage:1)"), + runEmptyVisitor(createVisitorCommand("bucketlowbit", id, nullId))); + + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("buckethighbitcount", + id, + nullId, + 8, + 500, + false, + false, + "dumpvisitor", + document::OrderingSpecification::ASCENDING, + "true"), + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(WRONG_DISTRIBUTION, distributor:1 storage:1)"), + _sender.getLastReply()); +} + +void +VisitorOperationTest::testParallelVisitorsToOneStorageNode() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:1")); + + // Create buckets in bucketdb + for (int i=0; i<32; i++) { + document::BucketId id(21, i*0x10000 + 0x0001); + addNodesToBucketDB(id, "0=1/1/1/t"); + } + + document::BucketId id(16, 1); + + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("multiplebuckets", + id, + nullId, + 31), + VisitorOperation::Config( + framework::MilliSecTime(0), + 1, + 4)); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0,Visitor Create => 0," + "Visitor Create => 0,Visitor Create => 0"), + _sender.getCommands(true)); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorCommand(dumpvisitor, , 8 buckets) Buckets: [ " + "BucketId(0x5400000000000001) BucketId(0x5400000000040001) " + "BucketId(0x5400000000020001) BucketId(0x5400000000060001) " + "BucketId(0x5400000000010001) BucketId(0x5400000000050001) " + "BucketId(0x5400000000030001) BucketId(0x5400000000070001) ]"), + serializeVisitorCommand(0)); + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorCommand(dumpvisitor, , 8 buckets) Buckets: [ " + "BucketId(0x5400000000100001) BucketId(0x5400000000140001) " + "BucketId(0x5400000000120001) BucketId(0x5400000000160001) " + "BucketId(0x5400000000110001) BucketId(0x5400000000150001) " + "BucketId(0x5400000000130001) BucketId(0x5400000000170001) ]"), + serializeVisitorCommand(1)); + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorCommand(dumpvisitor, , 8 buckets) Buckets: [ " + "BucketId(0x5400000000080001) BucketId(0x54000000000c0001) " + "BucketId(0x54000000000a0001) BucketId(0x54000000000e0001) " + "BucketId(0x5400000000090001) BucketId(0x54000000000d0001) " + "BucketId(0x54000000000b0001) BucketId(0x54000000000f0001) ]"), + serializeVisitorCommand(2)); + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorCommand(dumpvisitor, , 7 buckets) Buckets: [ " + "BucketId(0x5400000000180001) BucketId(0x54000000001c0001) " + "BucketId(0x54000000001a0001) BucketId(0x54000000001e0001) " + "BucketId(0x5400000000190001) BucketId(0x54000000001d0001) " + "BucketId(0x54000000001b0001) ]"), + serializeVisitorCommand(3)); + + for (uint32_t i = 0; i < 4; ++i) { + sendReply(op, i); + } + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x54000000000f0001)) " + "ReturnCode(NONE)"), + _sender.getLastReply()); + + _sender.clear(); + + uint32_t minBucketsPerVisitor = 1; + uint32_t maxVisitorsPerNode = 4; + VisitorOperation op2(getExternalOperationHandler(), + createVisitorCommand("multiplebuckets", + id, + document::BucketId(0x54000000000f0001), + 31), + VisitorOperation::Config( + framework::MilliSecTime(0), + minBucketsPerVisitor, + maxVisitorsPerNode)); + + op2.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0"), + _sender.getCommands(true)); + + sendReply(op2); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x000000007fffffff)) " + "ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +VisitorOperationTest::testParallelVisitorsResendOnlyFailing() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:2")); + + // Create buckets in bucketdb + for (int i=0; i<32; i++) { + document::BucketId id(21, i*0x10000 + 0x0001); + addNodesToBucketDB(id, "0=1/1/1/t,1=1/1/1/t"); + } + + document::BucketId id(16, 1); + + uint32_t minBucketsPerVisitor = 5; + uint32_t maxVisitorsPerNode = 4; + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("multiplebuckets", + id, + nullId, + 31), + VisitorOperation::Config( + framework::MilliSecTime(0), + minBucketsPerVisitor, + maxVisitorsPerNode)); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0,Visitor Create => 0," + "Visitor Create => 0,Visitor Create => 0"), + _sender.getCommands(true)); + + for (uint32_t i = 0; i < 2; ++i) { + sendReply(op, i, api::ReturnCode::NOT_CONNECTED); + } + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0,Visitor Create => 0," + "Visitor Create => 0,Visitor Create => 0," + "Visitor Create => 1,Visitor Create => 1"), + _sender.getCommands(true)); + + for (uint32_t i = 2; i < 6; ++i) { + sendReply(op, i); + } + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x54000000000f0001)) " + "ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +VisitorOperationTest::testParallelVisitorsToOneStorageNodeOneSuperBucket() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:1")); + + // Create buckets in bucketdb + for (int i=0; i<8; i++) { + document::BucketId id(0x8c000000e3362b6aULL+i*0x100000000ull); + addNodesToBucketDB(id, "0=1/1/1/t"); + } + + document::BucketId id(16, 0x2b6a); + + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("multiplebucketsonesuper", + id, + nullId), + VisitorOperation::Config( + framework::MilliSecTime(0), + 5, + 4)); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0"), + _sender.getCommands(true)); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorCommand(dumpvisitor, , 8 buckets) Buckets: [ " + "BucketId(0x8c000000e3362b6a) BucketId(0x8c000004e3362b6a) " + "BucketId(0x8c000002e3362b6a) BucketId(0x8c000006e3362b6a) " + "BucketId(0x8c000001e3362b6a) BucketId(0x8c000005e3362b6a) " + "BucketId(0x8c000003e3362b6a) BucketId(0x8c000007e3362b6a) ]"), + serializeVisitorCommand(0)); + + sendReply(op); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x000000007fffffff)) " + "ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +VisitorOperationTest::testVisitWhenOneBucketCopyIsInvalid() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:2")); + + document::BucketId id(16, 0); + + addNodesToBucketDB(id, "0=100,1=0/0/1"); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(BUCKET_NOT_FOUND)"), + runEmptyVisitor(createVisitorCommand("incompletehandling", + id, + nullId))); +} + +void +VisitorOperationTest::testVisitingWhenAllBucketsAreInvalid() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:2")); + + document::BucketId id(16, 0); + + addNodesToBucketDB(id, "0=0/0/1,1=0/0/1"); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(BUCKET_NOT_FOUND)"), + runEmptyVisitor(createVisitorCommand("allincompletehandling", + id, + nullId))); +} + +void +VisitorOperationTest::testInconsistencyHandling() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:2")); + + document::BucketId id(16, 0); + + addNodesToBucketDB(id, "0=1/1/1,1=2/2/2"); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(BUCKET_NOT_FOUND)"), + runEmptyVisitor(createVisitorCommand("testinconsistencyhandling", + id, + nullId))); + _sender.clear(); + + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("multiplebucketsonesuper", + id, + nullId, + 8, + 500, + true), + VisitorOperation::Config( + framework::MilliSecTime(0), + 5, + 4)); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 1"), + _sender.getCommands(true)); + + sendReply(op); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x000000007fffffff)) " + "ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +VisitorOperationTest::testVisitIdealNode() +{ + ClusterState state("distributor:1 storage:3"); + _distributor->enableClusterState(state); + + // Create buckets in bucketdb + for (int i=0; i<32; i++ ) { + document::BucketId id(21, i*0x10000 + 0x0001); + addIdealNodes(state, id); + } + + document::BucketId id(16, 1); + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("multinode", + id, + nullId, + 8), + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0"), + _sender.getCommands(true)); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorCommand(dumpvisitor, , 8 buckets) Buckets: [ " + "BucketId(0x5400000000000001) BucketId(0x5400000000100001) " + "BucketId(0x5400000000080001) BucketId(0x5400000000180001) " + "BucketId(0x5400000000040001) BucketId(0x5400000000140001) " + "BucketId(0x54000000000c0001) BucketId(0x54000000001c0001) ]"), + serializeVisitorCommand(0)); + + sendReply(op); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x54000000001c0001)) " + "ReturnCode(NONE)"), + _sender.getLastReply()); +} + +void +VisitorOperationTest::testNoResendingOnCriticalFailure() +{ + ClusterState state("distributor:1 storage:3"); + _distributor->enableClusterState(state); + + // Create buckets in bucketdb + for (int i=0; i<32; i++ ) { + document::BucketId id(21, i*0x10000 + 0x0001); + addNodesToBucketDB(id, "0=1/1/1/t,1=1/1/1/t"); + } + + document::BucketId id(16, 1); + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("multinodefailurecritical", + id, + nullId, + 8), + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0"), + _sender.getCommands(true)); + + sendReply(op, -1, api::ReturnCode::ILLEGAL_PARAMETERS); + + CPPUNIT_ASSERT_EQUAL( + "CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(ILLEGAL_PARAMETERS, [from content node 0] )"s, + _sender.getLastReply()); +} + +void +VisitorOperationTest::testFailureOnAllNodes() +{ + ClusterState state("distributor:1 storage:3"); + _distributor->enableClusterState(state); + + // Create buckets in bucketdb + for (int i=0; i<32; i++ ) { + document::BucketId id(21, i*0x10000 + 0x0001); + addNodesToBucketDB(id, "0=1/1/1/t,1=1/1/1/t"); + } + + document::BucketId id(16, 1); + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand("multinodefailurecritical", + id, + nullId, + 8), + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0"), + _sender.getCommands(true)); + + sendReply(op, -1, api::ReturnCode::NOT_CONNECTED); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0,Visitor Create => 1"), + _sender.getCommands(true)); + + sendReply(op, -1, api::ReturnCode::NOT_CONNECTED); + + CPPUNIT_ASSERT_EQUAL( + std::string("CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(BUCKET_NOT_FOUND)"), + _sender.getLastReply()); +} + + +void +VisitorOperationTest::testVisitOrder() +{ + std::vector<document::BucketId> buckets; + + document::BucketId id000(35, 0x0000004d2); + buckets.push_back(id000); + document::BucketId id001(35, 0x4000004d2); + buckets.push_back(id001); + document::BucketId id01(34, 0x2000004d2); + buckets.push_back(id01); + document::BucketId id1(33, 0x1000004d2); + buckets.push_back(id1); + + std::sort(buckets.begin(), + buckets.end(), + VisitorOrder(document::OrderingSpecification( + document::OrderingSpecification::ASCENDING, 0x0, 6, 2))); + + CPPUNIT_ASSERT_EQUAL(buckets[0], id000); + CPPUNIT_ASSERT_EQUAL(buckets[1], id001); + CPPUNIT_ASSERT_EQUAL(buckets[2], id01); + CPPUNIT_ASSERT_EQUAL(buckets[3], id1); + + std::sort(buckets.begin(), + buckets.end(), + VisitorOrder(document::OrderingSpecification( + document::OrderingSpecification::DESCENDING, 0xFF, 6, 2))); + CPPUNIT_ASSERT_EQUAL(buckets[0], id1); + CPPUNIT_ASSERT_EQUAL(buckets[1], id01); + CPPUNIT_ASSERT_EQUAL(buckets[2], id001); + CPPUNIT_ASSERT_EQUAL(buckets[3], id000); + + std::sort(buckets.begin(), + buckets.end(), + VisitorOrder(document::OrderingSpecification( + document::OrderingSpecification::ASCENDING, 0x14, 6, 2))); + CPPUNIT_ASSERT_EQUAL(buckets[0], id01); + CPPUNIT_ASSERT_EQUAL(buckets[1], id1); + CPPUNIT_ASSERT_EQUAL(buckets[2], id000); + CPPUNIT_ASSERT_EQUAL(buckets[3], id001); + + std::sort(buckets.begin(), + buckets.end(), + VisitorOrder(document::OrderingSpecification( + document::OrderingSpecification::DESCENDING, 0x14, 6, 2))); + CPPUNIT_ASSERT_EQUAL(buckets[0], id01); + CPPUNIT_ASSERT_EQUAL(buckets[1], id001); + CPPUNIT_ASSERT_EQUAL(buckets[2], id000); + CPPUNIT_ASSERT_EQUAL(buckets[3], id1); +} + +void +VisitorOperationTest::testVisitInChunks() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:1")); + + for (int i = 0; i < 9; ++i) { + addNodesToBucketDB(document::BucketId(30, i << 16), "0=1/1/1/t"); + } + + document::BucketId id(16, 0); + + std::pair<std::string, std::string> val(runVisitor(id, nullId, 3)); + CPPUNIT_ASSERT_EQUAL(std::string( + "CreateVisitorCommand(dumpvisitor, true, 3 buckets) " + "Buckets: [ BucketId(0x7800000000000000) " + "BucketId(0x7800000000080000) " + "BucketId(0x7800000000040000) ]"), + val.first); + + CPPUNIT_ASSERT_EQUAL(std::string( + "CreateVisitorReply(last=BucketId(0x7800000000040000)) " + "ReturnCode(NONE)"), + val.second); + + val = runVisitor(id, document::BucketId(0x7800000000040000), 3); + CPPUNIT_ASSERT_EQUAL(std::string( + "CreateVisitorCommand(dumpvisitor, true, 3 buckets) " + "Buckets: [ BucketId(0x7800000000020000) " + "BucketId(0x7800000000060000) " + "BucketId(0x7800000000010000) ]"), + val.first); + + CPPUNIT_ASSERT_EQUAL(std::string( + "CreateVisitorReply(last=BucketId(0x7800000000010000)) " + "ReturnCode(NONE)"), + val.second); + + val = runVisitor(id, document::BucketId(0x7800000000010000), 3); + CPPUNIT_ASSERT_EQUAL(std::string( + "CreateVisitorCommand(dumpvisitor, true, 3 buckets) " + "Buckets: [ BucketId(0x7800000000050000) " + "BucketId(0x7800000000030000) " + "BucketId(0x7800000000070000) ]"), + val.first); + + CPPUNIT_ASSERT_EQUAL(std::string( + "CreateVisitorReply(last=BucketId(0x000000007fffffff)) " + "ReturnCode(NONE)"), + val.second); +} + +void +VisitorOperationTest::testVisitOrderSplitPastOrderBits() +{ + std::vector<document::BucketId> buckets; + + document::BucketId max(INT_MAX); + buckets.push_back(max); + document::BucketId id1(33, 0x1000004d2); + buckets.push_back(id1); + document::BucketId id01(34, 0x2000004d2); + buckets.push_back(id01); + document::BucketId id00001(37, 0x10000004d2); + buckets.push_back(id00001); + document::BucketId id00000(37, 0x00000004d2); + buckets.push_back(id00000); + document::BucketId id0000(36, 0x0000004d2); + buckets.push_back(id0000); + document::BucketId null(0, 0); + buckets.push_back(null); + + std::sort(buckets.begin(), buckets.end(), VisitorOrder(document::OrderingSpecification(document::OrderingSpecification::ASCENDING, 0x0, 6, 2))); + CPPUNIT_ASSERT_EQUAL(buckets[0], null); + CPPUNIT_ASSERT_EQUAL(buckets[1], id0000); + CPPUNIT_ASSERT_EQUAL(buckets[2], id00000); + CPPUNIT_ASSERT_EQUAL(buckets[3], id00001); + CPPUNIT_ASSERT_EQUAL(buckets[4], id01); + CPPUNIT_ASSERT_EQUAL(buckets[5], id1); + CPPUNIT_ASSERT_EQUAL(buckets[6], max); + + std::sort(buckets.begin(), buckets.end(), VisitorOrder(document::OrderingSpecification(document::OrderingSpecification::DESCENDING, 0xFF, 6, 2))); + CPPUNIT_ASSERT_EQUAL(buckets[0], null); + CPPUNIT_ASSERT_EQUAL(buckets[1], id1); + CPPUNIT_ASSERT_EQUAL(buckets[2], id01); + CPPUNIT_ASSERT_EQUAL(buckets[3], id0000); + CPPUNIT_ASSERT_EQUAL(buckets[4], id00000); + CPPUNIT_ASSERT_EQUAL(buckets[5], id00001); + CPPUNIT_ASSERT_EQUAL(buckets[6], max); + + std::sort(buckets.begin(), buckets.end(), VisitorOrder(document::OrderingSpecification(document::OrderingSpecification::ASCENDING, 0x14, 6, 2))); + CPPUNIT_ASSERT_EQUAL(buckets[0], null); + CPPUNIT_ASSERT_EQUAL(buckets[1], id01); + CPPUNIT_ASSERT_EQUAL(buckets[2], id1); + CPPUNIT_ASSERT_EQUAL(buckets[3], id0000); + CPPUNIT_ASSERT_EQUAL(buckets[4], id00000); + CPPUNIT_ASSERT_EQUAL(buckets[5], id00001); + CPPUNIT_ASSERT_EQUAL(buckets[6], max); + + std::sort(buckets.begin(), buckets.end(), VisitorOrder(document::OrderingSpecification(document::OrderingSpecification::DESCENDING, 0x14, 6, 2))); + CPPUNIT_ASSERT_EQUAL(buckets[0], null); + CPPUNIT_ASSERT_EQUAL(buckets[1], id01); + CPPUNIT_ASSERT_EQUAL(buckets[2], id0000); + CPPUNIT_ASSERT_EQUAL(buckets[3], id00000); + CPPUNIT_ASSERT_EQUAL(buckets[4], id00001); + CPPUNIT_ASSERT_EQUAL(buckets[5], id1); + CPPUNIT_ASSERT_EQUAL(buckets[6], max); +} + +void +VisitorOperationTest::testVisitOrderInconsistentlySplit() +{ + std::vector<document::BucketId> buckets; + + document::BucketId max(INT_MAX); + buckets.push_back(max); + document::BucketId id000(35, 0x0000004d2); + buckets.push_back(id000); + document::BucketId id001(35, 0x4000004d2); + buckets.push_back(id001); + document::BucketId id01(34, 0x2000004d2); + buckets.push_back(id01); + document::BucketId id1(33, 0x1000004d2); + buckets.push_back(id1); + document::BucketId idsuper(16, 0x04d2); + buckets.push_back(idsuper); + document::BucketId null(0, 0); + buckets.push_back(null); + + std::sort(buckets.begin(), buckets.end(), VisitorOrder(document::OrderingSpecification(document::OrderingSpecification::ASCENDING, 0x0, 6, 2))); + CPPUNIT_ASSERT_EQUAL(buckets[0], null); + CPPUNIT_ASSERT_EQUAL(buckets[1], idsuper); + CPPUNIT_ASSERT_EQUAL(buckets[2], id000); + CPPUNIT_ASSERT_EQUAL(buckets[3], id001); + CPPUNIT_ASSERT_EQUAL(buckets[4], id01); + CPPUNIT_ASSERT_EQUAL(buckets[5], id1); + CPPUNIT_ASSERT_EQUAL(buckets[6], max); + + std::sort(buckets.begin(), buckets.end(), VisitorOrder(document::OrderingSpecification(document::OrderingSpecification::DESCENDING, 0xFF, 6, 2))); + CPPUNIT_ASSERT_EQUAL(buckets[0], null); + CPPUNIT_ASSERT_EQUAL(buckets[1], idsuper); + CPPUNIT_ASSERT_EQUAL(buckets[2], id1); + CPPUNIT_ASSERT_EQUAL(buckets[3], id01); + CPPUNIT_ASSERT_EQUAL(buckets[4], id001); + CPPUNIT_ASSERT_EQUAL(buckets[5], id000); + CPPUNIT_ASSERT_EQUAL(buckets[6], max); + + std::sort(buckets.begin(), buckets.end(), VisitorOrder(document::OrderingSpecification(document::OrderingSpecification::ASCENDING, 0x14, 6, 2))); + CPPUNIT_ASSERT_EQUAL(buckets[0], null); + CPPUNIT_ASSERT_EQUAL(buckets[1], idsuper); + CPPUNIT_ASSERT_EQUAL(buckets[2], id01); + CPPUNIT_ASSERT_EQUAL(buckets[3], id1); + CPPUNIT_ASSERT_EQUAL(buckets[4], id000); + CPPUNIT_ASSERT_EQUAL(buckets[5], id001); + CPPUNIT_ASSERT_EQUAL(buckets[6], max); + + std::sort(buckets.begin(), buckets.end(), VisitorOrder(document::OrderingSpecification(document::OrderingSpecification::DESCENDING, 0x14, 6, 2))); + CPPUNIT_ASSERT_EQUAL(buckets[0], null); + CPPUNIT_ASSERT_EQUAL(buckets[1], idsuper); + CPPUNIT_ASSERT_EQUAL(buckets[2], id01); + CPPUNIT_ASSERT_EQUAL(buckets[3], id001); + CPPUNIT_ASSERT_EQUAL(buckets[4], id000); + CPPUNIT_ASSERT_EQUAL(buckets[5], id1); + CPPUNIT_ASSERT_EQUAL(buckets[6], max); +} + +std::string +VisitorOperationTest::doOrderedVisitor(document::BucketId startBucket) +{ + std::vector<document::BucketId> buckets; + + while (true) { + _sender.clear(); + + VisitorOperation op(getExternalOperationHandler(), + createVisitorCommand( + "uservisitororder", + startBucket, + buckets.size() ? buckets[buckets.size() - 1] : + nullId, + 1, + 500, + false, + false, + "dumpvisitor", + document::OrderingSpecification::DESCENDING, + "id.order(6,2)<= 20"), + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0"), + _sender.getCommands(true)); + + for (uint32_t i = 0; i < _sender.commands.size(); ++i) { + const api::CreateVisitorCommand cmd( + static_cast<const api::CreateVisitorCommand&>( + *_sender.commands[i])); + + for (uint32_t j = 0; j < cmd.getBuckets().size(); ++j) { + buckets.push_back(cmd.getBuckets()[j]); + } + } + + sendReply(op); + + CPPUNIT_ASSERT_EQUAL(1, (int)_sender.replies.size()); + + const api::CreateVisitorReply& reply( + static_cast<const api::CreateVisitorReply&>(*_sender.replies[0])); + + if (reply.getLastBucket() == document::BucketId(0x000000007fffffff)) { + break; + } + } + + std::ostringstream ost; + for (uint32_t i = 0; i < buckets.size(); ++i) { + ost << buckets[i] << "\n"; + } + + return ost.str(); +} + +void +VisitorOperationTest::testUserVisitorOrder() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:1")); + + // Create buckets in bucketdb + std::vector<document::BucketId> buckets; + document::BucketId id000(35, 0x0000004d2); + buckets.push_back(id000); + document::BucketId id001(35, 0x4000004d2); + buckets.push_back(id001); + document::BucketId id01(34, 0x2000004d2); + buckets.push_back(id01); + document::BucketId id1(33, 0x1000004d2); + buckets.push_back(id1); + + for (uint32_t i=0; i<buckets.size(); i++) { + addNodesToBucketDB(buckets[i], "0=1/1/1/t"); + } + + document::BucketId id(16, 0x04d2); + + CPPUNIT_ASSERT_EQUAL(std::string("BucketId(0x88000002000004d2)\n" + "BucketId(0x8c000004000004d2)\n" + "BucketId(0x8c000000000004d2)\n" + "BucketId(0x84000001000004d2)\n"), + doOrderedVisitor(id)); +} + +void +VisitorOperationTest::testUserVisitorOrderSplitPastOrderBits() +{ + _distributor->enableClusterState(ClusterState("distributor:1 storage:1")); + + // Create buckets in bucketdb + std::vector<document::BucketId> buckets; + document::BucketId id1(33, 0x1000004d2); + buckets.push_back(id1); + document::BucketId id01(34, 0x2000004d2); + buckets.push_back(id01); + document::BucketId id00001(37, 0x10000004d2); + buckets.push_back(id00001); + document::BucketId id00000(37, 0x00000004d2); + buckets.push_back(id00000); + document::BucketId id0000(36, 0x0000004d2); + buckets.push_back(id0000); + for (uint32_t i=0; i<buckets.size(); i++) { + addNodesToBucketDB(buckets[i], "0=1/1/1/t"); + } + + document::BucketId id(16, 0x04d2); + + CPPUNIT_ASSERT_EQUAL(std::string("BucketId(0x88000002000004d2)\n" + "BucketId(0x90000000000004d2)\n" + "BucketId(0x94000000000004d2)\n" + "BucketId(0x94000010000004d2)\n" + "BucketId(0x84000001000004d2)\n"), + doOrderedVisitor(id)); +} + +std::unique_ptr<VisitorOperation> +VisitorOperationTest::startOperationWith2StorageNodeVisitors(bool inconsistent) +{ + ClusterState state("distributor:1 storage:3"); + _distributor->enableClusterState(state); + + addNodesToBucketDB(document::BucketId(17, 1), "0=1/1/1/t"); + addNodesToBucketDB(document::BucketId(17, 1 << 16 | 1), + "1=1/1/1/t"); + + document::BucketId id(16, 1); + auto op = std::make_unique<VisitorOperation>( + getExternalOperationHandler(), + createVisitorCommand( + "multinodefailurecritical", + id, + nullId, + 8, + 500, + inconsistent), + defaultConfig); + + op->start(_sender, framework::MilliSecTime(0)); + + CPPUNIT_ASSERT_EQUAL("Visitor Create => 0,Visitor Create => 1"s, + _sender.getCommands(true)); + return op; +} + +void +VisitorOperationTest::testNoClientReplyBeforeAllStorageRepliesReceived() +{ + auto op = startOperationWith2StorageNodeVisitors(false); + + sendReply(*op, 0, api::ReturnCode::BUSY); + // We don't want to see a reply here until the other node has replied. + CPPUNIT_ASSERT_EQUAL(""s, _sender.getReplies(true)); + // OK reply from 1, but have to retry from client anyhow since one of + // the sub buckets failed to be processed and we don't have inconsistent + // visiting set in the client visitor command. + sendReply(*op, 1); + CPPUNIT_ASSERT_EQUAL( + "CreateVisitorReply(last=BucketId(0x0000000000000000)) " + "ReturnCode(BUCKET_NOT_FOUND)"s, + _sender.getLastReply()); + // XXX we should consider wether we want BUSY to be returned instead. + // Non-critical error codes are currently converted to a generic "not found" + // code to let the client silently retry until the bucket has hopefully + // become consistent/available. +} + +void +VisitorOperationTest::testSkipFailedSubBucketsWhenVisitingInconsistent() +{ + auto op = startOperationWith2StorageNodeVisitors(true); + + sendReply(*op, 0, api::ReturnCode::BUSY); + CPPUNIT_ASSERT_EQUAL(""s, _sender.getReplies(true)); + // Subset of buckets could not be visited, but visit inconsistent flag is + // set in the client visitor so we treat it as a success anyway. In this + // case we've expanded the entire superbucket sub-tree so return with magic + // number to signify this. + sendReply(*op, 1); + CPPUNIT_ASSERT_EQUAL( + "CreateVisitorReply(last=BucketId(0x000000007fffffff)) " + "ReturnCode(NONE)"s, + _sender.getLastReply()); +} + +// By default, queue timeout should be half of remaining visitor time. This +// is a highly un-scientific heuristic, but seems rather more reasonable than +// having it hard-coded to 2000 ms as was the case earlier. +void +VisitorOperationTest::testQueueTimeoutIsFactorOfTotalTimeout() +{ + document::BucketId id(uint64_t(0x400000000000007b)); + _distributor->enableClusterState(ClusterState("distributor:1 storage:2")); + addNodesToBucketDB(id, "0=1/1/1/t,1=1/1/1/t"); + + VisitorOperation op( + getExternalOperationHandler(), + createVisitorCommand("foo", id, nullId, 8, 10000), + defaultConfig); + + op.start(_sender, framework::MilliSecTime(0)); + CPPUNIT_ASSERT_EQUAL(std::string("Visitor Create => 0"), + _sender.getCommands(true)); + + auto& cmd(dynamic_cast<CreateVisitorCommand&>(*_sender.commands[0])); + CPPUNIT_ASSERT_EQUAL(uint32_t(5000), cmd.getQueueTimeout()); +} + +} // distributor +} // storage diff --git a/storage/src/tests/fastos.project.newcore b/storage/src/tests/fastos.project.newcore new file mode 100644 index 00000000000..7b5cad846b1 --- /dev/null +++ b/storage/src/tests/fastos.project.newcore @@ -0,0 +1,80 @@ +APPLICATION testrunner +OBJS storageserver/dummystoragelink +OBJS testrunner testhelper +LIBS tests/persistence/memfile/testmemfiletop +LIBS tests/serverapp/testserverapp +LIBS tests/storageserver/teststorageserver +LIBS tests/bucketmover/testbucketmover +LIBS tests/storageutil/teststorageutil +LIBS tests/visiting/testvisiting +LIBS tests/bucketdb/testbucketdb +LIBS tests/common/testcommon +LIBS tests/common/hostreporter/testhostreporter +LIBS tests/distributor/testdistributor +LIBS tests/persistence/testpersistence +LIBS tests/persistence/device/testdevice +LIBS tests/persistence/filestorage/testfilestorage +LIBS tests/persistence/filestorage/slotfile/testslotfile +LIBS tests/splitting/testsplitting +LIBS tests/memorymanager/testmemorymanager + +LIBS storage/storageserver/storageserver +LIBS storage/bucketmover/bucketmover +LIBS storage/visiting/visitor +LIBS storage/memorymanager/memorymanager +LIBS storage/persistence/persistence +LIBS storage/persistence/filestorage/filestorpersistence +LIBS storage/persistence/memfile/memfiletop +LIBS storage/persistence/memfile/common/memfilecommon +LIBS storage/persistence/memfile/mapper/memfilemapper +LIBS storage/persistence/memfile/handler/memfilehandler +LIBS storage/persistence/memfile/memfile/memfile +LIBS storage/persistence/memfile/common/memfilecommon +LIBS storage/persistence/memfile/memfiletop +LIBS storage/storageutil/storageutil +LIBS storage/persistence/device/device +LIBS storage/persistence/filestorage/slotfile/slotfile +LIBS storage/bucketdb/bucketdb +LIBS storage/distributor/distributor +LIBS storage/common/common +LIBS storage/config/storageconfig +EXTERNALLIBS cppunit vdslib storageapi +EXTERNALLIBS document metrics boost_regex-mt-d +EXTERNALLIBS fast iconv +EXTERNALLIBS vespa +EXTERNALLIBS config vespalog Judy vdslib documentapi vespalib +EXTERNALLIBS messagebus-test slobrokserver + +CUSTOMMAKE + +LIBDIR_TESTS=persistence/memfile:bucketdb:common:distributor:persistence:persistence/device:persistence/filestorage:persistence/filestorage/slotfile:serverapp:storageserver:storageutil:visiting:splitting:memorymanager:bucketmover + +test: all + rm -f test.vlog + VESPA_LOG_TARGET=file:test.vlog LD_LIBRARY_PATH=$(LIBDIR_BOOST_REGEX-MT-D):$(LIBDIR_DSTORE):$(LIBDIR_ICONV):$(LIBDIR_CPPUNIT):$(LIBDIR_TESTS):$(LD_LIBRARY_PATH) $(VALGRIND) ./testrunner --verbose $(TESTRUNARGS) + +vtest: all + rm -f test.vlog + VESPA_LOG_TARGET=file:test.vlog LD_LIBRARY_PATH=$(LIBDIR_BOOST_REGEX-MT-D):$(LIBDIR_DSTORE):$(LIBDIR_ICONV):$(LIBDIR_CPPUNIT):$(LIBDIR_TESTS):$(LD_LIBRARY_PATH) valgrind --leak-check=no ./testrunner --verbose + +testdebug: all + rm -f test.vlog + VESPA_LOG_TARGET=file:test.vlog LD_LIBRARY_PATH=$(LIBDIR_BOOST_REGEX-MT-D):$(LIBDIR_DSTORE):$(LIBDIR_ICONV):$(LIBDIR_CPPUNIT):$(LIBDIR_TESTS):$(LD_LIBRARY_PATH) gdb53 ./testrunner --verbose + +testwithlog: all + LD_LIBRARY_PATH=$(LIBDIR_BOOST_REGEX-MT-D):$(LIBDIR_DSTORE):$(LIBDIR_ICONV):$(LIBDIR_CPPUNIT):$(LIBDIR_TESTS):$(LD_LIBRARY_PATH) ./testrunner --verbose + +vtestwithlog: all + LD_LIBRARY_PATH=$(LIBDIR_BOOST_REGEX-MT-D):$(LIBDIR_DSTORE):$(LIBDIR_ICONV):$(LIBDIR_CPPUNIT):$(LIBDIR_TESTS):$(LD_LIBRARY_PATH) valgrind ./testrunner --verbose + +stresstest: all + rm -f test.vlog + VESPA_LOG_TARGET=file:test.vlog LD_LIBRARY_PATH=$(LIBDIR_BOOST_REGEX-MT-D):$(LIBDIR_DSTORE):$(LIBDIR_ICONV):$(LIBDIR_CPPUNIT):$(LIBDIR_TESTS):$(LD_LIBRARY_PATH) ./testrunner --verbose --includestress stress Stress + +testverbose: all + rm -f test.vlog + VESPA_LOG_TARGET=file:test.vlog LD_LIBRARY_PATH=$(LIBDIR_BOOST_REGEX-MT-D):$(LIBDIR_DSTORE):$(LIBDIR_ICONV):$(LIBDIR_CPPUNIT):$(LIBDIR_TESTS):$(LD_LIBRARY_PATH) ./testrunner --verbose + +testall: all + rm -f test.vlog + VESPA_LOG_TARGET=file:test.vlog LD_LIBRARY_PATH=$(LIBDIR_BOOST_REGEX-MT-D):$(LIBDIR_DSTORE):$(LIBDIR_ICONV):$(LIBDIR_CPPUNIT):$(LIBDIR_TESTS):$(LD_LIBRARY_PATH) $(VALGRIND) ./testrunner --verbose --includestress diff --git a/storage/src/tests/frameworkimpl/memory/CMakeLists.txt b/storage/src/tests/frameworkimpl/memory/CMakeLists.txt new file mode 100644 index 00000000000..da78716459f --- /dev/null +++ b/storage/src/tests/frameworkimpl/memory/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storage_testmemory + SOURCES + memorystatusviewertest.cpp + DEPENDS + AFTER + storage_storageconfig +) diff --git a/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp b/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp new file mode 100644 index 00000000000..cc7e98d8718 --- /dev/null +++ b/storage/src/tests/frameworkimpl/memory/memorystatusviewertest.cpp @@ -0,0 +1,168 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/metrics/metrics.h> +#include <vespa/storage/frameworkimpl/memory/memorystatusviewer.h> +#include <vespa/storageframework/defaultimplementation/memory/prioritymemorylogic.h> +#include <tests/common/teststorageapp.h> +#include <vespa/vdstestlib/cppunit/macros.h> + +namespace storage { + +struct MemoryStatusViewerTest : public CppUnit::TestFixture +{ + static const int maxMemory = 1000; + std::unique_ptr<TestServiceLayerApp> _node; + std::unique_ptr<framework::defaultimplementation::MemoryManager> _memMan; + + void setUp(); + + void testEmptyState(); + void testSnapshots(); + + CPPUNIT_TEST_SUITE(MemoryStatusViewerTest); + CPPUNIT_TEST(testEmptyState); + CPPUNIT_TEST(testSnapshots); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(MemoryStatusViewerTest); + +void +MemoryStatusViewerTest::setUp() +{ + _node.reset(new TestServiceLayerApp(DiskCount(2))); + framework::defaultimplementation::PriorityMemoryLogic* logic( + new framework::defaultimplementation::PriorityMemoryLogic( + _node->getClock(), maxMemory)); + logic->setMinJumpToUpdateMax(1); + _memMan.reset(new framework::defaultimplementation::MemoryManager( + framework::defaultimplementation::AllocationLogic::UP(logic))); +} + +void +MemoryStatusViewerTest::testEmptyState() +{ + // Add a memory manager, and add a bit of load to it, so it's not + // totally empty. + StorageComponent component(_node->getComponentRegister(), "test"); + + metrics::MetricManager mm; + MemoryStatusViewer viewer( + *_memMan, mm, _node->getComponentRegister()); + std::ostringstream actual; + viewer.reportStatus(actual, framework::HttpUrlPath("/")); + CPPUNIT_ASSERT_MATCH_REGEX(".*Plotr.LineChart.*", actual.str()); + CPPUNIT_ASSERT_MATCH_REGEX( + ".*Current: 1970-01-01 00:00:00 Max memory 1000 SnapShot\\(Used 0, w/o cache 0\\).*", + actual.str()); + CPPUNIT_ASSERT_MATCH_REGEX( + ".*Last hour: na.*", actual.str()); +} + +namespace { + void waitForProcessedTime( + const MemoryStatusViewer& viewer, framework::SecondTime time, + framework::SecondTime timeout = framework::SecondTime(30)) + { + framework::defaultimplementation::RealClock clock; + framework::MilliSecTime endTime( + clock.getTimeInMillis() + timeout.getMillis()); + framework::SecondTime processedTime(0); + while (clock.getTimeInMillis() < endTime) { + processedTime = viewer.getProcessedTime(); + if (processedTime >= time) return; + FastOS_Thread::Sleep(1); + } + std::ostringstream ost; + ost << "Timed out waiting " << timeout << " ms for time " << time + << " to be processed. Currently time is only processed up to " + << processedTime; + throw new vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); + } +} + +#define ASSERT_MEMORY(output, period, maxmem, used, usedwocache) \ +{ \ + std::string::size_type _pos1_(output.find(period)); \ + std::string::size_type _pos2_(output.find("Max memory", _pos1_)); \ + std::string::size_type _pos3_(output.find("SnapShot", _pos2_)); \ + std::string _maxMemory_(output.substr(_pos2_ + 11, _pos3_ - _pos2_ - 12)); \ + std::string::size_type _pos4_(output.find(",", _pos3_)); \ + std::string _used_(output.substr(_pos3_ + 14, _pos4_ - _pos3_ - 14)); \ + std::string::size_type _pos5_(output.find(")", _pos4_)); \ + std::string _usedwo_(output.substr(_pos4_ + 12, _pos5_ - _pos4_ - 12)); \ + std::ostringstream _failure_; \ + _failure_ << "Wrong match in period " << period << " in output:\n" \ + << output << "\nFor value: "; \ + \ + CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Max memory", \ + uint64_t(maxmem), boost::lexical_cast<uint64_t>(_maxMemory_)); \ + CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Used memory", \ + uint64_t(used), boost::lexical_cast<uint64_t>(_used_)); \ + CPPUNIT_ASSERT_EQUAL_MSG(_failure_.str() + "Used memory w/o cache", \ + uint64_t(usedwocache), boost::lexical_cast<uint64_t>(_usedwo_)); \ +} + +void +MemoryStatusViewerTest::testSnapshots() +{ + // Add a memory manager, and add a bit of load to it, so it's not + // totally empty. + StorageComponent component(_node->getComponentRegister(), "test"); + const framework::MemoryAllocationType putAlloc( + component.getMemoryManager().registerAllocationType( + framework::MemoryAllocationType("PUT"))); + const framework::MemoryAllocationType getAlloc( + component.getMemoryManager().registerAllocationType( + framework::MemoryAllocationType("GET"))); + + framework::MemoryToken::UP put = _memMan->allocate(putAlloc, 0, 100, 80); + framework::MemoryToken::UP get = _memMan->allocate(getAlloc, 30, 200, 50); + framework::MemoryToken::UP get2 = _memMan->allocate(getAlloc, 70, 150, 60); + + metrics::MetricManager mm; + MemoryStatusViewer viewer(*_memMan, mm, _node->getComponentRegister()); + + _node->getClock().addSecondsToTime(1000); + viewer.notifyThread(); + waitForProcessedTime(viewer, framework::SecondTime(1000)); + + std::ostringstream actual; + viewer.printDebugOutput(actual); + //std::cerr << actual.str() << "\n"; + ASSERT_MEMORY(actual.str(), "Current", 1000, 450, 450); + ASSERT_MEMORY(actual.str(), "Last hour", 1000, 450, 450); + ASSERT_MEMORY(actual.str(), "Last ever", 1000, 450, 450); + + put = _memMan->allocate(putAlloc, 0, 50, 80); + get = _memMan->allocate(getAlloc, 100, 140, 50); + get2 = _memMan->allocate(getAlloc, 20, 100, 70); + + _node->getClock().addSecondsToTime(3600); + viewer.notifyThread(); + waitForProcessedTime(viewer, framework::SecondTime(4600)); + + actual.str(""); + viewer.printDebugOutput(actual); + //std::cerr << actual.str() << "\n"; + ASSERT_MEMORY(actual.str(), "Current", 1000, 290, 290); + ASSERT_MEMORY(actual.str(), "Last hour", 1000, 540, 540); + ASSERT_MEMORY(actual.str(), "Last ever", 1000, 540, 540); + + get.reset(); + + _node->getClock().addSecondsToTime(3600); + viewer.notifyThread(); + waitForProcessedTime(viewer, framework::SecondTime(4600 + 3600)); + + actual.str(""); + viewer.printDebugOutput(actual); + //std::cerr << actual.str() << "\n"; + ASSERT_MEMORY(actual.str(), "Current", 1000, 150, 150); + ASSERT_MEMORY(actual.str(), "Last hour", 1000, 290, 290); + ASSERT_MEMORY(actual.str(), "Last ever", 1000, 540, 540); + +} + +} // storage diff --git a/storage/src/tests/frameworkimpl/status/CMakeLists.txt b/storage/src/tests/frameworkimpl/status/CMakeLists.txt new file mode 100644 index 00000000000..734be8e9998 --- /dev/null +++ b/storage/src/tests/frameworkimpl/status/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storage_teststatus + SOURCES + statustest.cpp + DEPENDS + AFTER + storage_storageconfig +) diff --git a/storage/src/tests/frameworkimpl/status/statustest.cpp b/storage/src/tests/frameworkimpl/status/statustest.cpp new file mode 100644 index 00000000000..0fc10e411cb --- /dev/null +++ b/storage/src/tests/frameworkimpl/status/statustest.cpp @@ -0,0 +1,222 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/document/util/stringutil.h> +#include <vespa/log/log.h> +#include <sstream> +#include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> +#include <vespa/storage/frameworkimpl/status/statuswebserver.h> +#include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> +#include <tests/common/teststorageapp.h> +#include <vespa/vdstestlib/cppunit/macros.h> + +LOG_SETUP(".test.status"); + +namespace storage { + +struct StatusTest : public CppUnit::TestFixture { + std::unique_ptr<TestServiceLayerApp> _node; + + void setUp(); + + void testIndexStatusPage(); + void testHtmlStatus(); + void testXmlStatus(); + void test404(); + void requireThatServerSpecIsConstructedCorrectly(); + + CPPUNIT_TEST_SUITE(StatusTest); + CPPUNIT_TEST(testIndexStatusPage); + CPPUNIT_TEST(testHtmlStatus); + CPPUNIT_TEST(testXmlStatus); + CPPUNIT_TEST(test404); + CPPUNIT_TEST(requireThatServerSpecIsConstructedCorrectly); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(StatusTest); + +namespace { + struct HtmlStatusReporter : public framework::HtmlStatusReporter { + std::string _headerAddition; + std::string _content; + + HtmlStatusReporter(const std::string& id, const std::string& name, + const std::string& content, + const std::string& headerAddition = "") + : framework::HtmlStatusReporter(id, name), + _headerAddition(headerAddition), + _content(content) + { + } + + virtual void reportHtmlHeaderAdditions( + std::ostream& out, const framework::HttpUrlPath&) const + { + out << _headerAddition; + } + + virtual void reportHtmlStatus( + std::ostream& out, const framework::HttpUrlPath&) const + { + out << _content; + } + }; + + struct XmlStatusReporter : public framework::XmlStatusReporter { + XmlStatusReporter(const std::string& id, const std::string& name) + : framework::XmlStatusReporter(id, name) {} + virtual vespalib::string reportXmlStatus( + vespalib::xml::XmlOutputStream& xos, + const framework::HttpUrlPath&) const + { + xos << vespalib::xml::XmlTag("mytag") + << vespalib::xml::XmlAttribute("foo", "bar") + << vespalib::xml::XmlContent("content") + << vespalib::xml::XmlEndTag(); + return ""; + } + }; + + struct StatusComponent : public framework::Component { + framework::StatusReporter* _reporter; + + StatusComponent(framework::ComponentRegister& reg, const char* name, + framework::StatusReporter* reporter) + : framework::Component(reg, name), + _reporter(reporter) + { + registerStatusPage(*_reporter); + } + ~StatusComponent() { delete _reporter; } + }; + +} + +void +StatusTest::setUp() +{ + _node.reset(new TestServiceLayerApp); +} + +void +StatusTest::testIndexStatusPage() +{ + StatusComponent rep1(_node->getComponentRegister(), "foo", + new HtmlStatusReporter( + "fooid", "Foo impl", "<p>info</p>")); + StatusComponent rep2(_node->getComponentRegister(), "bar", + new HtmlStatusReporter( + "barid", "Bar impl", "<p>info</p>")); + StatusWebServer webServer(_node->getComponentRegister(), + _node->getComponentRegister(), + "raw:httpport -1"); + std::ostringstream ss; + framework::HttpUrlPath path(""); + webServer.handlePage(path, ss); + std::string expected( + "HTTP\\/1.1 200 OK\r\n" + "Connection: Close\r\n" + "Content-type: text\\/html\r\n" + "\r\n" + "<html>\n" + "<head>\n" + " <title>Index page</title>\n" + "<\\/head>\n" + "<body>\n" + " <h1>Index page</h1>\n" + "<p><b>Binary version of Vespa:<\\/b> [0-9.]+<\\/p>\n" + "<a href=\"fooid\">Foo impl<\\/a><br>\n" + "<a href=\"barid\">Bar impl<\\/a><br>\n" + "<\\/body>\n" + "<\\/html>\n" + ); + CPPUNIT_ASSERT_MATCH_REGEX(expected, ss.str()); +} + +void +StatusTest::testHtmlStatus() +{ + StatusComponent rep1(_node->getComponentRegister(), "foo", + new HtmlStatusReporter( + "fooid", "Foo impl", "<p>info</p>", "<!-- script -->")); + StatusWebServer webServer(_node->getComponentRegister(), + _node->getComponentRegister(), + "raw:httpport -1"); + std::ostringstream ost; + framework::HttpUrlPath path("/fooid?unusedParam"); + webServer.handlePage(path, ost); + std::string expected( + "HTTP/1.1 200 OK\r\n" + "Connection: Close\r\n" + "Content-type: text/html\r\n" + "\r\n" + "<html>\n" + "<head>\n" + " <title>Foo impl</title>\n" + "<!-- script --></head>\n" + "<body>\n" + " <h1>Foo impl</h1>\n" + "<p>info</p></body>\n" + "</html>\n" + ); + CPPUNIT_ASSERT_EQUAL(expected, ost.str()); +} + +void +StatusTest::testXmlStatus() +{ + StatusComponent rep1(_node->getComponentRegister(), "foo", + new XmlStatusReporter( + "fooid", "Foo impl")); + StatusWebServer webServer(_node->getComponentRegister(), + _node->getComponentRegister(), + "raw:httpport -1"); + std::ostringstream ost; + framework::HttpUrlPath path("/fooid?unusedParam"); + webServer.handlePage(path, ost); + std::string expected( + "HTTP/1.1 200 OK\r\n" + "Connection: Close\r\n" + "Content-type: application/xml\r\n" + "\r\n" + "<?xml version=\"1.0\"?>\n" + "<status id=\"fooid\" name=\"Foo impl\">\n" + "<mytag foo=\"bar\">content</mytag>\n" + "</status>" + ); + CPPUNIT_ASSERT_EQUAL(expected, ost.str()); +} + +void +StatusTest::test404() +{ + StatusWebServer webServer(_node->getComponentRegister(), + _node->getComponentRegister(), + "raw:httpport -1"); + std::ostringstream ost; + framework::HttpUrlPath path("/fooid?unusedParam"); + webServer.handlePage(path, ost); + std::string expected( + "HTTP/1.1 404 Not found\r\n" + "Connection: Close\r\n" + "Content-type: text/html\r\n" + "\r\n" + "<html><head><title>404 Not found</title></head>\r\n" + "<body><h1>404 Not found</h1>\r\n" + "<p></p></body>\r\n" + "</html>\r\n" + ); + CPPUNIT_ASSERT_EQUAL_ESCAPED(expected, ost.str()); +} + +void +StatusTest::requireThatServerSpecIsConstructedCorrectly() +{ + CPPUNIT_ASSERT_EQUAL(vespalib::string("requesthost:10"), + StatusWebServer::getServerSpec("requesthost:10", "serverhost:20")); + CPPUNIT_ASSERT_EQUAL(vespalib::string("serverhost:20"), + StatusWebServer::getServerSpec("", "serverhost:20")); +} + +} // storage diff --git a/storage/src/tests/persistence/.gitignore b/storage/src/tests/persistence/.gitignore new file mode 100644 index 00000000000..184e5d1c936 --- /dev/null +++ b/storage/src/tests/persistence/.gitignore @@ -0,0 +1,12 @@ +*.So +*.lo +*.o +.*.swp +.config.log +.depend +.depend.NEW +.deps +.libs +Makefile +testrunner +testrunner.core diff --git a/storage/src/tests/persistence/CMakeLists.txt b/storage/src/tests/persistence/CMakeLists.txt new file mode 100644 index 00000000000..c065c3eef5b --- /dev/null +++ b/storage/src/tests/persistence/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storage_testpersistence + SOURCES + processalltest.cpp + persistencetestutils.cpp + splitbitdetectortest.cpp + legacyoperationhandlertest.cpp + persistenceproviderwrapper.cpp + diskmoveoperationhandlertest.cpp + providershutdownwrappertest.cpp + mergehandlertest.cpp + persistencethread_splittest.cpp + bucketownershipnotifiertest.cpp + persistencequeuetest.cpp + testandsettest.cpp + DEPENDS + AFTER + storage_storageconfig +) diff --git a/storage/src/tests/persistence/bucketownershipnotifiertest.cpp b/storage/src/tests/persistence/bucketownershipnotifiertest.cpp new file mode 100644 index 00000000000..ae54e629473 --- /dev/null +++ b/storage/src/tests/persistence/bucketownershipnotifiertest.cpp @@ -0,0 +1,162 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <tests/distributor/messagesenderstub.h> +#include <tests/common/teststorageapp.h> +#include <vespa/storage/persistence/bucketownershipnotifier.h> + +namespace storage { + +class BucketOwnershipNotifierTest : public CppUnit::TestFixture +{ + std::unique_ptr<TestServiceLayerApp> _app; + lib::ClusterState _clusterState; +public: + + BucketOwnershipNotifierTest() + : _app(), + _clusterState("distributor:2 storage:1") + {} + + void setUp(); + + CPPUNIT_TEST_SUITE(BucketOwnershipNotifierTest); + CPPUNIT_TEST(testSendNotifyBucketChangeIfOwningDistributorChanged); + CPPUNIT_TEST(testDoNotSendNotifyBucketChangeIfBucketOwnedByInitialSender); + CPPUNIT_TEST(testIgnoreIdealStateCalculationExceptions); + CPPUNIT_TEST(testGuardNotifyAlways); + CPPUNIT_TEST_SUITE_END(); + + bool ownsBucket(uint16_t distributorIndex, + const document::BucketId& bucket) const + { + uint16_t distributor = _app->getDistribution()->getIdealDistributorNode( + _clusterState, bucket); + return distributor == distributorIndex; + } + + document::BucketId getFirstNonOwnedBucket() { + for (int i = 0; i < 1000; ++i) { + if (!ownsBucket(0, document::BucketId(16, i))) { + return document::BucketId(16, i); + } + } + return document::BucketId(0); + } + + document::BucketId getFirstOwnedBucket() { + for (int i = 0; i < 1000; ++i) { + if (ownsBucket(0, document::BucketId(16, i))) { + return document::BucketId(16, i); + } + } + return document::BucketId(0); + } + + + void testSendNotifyBucketChangeIfOwningDistributorChanged(); + void testDoNotSendNotifyBucketChangeIfBucketOwnedByInitialSender(); + void testIgnoreIdealStateCalculationExceptions(); + void testGuardNotifyAlways(); + + void doTestNotification(const document::BucketId& bucket, + const api::BucketInfo& info, + const std::string& wantedSend); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(BucketOwnershipNotifierTest); + +void +BucketOwnershipNotifierTest::setUp() +{ + _app.reset(new TestServiceLayerApp); + _app->setDistribution(Redundancy(1), NodeCount(2)); + _app->setClusterState(_clusterState); +} + +void +BucketOwnershipNotifierTest::doTestNotification(const document::BucketId& bucket, + const api::BucketInfo& info, + const std::string& wantedSend) +{ + ServiceLayerComponent component(_app->getComponentRegister(), "dummy"); + MessageSenderStub sender; + + BucketOwnershipNotifier notifier(component, sender); + + notifier.notifyIfOwnershipChanged(bucket, 0, info); + + CPPUNIT_ASSERT_EQUAL(wantedSend, sender.getCommands(true, true)); +} + +void +BucketOwnershipNotifierTest::testSendNotifyBucketChangeIfOwningDistributorChanged() +{ + api::BucketInfo info(0x1, 2, 3); + document::BucketId bucket(getFirstNonOwnedBucket()); + CPPUNIT_ASSERT(bucket.getRawId() != 0); + + std::ostringstream wanted; + wanted << "NotifyBucketChangeCommand(" + << bucket + << ", " << info + << ") => 1"; + + doTestNotification(bucket, info, wanted.str()); +} + +void +BucketOwnershipNotifierTest::testDoNotSendNotifyBucketChangeIfBucketOwnedByInitialSender() +{ + api::BucketInfo info(0x1, 2, 3); + document::BucketId bucket(getFirstOwnedBucket()); + CPPUNIT_ASSERT(bucket.getRawId() != 0); + + doTestNotification(bucket, info, ""); +} + +void +BucketOwnershipNotifierTest::testIgnoreIdealStateCalculationExceptions() +{ + api::BucketInfo info(0x1, 2, 3); + document::BucketId bucket(getFirstNonOwnedBucket()); + CPPUNIT_ASSERT(bucket.getRawId() != 0); + + _app->setClusterState(lib::ClusterState("distributor:0 storage:1")); + + doTestNotification(bucket, info, ""); +} + +void +BucketOwnershipNotifierTest::testGuardNotifyAlways() +{ + ServiceLayerComponent component(_app->getComponentRegister(), "dummy"); + MessageSenderStub sender; + BucketOwnershipNotifier notifier(component, sender); + std::ostringstream wanted; + { + NotificationGuard guard(notifier); + + api::BucketInfo info(0x1, 2, 3); + document::BucketId bucket1(getFirstOwnedBucket()); + guard.notifyAlways(bucket1, info); + + document::BucketId bucket2(getFirstNonOwnedBucket()); + guard.notifyAlways(bucket2, info); + + wanted << "NotifyBucketChangeCommand(" + << bucket1 + << ", " << info + << ") => 0," + << "NotifyBucketChangeCommand(" + << bucket2 + << ", " << info + << ") => 1"; + } + + CPPUNIT_ASSERT_EQUAL(wanted.str(), sender.getCommands(true, true)); +} + +} // storage + diff --git a/storage/src/tests/persistence/diskmoveoperationhandlertest.cpp b/storage/src/tests/persistence/diskmoveoperationhandlertest.cpp new file mode 100644 index 00000000000..f47cc334e30 --- /dev/null +++ b/storage/src/tests/persistence/diskmoveoperationhandlertest.cpp @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storage/persistence/diskmoveoperationhandler.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/persistence/messages.h> +#include <tests/persistence/persistencetestutils.h> + +namespace storage { + +class DiskMoveOperationHandlerTest : public PersistenceTestUtils +{ + CPPUNIT_TEST_SUITE(DiskMoveOperationHandlerTest); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST_SUITE_END(); + +public: + void testSimple(); + void testTargetExists(); + void testTargetWithOverlap(); + + void insertDocumentInBucket(uint64_t location, uint64_t timestamp, document::BucketId bucket); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(DiskMoveOperationHandlerTest); + +void +DiskMoveOperationHandlerTest::testSimple() +{ + setupDisks(10); + + // Create bucket 16, 4 on disk 3. + { + StorBucketDatabase::WrappedEntry entry( + createBucket(document::BucketId(16, 4))); + entry->disk = 3; + entry.write(); + } + + for (uint32_t i = 0; i < 10; i++) { + doPutOnDisk(3, 4, spi::Timestamp(1000 + i)); + } + + DiskMoveOperationHandler diskMoveHandler( + getEnv(3), + getPersistenceProvider()); + BucketDiskMoveCommand move(document::BucketId(16, 4), 3, 4); + + spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); + diskMoveHandler.handleBucketDiskMove(move, context); + + CPPUNIT_ASSERT_EQUAL( + std::string("BucketId(0x4000000000000004): 10,4"), + getBucketStatus(document::BucketId(16,4))); +} + +} diff --git a/storage/src/tests/persistence/filestorage/.gitignore b/storage/src/tests/persistence/filestorage/.gitignore new file mode 100644 index 00000000000..cfeb99e9e3f --- /dev/null +++ b/storage/src/tests/persistence/filestorage/.gitignore @@ -0,0 +1,13 @@ +*.So +*.lo +*.o +.*.swp +.config.log +.depend +.depend.NEW +.deps +.libs +Makefile +persistence +testrunner +testrunner.core diff --git a/storage/src/tests/persistence/filestorage/CMakeLists.txt b/storage/src/tests/persistence/filestorage/CMakeLists.txt new file mode 100644 index 00000000000..b1314ca0537 --- /dev/null +++ b/storage/src/tests/persistence/filestorage/CMakeLists.txt @@ -0,0 +1,17 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storage_testfilestorage + SOURCES + filestormanagertest.cpp + operationabortingtest.cpp + filestortestfixture.cpp + mergeblockingtest.cpp + sanitycheckeddeletetest.cpp + deactivatebucketstest.cpp + modifiedbucketcheckertest.cpp + filestormodifiedbucketstest.cpp + deletebuckettest.cpp + singlebucketjointest.cpp + DEPENDS + AFTER + storage_storageconfig +) diff --git a/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp b/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp new file mode 100644 index 00000000000..6de67a3fec0 --- /dev/null +++ b/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/state.h> +#include <tests/persistence/persistenceproviderwrapper.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <tests/persistence/filestorage/filestortestfixture.h> + +namespace storage { + +class DeactivateBucketsTest : public FileStorTestFixture +{ + bool isActive(const document::BucketId&) const; +public: + void bucketsInDatabaseDeactivatedWhenNodeDownInClusterState(); + + CPPUNIT_TEST_SUITE(DeactivateBucketsTest); + CPPUNIT_TEST(bucketsInDatabaseDeactivatedWhenNodeDownInClusterState); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(DeactivateBucketsTest); + +bool +DeactivateBucketsTest::isActive(const document::BucketId& bucket) const +{ + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get(bucket, "foo")); + CPPUNIT_ASSERT(entry.exist()); + return entry->info.isActive(); +} + +void +DeactivateBucketsTest::bucketsInDatabaseDeactivatedWhenNodeDownInClusterState() +{ + TestFileStorComponents c(*this, "bucketsInDatabaseDeactivatedWhenNodeDownInClusterState"); + // Must set state to up first, or down-edge case won't trigger. + std::string upState("storage:2 distributor:2"); + _node->getStateUpdater().setClusterState( + lib::ClusterState::CSP(new lib::ClusterState(upState))); + + document::BucketId bucket(8, 123); + spi::Bucket spiBucket(bucket, spi::PartitionId(0)); + + createBucket(bucket); + api::BucketInfo serviceLayerInfo(1, 2, 3, 4, 5, true, true); + { + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get(bucket, "foo", + StorBucketDatabase::CREATE_IF_NONEXISTING)); + entry->disk = 0; + entry->info = serviceLayerInfo; + entry.write(); + } + CPPUNIT_ASSERT(isActive(bucket)); + std::string downState("storage:2 .1.s:d distributor:2"); + _node->getStateUpdater().setClusterState( + lib::ClusterState::CSP(new lib::ClusterState(downState))); + + // Buckets should have been deactivated in content layer + CPPUNIT_ASSERT(!isActive(bucket)); +} + +} // namespace storage diff --git a/storage/src/tests/persistence/filestorage/deletebuckettest.cpp b/storage/src/tests/persistence/filestorage/deletebuckettest.cpp new file mode 100644 index 00000000000..08ca9bc68fa --- /dev/null +++ b/storage/src/tests/persistence/filestorage/deletebuckettest.cpp @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storageapi/message/bucket.h> +#include <tests/persistence/persistenceproviderwrapper.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <tests/persistence/filestorage/filestortestfixture.h> + +LOG_SETUP(".deletebuckettest"); + +namespace storage { + +class DeleteBucketTest : public FileStorTestFixture +{ +public: + void testDeleteAbortsOperationsForBucket(); + + CPPUNIT_TEST_SUITE(DeleteBucketTest); + CPPUNIT_TEST(testDeleteAbortsOperationsForBucket); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(DeleteBucketTest); + +void +DeleteBucketTest::testDeleteAbortsOperationsForBucket() +{ + TestFileStorComponents c(*this, "testDeleteAbortsOperationsForBucket"); + document::BucketId bucket(16, 1); + + createBucket(bucket); + LOG(info, "TEST STAGE: taking resume guard"); + ResumeGuard rg(c.manager->getFileStorHandler().pause()); + // First put may or may not be queued, since pausing might race with + // an existing getNextMessage iteration (ugh...). + c.sendPut(bucket, DocumentIndex(0), PutTimestamp(1000)); + // Put will be queued since thread now must know it's paused. + c.sendPut(bucket, DocumentIndex(1), PutTimestamp(1000)); + + auto deleteMsg = std::make_shared<api::DeleteBucketCommand>(bucket); + c.top.sendDown(deleteMsg); + // We should now have two put replies. The first one will either be OK + // or BUCKET_DELETED depending on whether it raced. The second (which is + // the one we care about since it's deterministic) must be BUCKET_DELETED. + // Problem is, their returned ordering is not deterministic so we're left + // with having to check that _at least_ 1 reply had BUCKET_DELETED. Joy! + c.top.waitForMessages(2, 60*2); + std::vector<api::StorageMessage::SP> msgs(c.top.getRepliesOnce()); + CPPUNIT_ASSERT_EQUAL(size_t(2), msgs.size()); + int numDeleted = 0; + for (uint32_t i = 0; i < 2; ++i) { + api::StorageReply& reply(dynamic_cast<api::StorageReply&>(*msgs[i])); + if (reply.getResult().getResult() == api::ReturnCode::BUCKET_DELETED) { + ++numDeleted; + } + } + CPPUNIT_ASSERT(numDeleted >= 1); + LOG(info, "TEST STAGE: done, releasing resume guard"); +} + +} // namespace storage diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp new file mode 100644 index 00000000000..0ffbe9fa440 --- /dev/null +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -0,0 +1,3150 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/document/update/assignvalueupdate.h> +#include <vespa/document/datatype/datatype.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/update/documentupdate.h> +#include <vespa/document/fieldvalue/rawfieldvalue.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/document/select/parser.h> +#include <fstream> +#include <memory> +#include <atomic> +#include <vespa/vdslib/state/random.h> +#include <vespa/vdslib/container/mutabledocumentlist.h> +#include <vespa/vdslib/container/operationlist.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storageapi/message/multioperation.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/removelocation.h> +#include <vespa/storage/bucketdb/bucketmanager.h> +#include <vespa/storage/bucketdb/storbucketdb.h> +#include <vespa/storage/common/bucketmessages.h> +#include <vespa/storageframework/storageframework.h> +#include <vespa/storage/persistence/persistencethread.h> +#include <vespa/storage/persistence/messages.h> +#include <vespa/storage/persistence/filestorage/filestormanager.h> +#include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> +#include <tests/common/testhelper.h> +#include <tests/common/storagelinktest.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/dummystoragelink.h> +#include <tests/persistence/filestorage/forwardingmessagesender.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <vespa/log/log.h> +#include <vespa/storageapi/message/batch.h> +#include <vespa/storage/storageserver/statemanager.h> + +LOG_SETUP(".filestormanagertest"); + +using std::unique_ptr; +using document::Document; +using namespace storage::api; + +#define ASSERT_SINGLE_REPLY(replytype, reply, link, time) \ +reply = 0; \ +try{ \ + link.waitForMessages(1, time); \ + CPPUNIT_ASSERT_EQUAL((size_t)1, link.getNumReplies()); \ + reply = dynamic_cast<replytype*>(link.getReply(0).get()); \ + if (reply == 0) { \ + CPPUNIT_FAIL("Got reply of unexpected type: " \ + + link.getReply(0)->getType().toString()); \ + } \ +} catch (vespalib::Exception& e) { \ + reply = 0; \ + CPPUNIT_FAIL("Failed to find single reply in time"); \ +} + +namespace storage { + +namespace { + spi::LoadType defaultLoadType(0, "default"); +} + +struct FileStorManagerTest : public CppUnit::TestFixture { + enum {LONG_WAITTIME=60}; + unique_ptr<TestServiceLayerApp> _node; + std::unique_ptr<vdstestlib::DirConfig> config; + std::unique_ptr<vdstestlib::DirConfig> config2; + std::unique_ptr<vdstestlib::DirConfig> smallConfig; + const uint32_t _waitTime; + const document::DocumentType* _testdoctype1; + + FileStorManagerTest() : _node(), _waitTime(LONG_WAITTIME) {} + + void setUp(); + void tearDown(); + + void testPut(); + void testHeaderOnlyPut(); + void testFlush(); + void testRemapSplit(); + void testHandlerPriority(); + void testHandlerPriorityBlocking(); + void testHandlerPriorityPreempt(); + void testHandlerMulti(); + void testHandlerTimeout(); + void testHandlerPause(); + void testHandlerPausedMultiThread(); + void testPriority(); + void testSplit1(); + void testSplitSingleGroup(); + void testSplitEmptyTargetWithRemappedOps(); + void testNotifyOnSplitSourceOwnershipChanged(); + void testJoin(); + void testVisiting(); + void testRemoveLocation(); + void testDeleteBucket(); + void testDeleteBucketRejectOutdatedBucketInfo(); + void testDeleteBucketWithInvalidBucketInfo(); + void testNoTimestamps(); + void testEqualTimestamps(); + void testMultiOp(); + void testGetIter(); + void testSetBucketActiveState(); + void testNotifyOwnerDistributorOnOutdatedSetBucketState(); + void testGetBucketDiffImplicitCreateBucket(); + void testMergeBucketImplicitCreateBucket(); + void testNewlyCreatedBucketIsReady(); + void testCreateBucketSetsActiveFlagInDatabaseAndReply(); + void testFileStorThreadLockingStressTest(); + void testStateChange(); + void testRepairNotifiesDistributorOnChange(); + void testDiskMove(); + + CPPUNIT_TEST_SUITE(FileStorManagerTest); + CPPUNIT_TEST(testPut); + CPPUNIT_TEST(testHeaderOnlyPut); + CPPUNIT_TEST(testFlush); + CPPUNIT_TEST(testRemapSplit); + CPPUNIT_TEST(testHandlerPriority); + CPPUNIT_TEST(testHandlerPriorityBlocking); + CPPUNIT_TEST(testHandlerPriorityPreempt); + CPPUNIT_TEST(testHandlerMulti); + CPPUNIT_TEST(testHandlerTimeout); + CPPUNIT_TEST(testHandlerPause); + CPPUNIT_TEST(testHandlerPausedMultiThread); + CPPUNIT_TEST(testPriority); + CPPUNIT_TEST(testSplit1); + CPPUNIT_TEST(testSplitSingleGroup); + CPPUNIT_TEST(testSplitEmptyTargetWithRemappedOps); + CPPUNIT_TEST(testNotifyOnSplitSourceOwnershipChanged); + CPPUNIT_TEST(testJoin); + CPPUNIT_TEST(testVisiting); + CPPUNIT_TEST(testRemoveLocation); + CPPUNIT_TEST(testDeleteBucket); + CPPUNIT_TEST(testDeleteBucketRejectOutdatedBucketInfo); + CPPUNIT_TEST(testDeleteBucketWithInvalidBucketInfo); + CPPUNIT_TEST(testNoTimestamps); + CPPUNIT_TEST(testEqualTimestamps); + CPPUNIT_TEST(testMultiOp); + CPPUNIT_TEST(testGetIter); + CPPUNIT_TEST(testSetBucketActiveState); + CPPUNIT_TEST(testNotifyOwnerDistributorOnOutdatedSetBucketState); + CPPUNIT_TEST(testGetBucketDiffImplicitCreateBucket); + CPPUNIT_TEST(testMergeBucketImplicitCreateBucket); + CPPUNIT_TEST(testNewlyCreatedBucketIsReady); + CPPUNIT_TEST(testCreateBucketSetsActiveFlagInDatabaseAndReply); + CPPUNIT_TEST(testStateChange); + CPPUNIT_TEST(testRepairNotifiesDistributorOnChange); + CPPUNIT_TEST(testDiskMove); + CPPUNIT_TEST_SUITE_END(); + + void createBucket(document::BucketId bid, uint16_t disk) + { + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + _node->getPersistenceProvider().createBucket( + spi::Bucket(bid, spi::PartitionId(disk)), context); + + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get(bid, "foo", + StorBucketDatabase::CREATE_IF_NONEXISTING)); + entry->disk = disk; + entry->info = api::BucketInfo(0, 0, 0, 0, 0, true, false); + entry.write(); + } + + document::Document::UP createDocument( + const std::string& content, const std::string& id) + { + return _node->getTestDocMan().createDocument(content, id); + } + + bool ownsBucket(uint16_t distributorIndex, + const document::BucketId& bucket) const + { + uint16_t distributor( + _node->getDistribution()->getIdealDistributorNode( + *_node->getStateUpdater().getSystemState(), bucket)); + return distributor == distributorIndex; + } + + document::BucketId getFirstBucketNotOwnedByDistributor(uint16_t distributor) { + for (int i = 0; i < 1000; ++i) { + if (!ownsBucket(distributor, document::BucketId(16, i))) { + return document::BucketId(16, i); + } + } + return document::BucketId(0); + } + + spi::dummy::DummyPersistence& getDummyPersistence() { + return static_cast<spi::dummy::DummyPersistence&> + (_node->getPersistenceProvider()); + } + + void setClusterState(const std::string& state) { + _node->getStateUpdater().setClusterState( + lib::ClusterState::CSP( + new lib::ClusterState(state))); + } + + void setupDisks(uint32_t diskCount) { + config.reset(new vdstestlib::DirConfig(getStandardConfig(true))); + + config2.reset(new vdstestlib::DirConfig(*config)); + config2->getConfig("stor-server").set("root_folder", "vdsroot.2"); + config2->getConfig("stor-devices").set("root_folder", "vdsroot.2"); + config2->getConfig("stor-server").set("node_index", "1"); + + smallConfig.reset(new vdstestlib::DirConfig(*config)); + vdstestlib::DirConfig::Config& c( + smallConfig->getConfig("stor-filestor", true)); + c.set("initial_index_read", "128"); + c.set("use_direct_io", "false"); + c.set("maximum_gap_to_read_through", "64"); + + assert(system("rm -rf vdsroot") == 0); + assert(system("rm -rf vdsroot.2") == 0); + assert(system("mkdir -p vdsroot/disks/d0") == 0); + assert(system("mkdir -p vdsroot.2/disks/d0") == 0); + try { + _node.reset(new TestServiceLayerApp(DiskCount(diskCount), NodeIndex(0), + config->getConfigId())); + _node->setupDummyPersistence(); + } catch (config::InvalidConfigException& e) { + fprintf(stderr, "%s\n", e.what()); + } + _testdoctype1 = _node->getTypeRepo()->getDocumentType("testdoctype1"); + _node->getMemoryManager().registerAllocationType( + framework::MemoryAllocationType("VISITOR_BUFFER")); + } + + void putDoc(DummyStorageLink& top, + FileStorHandler& filestorHandler, + const document::BucketId& bucket, + uint32_t docNum); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(FileStorManagerTest); + +std::string findFile(const std::string& path, const std::string& file) { + FastOS_DirectoryScan dirScan(path.c_str()); + while (dirScan.ReadNext()) { + if (dirScan.GetName()[0] == '.') { + // Ignore current and parent dir.. Ignores hidden files too, but + // that doesn't matter as we're not trying to find them. + continue; + } + std::string filename(dirScan.GetName()); + if (dirScan.IsDirectory()) { + std::string result = findFile(path + "/" + filename, file); + if (result != "") { + return result; + } + } + if (filename == file) { + return path + "/" + filename; + } + } + return ""; +} + +bool fileExistsWithin(const std::string& path, const std::string& file) { + return !(findFile(path, file) == ""); +} + +std::unique_ptr<DiskThread> createThread(vdstestlib::DirConfig& config, + TestServiceLayerApp& node, + spi::PersistenceProvider& provider, + FileStorHandler& filestorHandler, + FileStorThreadMetrics& metrics, + uint16_t deviceIndex, + uint8_t lowestPriority) +{ + (void) config; + std::unique_ptr<DiskThread> disk; + disk.reset(new PersistenceThread( + node.getComponentRegister(), config.getConfigId(), provider, + filestorHandler, metrics, + deviceIndex, lowestPriority)); + return disk; +} + +namespace { + +struct TestFileStorComponents +{ +private: + TestName _testName; +public: + DummyStorageLink top; + FileStorManager* manager; + + TestFileStorComponents(FileStorManagerTest& test, const char* testName) + : _testName(testName), + manager(new FileStorManager(test.config->getConfigId(), + test._node->getPartitions(), + test._node->getPersistenceProvider(), + test._node->getComponentRegister())) + { + top.push_back(unique_ptr<StorageLink>(manager)); + top.open(); + } +}; + +} + +void +FileStorManagerTest::setUp() +{ + setupDisks(1); +} + +void +FileStorManagerTest::tearDown() +{ + _node.reset(0); +} + +void +FileStorManagerTest::testHeaderOnlyPut() +{ + TestName testName("testHeaderOnlyPut"); + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = + new FileStorManager(config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + top.open(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); + // Creating a document to test with + Document::SP doc(createDocument( + "some content", "userdoc:crawler:4000:foo").release()); + + document::BucketId bid(16, 4000); + + createBucket(bid, 0); + + // Putting it + { + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, doc, 105)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + CPPUNIT_ASSERT_EQUAL(1, (int)reply->getBucketInfo().getDocumentCount()); + } + doc->setValue(doc->getField("headerval"), document::IntFieldValue(42)); + // Putting it again, this time with header only + { + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, doc, 124)); + cmd->setUpdateTimestamp(105); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode::OK, reply->getResult().getResult()); + } + // Getting it + { + std::shared_ptr<api::GetCommand> cmd(new api::GetCommand( + bid, doc->getId(), "[all]")); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::GetReply> reply2( + std::dynamic_pointer_cast<api::GetReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply2.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply2->getResult()); + CPPUNIT_ASSERT_EQUAL(doc->getId().toString(), + reply2->getDocumentId().toString()); + // Ensure partial update was done, but other things are equal + document::FieldValue::UP value( + reply2->getDocument()->getValue(doc->getField("headerval"))); + CPPUNIT_ASSERT(value.get()); + CPPUNIT_ASSERT_EQUAL(42, dynamic_cast<document::IntFieldValue&>( + *value).getAsInt()); + reply2->getDocument()->remove("headerval"); + doc->remove("headerval"); + CPPUNIT_ASSERT_EQUAL(*doc, *reply2->getDocument()); + } +} + +void +FileStorManagerTest::testPut() +{ + TestName testName("testPut"); + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = + new FileStorManager(config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + top.open(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); + // Creating a document to test with + Document::SP doc(createDocument( + "some content", "userdoc:crawler:4000:foo").release()); + + document::BucketId bid(16, 4000); + + createBucket(bid, 0); + + // Putting it + { + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, doc, 105)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + CPPUNIT_ASSERT_EQUAL(1, (int)reply->getBucketInfo().getDocumentCount()); + } +} + +void +FileStorManagerTest::testDiskMove() +{ + setupDisks(2); + + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = + new FileStorManager(config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + top.open(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); + // Creating a document to test with + Document::SP doc(createDocument( + "some content", "userdoc:crawler:4000:foo").release()); + + document::BucketId bid(16, 4000); + + createBucket(bid, 0); + + // Putting it + { + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, doc, 105)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + CPPUNIT_ASSERT_EQUAL(1, (int)reply->getBucketInfo().getDocumentCount()); + } + + { + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get(bid, "foo")); + + CPPUNIT_ASSERT_EQUAL(0, (int)entry->disk); + CPPUNIT_ASSERT_EQUAL( + vespalib::string( + "BucketInfo(crc 0x28cc441f, docCount 1, totDocSize 122, " + "ready true, active false)"), + entry->getBucketInfo().toString()); + } + + { + std::shared_ptr<BucketDiskMoveCommand> cmd( + new BucketDiskMoveCommand(bid, 0, 1)); + + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<BucketDiskMoveReply> reply( + std::dynamic_pointer_cast<BucketDiskMoveReply>(top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + CPPUNIT_ASSERT_EQUAL(1, (int)reply->getBucketInfo().getDocumentCount()); + } + + { + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get(bid, "foo")); + + CPPUNIT_ASSERT_EQUAL(1, (int)entry->disk); + CPPUNIT_ASSERT_EQUAL( + vespalib::string( + "BucketInfo(crc 0x28cc441f, docCount 1, totDocSize 122, " + "ready true, active false)"), + entry->getBucketInfo().toString()); + } +} + + +void +FileStorManagerTest::testStateChange() +{ + TestName testName("testStateChange"); + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = + new FileStorManager(config->getConfigId(), _node->getPartitions(), + _node->getPersistenceProvider(), + _node->getComponentRegister()))); + top.open(); + + setClusterState("storage:3 distributor:3"); + + CPPUNIT_ASSERT_EQUAL(true, getDummyPersistence().getClusterState().nodeUp()); + + setClusterState("storage:3 .0.s:d distributor:3"); + + CPPUNIT_ASSERT_EQUAL(false, getDummyPersistence().getClusterState().nodeUp()); +} + +void +FileStorManagerTest::testRepairNotifiesDistributorOnChange() +{ + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = + new FileStorManager(config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + setClusterState("storage:1 distributor:1"); + top.open(); + + createBucket(document::BucketId(16, 1), 0); + + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); + + // Creating a document to test with + + for (uint32_t i = 0; i < 3; ++i) { + document::DocumentId docId(vespalib::make_string("userdoc:ns:1:%d", i)); + Document::SP doc(new Document(*_testdoctype1, docId)); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(document::BucketId(16, 1), doc, i + 1)); + cmd->setAddress(address); + top.sendDown(cmd); + } + + top.waitForMessages(3, _waitTime); + top.reset(); + + getDummyPersistence().simulateMaintenanceFailure(); + + std::shared_ptr<RepairBucketCommand> cmd( + new RepairBucketCommand(document::BucketId(16, 1), 0)); + top.sendDown(cmd); + + top.waitForMessages(2, _waitTime); + + CPPUNIT_ASSERT_EQUAL( + std::string("NotifyBucketChangeCommand(BucketId(0x4000000000000001), " + "BucketInfo(crc 0x2625a314, docCount 2, totDocSize 170, " + "ready true, active false))"), top.getReply(0)->toString()); + + top.close(); +} + + +void +FileStorManagerTest::testFlush() +{ + TestName testName("testFlush"); + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = new FileStorManager( + config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + top.open(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); + // Creating a document to test with + + document::DocumentId docId("doc:crawler:http://www.ntnu.no/"); + Document::SP doc(new Document(*_testdoctype1, docId)); + document::BucketId bid(4000); + + static const uint32_t msgCount = 10; + + // Generating many put commands + std::vector<std::shared_ptr<api::StorageCommand> > _commands; + for (uint32_t i=0; i<msgCount; ++i) { + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, doc, i+1)); + cmd->setAddress(address); + _commands.push_back(cmd); + } + for (uint32_t i=0; i<msgCount; ++i) { + top.sendDown(_commands[i]); + } + top.close(); + top.flush(); + CPPUNIT_ASSERT_EQUAL((size_t) msgCount, top.getNumReplies()); +} + +void +FileStorManagerTest::testHandlerPriority() +{ + TestName testName("testHandlerPriority"); + // Setup a filestorthread to test + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>( + dummyManager = new DummyStorageLink)); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + // Since we fake time with small numbers, we need to make sure we dont + // compact them away, as they will seem to be from 1970 + + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1); + + FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), + _node->getComponentRegister(), 255, 0); + filestorHandler.setGetNextMessageTimeout(50); + + std::string content("Here is some content which is in all documents"); + std::ostringstream uri; + + Document::SP doc(createDocument( + content, "userdoc:footype:1234:bar").release()); + + document::BucketIdFactory factory; + document::BucketId bucket(16, factory.getBucketId( + doc->getId()).getRawId()); + + // Populate bucket with the given data + for (uint32_t i = 1; i < 6; i++) { + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bucket, doc, 100)); + std::unique_ptr<api::StorageMessageAddress> address( + new api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 3)); + cmd->setAddress(*address); + cmd->setPriority(i * 15); + filestorHandler.schedule(cmd, 0); + } + + CPPUNIT_ASSERT_EQUAL(15, (int)filestorHandler.getNextMessage(0, 20).second->getPriority()); + CPPUNIT_ASSERT(filestorHandler.getNextMessage(0, 20).second.get() == NULL); + CPPUNIT_ASSERT_EQUAL(30, (int)filestorHandler.getNextMessage(0, 50).second->getPriority()); + CPPUNIT_ASSERT_EQUAL(45, (int)filestorHandler.getNextMessage(0, 50).second->getPriority()); + CPPUNIT_ASSERT(filestorHandler.getNextMessage(0, 50).second.get() == NULL); + CPPUNIT_ASSERT_EQUAL(60, (int)filestorHandler.getNextMessage(0, 255).second->getPriority()); + CPPUNIT_ASSERT_EQUAL(75, (int)filestorHandler.getNextMessage(0, 255).second->getPriority()); +} + +class MessagePusherThread : public document::Runnable +{ +public: + FileStorHandler& _handler; + Document::SP _doc; + bool _done; + bool _threadDone; + + MessagePusherThread(FileStorHandler& handler, Document::SP doc) + : _handler(handler), _doc(doc), _done(false), _threadDone(false) {} + + void run() { + while (!_done) { + document::BucketIdFactory factory; + document::BucketId bucket(16, factory.getBucketId( + _doc->getId()).getRawId()); + + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bucket, _doc, 100)); + _handler.schedule(cmd, 0); + FastOS_Thread::Sleep(1); + } + + _threadDone = true; + } +}; + +class MessageFetchingThread : public document::Runnable { +public: + FileStorHandler& _handler; + std::atomic<uint32_t> _config; + uint32_t _fetchedCount; + bool _done; + bool _failed; + bool _threadDone; + + MessageFetchingThread(FileStorHandler& handler) + : _handler(handler), _config(0), _fetchedCount(0), _done(false), + _failed(false), _threadDone(false) {} + + void run() { + while (!_done) { + FileStorHandler::LockedMessage msg = _handler.getNextMessage(0, 255); + if (msg.second.get()) { + uint32_t originalConfig = _config.load(); + _fetchedCount++; + FastOS_Thread::Sleep(5); + + if (_config.load() != originalConfig) { + _failed = true; + } + } else { + FastOS_Thread::Sleep(1); + } + } + + _threadDone = true; + }; +}; + +void +FileStorManagerTest::testHandlerPausedMultiThread() +{ + TestName testName("testHandlerPausedMultiThread"); + // Setup a filestorthread to test + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>( + dummyManager = new DummyStorageLink)); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + // Since we fake time with small numbers, we need to make sure we dont + // compact them away, as they will seem to be from 1970 + + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1); + + FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), + _node->getComponentRegister(), 255, 0); + filestorHandler.setGetNextMessageTimeout(50); + + std::string content("Here is some content which is in all documents"); + std::ostringstream uri; + + Document::SP doc(createDocument(content, "userdoc:footype:1234:bar").release()); + + FastOS_ThreadPool pool(512 * 1024); + MessagePusherThread pushthread(filestorHandler, doc); + pushthread.start(pool); + + MessageFetchingThread thread(filestorHandler); + thread.start(pool); + + for (uint32_t i = 0; i < 50; ++i) { + FastOS_Thread::Sleep(2); + ResumeGuard guard = filestorHandler.pause(); + thread._config.fetch_add(1); + uint32_t count = thread._fetchedCount; + CPPUNIT_ASSERT_EQUAL(count, thread._fetchedCount); + } + + pushthread._done = true; + thread._done = true; + CPPUNIT_ASSERT(!thread._failed); + + while (!pushthread._threadDone || !thread._threadDone) { + FastOS_Thread::Sleep(1); + } +} + + +void +FileStorManagerTest::testHandlerPause() +{ + TestName testName("testHandlerPriority"); + // Setup a filestorthread to test + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>( + dummyManager = new DummyStorageLink)); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + // Since we fake time with small numbers, we need to make sure we dont + // compact them away, as they will seem to be from 1970 + + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1); + + FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), + _node->getComponentRegister(), 255, 0); + filestorHandler.setGetNextMessageTimeout(50); + + std::string content("Here is some content which is in all documents"); + std::ostringstream uri; + + Document::SP doc(createDocument(content, "userdoc:footype:1234:bar").release()); + + document::BucketIdFactory factory; + document::BucketId bucket(16, factory.getBucketId( + doc->getId()).getRawId()); + + // Populate bucket with the given data + for (uint32_t i = 1; i < 6; i++) { + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bucket, doc, 100)); + std::unique_ptr<api::StorageMessageAddress> address( + new api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 3)); + cmd->setAddress(*address); + cmd->setPriority(i * 15); + filestorHandler.schedule(cmd, 0); + } + + CPPUNIT_ASSERT_EQUAL(15, (int)filestorHandler.getNextMessage(0, 255).second->getPriority()); + + { + ResumeGuard guard = filestorHandler.pause(); + (void)guard; + CPPUNIT_ASSERT(filestorHandler.getNextMessage(0, 255).second.get() == NULL); + } + + CPPUNIT_ASSERT_EQUAL(30, (int)filestorHandler.getNextMessage(0, 255).second->getPriority()); +} + +namespace { + +uint64_t getPutTime(api::StorageMessage::SP& msg) +{ + if (!msg.get()) { + return (uint64_t)-1; + } + + return static_cast<api::PutCommand*>(msg.get())->getTimestamp(); +}; + +} + +void +FileStorManagerTest::testRemapSplit() +{ + TestName testName("testRemapSplit"); + // Setup a filestorthread to test + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>( + dummyManager = new DummyStorageLink)); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + // Since we fake time with small numbers, we need to make sure we dont + // compact them away, as they will seem to be from 1970 + + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1); + + FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), + _node->getComponentRegister(), 255, 0); + filestorHandler.setGetNextMessageTimeout(50); + + std::string content("Here is some content which is in all documents"); + + Document::SP doc1(createDocument(content, "userdoc:footype:1234:bar").release()); + + Document::SP doc2(createDocument(content, "userdoc:footype:4567:bar").release()); + + document::BucketIdFactory factory; + document::BucketId bucket1(16, 1234); + document::BucketId bucket2(16, 4567); + + // Populate bucket with the given data + for (uint32_t i = 1; i < 4; i++) { + filestorHandler.schedule( + api::StorageMessage::SP(new api::PutCommand(bucket1, doc1, i)), 0); + filestorHandler.schedule( + api::StorageMessage::SP(new api::PutCommand(bucket2, doc2, i + 10)), 0); + } + + CPPUNIT_ASSERT_EQUAL(std::string("BucketId(0x40000000000004d2): Put(BucketId(0x40000000000004d2), userdoc:footype:1234:bar, timestamp 1, size 108) (priority: 127)\n" + "BucketId(0x40000000000011d7): Put(BucketId(0x40000000000011d7), userdoc:footype:4567:bar, timestamp 11, size 108) (priority: 127)\n" + "BucketId(0x40000000000004d2): Put(BucketId(0x40000000000004d2), userdoc:footype:1234:bar, timestamp 2, size 108) (priority: 127)\n" + "BucketId(0x40000000000011d7): Put(BucketId(0x40000000000011d7), userdoc:footype:4567:bar, timestamp 12, size 108) (priority: 127)\n" + "BucketId(0x40000000000004d2): Put(BucketId(0x40000000000004d2), userdoc:footype:1234:bar, timestamp 3, size 108) (priority: 127)\n" + "BucketId(0x40000000000011d7): Put(BucketId(0x40000000000011d7), userdoc:footype:4567:bar, timestamp 13, size 108) (priority: 127)\n"), + filestorHandler.dumpQueue(0)); + + FileStorHandler::RemapInfo a(document::BucketId(17, 1234), 0); + FileStorHandler::RemapInfo b(document::BucketId(17, 1234 | 1 << 16), 0); + filestorHandler.remapQueueAfterSplit(FileStorHandler::RemapInfo(bucket1, 0), a, b); + + CPPUNIT_ASSERT(a.foundInQueue); + CPPUNIT_ASSERT(!b.foundInQueue); + + CPPUNIT_ASSERT_EQUAL(std::string( + "BucketId(0x40000000000011d7): Put(BucketId(0x40000000000011d7), userdoc:footype:4567:bar, timestamp 11, size 108) (priority: 127)\n" + "BucketId(0x40000000000011d7): Put(BucketId(0x40000000000011d7), userdoc:footype:4567:bar, timestamp 12, size 108) (priority: 127)\n" + "BucketId(0x40000000000011d7): Put(BucketId(0x40000000000011d7), userdoc:footype:4567:bar, timestamp 13, size 108) (priority: 127)\n" + "BucketId(0x44000000000004d2): Put(BucketId(0x44000000000004d2), userdoc:footype:1234:bar, timestamp 1, size 108) (priority: 127)\n" + "BucketId(0x44000000000004d2): Put(BucketId(0x44000000000004d2), userdoc:footype:1234:bar, timestamp 2, size 108) (priority: 127)\n" + "BucketId(0x44000000000004d2): Put(BucketId(0x44000000000004d2), userdoc:footype:1234:bar, timestamp 3, size 108) (priority: 127)\n"), + filestorHandler.dumpQueue(0)); + +} + +void +FileStorManagerTest::testHandlerMulti() +{ + TestName testName("testHandlerMulti"); + // Setup a filestorthread to test + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>( + dummyManager = new DummyStorageLink)); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + // Since we fake time with small numbers, we need to make sure we dont + // compact them away, as they will seem to be from 1970 + + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1); + + FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), + _node->getComponentRegister(), 255, 0); + filestorHandler.setGetNextMessageTimeout(50); + + std::string content("Here is some content which is in all documents"); + + Document::SP doc1(createDocument(content, "userdoc:footype:1234:bar").release()); + + Document::SP doc2(createDocument(content, "userdoc:footype:4567:bar").release()); + + document::BucketIdFactory factory; + document::BucketId bucket1(16, factory.getBucketId( + doc1->getId()).getRawId()); + document::BucketId bucket2(16, factory.getBucketId( + doc2->getId()).getRawId()); + + // Populate bucket with the given data + for (uint32_t i = 1; i < 10; i++) { + filestorHandler.schedule( + api::StorageMessage::SP(new api::PutCommand(bucket1, doc1, i)), 0); + filestorHandler.schedule( + api::StorageMessage::SP(new api::PutCommand(bucket2, doc2, i + 10)), 0); + } + + { + FileStorHandler::LockedMessage lock = filestorHandler.getNextMessage(0, 255); + CPPUNIT_ASSERT_EQUAL((uint64_t)1, getPutTime(lock.second)); + + lock = filestorHandler.getNextMessage(0, lock, 255); + CPPUNIT_ASSERT_EQUAL((uint64_t)2, getPutTime(lock.second)); + + lock = filestorHandler.getNextMessage(0, lock, 255); + CPPUNIT_ASSERT_EQUAL((uint64_t)3, getPutTime(lock.second)); + } + + { + FileStorHandler::LockedMessage lock = filestorHandler.getNextMessage(0, 255); + CPPUNIT_ASSERT_EQUAL((uint64_t)11, getPutTime(lock.second)); + + lock = filestorHandler.getNextMessage(0, lock, 255); + CPPUNIT_ASSERT_EQUAL((uint64_t)12, getPutTime(lock.second)); + } +} + + +void +FileStorManagerTest::testHandlerTimeout() +{ + TestName testName("testHandlerTimeout"); + // Setup a filestorthread to test + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>( + dummyManager = new DummyStorageLink)); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + + // Since we fake time with small numbers, we need to make sure we dont + // compact them away, as they will seem to be from 1970 + + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1); + + FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), + _node->getComponentRegister(), 255, 0); + filestorHandler.setGetNextMessageTimeout(50); + + std::string content("Here is some content which is in all documents"); + std::ostringstream uri; + + Document::SP doc(createDocument(content, "userdoc:footype:1234:bar").release()); + + document::BucketIdFactory factory; + document::BucketId bucket(16, factory.getBucketId( + doc->getId()).getRawId()); + + // Populate bucket with the given data + { + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bucket, doc, 100)); + std::unique_ptr<api::StorageMessageAddress> address( + new api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 3)); + cmd->setAddress(*address); + cmd->setPriority(0); + cmd->setTimeout(50); + filestorHandler.schedule(cmd, 0); + } + + { + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bucket, doc, 100)); + std::unique_ptr<api::StorageMessageAddress> address( + new api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 3)); + cmd->setAddress(*address); + cmd->setPriority(200); + cmd->setTimeout(10000); + filestorHandler.schedule(cmd, 0); + } + + FastOS_Thread::Sleep(51); + for (;;) { + auto lock = filestorHandler.getNextMessage(0, 255); + if (lock.first.get()) { + CPPUNIT_ASSERT_EQUAL(uint8_t(200), lock.second->getPriority()); + break; + } + } + + CPPUNIT_ASSERT_EQUAL(size_t(1), top.getNumReplies()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::TIMEOUT, + static_cast<api::StorageReply&>(*top.getReply(0)) + .getResult().getResult()); +} + +void +FileStorManagerTest::testHandlerPriorityBlocking() +{ + TestName testName("testHandlerPriorityBlocking"); + // Setup a filestorthread to test + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>( + dummyManager = new DummyStorageLink)); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + // Since we fake time with small numbers, we need to make sure we dont + // compact them away, as they will seem to be from 1970 + + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1); + + FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), + _node->getComponentRegister(), 21, 21); + filestorHandler.setGetNextMessageTimeout(50); + + std::string content("Here is some content which is in all documents"); + std::ostringstream uri; + + document::BucketIdFactory factory; + + // Populate bucket with the given data + for (uint32_t i = 1; i < 6; i++) { + Document::SP doc(createDocument(content, vespalib::make_string("doc:foo:%d",i)).release()); + document::BucketId bucket(16, factory.getBucketId( + doc->getId()).getRawId()); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bucket, doc, 100)); + std::unique_ptr<api::StorageMessageAddress> address( + new api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 3)); + cmd->setAddress(*address); + cmd->setPriority(i * 15); + filestorHandler.schedule(cmd, 0); + } + + { + FileStorHandler::LockedMessage lock1 = filestorHandler.getNextMessage(0, 20); + CPPUNIT_ASSERT_EQUAL(15, (int)lock1.second->getPriority()); + + LOG(debug, "Waiting for request that should time out"); + FileStorHandler::LockedMessage lock2 = filestorHandler.getNextMessage(0, 30); + LOG(debug, "Got request that should time out"); + CPPUNIT_ASSERT(lock2.second.get() == NULL); + } + + { + FileStorHandler::LockedMessage lock1 = filestorHandler.getNextMessage(0, 40); + CPPUNIT_ASSERT_EQUAL(30, (int)lock1.second->getPriority()); + + // New high-pri message comes in + Document::SP doc(createDocument(content, vespalib::make_string("doc:foo:%d", 100)).release()); + document::BucketId bucket(16, factory.getBucketId( + doc->getId()).getRawId()); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bucket, doc, 100)); + std::unique_ptr<api::StorageMessageAddress> address( + new api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 3)); + cmd->setAddress(*address); + cmd->setPriority(15); + filestorHandler.schedule(cmd, 0); + + FileStorHandler::LockedMessage lock2 = filestorHandler.getNextMessage(0, 20); + CPPUNIT_ASSERT_EQUAL(15, (int)lock2.second->getPriority()); + + LOG(debug, "Waiting for request that should time out"); + FileStorHandler::LockedMessage lock3 = filestorHandler.getNextMessage(0, 255); + LOG(debug, "Got request that should time out"); + CPPUNIT_ASSERT(lock3.second.get() == NULL); + } + + { + FileStorHandler::LockedMessage lock1 = filestorHandler.getNextMessage(0, 255); + CPPUNIT_ASSERT_EQUAL(45, (int)lock1.second->getPriority()); + + FileStorHandler::LockedMessage lock = filestorHandler.getNextMessage(0, 255); + CPPUNIT_ASSERT_EQUAL(60, (int)lock.second->getPriority()); + } + LOG(debug, "Test done"); +} + +class PausedThread : public document::Runnable { +private: + FileStorHandler& _handler; + +public: + bool pause; + bool done; + bool gotoperation; + + PausedThread(FileStorHandler& handler) + : _handler(handler), pause(false), done(false), gotoperation(false) {} + + void run() { + FileStorHandler::LockedMessage msg = _handler.getNextMessage(0, 255); + gotoperation = true; + + while (!done) { + if (pause) { + _handler.pause(0, msg.second->getPriority()); + pause = false; + } + FastOS_Thread::Sleep(10); + } + + done = false; + }; +}; + +void +FileStorManagerTest::testHandlerPriorityPreempt() +{ + TestName testName("testHandlerPriorityPreempt"); + // Setup a filestorthread to test + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>( + dummyManager = new DummyStorageLink)); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + // Since we fake time with small numbers, we need to make sure we dont + // compact them away, as they will seem to be from 1970 + + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1); + + FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), + _node->getComponentRegister(), 21, 21); + filestorHandler.setGetNextMessageTimeout(50); + + std::string content("Here is some content which is in all documents"); + std::ostringstream uri; + + document::BucketIdFactory factory; + + { + Document::SP doc(createDocument(content, "doc:foo:1").release()); + document::BucketId bucket(16, factory.getBucketId( + doc->getId()).getRawId()); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bucket, doc, 100)); + std::unique_ptr<api::StorageMessageAddress> address( + new api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 3)); + cmd->setAddress(*address); + cmd->setPriority(60); + filestorHandler.schedule(cmd, 0); + } + + PausedThread thread(filestorHandler); + FastOS_ThreadPool pool(512 * 1024); + thread.start(pool); + + while (!thread.gotoperation) { + FastOS_Thread::Sleep(10); + } + + { + Document::SP doc(createDocument(content, "doc:foo:2").release()); + document::BucketId bucket(16, factory.getBucketId( + doc->getId()).getRawId()); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bucket, doc, 100)); + std::unique_ptr<api::StorageMessageAddress> address( + new api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 3)); + cmd->setAddress(*address); + cmd->setPriority(20); + filestorHandler.schedule(cmd, 0); + } + + { + FileStorHandler::LockedMessage lock1 = filestorHandler.getNextMessage(0, 20); + CPPUNIT_ASSERT_EQUAL(20, (int)lock1.second->getPriority()); + + thread.pause = true; + + for (uint32_t i = 0; i < 10; i++) { + CPPUNIT_ASSERT(thread.pause); + FastOS_Thread::Sleep(100); + } + } + + while (thread.pause) { + FastOS_Thread::Sleep(10); + } + + thread.done = true; + + while (thread.done) { + FastOS_Thread::Sleep(10); + } +} + +void +FileStorManagerTest::testPriority() +{ + TestName testName("testPriority"); + // Setup a filestorthread to test + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>( + dummyManager = new DummyStorageLink)); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + // Since we fake time with small numbers, we need to make sure we dont + // compact them away, as they will seem to be from 1970 + + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 2); + + FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), + _node->getComponentRegister(), 255, 0); + std::unique_ptr<DiskThread> thread(createThread( + *config, *_node, _node->getPersistenceProvider(), + filestorHandler, *metrics.disks[0]->threads[0], 0, 25)); + std::unique_ptr<DiskThread> thread2(createThread( + *config, *_node, _node->getPersistenceProvider(), + filestorHandler, *metrics.disks[0]->threads[1], 0, 255)); + + // Creating documents to test with. Different gids, 2 locations. + std::vector<document::Document::SP > documents; + for (uint32_t i=0; i<50; ++i) { + std::string content("Here is some content which is in all documents"); + std::ostringstream uri; + + uri << "userdoc:footype:" << (i % 3 == 0 ? 0x10001 : 0x0100001) + << ":mydoc-" << i; + Document::SP doc(createDocument(content, uri.str()).release()); + documents.push_back(doc); + } + + document::BucketIdFactory factory; + + // Create buckets in separate, initial pass to avoid races with puts + for (uint32_t i=0; i<documents.size(); ++i) { + document::BucketId bucket(16, factory.getBucketId( + documents[i]->getId()).getRawId()); + + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + + _node->getPersistenceProvider().createBucket( + spi::Bucket(bucket, spi::PartitionId(0)), context); + } + + // Populate bucket with the given data + for (uint32_t i=0; i<documents.size(); ++i) { + document::BucketId bucket(16, factory.getBucketId( + documents[i]->getId()).getRawId()); + + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bucket, documents[i], 100 + i)); + std::unique_ptr<api::StorageMessageAddress> address( + new api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 3)); + cmd->setAddress(*address); + cmd->setPriority(i * 2); + filestorHandler.schedule(cmd, 0); + } + + filestorHandler.flush(true); + + // Wait until everything is done. + int count = 0; + while (documents.size() != top.getNumReplies() && count < 1000) { + FastOS_Thread::Sleep(100); + count++; + } + CPPUNIT_ASSERT(count < 1000); + + for (uint32_t i = 0; i < documents.size(); i++) { + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(i))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + } + + // Verify that thread 1 gets documents over 50 pri + CPPUNIT_ASSERT_EQUAL(uint64_t(documents.size()), + metrics.disks[0]->threads[0]->operations.getValue() + + metrics.disks[0]->threads[1]->operations.getValue()); + CPPUNIT_ASSERT(metrics.disks[0]->threads[0]->operations.getValue() <= 13); + // Closing file stor handler before threads are deleted, such that + // file stor threads getNextMessage calls returns. + filestorHandler.close(); +} + +void +FileStorManagerTest::testSplit1() +{ + TestName testName("testSplit1"); + // Setup a filestorthread to test + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>( + dummyManager = new DummyStorageLink)); + setClusterState("storage:2 distributor:1"); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1); + FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), + _node->getComponentRegister(), 255, 0); + std::unique_ptr<DiskThread> thread(createThread( + *config, *_node, _node->getPersistenceProvider(), + filestorHandler, *metrics.disks[0]->threads[0], 0, 255)); + // Creating documents to test with. Different gids, 2 locations. + std::vector<document::Document::SP > documents; + for (uint32_t i=0; i<20; ++i) { + std::string content("Here is some content which is in all documents"); + std::ostringstream uri; + + uri << "userdoc:footype:" << (i % 3 == 0 ? 0x10001 : 0x0100001) + << ":mydoc-" << i; + Document::SP doc(createDocument( + content, uri.str()).release()); + documents.push_back(doc); + } + document::BucketIdFactory factory; + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + { + // Populate bucket with the given data + for (uint32_t i=0; i<documents.size(); ++i) { + document::BucketId bucket(16, factory.getBucketId( + documents[i]->getId()).getRawId()); + + _node->getPersistenceProvider().createBucket( + spi::Bucket(bucket, spi::PartitionId(0)), context); + + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bucket, documents[i], 100 + i)); + std::unique_ptr<api::StorageMessageAddress> address( + new api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 3)); + cmd->setAddress(*address); + cmd->setSourceIndex(0); + + filestorHandler.schedule(cmd, 0); + filestorHandler.flush(true); + LOG(debug, "Got %" PRIu64 " replies", top.getNumReplies()); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + top.reset(); + + // Delete every 5th document to have delete entries in file too + if (i % 5 == 0) { + std::shared_ptr<api::RemoveCommand> rcmd( + new api::RemoveCommand( + bucket, documents[i]->getId(), 1000000 + 100 + i)); + rcmd->setAddress(*address); + filestorHandler.schedule(rcmd, 0); + filestorHandler.flush(true); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::RemoveReply> rreply( + std::dynamic_pointer_cast<api::RemoveReply>( + top.getReply(0))); + CPPUNIT_ASSERT_MSG(top.getReply(0)->getType().toString(), + rreply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + rreply->getResult()); + top.reset(); + } + } + + // Perform a split, check that locations are split + { + std::shared_ptr<api::SplitBucketCommand> cmd( + new api::SplitBucketCommand(document::BucketId(16, 1))); + cmd->setSourceIndex(0); + filestorHandler.schedule(cmd, 0); + filestorHandler.flush(true); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::SplitBucketReply> reply( + std::dynamic_pointer_cast<api::SplitBucketReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + top.reset(); + } + + // Test that the documents have gotten into correct parts. + for (uint32_t i=0; i<documents.size(); ++i) { + document::BucketId bucket( + 17, i % 3 == 0 ? 0x10001 : 0x0100001); + std::shared_ptr<api::GetCommand> cmd( + new api::GetCommand(bucket, documents[i]->getId(), "[all]")); + api::StorageMessageAddress address( + "storage", lib::NodeType::STORAGE, 3); + cmd->setAddress(address); + filestorHandler.schedule(cmd, 0); + filestorHandler.flush(true); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::GetReply> reply( + std::dynamic_pointer_cast<api::GetReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(i % 5 != 0 ? true : false, reply->wasFound()); + top.reset(); + } + + // Keep splitting location 1 until we gidsplit + for (int i=17; i<=32; ++i) { + std::shared_ptr<api::SplitBucketCommand> cmd( + new api::SplitBucketCommand( + document::BucketId(i, 0x0100001))); + cmd->setSourceIndex(0); + filestorHandler.schedule(cmd, 0); + filestorHandler.flush(true); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::SplitBucketReply> reply( + std::dynamic_pointer_cast<api::SplitBucketReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + top.reset(); + } + + // Test that the documents have gotten into correct parts. + for (uint32_t i=0; i<documents.size(); ++i) { + document::BucketId bucket; + if (i % 3 == 0) { + bucket = document::BucketId(17, 0x10001); + } else { + bucket = document::BucketId(33, factory.getBucketId( + documents[i]->getId()).getRawId()); + } + std::shared_ptr<api::GetCommand> cmd( + new api::GetCommand(bucket, documents[i]->getId(), "[all]")); + api::StorageMessageAddress address( + "storage", lib::NodeType::STORAGE, 3); + cmd->setAddress(address); + filestorHandler.schedule(cmd, 0); + filestorHandler.flush(true); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::GetReply> reply( + std::dynamic_pointer_cast<api::GetReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(i % 5 != 0 ? true : false, reply->wasFound()); + top.reset(); + } + } + // Closing file stor handler before threads are deleted, such that + // file stor threads getNextMessage calls returns. + filestorHandler.close(); +} + +void +FileStorManagerTest::testSplitSingleGroup() +{ + TestName testName("testSplitSingleGroup"); + // Setup a filestorthread to test + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>( + dummyManager = new DummyStorageLink)); + setClusterState("storage:2 distributor:1"); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1); + FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), + _node->getComponentRegister(), 255, 0); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + for (uint32_t j=0; j<1; ++j) { + // Test this twice, once where all the data ends up in file with + // splitbit set, and once where all the data ends up in file with + // splitbit unset + bool state = (j == 0); + + std::unique_ptr<DiskThread> thread(createThread( + *config, *_node, _node->getPersistenceProvider(), + filestorHandler, *metrics.disks[0]->threads[0], 0, 255)); + // Creating documents to test with. Different gids, 2 locations. + std::vector<document::Document::SP > documents; + for (uint32_t i=0; i<20; ++i) { + std::string content("Here is some content for all documents"); + std::ostringstream uri; + + uri << "userdoc:footype:" << (state ? 0x10001 : 0x0100001) + << ":mydoc-" << i; + Document::SP doc(createDocument( + content, uri.str()).release()); + documents.push_back(doc); + } + document::BucketIdFactory factory; + + // Populate bucket with the given data + for (uint32_t i=0; i<documents.size(); ++i) { + document::BucketId bucket(16, factory.getBucketId( + documents[i]->getId()).getRawId()); + + _node->getPersistenceProvider().createBucket( + spi::Bucket(bucket, spi::PartitionId(0)), context); + + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bucket, documents[i], 100 + i)); + api::StorageMessageAddress address( + "storage", lib::NodeType::STORAGE, 3); + cmd->setAddress(address); + filestorHandler.schedule(cmd, 0); + filestorHandler.flush(true); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + top.reset(); + } + // Perform a split, check that locations are split + { + std::shared_ptr<api::SplitBucketCommand> cmd( + new api::SplitBucketCommand(document::BucketId(16, 1))); + cmd->setSourceIndex(0); + filestorHandler.schedule(cmd, 0); + filestorHandler.flush(true); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::SplitBucketReply> reply( + std::dynamic_pointer_cast<api::SplitBucketReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + top.reset(); + } + + + // Test that the documents are all still there + for (uint32_t i=0; i<documents.size(); ++i) { + document::BucketId bucket(17, state ? 0x10001 : 0x00001); + std::shared_ptr<api::GetCommand> cmd( + new api::GetCommand(bucket, documents[i]->getId(), "[all]")); + api::StorageMessageAddress address( + "storage", lib::NodeType::STORAGE, 3); + cmd->setAddress(address); + filestorHandler.schedule(cmd, 0); + filestorHandler.flush(true); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::GetReply> reply( + std::dynamic_pointer_cast<api::GetReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + top.reset(); + } + // Closing file stor handler before threads are deleted, such that + // file stor threads getNextMessage calls returns. + filestorHandler.close(); + } +} + +void +FileStorManagerTest::putDoc(DummyStorageLink& top, + FileStorHandler& filestorHandler, + const document::BucketId& target, + uint32_t docNum) +{ + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + document::BucketIdFactory factory; + document::DocumentId docId(vespalib::make_string("userdoc:ns:%zu:%d", target.getId(), docNum)); + document::BucketId bucket(16, factory.getBucketId(docId).getRawId()); + //std::cerr << "doc bucket is " << bucket << " vs source " << source << "\n"; + _node->getPersistenceProvider().createBucket( + spi::Bucket(target, spi::PartitionId(0)), context); + Document::SP doc(new Document(*_testdoctype1, docId)); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(target, doc, docNum+1)); + cmd->setAddress(address); + cmd->setPriority(120); + filestorHandler.schedule(cmd, 0); + filestorHandler.flush(true); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + top.reset(); +} + +void +FileStorManagerTest::testSplitEmptyTargetWithRemappedOps() +{ + TestName testName("testSplitEmptyTargetWithRemappedOps"); + + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>( + dummyManager = new DummyStorageLink)); + setClusterState("storage:2 distributor:1"); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1); + FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), + _node->getComponentRegister(), 255, 0); + std::unique_ptr<DiskThread> thread(createThread( + *config, *_node, _node->getPersistenceProvider(), + filestorHandler, *metrics.disks[0]->threads[0], 0, 255)); + + document::BucketId source(16, 0x10001); + + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); + + for (uint32_t i=0; i<10; ++i) { + putDoc(top, filestorHandler, source, i); + } + + // Send split followed by a put that is bound for a target bucket that + // will end up empty in the split itself. The split should notice this + // and create the bucket explicitly afterwards in order to compensate for + // the persistence provider deleting it internally. + // Make sure we block the operation queue until we've scheduled all + // the operations. + std::unique_ptr<ResumeGuard> resumeGuard( + new ResumeGuard(filestorHandler.pause())); + + std::shared_ptr<api::SplitBucketCommand> splitCmd( + new api::SplitBucketCommand(source)); + splitCmd->setPriority(120); + splitCmd->setSourceIndex(0); + + document::DocumentId docId( + vespalib::make_string("userdoc:ns:%d:1234", 0x100001)); + Document::SP doc(new Document(*_testdoctype1, docId)); + std::shared_ptr<api::PutCommand> putCmd( + new api::PutCommand(source, doc, 1001)); + putCmd->setAddress(address); + putCmd->setPriority(120); + + filestorHandler.schedule(splitCmd, 0); + filestorHandler.schedule(putCmd, 0); + resumeGuard.reset(0); // Unpause + filestorHandler.flush(true); + + top.waitForMessages(2, _waitTime); + + CPPUNIT_ASSERT_EQUAL((size_t) 2, top.getNumReplies()); + { + std::shared_ptr<api::SplitBucketReply> reply( + std::dynamic_pointer_cast<api::SplitBucketReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + } + { + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(1))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + } + + top.reset(); +} + +void +FileStorManagerTest::testNotifyOnSplitSourceOwnershipChanged() +{ + TestName testName("testSplit1"); + // Setup a filestorthread to test + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>(dummyManager = new DummyStorageLink)); + setClusterState("storage:2 distributor:2"); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1); + FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), + _node->getComponentRegister(), 255, 0); + std::unique_ptr<DiskThread> thread(createThread( + *config, *_node, _node->getPersistenceProvider(), + filestorHandler, *metrics.disks[0]->threads[0], 0, 255)); + + document::BucketId source(getFirstBucketNotOwnedByDistributor(0)); + createBucket(source, 0); + for (uint32_t i=0; i<10; ++i) { + putDoc(top, filestorHandler, source, i); + } + + std::shared_ptr<api::SplitBucketCommand> splitCmd( + new api::SplitBucketCommand(source)); + splitCmd->setPriority(120); + splitCmd->setSourceIndex(0); // Source not owned by this distributor. + + filestorHandler.schedule(splitCmd, 0); + filestorHandler.flush(true); + top.waitForMessages(4, _waitTime); // 3 notify cmds + split reply + + CPPUNIT_ASSERT_EQUAL(size_t(4), top.getNumReplies()); + for (int i = 0; i < 3; ++i) { + CPPUNIT_ASSERT_EQUAL(api::MessageType::NOTIFYBUCKETCHANGE, + top.getReply(i)->getType()); + } + + std::shared_ptr<api::SplitBucketReply> reply( + std::dynamic_pointer_cast<api::SplitBucketReply>( + top.getReply(3))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); +} + +void +FileStorManagerTest::testJoin() +{ + TestName testName("testJoin"); + // Setup a filestorthread to test + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>( + dummyManager = new DummyStorageLink)); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1); + FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), + _node->getComponentRegister(), 255, 0); + std::unique_ptr<DiskThread> thread(createThread( + *config, *_node, _node->getPersistenceProvider(), + filestorHandler, *metrics.disks[0]->threads[0], 0, 255)); + // Creating documents to test with. Different gids, 2 locations. + std::vector<document::Document::SP > documents; + for (uint32_t i=0; i<20; ++i) { + std::string content("Here is some content which is in all documents"); + std::ostringstream uri; + + uri << "userdoc:footype:" << (i % 3 == 0 ? 0x10001 : 0x0100001) + << ":mydoc-" << i; + Document::SP doc(createDocument( + content, uri.str()).release()); + documents.push_back(doc); + } + document::BucketIdFactory factory; + + createBucket(document::BucketId(17, 0x00001), 0); + createBucket(document::BucketId(17, 0x10001), 0); + + { + // Populate bucket with the given data + for (uint32_t i=0; i<documents.size(); ++i) { + document::BucketId bucket(17, factory.getBucketId( + documents[i]->getId()).getRawId()); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bucket, documents[i], 100 + i)); + std::unique_ptr<api::StorageMessageAddress> address( + new api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 3)); + cmd->setAddress(*address); + filestorHandler.schedule(cmd, 0); + filestorHandler.flush(true); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + top.reset(); + // Delete every 5th document to have delete entries in file too + if (i % 5 == 0) { + std::shared_ptr<api::RemoveCommand> rcmd( + new api::RemoveCommand( + bucket, documents[i]->getId(), 1000000 + 100 + i)); + rcmd->setAddress(*address); + filestorHandler.schedule(rcmd, 0); + filestorHandler.flush(true); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::RemoveReply> rreply( + std::dynamic_pointer_cast<api::RemoveReply>( + top.getReply(0))); + CPPUNIT_ASSERT_MSG(top.getReply(0)->getType().toString(), + rreply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + rreply->getResult()); + top.reset(); + } + } + LOG(debug, "Starting the actual join after populating data"); + // Perform a join, check that other files are gone + { + std::shared_ptr<api::JoinBucketsCommand> cmd( + new api::JoinBucketsCommand(document::BucketId(16, 1))); + cmd->getSourceBuckets().push_back(document::BucketId(17, 0x00001)); + cmd->getSourceBuckets().push_back(document::BucketId(17, 0x10001)); + filestorHandler.schedule(cmd, 0); + filestorHandler.flush(true); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::JoinBucketsReply> reply( + std::dynamic_pointer_cast<api::JoinBucketsReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + top.reset(); + } + // Test that the documents have gotten into the file. + for (uint32_t i=0; i<documents.size(); ++i) { + document::BucketId bucket(16, 1); + std::shared_ptr<api::GetCommand> cmd( + new api::GetCommand(bucket, documents[i]->getId(), "[all]")); + api::StorageMessageAddress address( + "storage", lib::NodeType::STORAGE, 3); + cmd->setAddress(address); + filestorHandler.schedule(cmd, 0); + filestorHandler.flush(true); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::GetReply> reply( + std::dynamic_pointer_cast<api::GetReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(i % 5 != 0 ? true : false, reply->wasFound()); + top.reset(); + } + } + // Closing file stor handler before threads are deleted, such that + // file stor threads getNextMessage calls returns. + filestorHandler.close(); +} + +namespace { + +spi::IteratorId +createIterator(DummyStorageLink& link, + const document::BucketId& bucketId, + const std::string& docSel, + framework::MicroSecTime fromTime = framework::MicroSecTime(0), + framework::MicroSecTime toTime = framework::MicroSecTime::max(), + bool headerOnly = false) +{ + spi::Bucket bucket(bucketId, spi::PartitionId(0)); + + spi::Selection selection = + spi::Selection(spi::DocumentSelection(docSel)); + selection.setFromTimestamp(spi::Timestamp(fromTime.getTime())); + selection.setToTimestamp(spi::Timestamp(toTime.getTime())); + CreateIteratorCommand::SP createIterCmd( + new CreateIteratorCommand(bucket, + selection, + headerOnly ? "[header]" : "[all]", + spi::NEWEST_DOCUMENT_ONLY)); + link.sendDown(createIterCmd); + link.waitForMessages(1, FileStorManagerTest::LONG_WAITTIME); + CPPUNIT_ASSERT_EQUAL(size_t(1), link.getNumReplies()); + std::shared_ptr<CreateIteratorReply> reply( + std::dynamic_pointer_cast<CreateIteratorReply>( + link.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + link.reset(); + CPPUNIT_ASSERT(reply->getResult().success()); + return reply->getIteratorId(); +} + +} + +void +FileStorManagerTest::testVisiting() +{ + TestName testName("testVisiting"); + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = new FileStorManager( + smallConfig->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + top.open(); + // Adding documents to two buckets which we are going to visit + // We want one bucket in one slotfile, and one bucket with a file split + uint32_t docCount = 50; + std::vector<document::BucketId> ids(2); + ids[0] = document::BucketId(16, 1); + ids[1] = document::BucketId(16, 2); + + createBucket(ids[0], 0); + createBucket(ids[1], 0); + + lib::RandomGen randomizer(523); + for (uint32_t i=0; i<docCount; ++i) { + std::string content("Here is some content which is in all documents"); + std::ostringstream uri; + + uri << "userdoc:crawler:" << (i < 3 ? 1 : 2) << ":" + << randomizer.nextUint32() << ".html"; + Document::SP doc(createDocument( + content, uri.str()).release()); + const document::DocumentType& type(doc->getType()); + if (i < 30) { + doc->setValue(type.getField("hstringval"), + document::StringFieldValue("John Doe")); + } else { + doc->setValue(type.getField("hstringval"), + document::StringFieldValue("Jane Doe")); + } + std::shared_ptr<api::PutCommand> cmd(new api::PutCommand( + ids[i < 3 ? 0 : 1], doc, i+1)); + top.sendDown(cmd); + } + top.waitForMessages(docCount, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) docCount, top.getNumReplies()); + // Check nodestate with splitting + { + api::BucketInfo info; + for (uint32_t i=3; i<docCount; ++i) { + std::shared_ptr<api::BucketInfoReply> reply( + std::dynamic_pointer_cast<api::BucketInfoReply>( + top.getReply(i))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_MESSAGE(reply->getResult().toString(), + reply->getResult().success()); + + info = reply->getBucketInfo(); + } + CPPUNIT_ASSERT_EQUAL(docCount-3, info.getDocumentCount()); + } + top.reset(); + // Visit bucket with no split, using no selection + { + framework::MemoryToken::UP token( + _node->getMemoryManager().allocate( + _node->getMemoryManager().getAllocationType( + "VISITOR_BUFFER"), + 16*1024, + 16*1024, + 127)); + spi::IteratorId iterId(createIterator(top, ids[0], "true")); + std::shared_ptr<GetIterCommand> cmd( + new GetIterCommand(std::move(token), ids[0], iterId, 16*1024)); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<GetIterReply> reply( + std::dynamic_pointer_cast<GetIterReply>(top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + CPPUNIT_ASSERT_EQUAL(ids[0], reply->getBucketId()); + CPPUNIT_ASSERT_EQUAL(size_t(3), reply->getEntries().size()); + top.reset(); + } + // Visit bucket with split, using selection + { + uint32_t totalDocs = 0; + spi::IteratorId iterId( + createIterator(top, + ids[1], + "testdoctype1.hstringval = \"John Doe\"")); + while (true) { + framework::MemoryToken::UP token( + _node->getMemoryManager().allocate( + _node->getMemoryManager().getAllocationType( + "VISITOR_BUFFER"), + 16*1024, + 16*1024, + 127)); + std::shared_ptr<GetIterCommand> cmd( + new GetIterCommand(std::move(token), ids[1], iterId, 16*1024)); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<GetIterReply> reply( + std::dynamic_pointer_cast<GetIterReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + CPPUNIT_ASSERT_EQUAL(ids[1], reply->getBucketId()); + totalDocs += reply->getEntries().size(); + top.reset(); + if (reply->isCompleted()) { + break; + } + } + CPPUNIT_ASSERT_EQUAL(27u, totalDocs); + } + // Visit bucket with min and max timestamps set, headers only + { + document::BucketId bucket(16, 2); + spi::IteratorId iterId( + createIterator(top, + ids[1], + "", + framework::MicroSecTime(30), + framework::MicroSecTime(40), + true)); + uint32_t totalDocs = 0; + while (true) { + framework::MemoryToken::UP token( + _node->getMemoryManager().allocate( + _node->getMemoryManager().getAllocationType( + "VISITOR_BUFFER"), + 16*1024, + 16*1024, + 127)); + std::shared_ptr<GetIterCommand> cmd( + new GetIterCommand(std::move(token), ids[1], iterId, 16*1024)); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<GetIterReply> reply( + std::dynamic_pointer_cast<GetIterReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + CPPUNIT_ASSERT_EQUAL(bucket, reply->getBucketId()); +/* Header only is a VDS-specific thing. + + for (size_t i = 0; i < reply->getEntries().size(); ++i) { + CPPUNIT_ASSERT(reply->getEntries()[i]->getDocument() + ->getBody().empty()); + } +*/ + totalDocs += reply->getEntries().size(); + top.reset(); + if (reply->isCompleted()) { + break; + } + } + CPPUNIT_ASSERT_EQUAL(11u, totalDocs); + } + +} + +void +FileStorManagerTest::testRemoveLocation() +{ + TestName testName("testRemoveLocation"); + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = + new FileStorManager(config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + top.open(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); + document::BucketId bid(8, 0); + + createBucket(bid, 0); + + // Adding some documents to be removed later + for (uint32_t i=0; i<=10; ++i) { + std::ostringstream docid; + docid << "userdoc:ns:" << (i << 8) << ":foo"; + Document::SP doc(createDocument( + "some content", docid.str()).release()); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, doc, 1000 + i)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + CPPUNIT_ASSERT_EQUAL(i + 1u, reply->getBucketInfo().getDocumentCount()); + } + // Issuing remove location command + { + std::shared_ptr<api::RemoveLocationCommand> cmd( + new api::RemoveLocationCommand("id.user % 512 == 0", bid)); + //new api::RemoveLocationCommand("id.user == 1", bid)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::RemoveLocationReply> reply( + std::dynamic_pointer_cast<api::RemoveLocationReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + CPPUNIT_ASSERT_EQUAL(5u, reply->getBucketInfo().getDocumentCount()); + } +} + +void FileStorManagerTest::testDeleteBucket() +{ + TestName testName("testDeleteBucket"); + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = new FileStorManager( + config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + top.open(); + api::StorageMessageAddress address( + "storage", lib::NodeType::STORAGE, 2); + // Creating a document to test with + document::DocumentId docId("userdoc:crawler:4000:http://www.ntnu.no/"); + Document::SP doc(new Document(*_testdoctype1, docId)); + document::BucketId bid(16, 4000); + + createBucket(bid, 0); + + api::BucketInfo bucketInfo; + // Putting it + { + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, doc, 105)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + + CPPUNIT_ASSERT_EQUAL(1, (int)reply->getBucketInfo().getDocumentCount()); + bucketInfo = reply->getBucketInfo(); + top.reset(); + } + + // Delete bucket + { + std::shared_ptr<api::DeleteBucketCommand> cmd( + new api::DeleteBucketCommand(bid)); + cmd->setAddress(address); + cmd->setBucketInfo(bucketInfo); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::DeleteBucketReply> reply( + std::dynamic_pointer_cast<api::DeleteBucketReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + } +} + +void +FileStorManagerTest::testDeleteBucketRejectOutdatedBucketInfo() +{ + TestName testName("testDeleteBucketRejectOutdatedBucketInfo"); + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = new FileStorManager( + config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + top.open(); + api::StorageMessageAddress address( + "storage", lib::NodeType::STORAGE, 2); + // Creating a document to test with + document::DocumentId docId("userdoc:crawler:4000:http://www.ntnu.no/"); + Document::SP doc(new Document(*_testdoctype1, docId)); + document::BucketId bid(16, 4000); + + createBucket(bid, 0); + + api::BucketInfo bucketInfo; + + // Putting it + { + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, doc, 105)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + + CPPUNIT_ASSERT_EQUAL(1, (int)reply->getBucketInfo().getDocumentCount()); + bucketInfo = reply->getBucketInfo(); + top.reset(); + } + + // Attempt to delete bucket, but with non-matching bucketinfo + { + std::shared_ptr<api::DeleteBucketCommand> cmd( + new api::DeleteBucketCommand(bid)); + cmd->setBucketInfo(BucketInfo(0xf000baaa, 1, 123, 1, 456)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::DeleteBucketReply> reply( + std::dynamic_pointer_cast<api::DeleteBucketReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL( + ReturnCode::REJECTED, + reply->getResult().getResult()); + CPPUNIT_ASSERT_EQUAL(bucketInfo, reply->getBucketInfo()); + } +} + +/** + * Test that receiving a DeleteBucketCommand with invalid + * BucketInfo deletes the bucket and does not fail the operation. + */ +void +FileStorManagerTest::testDeleteBucketWithInvalidBucketInfo() +{ + TestName testName("testDeleteBucketWithInvalidBucketInfo"); + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = new FileStorManager( + config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + top.open(); + api::StorageMessageAddress address( + "storage", lib::NodeType::STORAGE, 2); + // Creating a document to test with + document::DocumentId docId("userdoc:crawler:4000:http://www.ntnu.no/"); + Document::SP doc(new Document(*_testdoctype1, docId)); + document::BucketId bid(16, 4000); + + createBucket(bid, 0); + + // Putting it + { + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, doc, 105)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + CPPUNIT_ASSERT_EQUAL(1, (int)reply->getBucketInfo().getDocumentCount()); + top.reset(); + } + + // Attempt to delete bucket with invalid bucketinfo + { + std::shared_ptr<api::DeleteBucketCommand> cmd( + new api::DeleteBucketCommand(bid)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::DeleteBucketReply> reply( + std::dynamic_pointer_cast<api::DeleteBucketReply>( + top.getReply(0))); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL( + ReturnCode::OK, + reply->getResult().getResult()); + CPPUNIT_ASSERT_EQUAL(api::BucketInfo(), reply->getBucketInfo()); + } +} + +namespace { + + /** + * Utility storage link, sending data to the given links instead of through + * a regular chain. + */ + struct MidLink : public StorageLink { + StorageLink& _up; + + public: + MidLink(std::unique_ptr<StorageLink> down, StorageLink& up) + : StorageLink("MidLink"), _up(up) + { + push_back(std::move(down)); + } + ~MidLink() { + closeNextLink(); + } + + virtual void print(std::ostream& out, bool, const std::string&) const + { out << "MidLink"; } + + virtual bool onUp(const std::shared_ptr<api::StorageMessage> & msg) { + if (!StorageLinkTest::callOnUp(_up, msg)) _up.sendUp(msg); + return true; + } + + }; + + /** + * Utility class, connecting two storage links below it, sending + * messages coming up from one down the other (providing address is set + * correctly.) + */ + class BinaryStorageLink : public DummyStorageLink { + vespalib::Lock _lock; + std::set<api::StorageMessage::Id> _seen; + MidLink _left; + MidLink _right; + uint16_t _leftAddr; + uint16_t _rightAddr; + + public: + BinaryStorageLink(uint16_t leftAddr, std::unique_ptr<StorageLink> left, + uint16_t rightAddr, std::unique_ptr<StorageLink> right) + : _left(std::move(left), *this), + _right(std::move(right), *this), + _leftAddr(leftAddr), + _rightAddr(rightAddr) {} + + virtual void print(std::ostream& out, bool, const std::string&) const + { out << "BinaryStorageLink"; } + + virtual bool onDown(const std::shared_ptr<api::StorageMessage> & msg) { +// LOG(debug, "onDown Received msg: ->%s, %s %llu\n", msg->getAddress() ? msg->getAddress()->toString().c_str() : "(null)", msg->toString().c_str(), msg->getMsgId()); + + vespalib::LockGuard lock(_lock); + _seen.insert(msg->getMsgId()); + return sendOn(msg); + } + + bool sendOn(const std::shared_ptr<api::StorageMessage> & msg) { + if (msg->getAddress()) { + uint16_t address = msg->getAddress()->getIndex(); + if ((address == _leftAddr && !msg->getType().isReply()) || + (address == _rightAddr && msg->getType().isReply())) + { + if (!StorageLinkTest::callOnDown(_left, msg)) { + _left.sendDown(msg); + } + } else if ((address == _rightAddr && !msg->getType().isReply()) || + (address == _leftAddr && msg->getType().isReply())) + { + if (!StorageLinkTest::callOnDown(_right, msg)) { + _right.sendDown(msg); + } + } else { + std::ostringstream ost; + ost << "Address " << address << " is neither " << _leftAddr + << " or " << _rightAddr << " in message " << *msg + << ".\n"; + CPPUNIT_FAIL(ost.str()); + } + } + return true; + } + + virtual bool onUp(const std::shared_ptr<api::StorageMessage> & msg) { // LOG(debug, "onUp Received msg: ->%s, %s %llu\n", msg->getAddress() ? msg->getAddress()->toString().c_str() : "(null)", msg->toString().c_str(), msg->getMsgId()); + + vespalib::LockGuard lock(_lock); + std::set<api::StorageMessage::Id>::iterator it + = _seen.find(msg->getMsgId()); + // If message originated from the outside + if (it != _seen.end()) { + LOG(debug, "Have seen this message before, storing"); + + _seen.erase(it); + return DummyStorageLink::onUp(msg); + // If it originated from below, send it down again. + } else if (msg->getType() == api::MessageType::NOTIFYBUCKETCHANGE) { + // Just throw away notify bucket change + return true; + } else { + LOG(debug, "Never seen %s, sending on!", + msg->toString().c_str()); + + return sendOn(msg); + } + } + + void onFlush(bool downwards) { + if (downwards) { + _left.flush(); + _right.flush(); + } + } + void onOpen() { + _left.open(); + _right.open(); + } + void onClose() { + _left.close(); + _right.close(); + } + }; +} + +void +FileStorManagerTest::testNoTimestamps() +{ + TestName testName("testNoTimestamps"); + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = + new FileStorManager(config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + top.open(); + api::StorageMessageAddress address( + "storage", lib::NodeType::STORAGE, 3); + // Creating a document to test with + Document::SP doc(createDocument( + "some content", "doc:crawler:http://www.ntnu.no/").release()); + document::BucketId bid(16, 4000); + + createBucket(bid, 0); + + // Putting it + { + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, doc, 0)); + cmd->setAddress(address); + CPPUNIT_ASSERT_EQUAL((api::Timestamp)0, cmd->getTimestamp()); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode::REJECTED, + reply->getResult().getResult()); + } + // Removing it + { + std::shared_ptr<api::RemoveCommand> cmd( + new api::RemoveCommand(bid, doc->getId(), 0)); + cmd->setAddress(address); + CPPUNIT_ASSERT_EQUAL((api::Timestamp)0, cmd->getTimestamp()); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::RemoveReply> reply( + std::dynamic_pointer_cast<api::RemoveReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode::REJECTED, + reply->getResult().getResult()); + } +} + +void +FileStorManagerTest::testEqualTimestamps() +{ + TestName testName("testEqualTimestamps"); + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = + new FileStorManager(config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + top.open(); + api::StorageMessageAddress address( + "storage", lib::NodeType::STORAGE, 3); + // Creating a document to test with + document::BucketId bid(16, 4000); + + createBucket(bid, 0); + + // Putting it + { + Document::SP doc(createDocument( + "some content", "userdoc:crawler:4000:http://www.ntnu.no/") + .release()); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, doc, 100)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode::OK, reply->getResult().getResult()); + } + + // Putting it on same timestamp again + // (ok as doc is the same. Since merge can move doc to other copy we + // have to accept this) + { + Document::SP doc(createDocument( + "some content", "userdoc:crawler:4000:http://www.ntnu.no/") + .release()); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, doc, 100)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode::OK, reply->getResult().getResult()); + } + + // Putting the doc with other id. Now we should fail + { + Document::SP doc(createDocument( + "some content", "userdoc:crawler:4000:http://www.ntnu.nu/") + .release()); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, doc, 100)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode::TIMESTAMP_EXIST, + reply->getResult().getResult()); + } +} + +void +FileStorManagerTest::testMultiOp() +{ + TestName testName("testMultiOp"); + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = + new FileStorManager(config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + top.open(); + api::StorageMessageAddress address( + "storage", lib::NodeType::STORAGE, 3); + + createBucket(document::BucketId(16, 0), 0); + + // Add some documents to remove/update later + for (uint32_t i=0; i<10; ++i) { + std::ostringstream did; + did << "userdoc:crawler:0:http://www.ntnu.no/" << i; + Document::SP doc(createDocument( + "some content", did.str()).release()); + doc->set("headerval", (int) i); + doc->set("content", "some content"); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(document::BucketId(16, 0), doc, 100 + i)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + } + document::DocumentTypeRepo::SP repo = _node->getTypeRepo(); + + // Create operation list + std::vector<char> buffer(1024 * 1024); + vdslib::WritableDocumentList mdl(repo, &buffer[0], buffer.size()); + for (uint32_t i=10; i<15; ++i) { + std::ostringstream did; + did << "userdoc:crawler:0:http://www.ntnu.no/" << i; + mdl.addPut(*createDocument("foo bar", did.str()), + 1000 + i); + } + for (uint32_t i=4; i<8; ++i) { + std::ostringstream did; + did << "userdoc:crawler:0:http://www.ntnu.no/" << i; + mdl.addRemove(document::DocumentId(did.str()), 2000 + i); + } + for (uint32_t i=1; i<3; ++i) { + std::ostringstream did; + did << "userdoc:crawler:0:http://www.ntnu.no/" << i; + document::DocumentUpdate update(*_testdoctype1, + document::DocumentId(did.str())); + if (i % 2 == 0) { + document::FieldUpdate fupd(_testdoctype1->getField("content")); + fupd.addUpdate(document::AssignValueUpdate( + document::StringFieldValue("baah"))); + update.addUpdate(fupd); + } else { + document::FieldUpdate fupd(_testdoctype1->getField("headerval")); + fupd.addUpdate(document::AssignValueUpdate( + document::IntFieldValue(i + 100))); + update.addUpdate(fupd); + } + mdl.addUpdate(update, 3000 + i); + } + // Add a non-existing update + { + std::ostringstream did; + did << "userdoc:crawler:0:http://www.ntnu.no/nonexisting1"; + document::DocumentUpdate update(*_testdoctype1, + document::DocumentId(did.str())); + document::FieldUpdate fupd(_testdoctype1->getField("content")); + fupd.addUpdate(document::AssignValueUpdate( + document::StringFieldValue("baah"))); + update.addUpdate(fupd); + mdl.addUpdate(update, 4000); + } + + // Issue operation. + { + std::shared_ptr<api::MultiOperationCommand> cmd( + new api::MultiOperationCommand( + repo, document::BucketId(16, 0), buffer)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::MultiOperationReply> reply( + std::dynamic_pointer_cast<api::MultiOperationReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply->getResult()); + } + // Verify that new documents exist and that removed are gone. + // Removing it + for (uint32_t i=0; i<16; ++i) { + std::ostringstream did; + did << "userdoc:crawler:0:http://www.ntnu.no/" << i; + std::shared_ptr<api::GetCommand> cmd(new api::GetCommand( + document::BucketId(16, 0), document::DocumentId(did.str()), + "[all]")); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::GetReply> reply3( + std::dynamic_pointer_cast<api::GetReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply3.get()); + if (i < 4 || (i >= 8 && i < 15)) { + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + reply3->getResult()); + CPPUNIT_ASSERT_EQUAL(vespalib::string(did.str()), + reply3->getDocumentId().toString()); + if (i >= 10) { + CPPUNIT_ASSERT(!reply3->getDocument()->hasValue("headerval")); + CPPUNIT_ASSERT(reply3->getDocument()->hasValue("content")); + } else if (i >= 1 && i <3) { + CPPUNIT_ASSERT(reply3->getDocument()->hasValue("headerval")); + CPPUNIT_ASSERT(reply3->getDocument()->hasValue("content")); + CPPUNIT_ASSERT_EQUAL( + static_cast<const document::FieldValue&>( + document::IntFieldValue(i % 2 == 0 ? i : i + 100)), + *reply3->getDocument()->getValue("headerval")); + CPPUNIT_ASSERT_EQUAL( + static_cast<const document::FieldValue&>( + document::StringFieldValue(i % 2 == 0 ? "baah" : "some content")), + *reply3->getDocument()->getValue("content")); + } else { + CPPUNIT_ASSERT(reply3->getDocument()->hasValue("headerval")); + CPPUNIT_ASSERT(reply3->getDocument()->hasValue("content")); + CPPUNIT_ASSERT_EQUAL( + static_cast<const document::FieldValue&>( + document::IntFieldValue(i)), + *reply3->getDocument()->getValue("headerval")); + } + } else { + CPPUNIT_ASSERT_EQUAL(false, reply3->wasFound()); + CPPUNIT_ASSERT_EQUAL(vespalib::string(did.str()), + reply3->getDocumentId().toString()); + } + } +} + +void +FileStorManagerTest::testGetIter() +{ + TestName testName("testGetIter"); + // Setting up manager + DummyStorageLink top; + FileStorManager *manager; + top.push_back(unique_ptr<StorageLink>(manager = + new FileStorManager(config->getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + top.open(); + api::StorageMessageAddress address( + "storage", lib::NodeType::STORAGE, 3); + document::BucketId bid(16, 4000); + + createBucket(bid, 0); + + std::vector<Document::SP > docs; + // Creating some documents to test with + for (uint32_t i=0; i<10; ++i) { + std::ostringstream id; + id << "userdoc:crawler:4000:http://www.ntnu.no/" << i; + docs.push_back( + Document::SP( + _node->getTestDocMan().createRandomDocumentAtLocation( + 4000, i, 400, 400))); + } + BucketInfo bucketInfo; + // Putting all docs to have something to visit + for (uint32_t i=0; i<docs.size(); ++i) { + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, docs[i], 100 + i)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + bucketInfo = reply->getBucketInfo(); + } + // Sending a getiter request that will only visit some of the docs + spi::IteratorId iterId(createIterator(top, bid, "")); + { + framework::MemoryToken::UP token( + _node->getMemoryManager().allocate( + _node->getMemoryManager().getAllocationType( + "VISITOR_BUFFER"), + 2048, + 2048, + 127)); + std::shared_ptr<GetIterCommand> cmd( + new GetIterCommand(std::move(token), bid, iterId, 2048)); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<GetIterReply> reply( + std::dynamic_pointer_cast<GetIterReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + CPPUNIT_ASSERT(reply->getEntries().size() > 0); + CPPUNIT_ASSERT(reply->getEntries().size() < docs.size()); + } + // Normal case of get iter is testing through visitor tests. + // Testing specific situation where file is deleted while visiting here + { + std::shared_ptr<api::DeleteBucketCommand> cmd( + new api::DeleteBucketCommand(bid)); + cmd->setBucketInfo(bucketInfo); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::DeleteBucketReply> reply( + std::dynamic_pointer_cast<api::DeleteBucketReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + } + { + framework::MemoryToken::UP token( + _node->getMemoryManager().allocate( + _node->getMemoryManager().getAllocationType( + "VISITOR_BUFFER"), + 2048, + 2048, + 127)); + std::shared_ptr<GetIterCommand> cmd( + new GetIterCommand(std::move(token), bid, iterId, 2048)); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<GetIterReply> reply( + std::dynamic_pointer_cast<GetIterReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode::BUCKET_NOT_FOUND, + reply->getResult().getResult()); + CPPUNIT_ASSERT(reply->getEntries().empty()); + } +} + +void +FileStorManagerTest::testSetBucketActiveState() +{ + TestName testName("testSetBucketActiveState"); + DummyStorageLink top; + FileStorManager* manager( + new FileStorManager(config->getConfigId(), + _node->getPartitions(), + _node->getPersistenceProvider(), + _node->getComponentRegister())); + top.push_back(unique_ptr<StorageLink>(manager)); + setClusterState("storage:4 distributor:1"); + top.open(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); + + document::BucketId bid(16, 4000); + + const uint16_t disk = 0; + createBucket(bid, disk); + spi::dummy::DummyPersistence& provider( + dynamic_cast<spi::dummy::DummyPersistence&>(_node->getPersistenceProvider())); + CPPUNIT_ASSERT(!provider.isActive(spi::Bucket(bid, spi::PartitionId(disk)))); + + { + std::shared_ptr<api::SetBucketStateCommand> cmd( + new api::SetBucketStateCommand( + bid, api::SetBucketStateCommand::ACTIVE)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::SetBucketStateReply> reply( + std::dynamic_pointer_cast<api::SetBucketStateReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + } + + CPPUNIT_ASSERT(provider.isActive(spi::Bucket(bid, spi::PartitionId(disk)))); + { + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get( + bid, "foo")); + CPPUNIT_ASSERT(entry->info.isActive()); + } + // Trigger bucket info to be read back into the database + { + std::shared_ptr<ReadBucketInfo> cmd( + new ReadBucketInfo(bid)); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<ReadBucketInfoReply> reply( + std::dynamic_pointer_cast<ReadBucketInfoReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + } + // Should not have lost active flag + { + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get( + bid, "foo")); + CPPUNIT_ASSERT(entry->info.isActive()); + } + + { + std::shared_ptr<api::SetBucketStateCommand> cmd( + new api::SetBucketStateCommand( + bid, api::SetBucketStateCommand::INACTIVE)); + cmd->setAddress(address); + top.sendDown(cmd); + top.waitForMessages(1, _waitTime); + CPPUNIT_ASSERT_EQUAL((size_t) 1, top.getNumReplies()); + std::shared_ptr<api::SetBucketStateReply> reply( + std::dynamic_pointer_cast<api::SetBucketStateReply>( + top.getReply(0))); + top.reset(); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), reply->getResult()); + } + + CPPUNIT_ASSERT(!provider.isActive(spi::Bucket(bid, spi::PartitionId(disk)))); + { + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get( + bid, "foo")); + CPPUNIT_ASSERT(!entry->info.isActive()); + } +} + +void +FileStorManagerTest::testNotifyOwnerDistributorOnOutdatedSetBucketState() +{ + TestName testName("testNotifyOwnerDistributorOnOutdatedSetBucketState"); + DummyStorageLink top; + FileStorManager* manager( + new FileStorManager(config->getConfigId(), + _node->getPartitions(), + _node->getPersistenceProvider(), + _node->getComponentRegister())); + top.push_back(unique_ptr<StorageLink>(manager)); + + setClusterState("storage:2 distributor:2"); + top.open(); + + document::BucketId bid(getFirstBucketNotOwnedByDistributor(0)); + CPPUNIT_ASSERT(bid.getRawId() != 0); + createBucket(bid, 0); + + std::shared_ptr<api::SetBucketStateCommand> cmd( + new api::SetBucketStateCommand( + bid, api::SetBucketStateCommand::ACTIVE)); + cmd->setAddress(api::StorageMessageAddress( + "cluster", lib::NodeType::STORAGE, 1)); + cmd->setSourceIndex(0); + + top.sendDown(cmd); + top.waitForMessages(2, _waitTime); + + CPPUNIT_ASSERT_EQUAL(size_t(2), top.getNumReplies()); + // Not necessarily deterministic order. + int idxOffset = 0; + if (top.getReply(0)->getType() != api::MessageType::NOTIFYBUCKETCHANGE) { + ++idxOffset; + } + std::shared_ptr<api::NotifyBucketChangeCommand> notifyCmd( + std::dynamic_pointer_cast<api::NotifyBucketChangeCommand>( + top.getReply(idxOffset))); + std::shared_ptr<api::SetBucketStateReply> stateReply( + std::dynamic_pointer_cast<api::SetBucketStateReply>( + top.getReply(1 - idxOffset))); + + CPPUNIT_ASSERT(stateReply.get()); + CPPUNIT_ASSERT_EQUAL(ReturnCode(ReturnCode::OK), + stateReply->getResult()); + + CPPUNIT_ASSERT(notifyCmd.get()); + CPPUNIT_ASSERT_EQUAL(uint16_t(1), notifyCmd->getAddress()->getIndex()); + // Not necessary for this to be set since distributor does not insert this + // info into its db, but useful for debugging purposes. + CPPUNIT_ASSERT(notifyCmd->getBucketInfo().isActive()); +} + +void +FileStorManagerTest::testGetBucketDiffImplicitCreateBucket() +{ + TestName testName("testGetBucketDiffImplicitCreateBucket"); + DummyStorageLink top; + FileStorManager* manager( + new FileStorManager(config->getConfigId(), + _node->getPartitions(), + _node->getPersistenceProvider(), + _node->getComponentRegister())); + top.push_back(unique_ptr<StorageLink>(manager)); + setClusterState("storage:2 distributor:1"); + top.open(); + + document::BucketId bid(16, 4000); + + std::vector<api::MergeBucketCommand::Node> nodes; + nodes.push_back(1); + nodes.push_back(0); + + std::shared_ptr<api::GetBucketDiffCommand> cmd( + new api::GetBucketDiffCommand(bid, nodes, Timestamp(1000))); + cmd->setAddress(api::StorageMessageAddress( + "cluster", lib::NodeType::STORAGE, 1)); + cmd->setSourceIndex(0); + top.sendDown(cmd); + + api::GetBucketDiffReply* reply; + ASSERT_SINGLE_REPLY(api::GetBucketDiffReply, reply, top, _waitTime); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), + reply->getResult()); + { + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get( + bid, "foo")); + CPPUNIT_ASSERT(entry.exist()); + CPPUNIT_ASSERT(entry->info.isReady()); + } +} + +void +FileStorManagerTest::testMergeBucketImplicitCreateBucket() +{ + TestName testName("testMergeBucketImplicitCreateBucket"); + DummyStorageLink top; + FileStorManager* manager( + new FileStorManager(config->getConfigId(), + _node->getPartitions(), + _node->getPersistenceProvider(), + _node->getComponentRegister())); + top.push_back(unique_ptr<StorageLink>(manager)); + setClusterState("storage:3 distributor:1"); + top.open(); + + document::BucketId bid(16, 4000); + + std::vector<api::MergeBucketCommand::Node> nodes; + nodes.push_back(1); + nodes.push_back(2); + + std::shared_ptr<api::MergeBucketCommand> cmd( + new api::MergeBucketCommand(bid, nodes, Timestamp(1000))); + cmd->setAddress(api::StorageMessageAddress( + "cluster", lib::NodeType::STORAGE, 1)); + cmd->setSourceIndex(0); + top.sendDown(cmd); + + api::GetBucketDiffCommand* diffCmd; + ASSERT_SINGLE_REPLY(api::GetBucketDiffCommand, diffCmd, top, _waitTime); + { + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get( + bid, "foo")); + CPPUNIT_ASSERT(entry.exist()); + CPPUNIT_ASSERT(entry->info.isReady()); + } +} + +void +FileStorManagerTest::testNewlyCreatedBucketIsReady() +{ + TestName testName("testNewlyCreatedBucketIsReady"); + DummyStorageLink top; + FileStorManager* manager( + new FileStorManager(config->getConfigId(), + _node->getPartitions(), + _node->getPersistenceProvider(), + _node->getComponentRegister())); + top.push_back(unique_ptr<StorageLink>(manager)); + setClusterState("storage:2 distributor:1"); + top.open(); + + document::BucketId bid(16, 4000); + + std::shared_ptr<api::CreateBucketCommand> cmd( + new api::CreateBucketCommand(bid)); + cmd->setAddress(api::StorageMessageAddress( + "cluster", lib::NodeType::STORAGE, 1)); + cmd->setSourceIndex(0); + top.sendDown(cmd); + + api::CreateBucketReply* reply; + ASSERT_SINGLE_REPLY(api::CreateBucketReply, reply, top, _waitTime); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), + reply->getResult()); + { + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get( + bid, "foo")); + CPPUNIT_ASSERT(entry.exist()); + CPPUNIT_ASSERT(entry->info.isReady()); + CPPUNIT_ASSERT(!entry->info.isActive()); + } +} + +void +FileStorManagerTest::testCreateBucketSetsActiveFlagInDatabaseAndReply() +{ + TestFileStorComponents c(*this, "testNotifyOnSplitSourceOwnershipChanged"); + setClusterState("storage:2 distributor:1"); + + document::BucketId bid(16, 4000); + std::shared_ptr<api::CreateBucketCommand> cmd( + new api::CreateBucketCommand(bid)); + cmd->setAddress(api::StorageMessageAddress( + "cluster", lib::NodeType::STORAGE, 1)); + cmd->setSourceIndex(0); + cmd->setActive(true); + c.top.sendDown(cmd); + + api::CreateBucketReply* reply; + ASSERT_SINGLE_REPLY(api::CreateBucketReply, reply, c.top, _waitTime); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), + reply->getResult()); + { + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get( + bid, "foo")); + CPPUNIT_ASSERT(entry.exist()); + CPPUNIT_ASSERT(entry->info.isReady()); + CPPUNIT_ASSERT(entry->info.isActive()); + } +} + +} // storage diff --git a/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp b/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp new file mode 100644 index 00000000000..19b84ef475b --- /dev/null +++ b/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp @@ -0,0 +1,142 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <memory> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> +#include <tests/persistence/persistenceproviderwrapper.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <tests/persistence/filestorage/filestortestfixture.h> + +namespace storage { + +/** + * Effectively an integration test between the ModifiedBucketChecker storage + * link and the behavior of the filestor component. + */ +class FileStorModifiedBucketsTest : public FileStorTestFixture +{ +public: + void modifiedBucketsSendNotifyBucketChange(); + void fileStorRepliesToRecheckBucketCommands(); + + void modifyBuckets(uint32_t first, uint32_t count); + + spi::dummy::DummyPersistence& getDummyPersistence() { + return dynamic_cast<spi::dummy::DummyPersistence&>(_node->getPersistenceProvider()); + } + + CPPUNIT_TEST_SUITE(FileStorModifiedBucketsTest); + CPPUNIT_TEST(modifiedBucketsSendNotifyBucketChange); + CPPUNIT_TEST(fileStorRepliesToRecheckBucketCommands); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(FileStorModifiedBucketsTest); + +namespace { + +struct BucketCheckerInjector : FileStorTestFixture::StorageLinkInjector +{ + TestServiceLayerApp& _node; + FileStorTestFixture& _fixture; + BucketCheckerInjector(TestServiceLayerApp& node, + FileStorTestFixture& fixture) + : _node(node), + _fixture(fixture) + {} + void inject(DummyStorageLink& link) const { + link.push_back(std::unique_ptr<ModifiedBucketChecker>( + new ModifiedBucketChecker(_node.getComponentRegister(), + _node.getPersistenceProvider(), + _fixture._config->getConfigId()))); + } +}; + +void +assertIsNotifyCommandWithActiveBucket(api::StorageMessage& msg) +{ + api::NotifyBucketChangeCommand& cmd( + dynamic_cast<api::NotifyBucketChangeCommand&>(msg)); + CPPUNIT_ASSERT(cmd.getBucketInfo().isActive()); + CPPUNIT_ASSERT_EQUAL( + vespalib::string("StorageMessageAddress(Storage protocol, " + "cluster storage, nodetype distributor, index 0)"), + cmd.getAddress()->toString()); +} + +} + +void +FileStorModifiedBucketsTest::modifyBuckets(uint32_t first, uint32_t count) +{ + spi::BucketIdListResult::List buckets; + for (uint32_t i = 0; i < count; ++i) { + buckets.push_back(document::BucketId(16, first + i)); + _node->getPersistenceProvider().setActiveState( + spi::Bucket(buckets[i], spi::PartitionId(0)), + spi::BucketInfo::ACTIVE); + } + + getDummyPersistence().setModifiedBuckets(buckets); +} + +void +FileStorModifiedBucketsTest::modifiedBucketsSendNotifyBucketChange() +{ + BucketCheckerInjector bcj(*_node, *this); + TestFileStorComponents c(*this, "modifiedBucketsSendNotifyBucketChange", bcj); + setClusterState("storage:1 distributor:1"); + + uint32_t numBuckets = 10; + + for (uint32_t i = 0; i < numBuckets; ++i) { + document::BucketId bucket(16, i); + createBucket(spi::Bucket(bucket, spi::PartitionId(0))); + c.sendPut(bucket, DocumentIndex(0), PutTimestamp(1000)); + } + c.top.waitForMessages(numBuckets, MSG_WAIT_TIME); + c.top.reset(); + + modifyBuckets(0, numBuckets); + c.top.waitForMessages(numBuckets, MSG_WAIT_TIME); + + for (uint32_t i = 0; i < 10; ++i) { + assertIsNotifyCommandWithActiveBucket(*c.top.getReply(i)); + + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get( + document::BucketId(16, i), "foo")); + + CPPUNIT_ASSERT(entry->info.isActive()); + } +} + +void +FileStorModifiedBucketsTest::fileStorRepliesToRecheckBucketCommands() +{ + BucketCheckerInjector bcj(*_node, *this); + TestFileStorComponents c(*this, "fileStorRepliesToRecheckBucketCommands", bcj); + setClusterState("storage:1 distributor:1"); + + document::BucketId bucket(16, 0); + createBucket(spi::Bucket(bucket, spi::PartitionId(0))); + c.sendPut(bucket, DocumentIndex(0), PutTimestamp(1000)); + c.top.waitForMessages(1, MSG_WAIT_TIME); + c.top.reset(); + + modifyBuckets(0, 1); + c.top.waitForMessages(1, MSG_WAIT_TIME); + assertIsNotifyCommandWithActiveBucket(*c.top.getReply(0)); + + // If we don't reply to the recheck bucket commands, we won't trigger + // a new round of getModifiedBuckets and recheck commands. + c.top.reset(); + createBucket(spi::Bucket(document::BucketId(16, 1), spi::PartitionId(0))); + modifyBuckets(1, 1); + c.top.waitForMessages(1, MSG_WAIT_TIME); + assertIsNotifyCommandWithActiveBucket(*c.top.getReply(0)); +} + +} // storage + diff --git a/storage/src/tests/persistence/filestorage/filestortestfixture.cpp b/storage/src/tests/persistence/filestorage/filestortestfixture.cpp new file mode 100644 index 00000000000..69b109b5cfc --- /dev/null +++ b/storage/src/tests/persistence/filestorage/filestortestfixture.cpp @@ -0,0 +1,143 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <sstream> +#include <vespa/storage/persistence/messages.h> +#include <vespa/storage/persistence/filestorage/filestormanager.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <tests/persistence/filestorage/filestortestfixture.h> + +namespace storage { + +spi::LoadType FileStorTestFixture::defaultLoadType = spi::LoadType(0, "default"); +const uint32_t FileStorTestFixture::MSG_WAIT_TIME; + +void +FileStorTestFixture::setupDisks(uint32_t diskCount) +{ + _config.reset(new vdstestlib::DirConfig(getStandardConfig(true))); + + _config2.reset(new vdstestlib::DirConfig(*_config)); + _config2->getConfig("stor-server").set("root_folder", "vdsroot.2"); + _config2->getConfig("stor-devices").set("root_folder", "vdsroot.2"); + _config2->getConfig("stor-server").set("node_index", "1"); + + _smallConfig.reset(new vdstestlib::DirConfig(*_config)); + _node.reset(new TestServiceLayerApp(DiskCount(diskCount), NodeIndex(1), + _config->getConfigId())); + _testdoctype1 = _node->getTypeRepo()->getDocumentType("testdoctype1"); +} + +// Default provider setup which should work out of the box for most tests. +void +FileStorTestFixture::setUp() +{ + setupDisks(1); + _node->setPersistenceProvider( + spi::PersistenceProvider::UP( + new spi::dummy::DummyPersistence(_node->getTypeRepo(), 1))); +} + +void +FileStorTestFixture::tearDown() +{ + _node.reset(0); +} + +void +FileStorTestFixture::createBucket(const document::BucketId& bid) +{ + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + _node->getPersistenceProvider().createBucket( + spi::Bucket(bid, spi::PartitionId(0)), context); + + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get(bid, "foo", + StorBucketDatabase::CREATE_IF_NONEXISTING)); + entry->disk = 0; + entry->info = api::BucketInfo(0, 0, 0, 0, 0, true, false); + entry.write(); +} + +bool +FileStorTestFixture::bucketExistsInDb(const document::BucketId& bucket) const +{ + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get(bucket, "bucketExistsInDb")); + return entry.exist(); +} + +FileStorTestFixture::TestFileStorComponents::TestFileStorComponents( + FileStorTestFixture& fixture, + const char* testName, + const StorageLinkInjector& injector) + : _testName(testName), + _fixture(fixture), + manager(new FileStorManager(fixture._config->getConfigId(), + fixture._node->getPartitions(), + fixture._node->getPersistenceProvider(), + fixture._node->getComponentRegister())) +{ + injector.inject(top); + top.push_back(StorageLink::UP(manager)); + top.open(); +} + +api::StorageMessageAddress +FileStorTestFixture::TestFileStorComponents::makeSelfAddress() const { + return api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 0); +} + +void +FileStorTestFixture::TestFileStorComponents::sendDummyGet( + const document::BucketId& bid) +{ + std::ostringstream id; + id << "id:foo:testdoctype1:n=" << bid.getId() << ":0"; + std::shared_ptr<api::GetCommand> cmd( + new api::GetCommand(bid, document::DocumentId(id.str()), "[all]")); + cmd->setAddress(makeSelfAddress()); + cmd->setPriority(255); + top.sendDown(cmd); +} + +void +FileStorTestFixture::TestFileStorComponents::sendDummyGetDiff( + const document::BucketId& bid) +{ + std::vector<api::GetBucketDiffCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + std::shared_ptr<api::GetBucketDiffCommand> cmd( + new api::GetBucketDiffCommand(bid, nodes, 12345)); + cmd->setAddress(makeSelfAddress()); + cmd->setPriority(255); + top.sendDown(cmd); +} + +void +FileStorTestFixture::TestFileStorComponents::sendPut( + const document::BucketId& bid, + uint32_t docIdx, + uint64_t timestamp) +{ + std::ostringstream id; + id << "id:foo:testdoctype1:n=" << bid.getId() << ":" << docIdx; + document::Document::SP doc( + _fixture._node->getTestDocMan().createDocument("foobar", id.str())); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, doc, timestamp)); + cmd->setAddress(makeSelfAddress()); + top.sendDown(cmd); +} + +void +FileStorTestFixture::setClusterState(const std::string& state) +{ + _node->getStateUpdater().setClusterState( + lib::ClusterState::CSP(new lib::ClusterState(state))); +} + + +} // ns storage diff --git a/storage/src/tests/persistence/filestorage/filestortestfixture.h b/storage/src/tests/persistence/filestorage/filestortestfixture.h new file mode 100644 index 00000000000..4f1de549f47 --- /dev/null +++ b/storage/src/tests/persistence/filestorage/filestortestfixture.h @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vdstestlib/cppunit/macros.h> +#include <tests/common/testhelper.h> +#include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/storage/persistence/filestorage/filestormanager.h> +#include <vespa/storageapi/message/persistence.h> +#include <tests/common/dummystoragelink.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/testhelper.h> + +namespace storage { + +class FileStorTestFixture : public CppUnit::TestFixture +{ +public: + static spi::LoadType defaultLoadType; + + std::unique_ptr<TestServiceLayerApp> _node; + std::unique_ptr<vdstestlib::DirConfig> _config; + std::unique_ptr<vdstestlib::DirConfig> _config2; + std::unique_ptr<vdstestlib::DirConfig> _smallConfig; + const document::DocumentType* _testdoctype1; + + static const uint32_t MSG_WAIT_TIME = 60 * 1000; + + typedef uint32_t DocumentIndex; + typedef uint64_t PutTimestamp; + + void setUp() override; + void tearDown() override; + void setupDisks(uint32_t diskCount); + void createBucket(const document::BucketId& bid); + bool bucketExistsInDb(const document::BucketId& bucket) const; + + api::ReturnCode::Result resultOf(const api::StorageReply& reply) const { + return reply.getResult().getResult(); + } + void setClusterState(const std::string&); + + struct StorageLinkInjector + { + virtual ~StorageLinkInjector() {} + + virtual void inject(DummyStorageLink&) const = 0; + }; + + struct NoOpStorageLinkInjector : StorageLinkInjector + { + void inject(DummyStorageLink&) const {} + }; + + void + expectNoReplies(DummyStorageLink& link) { + CPPUNIT_ASSERT_EQUAL(size_t(0), link.getNumReplies()); + } + + template <typename ReplyType> + void + expectReply(DummyStorageLink& link, + api::ReturnCode::Result result) + { + link.waitForMessages(1, 60*1000); + api::StorageReply* reply( + dynamic_cast<ReplyType*>(link.getReply(0).get())); + if (reply == 0) { + std::ostringstream ss; + ss << "got unexpected reply " + << link.getReply(0)->toString(true); + CPPUNIT_FAIL(ss.str()); + } + CPPUNIT_ASSERT_EQUAL(result, reply->getResult().getResult()); + } + + template <typename ReplyType> + void + expectAbortedReply(DummyStorageLink& link) { + expectReply<ReplyType>(link, api::ReturnCode::ABORTED); + } + + template <typename ReplyType> + void + expectOkReply(DummyStorageLink& link) { + expectReply<ReplyType>(link, api::ReturnCode::OK); + } + + + struct TestFileStorComponents + { + private: + TestName _testName; + FileStorTestFixture& _fixture; + public: + DummyStorageLink top; + FileStorManager* manager; + + TestFileStorComponents(FileStorTestFixture& fixture, + const char* testName, + const StorageLinkInjector& i = NoOpStorageLinkInjector()); + + api::StorageMessageAddress makeSelfAddress() const; + + void sendDummyGet(const document::BucketId& bid); + void sendPut(const document::BucketId& bid, + uint32_t docIdx, + uint64_t timestamp); + void sendDummyGetDiff(const document::BucketId& bid); + }; +}; + +} // ns storage diff --git a/storage/src/tests/persistence/filestorage/forwardingmessagesender.h b/storage/src/tests/persistence/filestorage/forwardingmessagesender.h new file mode 100644 index 00000000000..691e291e534 --- /dev/null +++ b/storage/src/tests/persistence/filestorage/forwardingmessagesender.h @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/storage/common/messagesender.h> +#include <vespa/storage/common/storagelink.h> + +namespace storage { + +/** + * Simple implementation of MessageSender which simply forwards all messages + * to a provided storage link. + */ +struct ForwardingMessageSender : public MessageSender { + StorageLink& link; + + ForwardingMessageSender(StorageLink& l) : link(l) {} + + void sendCommand(const std::shared_ptr<api::StorageCommand> & cmd) + { link.sendUp(cmd); } + + void sendReply(const std::shared_ptr<api::StorageReply> & reply) + { link.sendUp(reply); } +}; + +} // storage + diff --git a/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp b/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp new file mode 100644 index 00000000000..ff9ec063555 --- /dev/null +++ b/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp @@ -0,0 +1,239 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vector> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/persistence/messages.h> +#include <tests/persistence/persistenceproviderwrapper.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <tests/persistence/filestorage/filestortestfixture.h> + +namespace storage { + +class MergeBlockingTest : public FileStorTestFixture +{ +public: + void setupDisks() { + FileStorTestFixture::setupDisks(1); + _node->setPersistenceProvider( + spi::PersistenceProvider::UP( + new spi::dummy::DummyPersistence(_node->getTypeRepo(), 1))); + } + +public: + void testRejectMergeForInconsistentInnerBucket(); + void testRejectMergeForInconsistentLeafBucket(); + void testRejectGetBucketDiffWithInconsistentBucket(); + void testRejectApplyDiffWhenBucketHasBecomeInconsistent(); + void testRejectApplyReplyWhenBucketHasBecomeInconsistent(); + void testRejectGetDiffReplyWhenBucketHasBecomeInconsistent(); + void testRejectMergeWhenLowUsedBitCount(); + + void setUp() override; + + CPPUNIT_TEST_SUITE(MergeBlockingTest); + CPPUNIT_TEST(testRejectMergeForInconsistentInnerBucket); + CPPUNIT_TEST(testRejectMergeForInconsistentLeafBucket); + CPPUNIT_TEST(testRejectGetBucketDiffWithInconsistentBucket); + CPPUNIT_TEST(testRejectApplyDiffWhenBucketHasBecomeInconsistent); + CPPUNIT_TEST(testRejectApplyReplyWhenBucketHasBecomeInconsistent); + CPPUNIT_TEST(testRejectGetDiffReplyWhenBucketHasBecomeInconsistent); + CPPUNIT_TEST(testRejectMergeWhenLowUsedBitCount); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(MergeBlockingTest); + +void +MergeBlockingTest::setUp() +{ + setupDisks(); +} + +namespace { + +api::StorageMessageAddress +makeAddress() { + return api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 0); +} + +void +assignCommandMeta(api::StorageCommand& msg) { + msg.setAddress(makeAddress()); + msg.setSourceIndex(0); +} + +std::vector<api::MergeBucketCommand::Node> +getNodes() { + std::vector<api::MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + return nodes; +} + +std::vector<api::MergeBucketCommand::Node> +getNodesWithForwarding() { + std::vector<api::MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + return nodes; +} + +std::shared_ptr<api::MergeBucketCommand> +createMerge(const document::BucketId& bucket) { + std::shared_ptr<api::MergeBucketCommand> cmd( + new api::MergeBucketCommand(bucket, getNodes(), api::Timestamp(1000))); + assignCommandMeta(*cmd); + return cmd; +} + +std::shared_ptr<api::GetBucketDiffCommand> +createGetDiff(const document::BucketId& bucket, + const std::vector<api::MergeBucketCommand::Node>& nodes) +{ + std::shared_ptr<api::GetBucketDiffCommand> cmd( + new api::GetBucketDiffCommand(bucket, nodes, api::Timestamp(1000))); + assignCommandMeta(*cmd); + return cmd; +} + +std::shared_ptr<api::ApplyBucketDiffCommand> +createApplyDiff(const document::BucketId& bucket, + const std::vector<api::MergeBucketCommand::Node>& nodes) { + std::shared_ptr<api::ApplyBucketDiffCommand> cmd( + new api::ApplyBucketDiffCommand(bucket, nodes, 1024*1024)); + assignCommandMeta(*cmd); + return cmd; +} + +const document::BucketId leafBucket(17, 1); +const document::BucketId innerBucket(16, 1); +const document::BucketId innerBucket2(15, 1); + +} + +void +MergeBlockingTest::testRejectMergeForInconsistentInnerBucket() +{ + TestFileStorComponents c(*this, "testRejectMergeForInconsistentInnerBucket"); + createBucket(leafBucket); + + std::shared_ptr<api::MergeBucketCommand> cmd(createMerge(innerBucket)); + c.top.sendDown(cmd); + + expectAbortedReply<api::MergeBucketReply>(c.top); + CPPUNIT_ASSERT(!bucketExistsInDb(innerBucket)); +} + +void +MergeBlockingTest::testRejectMergeForInconsistentLeafBucket() +{ + TestFileStorComponents c(*this, "testRejectMergeForInconsistentInnerBucket"); + createBucket(innerBucket); + + std::shared_ptr<api::MergeBucketCommand> cmd(createMerge(leafBucket)); + c.top.sendDown(cmd); + + expectAbortedReply<api::MergeBucketReply>(c.top); + CPPUNIT_ASSERT(!bucketExistsInDb(leafBucket)); +} + +void +MergeBlockingTest::testRejectGetBucketDiffWithInconsistentBucket() +{ + TestFileStorComponents c(*this, "testRejectGetBucketDiffWithInconsistentBucket"); + CPPUNIT_ASSERT(innerBucket.contains(leafBucket)); + createBucket(innerBucket); + + std::shared_ptr<api::GetBucketDiffCommand> cmd(createGetDiff(leafBucket, getNodes())); + c.top.sendDown(cmd); + + expectAbortedReply<api::GetBucketDiffReply>(c.top); + CPPUNIT_ASSERT(!bucketExistsInDb(leafBucket)); +} + +void +MergeBlockingTest::testRejectApplyDiffWhenBucketHasBecomeInconsistent() +{ + TestFileStorComponents c(*this, "testRejectApplyDiffWhenBucketHasBecomeInconsistent"); + createBucket(leafBucket); + createBucket(innerBucket); + + std::shared_ptr<api::ApplyBucketDiffCommand> applyDiff( + createApplyDiff(innerBucket, getNodes())); + c.top.sendDown(applyDiff); + + expectAbortedReply<api::ApplyBucketDiffReply>(c.top); +} + +void +MergeBlockingTest::testRejectApplyReplyWhenBucketHasBecomeInconsistent() +{ + TestFileStorComponents c(*this, "testRejectApplyReplyWhenBucketHasBecomeInconsistent"); + createBucket(innerBucket); + + std::shared_ptr<api::ApplyBucketDiffCommand> applyDiff( + createApplyDiff(innerBucket, getNodesWithForwarding())); + c.top.sendDown(applyDiff); + c.top.waitForMessages(1, MSG_WAIT_TIME); + + api::StorageMessage::SP fwdDiff( + c.top.getAndRemoveMessage(api::MessageType::APPLYBUCKETDIFF)); + api::ApplyBucketDiffCommand& diffCmd( + dynamic_cast<api::ApplyBucketDiffCommand&>(*fwdDiff)); + + api::ApplyBucketDiffReply::SP diffReply( + new api::ApplyBucketDiffReply(diffCmd)); + createBucket(leafBucket); + c.top.sendDown(diffReply); + + expectAbortedReply<api::ApplyBucketDiffReply>(c.top); +} + +void +MergeBlockingTest::testRejectGetDiffReplyWhenBucketHasBecomeInconsistent() +{ + TestFileStorComponents c(*this, "testRejectGetDiffReplyWhenBucketHasBecomeInconsistent"); + createBucket(innerBucket); + + std::shared_ptr<api::GetBucketDiffCommand> getDiff( + createGetDiff(innerBucket, getNodesWithForwarding())); + c.top.sendDown(getDiff); + c.top.waitForMessages(1, MSG_WAIT_TIME); + + api::StorageMessage::SP fwdDiff( + c.top.getAndRemoveMessage(api::MessageType::GETBUCKETDIFF)); + api::GetBucketDiffCommand& diffCmd( + dynamic_cast<api::GetBucketDiffCommand&>(*fwdDiff)); + + api::GetBucketDiffReply::SP diffReply( + new api::GetBucketDiffReply(diffCmd)); + createBucket(innerBucket2); + c.top.sendDown(diffReply); + + expectAbortedReply<api::GetBucketDiffReply>(c.top); +} + +/** + * Test case for buckets in ticket 6389558, comment #4. + */ +void +MergeBlockingTest::testRejectMergeWhenLowUsedBitCount() +{ + document::BucketId superBucket(1, 0x1); + document::BucketId subBucket(2, 0x1); + + CPPUNIT_ASSERT(superBucket.contains(subBucket)); + + TestFileStorComponents c(*this, "testRejectMergeWithInconsistentBucket"); + createBucket(superBucket); + + std::shared_ptr<api::MergeBucketCommand> cmd(createMerge(subBucket)); + c.top.sendDown(cmd); + + expectAbortedReply<api::MergeBucketReply>(c.top); + CPPUNIT_ASSERT(!bucketExistsInDb(subBucket)); +} + +} // ns storage diff --git a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp new file mode 100644 index 00000000000..848799fde95 --- /dev/null +++ b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp @@ -0,0 +1,214 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> + +#include <vespa/vdstestlib/cppunit/macros.h> +#include <tests/common/testhelper.h> +#include <tests/common/storagelinktest.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> +#include <vespa/storage/persistence/messages.h> + +namespace storage { + +class ModifiedBucketCheckerTest : public CppUnit::TestFixture +{ +public: + enum { + MESSAGE_WAIT_TIME = 60*2 + }; + + void setUp(); + void tearDown(); + + void testModifiedBucketThreadSendsRecheckBucketCommands(); + void testDoNotCheckModifiedBucketsIfAlreadyPending(); + void testBucketCheckerOnlySwallowsRecheckBucketReplies(); + void testRecheckRequestsAreChunked(); + void testInvalidChunkSizeConfigIsRejected(); + + CPPUNIT_TEST_SUITE(ModifiedBucketCheckerTest); + CPPUNIT_TEST(testModifiedBucketThreadSendsRecheckBucketCommands); + CPPUNIT_TEST(testDoNotCheckModifiedBucketsIfAlreadyPending); + CPPUNIT_TEST(testBucketCheckerOnlySwallowsRecheckBucketReplies); + CPPUNIT_TEST(testRecheckRequestsAreChunked); + CPPUNIT_TEST(testInvalidChunkSizeConfigIsRejected); + CPPUNIT_TEST_SUITE_END(); +private: + spi::dummy::DummyPersistence& getDummyPersistence() { + return static_cast<spi::dummy::DummyPersistence&>( + _node->getPersistenceProvider()); + } + void expectCommandsAndSendReplies(uint32_t count, uint32_t firstBucket); + void modifyBuckets(uint32_t count, uint32_t firstBucket); + void replyToAll(const std::vector<api::StorageMessage::SP>& messages, + uint32_t firstBucket); + + std::unique_ptr<DummyStorageLink> _top; + ModifiedBucketChecker* _handler; + DummyStorageLink* _bottom; + + std::unique_ptr<TestServiceLayerApp> _node; + std::unique_ptr<vdstestlib::DirConfig> _config; +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(ModifiedBucketCheckerTest); + +void +ModifiedBucketCheckerTest::setUp() +{ + _config.reset(new vdstestlib::DirConfig(getStandardConfig(true))); + _node.reset(new TestServiceLayerApp(DiskCount(1), NodeIndex(0), + _config->getConfigId())); + _node->setupDummyPersistence(); + + _top.reset(new DummyStorageLink); + _handler = new ModifiedBucketChecker(_node->getComponentRegister(), + _node->getPersistenceProvider(), + _config->getConfigId()); + _top->push_back(std::unique_ptr<StorageLink>(_handler)); + _bottom = new DummyStorageLink; + _handler->push_back(std::unique_ptr<StorageLink>(_bottom)); +} + +void +ModifiedBucketCheckerTest::tearDown() +{ + _top->close(); + _top.reset(0); + _node.reset(0); + _config.reset(0); +} + +void +ModifiedBucketCheckerTest::modifyBuckets(uint32_t count, uint32_t firstBucket) +{ + spi::BucketIdListResult::List buckets; + for (uint32_t i = firstBucket; i < firstBucket + count; ++i) { + buckets.push_back(document::BucketId(16, i)); + } + getDummyPersistence().setModifiedBuckets(buckets); +} + +void +ModifiedBucketCheckerTest::replyToAll( + const std::vector<api::StorageMessage::SP>& messages, + uint32_t firstBucket) +{ + for (uint32_t i = 0; i < messages.size(); ++i) { + RecheckBucketInfoCommand& cmd( + dynamic_cast<RecheckBucketInfoCommand&>(*messages[i])); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, i+firstBucket), + cmd.getBucketId()); + _bottom->sendUp(cmd.makeReply()); + } +} + +void +ModifiedBucketCheckerTest::expectCommandsAndSendReplies( + uint32_t count, uint32_t firstBucket) +{ + std::vector<api::StorageMessage::SP> messages(_bottom->getCommandsOnce()); + CPPUNIT_ASSERT_EQUAL(size_t(count), messages.size()); + replyToAll(messages, firstBucket); +} + +void +ModifiedBucketCheckerTest::testModifiedBucketThreadSendsRecheckBucketCommands() +{ + _top->open(); // Multi-threaded test + modifyBuckets(3, 0); + // Should now get 3 RecheckBucketInfo commands down the dummy link. + _bottom->waitForMessages(3, MESSAGE_WAIT_TIME); + expectCommandsAndSendReplies(3, 0); + // No replies should reach top link + CPPUNIT_ASSERT_EQUAL(size_t(0), _top->getNumReplies()); +} + +void +ModifiedBucketCheckerTest::testDoNotCheckModifiedBucketsIfAlreadyPending() +{ + _handler->setUnitTestingSingleThreadedMode(); + _top->open(); + modifyBuckets(3, 0); + _handler->tick(); + + std::vector<api::StorageMessage::SP> messages(_bottom->getCommandsOnce()); + CPPUNIT_ASSERT_EQUAL(size_t(3), messages.size()); + + modifyBuckets(3, 3); + _handler->tick(); + expectCommandsAndSendReplies(0, 0); + // After replies received, tick should send new requests again. + replyToAll(messages, 0); + _handler->tick(); + expectCommandsAndSendReplies(3, 3); +} + +void +ModifiedBucketCheckerTest::testBucketCheckerOnlySwallowsRecheckBucketReplies() +{ + _top->open(); + DestroyIteratorCommand cmd(spi::IteratorId(123)); + _bottom->sendUp(api::StorageMessage::SP(cmd.makeReply())); + CPPUNIT_ASSERT_EQUAL(size_t(1), _top->getNumReplies()); +} + +void +ModifiedBucketCheckerTest::testRecheckRequestsAreChunked() +{ + namespace cfgns = vespa::config::content::core; + _handler->setUnitTestingSingleThreadedMode(); + _top->open(); + cfgns::StorServerConfigBuilder cfgBuilder; + cfgBuilder.bucketRecheckingChunkSize = 2; + _handler->configure(std::unique_ptr<cfgns::StorServerConfig>( + new cfgns::StorServerConfig(cfgBuilder))); + + modifyBuckets(5, 0); + _handler->tick(); + + modifyBuckets(1, 10); // should not be checked yet; + // Rechecks should now be done in 3 chunks of 2, 2 and 1 each, respectively. + expectCommandsAndSendReplies(2, 0); + + _handler->tick(); + expectCommandsAndSendReplies(2, 2); + + _handler->tick(); + expectCommandsAndSendReplies(1, 4); + + // New round of fetching + _handler->tick(); + expectCommandsAndSendReplies(1, 10); + + // And done! + _handler->tick(); + expectCommandsAndSendReplies(0, 0); +} + +void +ModifiedBucketCheckerTest::testInvalidChunkSizeConfigIsRejected() +{ + namespace cfgns = vespa::config::content::core; + _handler->setUnitTestingSingleThreadedMode(); + _top->open(); + cfgns::StorServerConfigBuilder cfgBuilder; + cfgBuilder.bucketRecheckingChunkSize = 0; + try { + _handler->configure(std::unique_ptr<cfgns::StorServerConfig>( + new cfgns::StorServerConfig(cfgBuilder))); + CPPUNIT_FAIL("Expected bad config to be rejected"); + } catch (const config::InvalidConfigException&) { + // Happy days + } catch (...) { + CPPUNIT_FAIL("Got unexpected exception"); + } +} + +// RecheckBucketInfoCommand handling is done in persistence threads, +// so that functionality is tested in the filestor tests. + +} // ns storage + diff --git a/storage/src/tests/persistence/filestorage/operationabortingtest.cpp b/storage/src/tests/persistence/filestorage/operationabortingtest.cpp new file mode 100644 index 00000000000..0d6583cacdb --- /dev/null +++ b/storage/src/tests/persistence/filestorage/operationabortingtest.cpp @@ -0,0 +1,470 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vector> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/persistence/messages.h> +#include <tests/persistence/persistenceproviderwrapper.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <tests/persistence/filestorage/filestortestfixture.h> +#include <vespa/vespalib/util/barrier.h> +#include <vespa/vespalib/util/thread.h> + +#include <vespa/log/log.h> +LOG_SETUP(".operationabortingtest"); + +namespace storage { + +namespace { + +// Exploit the fact that PersistenceProviderWrapper already provides a forwarding +// implementation of all SPI calls, so we can selectively override. +class BlockingMockProvider : public PersistenceProviderWrapper +{ + vespalib::Barrier& _queueBarrier; + vespalib::Barrier& _completionBarrier; +public: + typedef std::unique_ptr<BlockingMockProvider> UP; + + mutable uint32_t _bucketInfoInvocations; + uint32_t _createBucketInvocations; + uint32_t _deleteBucketInvocations; + + BlockingMockProvider(spi::PersistenceProvider& wrappedProvider, + vespalib::Barrier& queueBarrier, + vespalib::Barrier& completionBarrier) + : PersistenceProviderWrapper(wrappedProvider), + _queueBarrier(queueBarrier), + _completionBarrier(completionBarrier), + _bucketInfoInvocations(0), + _createBucketInvocations(0), + _deleteBucketInvocations(0) + {} + + spi::Result put(const spi::Bucket& bucket, + spi::Timestamp timestamp, + const document::Document::SP& doc, + spi::Context& context) override + { + (void) bucket; + (void) timestamp; + (void) doc; + (void) context; + _queueBarrier.await(); + // message abort stage with active opertion in disk queue + FastOS_Thread::Sleep(75); + _completionBarrier.await(); + // test finished + return spi::Result(); + } + + spi::BucketInfoResult getBucketInfo(const spi::Bucket& bucket) const override { + ++_bucketInfoInvocations; + return PersistenceProviderWrapper::getBucketInfo(bucket); + } + + spi::Result createBucket(const spi::Bucket& bucket, spi::Context& ctx) override { + ++_createBucketInvocations; + return PersistenceProviderWrapper::createBucket(bucket, ctx); + } + + spi::Result deleteBucket(const spi::Bucket& bucket, spi::Context& ctx) override { + ++_deleteBucketInvocations; + return PersistenceProviderWrapper::deleteBucket(bucket, ctx); + } +}; + +spi::LoadType defaultLoadType(0, "default"); + +} + +class OperationAbortingTest : public FileStorTestFixture +{ +public: + spi::PersistenceProvider::UP _dummyProvider; + BlockingMockProvider* _blockingProvider; + std::unique_ptr<vespalib::Barrier> _queueBarrier; + std::unique_ptr<vespalib::Barrier> _completionBarrier; + + void setupDisks(uint32_t diskCount, uint32_t queueBarrierThreads) { + FileStorTestFixture::setupDisks(diskCount); + _dummyProvider.reset(new spi::dummy::DummyPersistence( + _node->getTypeRepo(), diskCount)); + _queueBarrier.reset(new vespalib::Barrier(queueBarrierThreads)); + _completionBarrier.reset(new vespalib::Barrier(2)); + _blockingProvider = new BlockingMockProvider(*_dummyProvider, + *_queueBarrier, *_completionBarrier); + + _node->setPersistenceProvider( + spi::PersistenceProvider::UP(_blockingProvider)); + } + + void validateReplies(DummyStorageLink& link, + size_t repliesTotal, + const std::vector<document::BucketId>& okReplies, + const std::vector<document::BucketId>& abortedGetDiffs); + + void doTestSpecificOperationsNotAborted( + const char* testName, + const std::vector<api::StorageMessage::SP>& msgs, + bool shouldCreateBucketInitially); + + api::BucketInfo getBucketInfoFromDB(const document::BucketId&) const; + +public: + void testAbortMessageClearsRelevantQueuedOperations(); + void testWaitForCurrentOperationCompletionForAbortedBucket(); + void testDoNotAbortCreateBucketCommands(); + void testDoNotAbortRecheckBucketCommands(); + void testDoNotAbortDeleteBucketCommands(); + + void setUp() override; + + CPPUNIT_TEST_SUITE(OperationAbortingTest); + CPPUNIT_TEST(testAbortMessageClearsRelevantQueuedOperations); + CPPUNIT_TEST(testWaitForCurrentOperationCompletionForAbortedBucket); + CPPUNIT_TEST(testDoNotAbortCreateBucketCommands); + CPPUNIT_TEST(testDoNotAbortRecheckBucketCommands); + CPPUNIT_TEST(testDoNotAbortDeleteBucketCommands); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(OperationAbortingTest); + +namespace { + +template <typename T, typename Collection> +bool +existsIn(const T& elem, const Collection& collection) { + return (std::find(collection.begin(), collection.end(), elem) + != collection.end()); +} + +} + +void +OperationAbortingTest::setUp() +{ +} + +void +OperationAbortingTest::validateReplies( + DummyStorageLink& link, + size_t repliesTotal, + const std::vector<document::BucketId>& okReplies, + const std::vector<document::BucketId>& abortedGetDiffs) +{ + link.waitForMessages(repliesTotal, MSG_WAIT_TIME); + CPPUNIT_ASSERT_EQUAL(repliesTotal, link.getNumReplies()); + + for (uint32_t i = 0; i < repliesTotal; ++i) { + api::StorageReply& reply( + dynamic_cast<api::StorageReply&>(*link.getReply(i))); + LOG(info, "Checking reply %s", reply.toString(true).c_str()); + switch (static_cast<uint32_t>(reply.getType().getId())) { + case api::MessageType::PUT_REPLY_ID: + case api::MessageType::CREATEBUCKET_REPLY_ID: + case api::MessageType::DELETEBUCKET_REPLY_ID: + case api::MessageType::GET_REPLY_ID: + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::OK, resultOf(reply)); + break; + case api::MessageType::GETBUCKETDIFF_REPLY_ID: + { + api::GetBucketDiffReply& gr( + static_cast<api::GetBucketDiffReply&>(reply)); + if (existsIn(gr.getBucketId(), abortedGetDiffs)) { + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ABORTED, resultOf(reply)); + } else { + CPPUNIT_ASSERT(existsIn(gr.getBucketId(), okReplies)); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::OK, resultOf(reply)); + } + break; + } + case api::MessageType::INTERNAL_REPLY_ID: + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::OK, resultOf(reply)); + break; + default: + CPPUNIT_FAIL("got unknown reply type"); + } + } +} + +namespace { + +template <typename Container> +AbortBucketOperationsCommand::SP +makeAbortCmd(const Container& buckets) +{ + std::unique_ptr<AbortBucketOperationsCommand::AbortPredicate> pred( + new AbortBucketOperationsCommand::ExplicitBucketSetPredicate( + buckets.begin(), buckets.end())); + AbortBucketOperationsCommand::SP cmd( + new AbortBucketOperationsCommand(std::move(pred))); + return cmd; +} + +} + +void +OperationAbortingTest::testAbortMessageClearsRelevantQueuedOperations() +{ + uint32_t queueBarrierThreads = 2; + setupDisks(1, queueBarrierThreads); + TestFileStorComponents c(*this, "testAbortMessageClearsRelevantQueuedOperations"); + document::BucketId bucket(16, 1); + createBucket(bucket); + LOG(info, "Sending put to trigger thread barrier"); + c.sendPut(bucket, DocumentIndex(0), PutTimestamp(1000)); + LOG(info, "waiting for test and persistence thread to reach barriers"); + _queueBarrier->await(); + LOG(info, "barrier passed"); + /* + * All load we send down to filestor from now on wil be enqueued, as the + * persistence thread is blocked. + * + * Cannot abort the bucket we're blocking the thread on since we'll + * deadlock the test if we do. + */ + std::vector<document::BucketId> bucketsToAbort; + bucketsToAbort.push_back(document::BucketId(16, 3)); + bucketsToAbort.push_back(document::BucketId(16, 5)); + std::vector<document::BucketId> bucketsToKeep; + bucketsToKeep.push_back(document::BucketId(16, 2)); + bucketsToKeep.push_back(document::BucketId(16, 4)); + + for (uint32_t i = 0; i < bucketsToAbort.size(); ++i) { + createBucket(bucketsToAbort[i]); + c.sendDummyGetDiff(bucketsToAbort[i]); + } + for (uint32_t i = 0; i < bucketsToKeep.size(); ++i) { + createBucket(bucketsToKeep[i]); + c.sendDummyGetDiff(bucketsToKeep[i]); + } + + AbortBucketOperationsCommand::SP abortCmd(makeAbortCmd(bucketsToAbort)); + c.top.sendDown(abortCmd); + + LOG(info, "waiting on completion barrier"); + _completionBarrier->await(); + + // put+abort+get replies + size_t expectedMsgs(2 + bucketsToAbort.size() + bucketsToKeep.size()); + LOG(info, "barrier passed, waiting for %zu replies", expectedMsgs); + + validateReplies(c.top, expectedMsgs, bucketsToKeep, bucketsToAbort); +} + +namespace { + +/** + * Sending an abort while we're processing a message for a bucket in its set + * will block until the operation has completed. Therefore we logically cannot + * do any operations to trigger the operation to complete after the send in + * the same thread as we're sending in... + */ +class SendTask : public vespalib::Runnable +{ + AbortBucketOperationsCommand::SP _abortCmd; + vespalib::Barrier& _queueBarrier; + StorageLink& _downLink; +public: + SendTask(const AbortBucketOperationsCommand::SP& abortCmd, + vespalib::Barrier& queueBarrier, + StorageLink& downLink) + : _abortCmd(abortCmd), + _queueBarrier(queueBarrier), + _downLink(downLink) + {} + + void run() { + // Best-effort synchronized starting + _queueBarrier.await(); + _downLink.sendDown(_abortCmd); + } +}; + +} + +/** + * This test basically is not fully deterministic in that it tests cross-thread + * behavior on mutexes that are not visible to the thread itself and where there + * are no available side-effects to consistently sync around. However, it should + * impose sufficient ordering guarantees that it never provides false positives + * as long as the tested functionality is in fact correct. + */ +void +OperationAbortingTest::testWaitForCurrentOperationCompletionForAbortedBucket() +{ + uint32_t queueBarrierThreads = 3; + setupDisks(1, queueBarrierThreads); + TestFileStorComponents c(*this, "testWaitForCurrentOperationCompletionForAbortedBucket"); + + document::BucketId bucket(16, 1); + createBucket(bucket); + LOG(info, "Sending put to trigger thread barrier"); + c.sendPut(bucket, DocumentIndex(0), PutTimestamp(1000)); + + std::vector<document::BucketId> abortSet { bucket }; + AbortBucketOperationsCommand::SP abortCmd(makeAbortCmd(abortSet)); + + SendTask sendTask(abortCmd, *_queueBarrier, c.top); + vespalib::Thread thread(sendTask); + thread.start(); + + LOG(info, "waiting for threads to reach barriers"); + _queueBarrier->await(); + LOG(info, "barrier passed"); + + LOG(info, "waiting on completion barrier"); + _completionBarrier->await(); + + thread.stop(); + thread.join(); + + // If waiting works, put reply shall always be ordered before the internal + // reply, as it must finish processing fully before the abort returns. + c.top.waitForMessages(2, MSG_WAIT_TIME); + CPPUNIT_ASSERT_EQUAL(size_t(2), c.top.getNumReplies()); + CPPUNIT_ASSERT_EQUAL(api::MessageType::PUT_REPLY, + c.top.getReply(0)->getType()); + CPPUNIT_ASSERT_EQUAL(api::MessageType::INTERNAL_REPLY, + c.top.getReply(1)->getType()); +} + +void +OperationAbortingTest::testDoNotAbortCreateBucketCommands() +{ + document::BucketId bucket(16, 1); + std::vector<api::StorageMessage::SP> msgs; + msgs.push_back(api::StorageMessage::SP(new api::CreateBucketCommand(bucket))); + + bool shouldCreateBucketInitially(false); + doTestSpecificOperationsNotAborted( + "testDoNotAbortCreateBucketCommands", + msgs, + shouldCreateBucketInitially); +} + +void +OperationAbortingTest::testDoNotAbortRecheckBucketCommands() +{ + document::BucketId bucket(16, 1); + std::vector<api::StorageMessage::SP> msgs; + msgs.push_back(api::StorageMessage::SP(new RecheckBucketInfoCommand(bucket))); + + bool shouldCreateBucketInitially(true); + doTestSpecificOperationsNotAborted( + "testDoNotAbortRecheckBucketCommands", + msgs, + shouldCreateBucketInitially); +} + +api::BucketInfo +OperationAbortingTest::getBucketInfoFromDB(const document::BucketId& id) const +{ + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get(id, "foo", + StorBucketDatabase::CREATE_IF_NONEXISTING)); + CPPUNIT_ASSERT(entry.exist()); + return entry->info; +} + +void +OperationAbortingTest::testDoNotAbortDeleteBucketCommands() +{ + document::BucketId bucket(16, 1); + std::vector<api::StorageMessage::SP> msgs; + api::DeleteBucketCommand::SP cmd(new api::DeleteBucketCommand(bucket)); + msgs.push_back(cmd); + + bool shouldCreateBucketInitially(true); + doTestSpecificOperationsNotAborted( + "testDoNotAbortRecheckBucketCommands", + msgs, + shouldCreateBucketInitially); +} + +void +OperationAbortingTest::doTestSpecificOperationsNotAborted( + const char* testName, + const std::vector<api::StorageMessage::SP>& msgs, + bool shouldCreateBucketInitially) +{ + uint32_t queueBarrierThreads = 2; + setupDisks(1, queueBarrierThreads); + TestFileStorComponents c(*this, testName); + document::BucketId bucket(16, 1); + document::BucketId blockerBucket(16, 2); + + if (shouldCreateBucketInitially) { + createBucket(bucket); + } + createBucket(blockerBucket); + LOG(info, "Sending put to trigger thread barrier"); + c.sendPut(blockerBucket, DocumentIndex(0), PutTimestamp(1000)); + LOG(info, "waiting for test and persistence thread to reach barriers"); + _queueBarrier->await(); + LOG(info, "barrier passed"); + + uint32_t expectedCreateBuckets = 0; + uint32_t expectedDeleteBuckets = 0; + uint32_t expectedBucketInfoInvocations = 1; // from blocker put + uint32_t expectedRecheckReplies = 0; + + for (uint32_t i = 0; i < msgs.size(); ++i) { + switch (msgs[i]->getType().getId()) { + case api::MessageType::CREATEBUCKET_ID: + ++expectedCreateBuckets; + break; + case api::MessageType::DELETEBUCKET_ID: + { + api::DeleteBucketCommand& delCmd( + dynamic_cast<api::DeleteBucketCommand&>(*msgs[i])); + delCmd.setBucketInfo(getBucketInfoFromDB(delCmd.getBucketId())); + } + ++expectedDeleteBuckets; + ++expectedBucketInfoInvocations; + break; + case api::MessageType::INTERNAL_ID: + ++expectedRecheckReplies; + ++expectedBucketInfoInvocations; + break; + default: + CPPUNIT_FAIL("unsupported message type"); + } + c.top.sendDown(msgs[i]); + } + + std::vector<document::BucketId> abortSet { bucket }; + AbortBucketOperationsCommand::SP abortCmd(makeAbortCmd(abortSet)); + c.top.sendDown(abortCmd); + + LOG(info, "waiting on completion barrier"); + _completionBarrier->await(); + + // At this point, the recheck command is still either enqueued, is processing + // or has finished. Since it does not generate any replies, send a low priority + // get which will wait until it has finished processing. + c.sendDummyGet(blockerBucket); + + // put+abort+get + any other creates/deletes/rechecks + size_t expectedMsgs(3 + expectedCreateBuckets + expectedDeleteBuckets + + expectedRecheckReplies); + LOG(info, "barrier passed, waiting for %zu replies", expectedMsgs); + + std::vector<document::BucketId> okReplies; + okReplies.push_back(bucket); + okReplies.push_back(blockerBucket); + std::vector<document::BucketId> abortedGetDiffs; + validateReplies(c.top, expectedMsgs, okReplies, abortedGetDiffs); + + CPPUNIT_ASSERT_EQUAL(expectedBucketInfoInvocations, + _blockingProvider->_bucketInfoInvocations); + CPPUNIT_ASSERT_EQUAL(expectedCreateBuckets + (shouldCreateBucketInitially ? 2 : 1), + _blockingProvider->_createBucketInvocations); + CPPUNIT_ASSERT_EQUAL(expectedDeleteBuckets, + _blockingProvider->_deleteBucketInvocations); +} + + +} // storage diff --git a/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp b/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp new file mode 100644 index 00000000000..9b492a3aaa6 --- /dev/null +++ b/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storageapi/message/bucket.h> +#include <tests/persistence/persistenceproviderwrapper.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <tests/persistence/filestorage/filestortestfixture.h> + +namespace storage { + +class SanityCheckedDeleteTest : public FileStorTestFixture +{ +public: + void testDeleteBucketFailsWhenProviderOutOfSync(); + + CPPUNIT_TEST_SUITE(SanityCheckedDeleteTest); + CPPUNIT_TEST(testDeleteBucketFailsWhenProviderOutOfSync); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(SanityCheckedDeleteTest); + +void +SanityCheckedDeleteTest::testDeleteBucketFailsWhenProviderOutOfSync() +{ + TestFileStorComponents c(*this, "testDeleteBucketFailsWhenProviderOutOfSync"); + document::BucketId bucket(8, 123); + document::BucketId syncBucket(8, 234); + spi::Bucket spiBucket(bucket, spi::PartitionId(0)); + + createBucket(bucket); + // Send a put to ensure bucket isn't empty. + c.sendPut(bucket, DocumentIndex(0), PutTimestamp(1000)); + c.top.waitForMessages(1, MSG_WAIT_TIME); + c.top.getRepliesOnce(); + spi::BucketInfo infoBefore( + _node->getPersistenceProvider() + .getBucketInfo(spiBucket).getBucketInfo()); + + createBucket(syncBucket); + + api::BucketInfo serviceLayerInfo(1, 2, 3, 4, 5, true, false); + { + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get(bucket, "foo", + StorBucketDatabase::CREATE_IF_NONEXISTING)); + entry->disk = 0; + entry->info = serviceLayerInfo; + entry.write(); + } + + std::shared_ptr<api::DeleteBucketCommand> cmd( + new api::DeleteBucketCommand(bucket)); + cmd->setBucketInfo(serviceLayerInfo); + + c.top.sendDown(cmd); + c.top.waitForMessages(1, MSG_WAIT_TIME); + api::StorageMessage::SP reply(c.top.getReply(0)); + api::DeleteBucketReply& deleteReply( + dynamic_cast<api::DeleteBucketReply&>(*reply)); + // Reply happens in a filestor manager context and before the sanity + // check kicks in, meaning it will always be OK. + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::OK, resultOf(deleteReply)); + // At this point we do not know if the scheduled delete has been + // executed; it may still be in the persistence queue. + // Send a put to another bucket to serialize the operation (guaranteed + // since we only have 1 thread and the delete always has max priority). + c.sendPut(syncBucket, DocumentIndex(0), PutTimestamp(1001)); + c.top.waitForMessages(1, MSG_WAIT_TIME); + // Should still be able to get identical bucket info for bucket. + spi::BucketInfoResult infoResult( + _node->getPersistenceProvider().getBucketInfo(spiBucket)); + CPPUNIT_ASSERT_MSG(infoResult.getErrorMessage(), !infoResult.hasError()); + CPPUNIT_ASSERT(infoBefore == infoResult.getBucketInfo()); +} + +} // namespace storage diff --git a/storage/src/tests/persistence/filestorage/singlebucketjointest.cpp b/storage/src/tests/persistence/filestorage/singlebucketjointest.cpp new file mode 100644 index 00000000000..480652207d3 --- /dev/null +++ b/storage/src/tests/persistence/filestorage/singlebucketjointest.cpp @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <tests/persistence/persistenceproviderwrapper.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <tests/persistence/filestorage/filestortestfixture.h> + +LOG_SETUP(".singlebucketjointest"); + +namespace storage { + +class SingleBucketJoinTest : public FileStorTestFixture +{ +public: + void testPersistenceCanHandleSingleBucketJoin(); + + CPPUNIT_TEST_SUITE(SingleBucketJoinTest); + CPPUNIT_TEST(testPersistenceCanHandleSingleBucketJoin); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(SingleBucketJoinTest); + +void +SingleBucketJoinTest::testPersistenceCanHandleSingleBucketJoin() +{ + TestFileStorComponents c(*this, "testPersistenceCanHandleSingleBucketJoin"); + document::BucketId targetBucket(16, 1); + document::BucketId sourceBucket(17, 1); + + createBucket(sourceBucket); + // Make sure it's not empty + c.sendPut(sourceBucket, DocumentIndex(0), PutTimestamp(1000)); + expectOkReply<api::PutReply>(c.top); + c.top.getRepliesOnce(); + + auto cmd = std::make_shared<api::JoinBucketsCommand>(targetBucket); + cmd->getSourceBuckets().push_back(sourceBucket); + cmd->getSourceBuckets().push_back(sourceBucket); + + c.top.sendDown(cmd); + // If single bucket join locking is not working properly, this + // will hang forever. + expectOkReply<api::JoinBucketsReply>(c.top); +} + +} // namespace storage diff --git a/storage/src/tests/persistence/legacyoperationhandlertest.cpp b/storage/src/tests/persistence/legacyoperationhandlertest.cpp new file mode 100644 index 00000000000..ca496f4a260 --- /dev/null +++ b/storage/src/tests/persistence/legacyoperationhandlertest.cpp @@ -0,0 +1,190 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/document/base/testdocrepo.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/documentapi/loadtypes/loadtype.h> +#include <vespa/storage/persistence/messages.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storageapi/message/multioperation.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <tests/persistence/persistencetestutils.h> +#include <vespa/storage/persistence/types.h> + +using document::DocumentTypeRepo; +using document::TestDocRepo; + +namespace storage { + +class LegacyOperationHandlerTest : public SingleDiskPersistenceTestUtils +{ + CPPUNIT_TEST_SUITE(LegacyOperationHandlerTest); + CPPUNIT_TEST(testMultioperationSingleBodyPut); + CPPUNIT_TEST(testMultioperationSingleRemove); + CPPUNIT_TEST(testMultioperationSingleUpdate); + CPPUNIT_TEST(testMultioperationUpdateNotFound); + CPPUNIT_TEST(testMultioperationMixedOperations); + CPPUNIT_TEST_SUITE_END(); + +public: + void setUp() { + SingleDiskPersistenceTestUtils::setUp(); + createBucket(document::BucketId(16, 4)); + spi::Context context(spi::LoadType(0, "default"), spi::Priority(0), + spi::Trace::TraceLevel(0)); + getPersistenceProvider().createBucket( + spi::Bucket(document::BucketId(16, 4), spi::PartitionId(0)), + context); + } + + std::string stat() { + return dumpBucket(document::BucketId(16, 4), 0); + } + + void testMultioperationSingleBodyPut(); + void testMultioperationSingleRemove(); + void testMultioperationSingleUpdate(); + void testMultioperationUpdateNotFound(); + void testMultioperationMixedOperations(); + void testMultioperationMixedOperationsWrongBucket(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(LegacyOperationHandlerTest); + +void +LegacyOperationHandlerTest::testMultioperationSingleBodyPut() +{ + std::unique_ptr<PersistenceThread> thread(createPersistenceThread(0)); + document::BucketId bucketId(16, 4); + + document::Document::SP doc(createRandomDocumentAtLocation(4, 1234, 0, 128)); + + std::vector<char> buffer(1024); + vdslib::WritableDocumentList block(getTypeRepo(), &buffer[0], buffer.size()); + block.addPut(*doc, api::Timestamp(1234)); + + api::MultiOperationCommand cmd(getTypeRepo(), bucketId, 0); + cmd.setOperations(block); + + thread->handleMultiOperation(cmd); + + CPPUNIT_ASSERT_EQUAL( + std::string("DocEntry(1234, 0, Doc(id:mail:testdoctype1:n=4:3619.html))\n"), stat()); +} + +void +LegacyOperationHandlerTest::testMultioperationSingleRemove() +{ + std::unique_ptr<PersistenceThread> thread(createPersistenceThread(0)); + document::BucketId bucketId(16, 4); + + document::Document::SP doc = doPut(4, spi::Timestamp(1234)); + + std::vector<char> buffer(1024); + vdslib::WritableDocumentList block(getTypeRepo(), &buffer[0], buffer.size()); + block.addRemove(doc->getId(), spi::Timestamp(1235)); + + api::MultiOperationCommand cmd(getTypeRepo(), bucketId, 0); + cmd.setOperations(block); + + thread->handleMultiOperation(cmd); + + CPPUNIT_ASSERT_EQUAL( + std::string("DocEntry(1234, 0, Doc(id:mail:testdoctype1:n=4:3619.html))\n" + "DocEntry(1235, 1, id:mail:testdoctype1:n=4:3619.html)\n"), stat()); +} + +void +LegacyOperationHandlerTest::testMultioperationSingleUpdate() +{ + std::unique_ptr<PersistenceThread> thread(createPersistenceThread(0)); + document::BucketId bucketId(16, 4); + document::StringFieldValue updateValue("foo"); + + document::Document::SP doc = doPut(4, spi::Timestamp(1234)); + document::Document originalDoc(*doc); + + document::DocumentUpdate::SP update = createBodyUpdate( + doc->getId(), updateValue); + + std::vector<char> buffer(1024); + vdslib::WritableDocumentList block(getTypeRepo(), &buffer[0], buffer.size()); + block.addUpdate(*update, api::Timestamp(1235)); + + api::MultiOperationCommand cmd(getTypeRepo(), bucketId, 0); + cmd.setOperations(block); + + thread->handleMultiOperation(cmd); + + CPPUNIT_ASSERT_EQUAL( + std::string("DocEntry(1234, 0, Doc(id:mail:testdoctype1:n=4:3619.html))\n" + "DocEntry(1235, 0, Doc(id:mail:testdoctype1:n=4:3619.html))\n"), stat()); +} + +void +LegacyOperationHandlerTest::testMultioperationUpdateNotFound() +{ + std::unique_ptr<PersistenceThread> thread(createPersistenceThread(0)); + document::BucketId bucketId(16, 4); + document::DocumentId docId("userdoc:test:4:0"); + document::StringFieldValue updateValue("foo"); + + document::DocumentUpdate::SP update = createBodyUpdate( + docId, updateValue); + + std::vector<char> buffer(1024); + vdslib::WritableDocumentList block(getTypeRepo(), &buffer[0], buffer.size()); + block.addUpdate(*update, api::Timestamp(1235)); + + api::MultiOperationCommand cmd(getTypeRepo(), bucketId, 0); + cmd.setOperations(block); + + thread->handleMultiOperation(cmd); + + CPPUNIT_ASSERT_EQUAL(std::string(""), stat()); +} + +void +LegacyOperationHandlerTest::testMultioperationMixedOperations() +{ + std::unique_ptr<PersistenceThread> thread(createPersistenceThread(0)); + document::BucketId bucketId(16, 4); + document::StringFieldValue updateValue("bar"); + + document::Document::SP originalUpdateDoc = doPut(4, spi::Timestamp(1234)); + document::Document::SP originalRemoveDoc = doPut(4, spi::Timestamp(2345)); + + document::DocumentUpdate::SP update = createBodyUpdate( + originalUpdateDoc->getId(), updateValue); + + document::DocumentUpdate::SP nonExistingUpdate = createBodyUpdate( + document::DocumentId("id:test:testdoctype1:n=4:nonexisting1"), updateValue); + + document::Document::SP putDoc(createRandomDocumentAtLocation(4, 5678, 0, 128)); + + std::vector<char> buffer(1024); + vdslib::WritableDocumentList block(getTypeRepo(), &buffer[0], buffer.size()); + + block.addUpdate(*update, api::Timestamp(3456)); + block.addUpdate(*nonExistingUpdate, api::Timestamp(3457)); + block.addRemove(originalRemoveDoc->getId(), api::Timestamp(4567)); + block.addRemove(document::DocumentId("id:test:testdoctype1:n=4:nonexisting2"), + api::Timestamp(4568)); + block.addPut(*putDoc, api::Timestamp(5678)); + + api::MultiOperationCommand cmd(getTypeRepo(), bucketId, 0); + cmd.setOperations(block); + + thread->handleMultiOperation(cmd); + + CPPUNIT_ASSERT_EQUAL( + std::string("DocEntry(1234, 0, Doc(id:mail:testdoctype1:n=4:3619.html))\n" + "DocEntry(2345, 0, Doc(id:mail:testdoctype1:n=4:4008.html))\n" + "DocEntry(3456, 0, Doc(id:mail:testdoctype1:n=4:3619.html))\n" + "DocEntry(4567, 1, id:mail:testdoctype1:n=4:4008.html)\n" + "DocEntry(4568, 1, id:test:testdoctype1:n=4:nonexisting2)\n" + "DocEntry(5678, 0, Doc(id:mail:testdoctype1:n=4:5177.html))\n"), + stat()); +} + +} diff --git a/storage/src/tests/persistence/mergehandlertest.cpp b/storage/src/tests/persistence/mergehandlertest.cpp new file mode 100644 index 00000000000..3d3ce25a7d7 --- /dev/null +++ b/storage/src/tests/persistence/mergehandlertest.cpp @@ -0,0 +1,1494 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/document/base/testdocman.h> +#include <vespa/storage/persistence/mergehandler.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/log/log.h> +#include <tests/persistence/persistencetestutils.h> +#include <tests/persistence/persistenceproviderwrapper.h> +#include <tests/distributor/messagesenderstub.h> +#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> + +LOG_SETUP(".test.persistence.handler.merge"); + +namespace storage { + +struct MergeHandlerTest : public SingleDiskPersistenceTestUtils +{ + uint32_t _location; // Location used for all merge tests + document::BucketId _bucket; // Bucket used for all merge tests + uint64_t _maxTimestamp; + std::vector<api::MergeBucketCommand::Node> _nodes; + std::unique_ptr<spi::Context> _context; + + // Fetch a single command or reply; doesn't care which. + template <typename T> + std::shared_ptr<T> fetchSingleMessage(); + + void setUp(); + + enum ChainPos { FRONT, MIDDLE, BACK }; + void setUpChain(ChainPos); + + // Test a regular merge bucket command fetching data, including + // puts, removes, unrevertable removes & duplicates. + void testMergeBucketCommand(); + // Test that a simplistic merge with nothing to actually merge, + // sends get bucket diff through the entire chain of 3 nodes. + void testGetBucketDiffChain(bool midChain); + void testGetBucketDiffMidChain() { testGetBucketDiffChain(true); } + void testGetBucketDiffEndOfChain() { testGetBucketDiffChain(false); } + // Test that a simplistic merge with nothing to actually merge, + // sends apply bucket diff through the entire chain of 3 nodes. + void testApplyBucketDiffChain(bool midChain); + void testApplyBucketDiffMidChain() { testApplyBucketDiffChain(true); } + void testApplyBucketDiffEndOfChain() { testApplyBucketDiffChain(false); } + // Test that a simplistic merge with one thing to actually merge, + // sends correct commands and finish. + void testMasterMessageFlow(); + // Test that a simplistic merge with 1 doc to actually merge, + // sends apply bucket diff through the entire chain of 3 nodes. + void testApplyBucketDiffChain(); + void testMergeUnrevertableRemove(); + void testChunkedApplyBucketDiff(); + void testChunkLimitPartiallyFilledDiff(); + void testMaxTimestamp(); + void testSPIFlushGuard(); + void testBucketNotFoundInDb(); + void testMergeProgressSafeGuard(); + void testSafeGuardNotInvokedWhenHasMaskChanges(); + void testEntryRemovedAfterGetBucketDiff(); + + void testMergeBucketSPIFailures(); + void testGetBucketDiffSPIFailures(); + void testApplyBucketDiffSPIFailures(); + void testGetBucketDiffReplySPIFailures(); + void testApplyBucketDiffReplySPIFailures(); + + void testRemoveFromDiff(); + + void testRemovePutOnExistingTimestamp(); + + CPPUNIT_TEST_SUITE(MergeHandlerTest); + CPPUNIT_TEST(testMergeBucketCommand); + CPPUNIT_TEST(testGetBucketDiffMidChain); + CPPUNIT_TEST(testGetBucketDiffEndOfChain); + CPPUNIT_TEST(testApplyBucketDiffMidChain); + CPPUNIT_TEST(testApplyBucketDiffEndOfChain); + CPPUNIT_TEST(testMasterMessageFlow); + CPPUNIT_TEST(testMergeUnrevertableRemove); + CPPUNIT_TEST(testChunkedApplyBucketDiff); + CPPUNIT_TEST(testChunkLimitPartiallyFilledDiff); + CPPUNIT_TEST(testMaxTimestamp); + CPPUNIT_TEST(testSPIFlushGuard); + CPPUNIT_TEST(testBucketNotFoundInDb); + CPPUNIT_TEST(testMergeProgressSafeGuard); + CPPUNIT_TEST(testSafeGuardNotInvokedWhenHasMaskChanges); + CPPUNIT_TEST(testEntryRemovedAfterGetBucketDiff); + CPPUNIT_TEST(testMergeBucketSPIFailures); + CPPUNIT_TEST(testGetBucketDiffSPIFailures); + CPPUNIT_TEST(testApplyBucketDiffSPIFailures); + CPPUNIT_TEST(testGetBucketDiffReplySPIFailures); + CPPUNIT_TEST(testApplyBucketDiffReplySPIFailures); + CPPUNIT_TEST(testRemoveFromDiff); + CPPUNIT_TEST(testRemovePutOnExistingTimestamp); + CPPUNIT_TEST_SUITE_END(); + + // @TODO Add test to test that buildBucketInfo and mergeLists create minimal list (wrong sorting screws this up) +private: + void fillDummyApplyDiff(std::vector<api::ApplyBucketDiffCommand::Entry>& diff); + std::shared_ptr<api::ApplyBucketDiffCommand> createDummyApplyDiff( + int timestampOffset, + uint16_t hasMask = 0x1, + bool filled = true); + + std::shared_ptr<api::GetBucketDiffCommand> + createDummyGetBucketDiff(int timestampOffset, + uint16_t hasMask); + + struct ExpectedExceptionSpec // Try saying this out loud 3 times in a row. + { + uint32_t mask; + const char* expected; + }; + + class HandlerInvoker + { + public: + virtual ~HandlerInvoker() {} + virtual void beforeInvoke(MergeHandlerTest&, MergeHandler&, spi::Context&) {} + virtual void invoke(MergeHandlerTest&, MergeHandler&, spi::Context&) = 0; + virtual std::string afterInvoke(MergeHandlerTest&, MergeHandler&) = 0; + }; + friend class HandlerInvoker; + + class NoReplyHandlerInvoker + : public HandlerInvoker + { + public: + std::string afterInvoke(MergeHandlerTest&, MergeHandler&); + }; + + template <typename ExpectedMessage> + std::string checkMessage(api::ReturnCode::Result expectedResult); + + class HandleMergeBucketInvoker + : public NoReplyHandlerInvoker + { + public: + void invoke(MergeHandlerTest&, MergeHandler&, spi::Context&); + }; + + class HandleMergeBucketReplyInvoker + : public NoReplyHandlerInvoker + { + public: + void invoke(MergeHandlerTest&, MergeHandler&, spi::Context&); + }; + + class HandleGetBucketDiffInvoker + : public NoReplyHandlerInvoker + { + public: + void invoke(MergeHandlerTest&, MergeHandler&, spi::Context&); + }; + + class MultiPositionHandlerInvoker + : public HandlerInvoker + { + public: + MultiPositionHandlerInvoker() + : _pos(FRONT) + { + } + void setChainPos(ChainPos pos) { _pos = pos; } + ChainPos getChainPos() const { return _pos; } + private: + ChainPos _pos; + }; + + class HandleGetBucketDiffReplyInvoker + : public HandlerInvoker + { + public: + void beforeInvoke(MergeHandlerTest&, MergeHandler&, spi::Context&); + void invoke(MergeHandlerTest&, MergeHandler&, spi::Context&); + std::string afterInvoke(MergeHandlerTest&, MergeHandler&); + private: + MessageSenderStub _stub; + std::shared_ptr<api::GetBucketDiffCommand> _diffCmd; + }; + + class HandleApplyBucketDiffInvoker + : public NoReplyHandlerInvoker + { + public: + HandleApplyBucketDiffInvoker() : _counter(0) {} + void invoke(MergeHandlerTest&, MergeHandler&, spi::Context&); + private: + int _counter; + }; + + class HandleApplyBucketDiffReplyInvoker + : public MultiPositionHandlerInvoker + { + public: + HandleApplyBucketDiffReplyInvoker() + : _counter(0), + _stub(), + _applyCmd() + {} + void beforeInvoke(MergeHandlerTest&, MergeHandler&, spi::Context&); + void invoke(MergeHandlerTest&, MergeHandler&, spi::Context&); + std::string afterInvoke(MergeHandlerTest&, MergeHandler&); + private: + int _counter; + MessageSenderStub _stub; + std::shared_ptr<api::ApplyBucketDiffCommand> _applyCmd; + }; + + std::string + doTestSPIException(MergeHandler& handler, + PersistenceProviderWrapper& providerWrapper, + HandlerInvoker& invoker, + const ExpectedExceptionSpec& spec); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(MergeHandlerTest); + +void +MergeHandlerTest::setUp() { + _context.reset(new spi::Context(documentapi::LoadType::DEFAULT, 0, 0)); + SingleDiskPersistenceTestUtils::setUp(); + + _location = 1234; + _bucket = document::BucketId(16, _location); + _maxTimestamp = 11501; + + LOG(info, "Creating %s in bucket database", _bucket.toString().c_str()); + bucketdb::StorageBucketInfo bucketDBEntry; + bucketDBEntry.disk = 0; + getEnv().getBucketDatabase().insert(_bucket, bucketDBEntry, "mergetestsetup"); + + LOG(info, "Creating bucket to merge"); + createTestBucket(_bucket); + + setUpChain(FRONT); +} + +void +MergeHandlerTest::setUpChain(ChainPos pos) { + _nodes.clear(); + if (pos != FRONT) { + _nodes.push_back(api::MergeBucketCommand::Node(2, false)); + } + _nodes.push_back(api::MergeBucketCommand::Node(0, false)); + if (pos != BACK) { + _nodes.push_back(api::MergeBucketCommand::Node(1, false)); + } +} + +void +MergeHandlerTest::testMergeBucketCommand() +{ + MergeHandler handler(getPersistenceProvider(), getEnv()); + + LOG(info, "Handle a merge bucket command"); + api::MergeBucketCommand cmd(_bucket, _nodes, _maxTimestamp); + cmd.setSourceIndex(1234); + MessageTracker::UP tracker = handler.handleMergeBucket(cmd, *_context); + + LOG(info, "Check state"); + CPPUNIT_ASSERT_EQUAL(uint64_t(1), messageKeeper()._msgs.size()); + CPPUNIT_ASSERT_EQUAL(api::MessageType::GETBUCKETDIFF, + messageKeeper()._msgs[0]->getType()); + api::GetBucketDiffCommand& cmd2(dynamic_cast<api::GetBucketDiffCommand&>( + *messageKeeper()._msgs[0])); + CPPUNIT_ASSERT_EQUAL(_nodes, cmd2.getNodes()); + std::vector<api::GetBucketDiffCommand::Entry> diff(cmd2.getDiff()); + CPPUNIT_ASSERT_EQUAL(uint64_t(17), diff.size()); + CPPUNIT_ASSERT_EQUAL(uint16_t(1), cmd2.getAddress()->getIndex()); + CPPUNIT_ASSERT_EQUAL(uint16_t(1234), cmd2.getSourceIndex()); + + tracker->generateReply(cmd); + CPPUNIT_ASSERT(!tracker->getReply().get()); +} + +void +MergeHandlerTest::testGetBucketDiffChain(bool midChain) +{ + setUpChain(midChain ? MIDDLE : BACK); + MergeHandler handler(getPersistenceProvider(), getEnv()); + + LOG(info, "Verifying that get bucket diff is sent on"); + api::GetBucketDiffCommand cmd(_bucket, _nodes, _maxTimestamp); + MessageTracker::UP tracker1 = handler.handleGetBucketDiff(cmd, *_context); + api::StorageMessage::SP replySent = tracker1->getReply(); + + if (midChain) { + LOG(info, "Check state"); + CPPUNIT_ASSERT_EQUAL(uint64_t(1), messageKeeper()._msgs.size()); + CPPUNIT_ASSERT_EQUAL(api::MessageType::GETBUCKETDIFF, + messageKeeper()._msgs[0]->getType()); + api::GetBucketDiffCommand& cmd2( + dynamic_cast<api::GetBucketDiffCommand&>( + *messageKeeper()._msgs[0])); + CPPUNIT_ASSERT_EQUAL(_nodes, cmd2.getNodes()); + std::vector<api::GetBucketDiffCommand::Entry> diff(cmd2.getDiff()); + CPPUNIT_ASSERT_EQUAL(uint64_t(17), diff.size()); + CPPUNIT_ASSERT_EQUAL(uint16_t(1), cmd2.getAddress()->getIndex()); + + LOG(info, "Verifying that replying the diff sends on back"); + api::GetBucketDiffReply::UP reply(new api::GetBucketDiffReply(cmd2)); + + CPPUNIT_ASSERT(!replySent.get()); + + MessageSenderStub stub; + handler.handleGetBucketDiffReply(*reply, stub); + CPPUNIT_ASSERT_EQUAL(1, (int)stub.replies.size()); + replySent = stub.replies[0]; + } + api::GetBucketDiffReply::SP reply2( + std::dynamic_pointer_cast<api::GetBucketDiffReply>( + replySent)); + CPPUNIT_ASSERT(reply2.get()); + + CPPUNIT_ASSERT_EQUAL(_nodes, reply2->getNodes()); + std::vector<api::GetBucketDiffCommand::Entry> diff(reply2->getDiff()); + CPPUNIT_ASSERT_EQUAL(uint64_t(17), diff.size()); +} + +void +MergeHandlerTest::testApplyBucketDiffChain(bool midChain) +{ + setUpChain(midChain ? MIDDLE : BACK); + MergeHandler handler(getPersistenceProvider(), getEnv()); + + LOG(info, "Verifying that apply bucket diff is sent on"); + api::ApplyBucketDiffCommand cmd(_bucket, _nodes, _maxTimestamp); + MessageTracker::UP tracker1 = handler.handleApplyBucketDiff(cmd, *_context); + api::StorageMessage::SP replySent = tracker1->getReply(); + + if (midChain) { + LOG(info, "Check state"); + CPPUNIT_ASSERT_EQUAL(uint64_t(1), messageKeeper()._msgs.size()); + CPPUNIT_ASSERT_EQUAL(api::MessageType::APPLYBUCKETDIFF, + messageKeeper()._msgs[0]->getType()); + api::ApplyBucketDiffCommand& cmd2( + dynamic_cast<api::ApplyBucketDiffCommand&>( + *messageKeeper()._msgs[0])); + CPPUNIT_ASSERT_EQUAL(_nodes, cmd2.getNodes()); + std::vector<api::ApplyBucketDiffCommand::Entry> diff(cmd2.getDiff()); + CPPUNIT_ASSERT_EQUAL(uint64_t(0), diff.size()); + CPPUNIT_ASSERT_EQUAL(uint16_t(1), cmd2.getAddress()->getIndex()); + + CPPUNIT_ASSERT(!replySent.get()); + + LOG(info, "Verifying that replying the diff sends on back"); + api::ApplyBucketDiffReply::UP reply( + new api::ApplyBucketDiffReply(cmd2)); + + MessageSenderStub stub; + handler.handleApplyBucketDiffReply(*reply, stub); + CPPUNIT_ASSERT_EQUAL(1, (int)stub.replies.size()); + replySent = stub.replies[0]; + } + + api::ApplyBucketDiffReply::SP reply2( + std::dynamic_pointer_cast<api::ApplyBucketDiffReply>(replySent)); + CPPUNIT_ASSERT(reply2.get()); + + CPPUNIT_ASSERT_EQUAL(_nodes, reply2->getNodes()); + std::vector<api::ApplyBucketDiffCommand::Entry> diff(reply2->getDiff()); + CPPUNIT_ASSERT_EQUAL(uint64_t(0), diff.size()); +} + +void +MergeHandlerTest::testMasterMessageFlow() +{ + MergeHandler handler(getPersistenceProvider(), getEnv()); + + LOG(info, "Handle a merge bucket command"); + api::MergeBucketCommand cmd(_bucket, _nodes, _maxTimestamp); + + handler.handleMergeBucket(cmd, *_context); + LOG(info, "Check state"); + CPPUNIT_ASSERT_EQUAL(uint64_t(1), messageKeeper()._msgs.size()); + CPPUNIT_ASSERT_EQUAL(api::MessageType::GETBUCKETDIFF, + messageKeeper()._msgs[0]->getType()); + api::GetBucketDiffCommand& cmd2(dynamic_cast<api::GetBucketDiffCommand&>( + *messageKeeper()._msgs[0])); + + api::GetBucketDiffReply::UP reply(new api::GetBucketDiffReply(cmd2)); + // End of chain can remove entries all have. This should end up with + // one entry master node has other node don't have + reply->getDiff().resize(1); + + handler.handleGetBucketDiffReply(*reply, messageKeeper()); + + LOG(info, "Check state"); + CPPUNIT_ASSERT_EQUAL(uint64_t(2), messageKeeper()._msgs.size()); + CPPUNIT_ASSERT_EQUAL(api::MessageType::APPLYBUCKETDIFF, + messageKeeper()._msgs[1]->getType()); + api::ApplyBucketDiffCommand& cmd3( + dynamic_cast<api::ApplyBucketDiffCommand&>( + *messageKeeper()._msgs[1])); + api::ApplyBucketDiffReply::UP reply2(new api::ApplyBucketDiffReply(cmd3)); + CPPUNIT_ASSERT_EQUAL(size_t(1), reply2->getDiff().size()); + reply2->getDiff()[0]._entry._hasMask |= 2; + + MessageSenderStub stub; + handler.handleApplyBucketDiffReply(*reply2, stub); + + CPPUNIT_ASSERT_EQUAL(1, (int)stub.replies.size()); + + api::MergeBucketReply::SP reply3( + std::dynamic_pointer_cast<api::MergeBucketReply>(stub.replies[0])); + CPPUNIT_ASSERT(reply3.get()); + + CPPUNIT_ASSERT_EQUAL(_nodes, reply3->getNodes()); + CPPUNIT_ASSERT(reply3->getResult().success()); + CPPUNIT_ASSERT(!fsHandler().isMerging(_bucket)); +} + +void +MergeHandlerTest::testMergeUnrevertableRemove() +{ +/* + MergeHandler handler(getPersistenceProvider(), getEnv()); + + LOG(info, "Handle a merge bucket command"); + api::MergeBucketCommand cmd(_bucket, _nodes, _maxTimestamp); + { + MessageTracker tracker; + handler.handleMergeBucket(cmd, tracker); + } + + LOG(info, "Check state"); + CPPUNIT_ASSERT_EQUAL(uint64_t(1), messageKeeper()._msgs.size()); + CPPUNIT_ASSERT_EQUAL(api::MessageType::GETBUCKETDIFF, + messageKeeper()._msgs[0]->getType()); + api::GetBucketDiffCommand& cmd2( + dynamic_cast<api::GetBucketDiffCommand&>( + *messageKeeper()._msgs[0])); + + api::GetBucketDiffReply::UP reply(new api::GetBucketDiffReply(cmd2)); + + std::vector<Timestamp> docTimestamps; + for (int i = 0; i < 4; ++i) { + docTimestamps.push_back(Timestamp(reply->getDiff()[i]._timestamp)); + } + CPPUNIT_ASSERT(reply->getDiff().size() >= 4); + reply->getDiff().resize(4); + // Add one non-unrevertable entry for existing timestamp which + // should not be added + reply->getDiff()[0]._flags |= Types::DELETED; + reply->getDiff()[0]._bodySize = 0; + reply->getDiff()[0]._hasMask = 2; + // Add a unrevertable entry which should be modified + reply->getDiff()[1]._flags |= Types::DELETED | Types::DELETED_IN_PLACE; + reply->getDiff()[1]._bodySize = 0; + reply->getDiff()[1]._hasMask = 2; + // Add one non-unrevertable entry that is a duplicate put + // which should not be added or fail the merge. + LOG(info, "duplicate put has timestamp %zu and flags %u", + reply->getDiff()[2]._timestamp, + reply->getDiff()[2]._flags); + reply->getDiff()[2]._hasMask = 2; + // Add one unrevertable entry for a timestamp that does not exist + reply->getDiff()[3]._flags |= Types::DELETED | Types::DELETED_IN_PLACE; + reply->getDiff()[3]._timestamp = 12345678; + reply->getDiff()[3]._bodySize = 0; + reply->getDiff()[3]._hasMask = 2; + { + MessageTracker tracker; + handler.handleGetBucketDiffReply(*reply, tracker); + } + + LOG(info, "%s", reply->toString(true).c_str()); + + LOG(info, "Create bucket diff reply"); + CPPUNIT_ASSERT_EQUAL(uint64_t(2), messageKeeper()._msgs.size()); + CPPUNIT_ASSERT_EQUAL(api::MessageType::APPLYBUCKETDIFF, + messageKeeper()._msgs[1]->getType()); + api::ApplyBucketDiffCommand& cmd3( + dynamic_cast<api::ApplyBucketDiffCommand&>( + *messageKeeper()._msgs[1])); + api::ApplyBucketDiffReply::UP reply2( + new api::ApplyBucketDiffReply(cmd3)); + CPPUNIT_ASSERT_EQUAL(size_t(4), reply2->getDiff().size()); + + memfile::DataLocation headerLocs[4]; + std::vector<DocumentId> documentIds; + // So deserialization won't fail, we need some kind of header blob + // for each entry + + for (int i = 0; i < 4; ++i) { + api::ApplyBucketDiffReply::Entry& entry = reply2->getDiff()[i]; + CPPUNIT_ASSERT_EQUAL(uint16_t(2), entry._entry._hasMask); + + memfile::MemFilePtr file(getMemFile(_bucket)); + const memfile::MemSlot* slot = file->getSlotAtTime(docTimestamps[i]); + CPPUNIT_ASSERT(slot != NULL); + LOG(info, "Processing slot %s", slot->toString().c_str()); + CPPUNIT_ASSERT(slot->hasBodyContent()); + documentIds.push_back(file->getDocumentId(*slot)); + entry._docName = documentIds.back().toString(); + headerLocs[i] = slot->getLocation(HEADER); + + document::Document::UP doc(file->getDocument(*slot, ALL)); + { + vespalib::nbostream stream; + doc->serializeHeader(stream); + std::vector<char> buf( + stream.peek(), stream.peek() + stream.size()); + entry._headerBlob.swap(buf); + } + // Put duplicate needs body blob as well + if (i == 2) { + vespalib::nbostream stream; + doc->serializeBody(stream); + std::vector<char> buf( + stream.peek(), stream.peek() + stream.size()); + entry._bodyBlob.swap(buf); + } + } + + LOG(info, "%s", reply2->toString(true).c_str()); + + MessageTracker tracker; + handler.handleApplyBucketDiffReply(*reply2, tracker); + + CPPUNIT_ASSERT(tracker._sendReply); + api::MergeBucketReply::SP reply3( + std::dynamic_pointer_cast<api::MergeBucketReply>( + tracker._reply)); + CPPUNIT_ASSERT(reply3.get()); + + CPPUNIT_ASSERT_EQUAL(_nodes, reply3->getNodes()); + CPPUNIT_ASSERT(reply3->getResult().success()); + + memfile::MemFilePtr file(getMemFile(_bucket)); + // Existing timestamp should not be modified by + // non-unrevertable entry + { + const memfile::MemSlot* slot = file->getSlotAtTime( + Timestamp(reply->getDiff()[0]._timestamp)); + CPPUNIT_ASSERT(slot != NULL); + CPPUNIT_ASSERT(!slot->deleted()); + } + // Ensure unrevertable remove for existing put was merged in OK + { + const memfile::MemSlot* slot = file->getSlotAtTime( + Timestamp(reply->getDiff()[1]._timestamp)); + CPPUNIT_ASSERT(slot != NULL); + CPPUNIT_ASSERT(slot->deleted()); + CPPUNIT_ASSERT(slot->deletedInPlace()); + CPPUNIT_ASSERT(!slot->hasBodyContent()); + // Header location should not have changed + CPPUNIT_ASSERT_EQUAL(headerLocs[1], slot->getLocation(HEADER)); + } + + // Non-existing timestamp unrevertable remove should be added as + // entry with doc id-only header + { + const memfile::MemSlot* slot = file->getSlotAtTime( + Timestamp(reply->getDiff()[3]._timestamp)); + CPPUNIT_ASSERT(slot != NULL); + CPPUNIT_ASSERT(slot->deleted()); + CPPUNIT_ASSERT(slot->deletedInPlace()); + CPPUNIT_ASSERT(!slot->hasBodyContent()); + CPPUNIT_ASSERT_EQUAL(documentIds[3], file->getDocumentId(*slot)); + } + +*/ +} + +template <typename T> +std::shared_ptr<T> +MergeHandlerTest::fetchSingleMessage() +{ + std::vector<api::StorageMessage::SP>& msgs(messageKeeper()._msgs); + if (msgs.empty()) { + std::ostringstream oss; + oss << "No messages available to fetch (expected type " + << typeid(T).name() + << ")"; + throw std::runtime_error(oss.str()); + } + std::shared_ptr<T> ret(std::dynamic_pointer_cast<T>( + messageKeeper()._msgs.back())); + if (!ret) { + std::ostringstream oss; + oss << "Expected message of type " + << typeid(T).name() + << ", but got " + << messageKeeper()._msgs[0]->toString(); + throw std::runtime_error(oss.str()); + } + messageKeeper()._msgs.pop_back(); + + return ret; +} + +namespace { + +size_t +getFilledCount(const std::vector<api::ApplyBucketDiffCommand::Entry>& diff) +{ + size_t filledCount = 0; + for (size_t i=0; i<diff.size(); ++i) { + if (diff[i].filled()) { + ++filledCount; + } + } + return filledCount; +} + +size_t +getFilledDataSize(const std::vector<api::ApplyBucketDiffCommand::Entry>& diff) +{ + size_t filledSize = 0; + for (size_t i=0; i<diff.size(); ++i) { + filledSize += diff[i]._headerBlob.size(); + filledSize += diff[i]._bodyBlob.size(); + } + return filledSize; +} + +} + +void +MergeHandlerTest::testChunkedApplyBucketDiff() +{ + uint32_t docSize = 1024; + uint32_t docCount = 10; + uint32_t maxChunkSize = docSize * 3; + for (uint32_t i = 0; i < docCount; ++i) { + doPut(1234, spi::Timestamp(4000 + i), docSize, docSize); + } + + MergeHandler handler(getPersistenceProvider(), getEnv(), maxChunkSize); + + LOG(info, "Handle a merge bucket command"); + api::MergeBucketCommand cmd(_bucket, _nodes, _maxTimestamp); + handler.handleMergeBucket(cmd, *_context); + + std::shared_ptr<api::GetBucketDiffCommand> getBucketDiffCmd( + fetchSingleMessage<api::GetBucketDiffCommand>()); + api::GetBucketDiffReply::UP getBucketDiffReply( + new api::GetBucketDiffReply(*getBucketDiffCmd)); + + handler.handleGetBucketDiffReply(*getBucketDiffReply, messageKeeper()); + + uint32_t totalDiffs = getBucketDiffCmd->getDiff().size(); + std::set<spi::Timestamp> seen; + + api::MergeBucketReply::SP reply; + while (seen.size() != totalDiffs) { + std::shared_ptr<api::ApplyBucketDiffCommand> applyBucketDiffCmd( + fetchSingleMessage<api::ApplyBucketDiffCommand>()); + + LOG(info, "Test that we get chunked diffs in ApplyBucketDiff"); + std::vector<api::ApplyBucketDiffCommand::Entry>& diff( + applyBucketDiffCmd->getDiff()); + CPPUNIT_ASSERT(getFilledCount(diff) < totalDiffs); + CPPUNIT_ASSERT(getFilledDataSize(diff) <= maxChunkSize); + + // Include node 1 in hasmask for all diffs to indicate it's done + // Also remember the diffs we've seen thus far to ensure chunking + // does not send duplicates. + for (size_t i = 0; i < diff.size(); ++i) { + if (!diff[i].filled()) { + continue; + } + diff[i]._entry._hasMask |= 2; + std::pair<std::set<spi::Timestamp>::iterator, bool> inserted( + seen.insert(spi::Timestamp(diff[i]._entry._timestamp))); + if (!inserted.second) { + std::ostringstream ss; + ss << "Diff for " << diff[i] + << " has already been seen in another ApplyBucketDiff"; + CPPUNIT_FAIL(ss.str()); + } + } + + api::ApplyBucketDiffReply::UP applyBucketDiffReply( + new api::ApplyBucketDiffReply(*applyBucketDiffCmd)); + { + handler.handleApplyBucketDiffReply(*applyBucketDiffReply, messageKeeper()); + + if (messageKeeper()._msgs.size()) { + CPPUNIT_ASSERT(!reply.get()); + reply = std::dynamic_pointer_cast<api::MergeBucketReply>( + messageKeeper()._msgs[messageKeeper()._msgs.size() - 1]); + } + } + } + LOG(info, "Done with applying diff"); + + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(_nodes, reply->getNodes()); + CPPUNIT_ASSERT(reply->getResult().success()); +} + +void +MergeHandlerTest::testChunkLimitPartiallyFilledDiff() +{ + setUpChain(FRONT); + + uint32_t docSize = 1024; + uint32_t docCount = 3; + uint32_t maxChunkSize = 1024 + 1024 + 512; + + for (uint32_t i = 0; i < docCount; ++i) { + doPut(1234, spi::Timestamp(4000 + i), docSize, docSize); + } + + std::vector<api::ApplyBucketDiffCommand::Entry> applyDiff; + for (uint32_t i = 0; i < docCount; ++i) { + api::ApplyBucketDiffCommand::Entry e; + e._entry._timestamp = 4000 + i; + if (i == 0) { + e._headerBlob.resize(docSize); + } + e._entry._hasMask = 0x3; + e._entry._flags = MergeHandler::IN_USE; + applyDiff.push_back(e); + } + + setUpChain(MIDDLE); + std::shared_ptr<api::ApplyBucketDiffCommand> applyBucketDiffCmd( + new api::ApplyBucketDiffCommand(_bucket, _nodes, maxChunkSize)); + applyBucketDiffCmd->getDiff() = applyDiff; + + MergeHandler handler( + getPersistenceProvider(), getEnv(), maxChunkSize); + handler.handleApplyBucketDiff(*applyBucketDiffCmd, *_context); + + std::shared_ptr<api::ApplyBucketDiffCommand> fwdDiffCmd( + fetchSingleMessage<api::ApplyBucketDiffCommand>()); + // Should not fill up more than chunk size allows for + CPPUNIT_ASSERT_EQUAL(size_t(2), getFilledCount(fwdDiffCmd->getDiff())); + CPPUNIT_ASSERT(getFilledDataSize(fwdDiffCmd->getDiff()) <= maxChunkSize); +} + +void +MergeHandlerTest::testMaxTimestamp() +{ + doPut(1234, spi::Timestamp(_maxTimestamp + 10), 1024, 1024); + + MergeHandler handler(getPersistenceProvider(), getEnv()); + + api::MergeBucketCommand cmd(_bucket, _nodes, _maxTimestamp); + handler.handleMergeBucket(cmd, *_context); + + std::shared_ptr<api::GetBucketDiffCommand> getCmd( + fetchSingleMessage<api::GetBucketDiffCommand>()); + + CPPUNIT_ASSERT(!getCmd->getDiff().empty()); + CPPUNIT_ASSERT(getCmd->getDiff().back()._timestamp <= _maxTimestamp); +} + +void +MergeHandlerTest::fillDummyApplyDiff( + std::vector<api::ApplyBucketDiffCommand::Entry>& diff) +{ + document::TestDocMan docMan; + document::Document::SP doc( + docMan.createRandomDocumentAtLocation(_location)); + std::vector<char> headerBlob; + { + vespalib::nbostream stream; + doc->serializeHeader(stream); + headerBlob.resize(stream.size()); + memcpy(&headerBlob[0], stream.peek(), stream.size()); + } + + assert(diff.size() == 3); + diff[0]._headerBlob = headerBlob; + diff[1]._docName = doc->getId().toString(); + diff[2]._docName = doc->getId().toString(); +} + +std::shared_ptr<api::ApplyBucketDiffCommand> +MergeHandlerTest::createDummyApplyDiff(int timestampOffset, + uint16_t hasMask, + bool filled) +{ + + std::vector<api::ApplyBucketDiffCommand::Entry> applyDiff; + { + api::ApplyBucketDiffCommand::Entry e; + e._entry._timestamp = timestampOffset; + e._entry._hasMask = hasMask; + e._entry._flags = MergeHandler::IN_USE; + applyDiff.push_back(e); + } + { + api::ApplyBucketDiffCommand::Entry e; + e._entry._timestamp = timestampOffset + 1; + e._entry._hasMask = hasMask; + e._entry._flags = MergeHandler::IN_USE | MergeHandler::DELETED; + applyDiff.push_back(e); + } + { + api::ApplyBucketDiffCommand::Entry e; + e._entry._timestamp = timestampOffset + 2; + e._entry._hasMask = hasMask; + e._entry._flags = MergeHandler::IN_USE | + MergeHandler::DELETED | + MergeHandler::DELETED_IN_PLACE; + applyDiff.push_back(e); + } + + if (filled) { + fillDummyApplyDiff(applyDiff); + } + + std::shared_ptr<api::ApplyBucketDiffCommand> applyBucketDiffCmd( + new api::ApplyBucketDiffCommand(_bucket, _nodes, 1024*1024)); + applyBucketDiffCmd->getDiff() = applyDiff; + return applyBucketDiffCmd; +} + +// Must match up with diff used in createDummyApplyDiff +std::shared_ptr<api::GetBucketDiffCommand> +MergeHandlerTest::createDummyGetBucketDiff(int timestampOffset, + uint16_t hasMask) +{ + std::vector<api::GetBucketDiffCommand::Entry> diff; + { + api::GetBucketDiffCommand::Entry e; + e._timestamp = timestampOffset; + e._hasMask = hasMask; + e._flags = MergeHandler::IN_USE; + diff.push_back(e); + } + { + api::GetBucketDiffCommand::Entry e; + e._timestamp = timestampOffset + 1; + e._hasMask = hasMask; + e._flags = MergeHandler::IN_USE | MergeHandler::DELETED; + diff.push_back(e); + } + { + api::GetBucketDiffCommand::Entry e; + e._timestamp = timestampOffset + 2; + e._hasMask = hasMask; + e._flags = MergeHandler::IN_USE | + MergeHandler::DELETED | + MergeHandler::DELETED_IN_PLACE; + diff.push_back(e); + } + + std::shared_ptr<api::GetBucketDiffCommand> getBucketDiffCmd( + new api::GetBucketDiffCommand(_bucket, _nodes, 1024*1024)); + getBucketDiffCmd->getDiff() = diff; + return getBucketDiffCmd; +} + +void +MergeHandlerTest::testSPIFlushGuard() +{ + PersistenceProviderWrapper providerWrapper( + getPersistenceProvider()); + MergeHandler handler(providerWrapper, getEnv()); + + providerWrapper.setResult( + spi::Result(spi::Result::PERMANENT_ERROR, + "who you gonna call?")); + + setUpChain(MIDDLE); + // Fail applying unrevertable remove + providerWrapper.setFailureMask( + PersistenceProviderWrapper::FAIL_REMOVE); + providerWrapper.clearOperationLog(); + try { + handler.handleApplyBucketDiff(*createDummyApplyDiff(6000), *_context); + CPPUNIT_FAIL("No exception thrown on failing in-place remove"); + } catch (const std::runtime_error& e) { + CPPUNIT_ASSERT(std::string(e.what()).find("Failed remove") + != std::string::npos); + } + // Test that we always flush after applying diff locally, even when + // errors are encountered. + const std::vector<std::string>& opLog(providerWrapper.getOperationLog()); + CPPUNIT_ASSERT(!opLog.empty()); + CPPUNIT_ASSERT_EQUAL( + std::string("flush(Bucket(0x40000000000004d2, partition 0))"), + opLog.back()); +} + +void +MergeHandlerTest::testBucketNotFoundInDb() +{ + MergeHandler handler(getPersistenceProvider(), getEnv()); + // Send merge for unknown bucket + api::MergeBucketCommand cmd(document::BucketId(16, 6789), _nodes, _maxTimestamp); + MessageTracker::UP tracker = handler.handleMergeBucket(cmd, *_context); + CPPUNIT_ASSERT(tracker->getResult().isBucketDisappearance()); +} + +void +MergeHandlerTest::testMergeProgressSafeGuard() +{ + MergeHandler handler(getPersistenceProvider(), getEnv()); + api::MergeBucketCommand cmd(_bucket, _nodes, _maxTimestamp); + handler.handleMergeBucket(cmd, *_context); + + std::shared_ptr<api::GetBucketDiffCommand> getBucketDiffCmd( + fetchSingleMessage<api::GetBucketDiffCommand>()); + api::GetBucketDiffReply::UP getBucketDiffReply( + new api::GetBucketDiffReply(*getBucketDiffCmd)); + + handler.handleGetBucketDiffReply(*getBucketDiffReply, messageKeeper()); + + std::shared_ptr<api::ApplyBucketDiffCommand> applyBucketDiffCmd( + fetchSingleMessage<api::ApplyBucketDiffCommand>()); + api::ApplyBucketDiffReply::UP applyBucketDiffReply( + new api::ApplyBucketDiffReply(*applyBucketDiffCmd)); + + MessageSenderStub stub; + handler.handleApplyBucketDiffReply(*applyBucketDiffReply, stub); + + CPPUNIT_ASSERT_EQUAL(1, (int)stub.replies.size()); + + api::MergeBucketReply::SP mergeReply( + std::dynamic_pointer_cast<api::MergeBucketReply>( + stub.replies[0])); + CPPUNIT_ASSERT(mergeReply.get()); + CPPUNIT_ASSERT(mergeReply->getResult().getResult() + == api::ReturnCode::INTERNAL_FAILURE); +} + +void +MergeHandlerTest::testSafeGuardNotInvokedWhenHasMaskChanges() +{ + MergeHandler handler(getPersistenceProvider(), getEnv()); + _nodes.clear(); + _nodes.push_back(api::MergeBucketCommand::Node(0, false)); + _nodes.push_back(api::MergeBucketCommand::Node(1, false)); + _nodes.push_back(api::MergeBucketCommand::Node(2, false)); + api::MergeBucketCommand cmd(_bucket, _nodes, _maxTimestamp); + handler.handleMergeBucket(cmd, *_context); + + std::shared_ptr<api::GetBucketDiffCommand> getBucketDiffCmd( + fetchSingleMessage<api::GetBucketDiffCommand>()); + api::GetBucketDiffReply::UP getBucketDiffReply( + new api::GetBucketDiffReply(*getBucketDiffCmd)); + + handler.handleGetBucketDiffReply(*getBucketDiffReply, messageKeeper()); + + std::shared_ptr<api::ApplyBucketDiffCommand> applyBucketDiffCmd( + fetchSingleMessage<api::ApplyBucketDiffCommand>()); + api::ApplyBucketDiffReply::UP applyBucketDiffReply( + new api::ApplyBucketDiffReply(*applyBucketDiffCmd)); + CPPUNIT_ASSERT(!applyBucketDiffReply->getDiff().empty()); + // Change a hasMask to indicate something changed during merging. + applyBucketDiffReply->getDiff()[0]._entry._hasMask = 0x5; + + MessageSenderStub stub; + LOG(debug, "sending apply bucket diff reply"); + handler.handleApplyBucketDiffReply(*applyBucketDiffReply, stub); + + CPPUNIT_ASSERT_EQUAL(1, (int)stub.commands.size()); + + api::ApplyBucketDiffCommand::SP applyBucketDiffCmd2( + std::dynamic_pointer_cast<api::ApplyBucketDiffCommand>( + stub.commands[0])); + CPPUNIT_ASSERT(applyBucketDiffCmd2.get()); + CPPUNIT_ASSERT_EQUAL(applyBucketDiffCmd->getDiff().size(), + applyBucketDiffCmd2->getDiff().size()); + CPPUNIT_ASSERT_EQUAL(uint16_t(0x5), + applyBucketDiffCmd2->getDiff()[0]._entry._hasMask); +} + +void +MergeHandlerTest::testEntryRemovedAfterGetBucketDiff() +{ + MergeHandler handler(getPersistenceProvider(), getEnv()); + std::vector<api::ApplyBucketDiffCommand::Entry> applyDiff; + { + api::ApplyBucketDiffCommand::Entry e; + e._entry._timestamp = 13001; // Removed in persistence + e._entry._hasMask = 0x2; + e._entry._flags = MergeHandler::IN_USE; + applyDiff.push_back(e); + } + setUpChain(BACK); + std::shared_ptr<api::ApplyBucketDiffCommand> applyBucketDiffCmd( + new api::ApplyBucketDiffCommand(_bucket, _nodes, 1024*1024)); + applyBucketDiffCmd->getDiff() = applyDiff; + + MessageTracker::UP tracker = handler.handleApplyBucketDiff(*applyBucketDiffCmd, *_context); + + api::ApplyBucketDiffReply::SP applyBucketDiffReply( + std::dynamic_pointer_cast<api::ApplyBucketDiffReply>( + tracker->getReply())); + CPPUNIT_ASSERT(applyBucketDiffReply.get()); + + std::vector<api::ApplyBucketDiffCommand::Entry>& diff( + applyBucketDiffReply->getDiff()); + CPPUNIT_ASSERT_EQUAL(size_t(1), diff.size()); + CPPUNIT_ASSERT(!diff[0].filled()); + CPPUNIT_ASSERT_EQUAL(uint16_t(0x0), diff[0]._entry._hasMask); +} + +std::string +MergeHandlerTest::doTestSPIException(MergeHandler& handler, + PersistenceProviderWrapper& providerWrapper, + HandlerInvoker& invoker, + const ExpectedExceptionSpec& spec) +{ + providerWrapper.setFailureMask(0); + invoker.beforeInvoke(*this, handler, *_context); // Do any setup stuff first + + uint32_t failureMask = spec.mask; + const char* expectedSubstring = spec.expected; + providerWrapper.setFailureMask(failureMask); + try { + invoker.invoke(*this, handler, *_context); + if (failureMask != 0) { + return (std::string("No exception was thrown during handler " + "invocation. Expected exception containing '") + + expectedSubstring + "'"); + } + } catch (const std::runtime_error& e) { + if (std::string(e.what()).find(expectedSubstring) + == std::string::npos) + { + return (std::string("Expected exception to contain substring '") + + expectedSubstring + "', but message was: " + e.what()); + } + } + if (fsHandler().isMerging(_bucket)) { + return (std::string("After operation with expected exception '") + + expectedSubstring + "', merge state was not cleared"); + } + // Postcondition check. + std::string check = invoker.afterInvoke(*this, handler); + if (!check.empty()) { + return (std::string("Postcondition validation failed for operation " + "with expected exception '") + + expectedSubstring + "': " + check); + } + return ""; +} + +std::string +MergeHandlerTest::NoReplyHandlerInvoker::afterInvoke( + MergeHandlerTest& test, + MergeHandler& handler) +{ + (void) handler; + if (!test.messageKeeper()._msgs.empty()) { + std::ostringstream ss; + ss << "Expected 0 explicit replies, got " + << test.messageKeeper()._msgs.size(); + return ss.str(); + } + return ""; +} + +template <typename ExpectedMessage> +std::string +MergeHandlerTest::checkMessage(api::ReturnCode::Result expectedResult) +{ + try { + std::shared_ptr<ExpectedMessage> msg( + fetchSingleMessage<ExpectedMessage>()); + if (msg->getResult().getResult() != expectedResult) { + return "Got unexpected result: " + msg->getResult().toString(); + } + } catch (std::exception& e) { + return e.what(); + } + return ""; +} + +void +MergeHandlerTest::HandleMergeBucketInvoker::invoke( + MergeHandlerTest& test, + MergeHandler& handler, + spi::Context& context) +{ + api::MergeBucketCommand cmd(test._bucket, test._nodes, test._maxTimestamp); + handler.handleMergeBucket(cmd, context); +} + +void +MergeHandlerTest::testMergeBucketSPIFailures() +{ + PersistenceProviderWrapper providerWrapper( + getPersistenceProvider()); + MergeHandler handler(providerWrapper, getEnv()); + providerWrapper.setResult( + spi::Result(spi::Result::PERMANENT_ERROR, + "who you gonna call?")); + setUpChain(MIDDLE); + + ExpectedExceptionSpec exceptions[] = { + { PersistenceProviderWrapper::FAIL_CREATE_BUCKET, "create bucket" }, + { PersistenceProviderWrapper::FAIL_BUCKET_INFO, "get bucket info" }, + { PersistenceProviderWrapper::FAIL_CREATE_ITERATOR, "create iterator" }, + { PersistenceProviderWrapper::FAIL_ITERATE, "iterate" }, + }; + typedef ExpectedExceptionSpec* ExceptionIterator; + ExceptionIterator last = exceptions + sizeof(exceptions)/sizeof(exceptions[0]); + + for (ExceptionIterator it = exceptions; it != last; ++it) { + HandleMergeBucketInvoker invoker; + CPPUNIT_ASSERT_EQUAL(std::string(), + doTestSPIException(handler, + providerWrapper, + invoker, + *it)); + } +} + +void +MergeHandlerTest::HandleGetBucketDiffInvoker::invoke( + MergeHandlerTest& test, + MergeHandler& handler, + spi::Context& context) +{ + api::GetBucketDiffCommand cmd(test._bucket, test._nodes, test._maxTimestamp); + handler.handleGetBucketDiff(cmd, context); +} + +void +MergeHandlerTest::testGetBucketDiffSPIFailures() +{ + PersistenceProviderWrapper providerWrapper( + getPersistenceProvider()); + MergeHandler handler(providerWrapper, getEnv()); + providerWrapper.setResult( + spi::Result(spi::Result::PERMANENT_ERROR, + "who you gonna call?")); + setUpChain(MIDDLE); + + ExpectedExceptionSpec exceptions[] = { + { PersistenceProviderWrapper::FAIL_CREATE_BUCKET, "create bucket" }, + { PersistenceProviderWrapper::FAIL_BUCKET_INFO, "get bucket info" }, + { PersistenceProviderWrapper::FAIL_CREATE_ITERATOR, "create iterator" }, + { PersistenceProviderWrapper::FAIL_ITERATE, "iterate" }, + }; + + typedef ExpectedExceptionSpec* ExceptionIterator; + ExceptionIterator last = exceptions + sizeof(exceptions)/sizeof(exceptions[0]); + + for (ExceptionIterator it = exceptions; it != last; ++it) { + HandleGetBucketDiffInvoker invoker; + CPPUNIT_ASSERT_EQUAL(std::string(), + doTestSPIException(handler, + providerWrapper, + invoker, + *it)); + } +} + +void +MergeHandlerTest::HandleApplyBucketDiffInvoker::invoke( + MergeHandlerTest& test, + MergeHandler& handler, + spi::Context& context) +{ + ++_counter; + std::shared_ptr<api::ApplyBucketDiffCommand> cmd( + test.createDummyApplyDiff(100000 * _counter)); + handler.handleApplyBucketDiff(*cmd, context); +} + +void +MergeHandlerTest::testApplyBucketDiffSPIFailures() +{ + PersistenceProviderWrapper providerWrapper( + getPersistenceProvider()); + MergeHandler handler(providerWrapper, getEnv()); + providerWrapper.setResult( + spi::Result(spi::Result::PERMANENT_ERROR, + "who you gonna call?")); + setUpChain(MIDDLE); + + ExpectedExceptionSpec exceptions[] = { + { PersistenceProviderWrapper::FAIL_CREATE_ITERATOR, "create iterator" }, + { PersistenceProviderWrapper::FAIL_ITERATE, "iterate" }, + { PersistenceProviderWrapper::FAIL_PUT, "Failed put" }, + { PersistenceProviderWrapper::FAIL_REMOVE, "Failed remove" }, + { PersistenceProviderWrapper::FAIL_FLUSH, "Failed flush" }, + }; + + typedef ExpectedExceptionSpec* ExceptionIterator; + ExceptionIterator last = exceptions + sizeof(exceptions)/sizeof(exceptions[0]); + + for (ExceptionIterator it = exceptions; it != last; ++it) { + HandleApplyBucketDiffInvoker invoker; + CPPUNIT_ASSERT_EQUAL(std::string(), + doTestSPIException(handler, + providerWrapper, + invoker, + *it)); + // Casual, in-place testing of bug 6752085. + // This will fail if we give NaN to the metric in question. + CPPUNIT_ASSERT(std::isfinite(getEnv()._metrics + .mergeAverageDataReceivedNeeded.getLast())); + } +} + +void +MergeHandlerTest::HandleGetBucketDiffReplyInvoker::beforeInvoke( + MergeHandlerTest& test, + MergeHandler& handler, + spi::Context& context) +{ + api::MergeBucketCommand cmd(test._bucket, test._nodes, test._maxTimestamp); + handler.handleMergeBucket(cmd, context); + _diffCmd = test.fetchSingleMessage<api::GetBucketDiffCommand>(); +} + +void +MergeHandlerTest::HandleGetBucketDiffReplyInvoker::invoke( + MergeHandlerTest& test, + MergeHandler& handler, + spi::Context&) +{ + (void) test; + api::GetBucketDiffReply reply(*_diffCmd); + handler.handleGetBucketDiffReply(reply, _stub); +} + +std::string +MergeHandlerTest::HandleGetBucketDiffReplyInvoker::afterInvoke( + MergeHandlerTest& test, + MergeHandler& handler) +{ + (void) handler; + if (!_stub.commands.empty()) { + return "Unexpected commands in reply stub"; + } + if (!_stub.replies.empty()) { + return "Unexpected replies in reply stub"; + } + // Initial merge bucket should have been replied to by clearMergeStatus. + return test.checkMessage<api::MergeBucketReply>( + api::ReturnCode::INTERNAL_FAILURE); +} + +void +MergeHandlerTest::testGetBucketDiffReplySPIFailures() +{ + PersistenceProviderWrapper providerWrapper( + getPersistenceProvider()); + MergeHandler handler(providerWrapper, getEnv()); + providerWrapper.setResult( + spi::Result(spi::Result::PERMANENT_ERROR, + "who you gonna call?")); + HandleGetBucketDiffReplyInvoker invoker; + + setUpChain(FRONT); + + ExpectedExceptionSpec exceptions[] = { + { PersistenceProviderWrapper::FAIL_CREATE_ITERATOR, "create iterator" }, + { PersistenceProviderWrapper::FAIL_ITERATE, "iterate" }, + }; + + typedef ExpectedExceptionSpec* ExceptionIterator; + ExceptionIterator last = exceptions + sizeof(exceptions)/sizeof(exceptions[0]); + + for (ExceptionIterator it = exceptions; it != last; ++it) { + CPPUNIT_ASSERT_EQUAL(std::string(), + doTestSPIException(handler, + providerWrapper, + invoker, + *it)); + } +} + +void +MergeHandlerTest::HandleApplyBucketDiffReplyInvoker::beforeInvoke( + MergeHandlerTest& test, + MergeHandler& handler, + spi::Context& context) +{ + ++_counter; + _stub.clear(); + if (getChainPos() == FRONT) { + api::MergeBucketCommand cmd(test._bucket, test._nodes, test._maxTimestamp); + handler.handleMergeBucket(cmd, context); + std::shared_ptr<api::GetBucketDiffCommand> diffCmd( + test.fetchSingleMessage<api::GetBucketDiffCommand>()); + std::shared_ptr<api::GetBucketDiffCommand> dummyDiff( + test.createDummyGetBucketDiff(100000 * _counter, 0x4)); + diffCmd->getDiff() = dummyDiff->getDiff(); + + api::GetBucketDiffReply diffReply(*diffCmd); + handler.handleGetBucketDiffReply(diffReply, _stub); + + CPPUNIT_ASSERT_EQUAL(size_t(1), _stub.commands.size()); + _applyCmd = std::dynamic_pointer_cast<api::ApplyBucketDiffCommand>( + _stub.commands[0]); + } else { + // Pretend last node in chain has data and that it will be fetched when + // chain is unwinded. + std::shared_ptr<api::ApplyBucketDiffCommand> cmd( + test.createDummyApplyDiff(100000 * _counter, 0x4, false)); + handler.handleApplyBucketDiff(*cmd, context); + _applyCmd = test.fetchSingleMessage<api::ApplyBucketDiffCommand>(); + } +} + +void +MergeHandlerTest::HandleApplyBucketDiffReplyInvoker::invoke( + MergeHandlerTest& test, + MergeHandler& handler, + spi::Context&) +{ + (void) test; + api::ApplyBucketDiffReply reply(*_applyCmd); + test.fillDummyApplyDiff(reply.getDiff()); + _stub.clear(); + handler.handleApplyBucketDiffReply(reply, _stub); +} + +std::string +MergeHandlerTest::HandleApplyBucketDiffReplyInvoker::afterInvoke( + MergeHandlerTest& test, + MergeHandler& handler) +{ + (void) handler; + if (!_stub.commands.empty()) { + return "Unexpected commands in reply stub"; + } + if (!_stub.replies.empty()) { + return "Unexpected replies in reply stub"; + } + if (getChainPos() == FRONT) { + return test.checkMessage<api::MergeBucketReply>( + api::ReturnCode::INTERNAL_FAILURE); + } else { + return test.checkMessage<api::ApplyBucketDiffReply>( + api::ReturnCode::INTERNAL_FAILURE); + } +} + +void +MergeHandlerTest::testApplyBucketDiffReplySPIFailures() +{ + PersistenceProviderWrapper providerWrapper( + getPersistenceProvider()); + HandleApplyBucketDiffReplyInvoker invoker; + for (int i = 0; i < 2; ++i) { + ChainPos pos(i == 0 ? FRONT : MIDDLE); + setUpChain(pos); + invoker.setChainPos(pos); + MergeHandler handler(providerWrapper, getEnv()); + providerWrapper.setResult( + spi::Result(spi::Result::PERMANENT_ERROR, + "who you gonna call?")); + + ExpectedExceptionSpec exceptions[] = { + { PersistenceProviderWrapper::FAIL_CREATE_ITERATOR, "create iterator" }, + { PersistenceProviderWrapper::FAIL_ITERATE, "iterate" }, + { PersistenceProviderWrapper::FAIL_PUT, "Failed put" }, + { PersistenceProviderWrapper::FAIL_REMOVE, "Failed remove" }, + { PersistenceProviderWrapper::FAIL_FLUSH, "Failed flush" }, + }; + + typedef ExpectedExceptionSpec* ExceptionIterator; + ExceptionIterator last = exceptions + sizeof(exceptions)/sizeof(exceptions[0]); + + for (ExceptionIterator it = exceptions; it != last; ++it) { + CPPUNIT_ASSERT_EQUAL(std::string(), + doTestSPIException(handler, + providerWrapper, + invoker, + *it)); + } + } +} + +void +MergeHandlerTest::testRemoveFromDiff() +{ + framework::defaultimplementation::FakeClock clock; + MergeStatus status(clock, documentapi::LoadType::DEFAULT, 0, 0); + + std::vector<api::GetBucketDiffCommand::Entry> diff(2); + diff[0]._timestamp = 1234; + diff[0]._flags = 0x1; + diff[0]._hasMask = 0x2; + + diff[1]._timestamp = 5678; + diff[1]._flags = 0x3; + diff[1]._hasMask = 0x6; + + status.diff.insert(status.diff.end(), diff.begin(), diff.end()); + + { + std::vector<api::ApplyBucketDiffCommand::Entry> applyDiff(2); + applyDiff[0]._entry._timestamp = 1234; + applyDiff[0]._entry._flags = 0x1; + applyDiff[0]._entry._hasMask = 0x0; // Removed during merging + + applyDiff[1]._entry._timestamp = 5678; + applyDiff[1]._entry._flags = 0x3; + applyDiff[1]._entry._hasMask = 0x7; + + CPPUNIT_ASSERT(status.removeFromDiff(applyDiff, 0x7)); + CPPUNIT_ASSERT(status.diff.empty()); + } + + status.diff.insert(status.diff.end(), diff.begin(), diff.end()); + + { + std::vector<api::ApplyBucketDiffCommand::Entry> applyDiff(2); + applyDiff[0]._entry._timestamp = 1234; + applyDiff[0]._entry._flags = 0x1; + applyDiff[0]._entry._hasMask = 0x2; + + applyDiff[1]._entry._timestamp = 5678; + applyDiff[1]._entry._flags = 0x3; + applyDiff[1]._entry._hasMask = 0x6; + + CPPUNIT_ASSERT(!status.removeFromDiff(applyDiff, 0x7)); + CPPUNIT_ASSERT_EQUAL(size_t(2), status.diff.size()); + } + + status.diff.clear(); + status.diff.insert(status.diff.end(), diff.begin(), diff.end()); + + { + // Hasmasks have changed but diff still remains the same size. + std::vector<api::ApplyBucketDiffCommand::Entry> applyDiff(2); + applyDiff[0]._entry._timestamp = 1234; + applyDiff[0]._entry._flags = 0x1; + applyDiff[0]._entry._hasMask = 0x1; + + applyDiff[1]._entry._timestamp = 5678; + applyDiff[1]._entry._flags = 0x3; + applyDiff[1]._entry._hasMask = 0x5; + + CPPUNIT_ASSERT(status.removeFromDiff(applyDiff, 0x7)); + CPPUNIT_ASSERT_EQUAL(size_t(2), status.diff.size()); + } +} + +void +MergeHandlerTest::testRemovePutOnExistingTimestamp() +{ + setUpChain(BACK); + + document::TestDocMan docMan; + document::Document::SP doc( + docMan.createRandomDocumentAtLocation(_location)); + spi::Timestamp ts(10111); + doPut(doc, ts); + + MergeHandler handler(getPersistenceProvider(), getEnv()); + std::vector<api::ApplyBucketDiffCommand::Entry> applyDiff; + { + api::ApplyBucketDiffCommand::Entry e; + e._entry._timestamp = ts; + e._entry._hasMask = 0x1; + e._docName = doc->getId().toString(); + e._entry._flags = MergeHandler::IN_USE | MergeHandler::DELETED; + applyDiff.push_back(e); + } + + std::shared_ptr<api::ApplyBucketDiffCommand> applyBucketDiffCmd( + new api::ApplyBucketDiffCommand(_bucket, _nodes, 1024*1024)); + applyBucketDiffCmd->getDiff() = applyDiff; + + MessageTracker::UP tracker = handler.handleApplyBucketDiff(*applyBucketDiffCmd, *_context); + + api::ApplyBucketDiffReply::SP applyBucketDiffReply( + std::dynamic_pointer_cast<api::ApplyBucketDiffReply>( + tracker->getReply())); + CPPUNIT_ASSERT(applyBucketDiffReply.get()); + + api::MergeBucketCommand cmd(_bucket, _nodes, _maxTimestamp); + handler.handleMergeBucket(cmd, *_context); + + std::shared_ptr<api::GetBucketDiffCommand> getBucketDiffCmd( + fetchSingleMessage<api::GetBucketDiffCommand>()); + + // Timestamp should now be a regular remove + bool foundTimestamp = false; + for (size_t i = 0; i < getBucketDiffCmd->getDiff().size(); ++i) { + const api::GetBucketDiffCommand::Entry& e( + getBucketDiffCmd->getDiff()[i]); + if (e._timestamp == ts) { + CPPUNIT_ASSERT_EQUAL( + uint16_t(MergeHandler::IN_USE | MergeHandler::DELETED), + e._flags); + foundTimestamp = true; + break; + } + } + CPPUNIT_ASSERT(foundTimestamp); +} + +} // storage diff --git a/storage/src/tests/persistence/persistenceproviderwrapper.cpp b/storage/src/tests/persistence/persistenceproviderwrapper.cpp new file mode 100644 index 00000000000..4a09235ddce --- /dev/null +++ b/storage/src/tests/persistence/persistenceproviderwrapper.cpp @@ -0,0 +1,222 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <iostream> +#include <sstream> +#include <tests/persistence/persistenceproviderwrapper.h> + +#define LOG_SPI(ops) \ + { \ + std::ostringstream logStream; \ + logStream << ops; \ + _log.push_back(logStream.str()); \ + } + +#define CHECK_ERROR(className, failType) \ + { \ + if (_result.getErrorCode() != spi::Result::NONE && (_failureMask & (failType))) { \ + return className(_result.getErrorCode(), _result.getErrorMessage()); \ + } \ + } + +namespace storage { + +namespace { + +const char* +includedVersionsToString(spi::IncludedVersions versions) +{ + switch (versions) { + case spi::NEWEST_DOCUMENT_ONLY: + return "NEWEST_DOCUMENT_ONLY"; + case spi::NEWEST_DOCUMENT_OR_REMOVE: + return "NEWEST_DOCUMENT_OR_REMOVE"; + case spi::ALL_VERSIONS: + return "ALL_VERSIONS"; + } + return "!!UNKNOWN!!"; +} + +} // anon namespace + +std::string +PersistenceProviderWrapper::toString() const +{ + std::ostringstream ss; + for (size_t i = 0; i < _log.size(); ++i) { + ss << _log[i] << "\n"; + } + return ss.str(); +} + +spi::PartitionStateListResult +PersistenceProviderWrapper::getPartitionStates() const +{ + LOG_SPI("getPartitionStates()"); + return _spi.getPartitionStates(); +} + +spi::BucketIdListResult +PersistenceProviderWrapper::listBuckets(spi::PartitionId partitionId) const +{ + LOG_SPI("listBuckets(" << uint16_t(partitionId) << ")"); + CHECK_ERROR(spi::BucketIdListResult, FAIL_LIST_BUCKETS); + return _spi.listBuckets(partitionId); +} + +spi::Result +PersistenceProviderWrapper::createBucket(const spi::Bucket& bucket, + spi::Context& context) +{ + LOG_SPI("createBucket(" << bucket << ")"); + CHECK_ERROR(spi::Result, FAIL_CREATE_BUCKET); + return _spi.createBucket(bucket, context); +} + +spi::BucketInfoResult +PersistenceProviderWrapper::getBucketInfo(const spi::Bucket& bucket) const +{ + LOG_SPI("getBucketInfo(" << bucket << ")"); + CHECK_ERROR(spi::BucketInfoResult, FAIL_BUCKET_INFO); + return _spi.getBucketInfo(bucket); +} + +spi::Result +PersistenceProviderWrapper::put(const spi::Bucket& bucket, + spi::Timestamp timestamp, + const document::Document::SP& doc, + spi::Context& context) +{ + LOG_SPI("put(" << bucket << ", " << timestamp << ", " << doc->getId() << ")"); + CHECK_ERROR(spi::Result, FAIL_PUT); + return _spi.put(bucket, timestamp, doc, context); +} + +spi::RemoveResult +PersistenceProviderWrapper::remove(const spi::Bucket& bucket, + spi::Timestamp timestamp, + const spi::DocumentId& id, + spi::Context& context) +{ + LOG_SPI("remove(" << bucket << ", " << timestamp << ", " << id << ")"); + CHECK_ERROR(spi::RemoveResult, FAIL_REMOVE); + return _spi.remove(bucket, timestamp, id, context); +} + +spi::RemoveResult +PersistenceProviderWrapper::removeIfFound(const spi::Bucket& bucket, + spi::Timestamp timestamp, + const spi::DocumentId& id, + spi::Context& context) +{ + LOG_SPI("removeIfFound(" << bucket << ", " << timestamp << ", " << id << ")"); + CHECK_ERROR(spi::RemoveResult, FAIL_REMOVE_IF_FOUND); + return _spi.removeIfFound(bucket, timestamp, id, context); +} + +spi::UpdateResult +PersistenceProviderWrapper::update(const spi::Bucket& bucket, + spi::Timestamp timestamp, + const document::DocumentUpdate::SP& upd, + spi::Context& context) +{ + LOG_SPI("update(" << bucket << ", " << timestamp << ", " << upd->getId() << ")"); + CHECK_ERROR(spi::UpdateResult, FAIL_UPDATE); + return _spi.update(bucket, timestamp, upd, context); +} + +spi::GetResult +PersistenceProviderWrapper::get(const spi::Bucket& bucket, + const document::FieldSet& fieldSet, + const spi::DocumentId& id, + spi::Context& context) const +{ + LOG_SPI("get(" << bucket << ", " << id << ")"); + CHECK_ERROR(spi::GetResult, FAIL_GET); + return _spi.get(bucket, fieldSet, id, context); +} + +spi::Result +PersistenceProviderWrapper::flush(const spi::Bucket& bucket, + spi::Context& context) +{ + LOG_SPI("flush(" << bucket << ")"); + CHECK_ERROR(spi::Result, FAIL_FLUSH); + return _spi.flush(bucket, context); +} + +spi::CreateIteratorResult +PersistenceProviderWrapper::createIterator(const spi::Bucket& bucket, + const document::FieldSet& fields, + const spi::Selection& sel, + spi::IncludedVersions versions, + spi::Context& context) +{ + // TODO: proper printing of FieldSet and Selection + + LOG_SPI("createIterator(" << bucket << ", " + << includedVersionsToString(versions) << ")"); + CHECK_ERROR(spi::CreateIteratorResult, FAIL_CREATE_ITERATOR); + return _spi.createIterator(bucket, fields, sel, versions, context); +} + +spi::IterateResult +PersistenceProviderWrapper::iterate(spi::IteratorId iterId, + uint64_t maxByteSize, + spi::Context& context) const +{ + LOG_SPI("iterate(" << uint64_t(iterId) << ", " << maxByteSize << ")"); + CHECK_ERROR(spi::IterateResult, FAIL_ITERATE); + return _spi.iterate(iterId, maxByteSize, context); +} + +spi::Result +PersistenceProviderWrapper::destroyIterator(spi::IteratorId iterId, + spi::Context& context) +{ + LOG_SPI("destroyIterator(" << uint64_t(iterId) << ")"); + CHECK_ERROR(spi::Result, FAIL_DESTROY_ITERATOR); + return _spi.destroyIterator(iterId, context); +} + +spi::Result +PersistenceProviderWrapper::deleteBucket(const spi::Bucket& bucket, + spi::Context& context) +{ + LOG_SPI("deleteBucket(" << bucket << ")"); + CHECK_ERROR(spi::Result, FAIL_DELETE_BUCKET); + return _spi.deleteBucket(bucket, context); +} + +spi::Result +PersistenceProviderWrapper::split(const spi::Bucket& source, + const spi::Bucket& target1, + const spi::Bucket& target2, + spi::Context& context) +{ + LOG_SPI("split(" << source << ", " << target1 << ", " << target2 << ")"); + CHECK_ERROR(spi::Result, FAIL_SPLIT); + return _spi.split(source, target1, target2, context); +} + +spi::Result +PersistenceProviderWrapper::join(const spi::Bucket& source1, + const spi::Bucket& source2, + const spi::Bucket& target, + spi::Context& context) +{ + LOG_SPI("join(" << source1 << ", " << source2 << ", " << target << ")"); + CHECK_ERROR(spi::Result, FAIL_JOIN); + return _spi.join(source1, source2, target, context); +} + +spi::Result +PersistenceProviderWrapper::removeEntry(const spi::Bucket& bucket, + spi::Timestamp timestamp, + spi::Context& context) +{ + LOG_SPI("revert(" << bucket << ", " << timestamp << ")"); + CHECK_ERROR(spi::Result, FAIL_REVERT); + return _spi.removeEntry(bucket, timestamp, context); +} + +} diff --git a/storage/src/tests/persistence/persistenceproviderwrapper.h b/storage/src/tests/persistence/persistenceproviderwrapper.h new file mode 100644 index 00000000000..b115eb7ef3d --- /dev/null +++ b/storage/src/tests/persistence/persistenceproviderwrapper.h @@ -0,0 +1,153 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class storage::PersistenceProviderWrapper + * + * \brief Test utility class for intercepting all operations upon a + * persistence layer, injecting errors and performing logging. + * + * The PersistenceProviderWrapper class implements the basic SPI by + * logging all operations and then delegating handling the operation + * to the SPI instance given during construction. If an error result + * is specified and the operation invoked is tagged that it should be + * failed via setFailureMask(), the operation on the wrapped SPI will + * not be executed, but the given error result will be immediately + * returned instead (wrapped in the proper return type). + */ +#pragma once + +#include <vector> +#include <string> +#include <vespa/persistence/spi/abstractpersistenceprovider.h> + +namespace storage { + +class PersistenceProviderWrapper : public spi::AbstractPersistenceProvider +{ +public: + enum OPERATION_FAILURE_FLAGS + { + FAIL_LIST_BUCKETS = 1 << 0, + FAIL_BUCKET_INFO = 1 << 1, + FAIL_GET = 1 << 2, + FAIL_PUT = 1 << 3, + FAIL_REMOVE = 1 << 4, + FAIL_REMOVE_IF_FOUND = 1 << 5, + FAIL_REPLACE_WITH_REMOVE = 1 << 6, + FAIL_UPDATE = 1 << 7, + FAIL_REVERT = 1 << 8, + FAIL_FLUSH = 1 << 9, + FAIL_CREATE_ITERATOR = 1 << 10, + FAIL_ITERATE = 1 << 11, + FAIL_DESTROY_ITERATOR = 1 << 12, + FAIL_DELETE_BUCKET = 1 << 13, + FAIL_SPLIT = 1 << 14, + FAIL_JOIN = 1 << 15, + FAIL_CREATE_BUCKET = 1 << 16, + FAIL_BUCKET_PERSISTENCE = FAIL_PUT|FAIL_REMOVE|FAIL_UPDATE|FAIL_REVERT|FAIL_FLUSH, + FAIL_ALL_OPERATIONS = 0xffff, + // TODO: add more as needed + }; +private: + spi::PersistenceProvider& _spi; + spi::Result _result; + mutable std::vector<std::string> _log; + uint32_t _failureMask; +public: + PersistenceProviderWrapper(spi::PersistenceProvider& spi) + : _spi(spi), + _result(spi::Result(spi::Result::NONE, "")), + _log(), + _failureMask(0) + { + } + + /** + * Explicitly set result to anything != NONE to have all operations + * return the given error without the wrapped SPI ever being invoked. + */ + void setResult(const spi::Result& result) { + _result = result; + } + void clearResult() { + _result = spi::Result(spi::Result::NONE, ""); + } + const spi::Result& getResult() const { return _result; } + /** + * Set a mask for operations to fail with _result + */ + void setFailureMask(uint32_t mask) { _failureMask = mask; } + uint32_t getFailureMask() const { return _failureMask; } + + /** + * Get a string representation of all the operations performed on the + * SPI with a newline separating each operation. + */ + std::string toString() const; + /** + * Clear log of all operations performed. + */ + void clearOperationLog() { _log.clear(); } + const std::vector<std::string>& getOperationLog() const { return _log; } + + spi::Result createBucket(const spi::Bucket&, spi::Context&); + + spi::PartitionStateListResult getPartitionStates() const; + + spi::BucketIdListResult listBuckets(spi::PartitionId) const; + + spi::BucketInfoResult getBucketInfo(const spi::Bucket&) const; + + spi::Result put(const spi::Bucket&, spi::Timestamp, const document::Document::SP&, spi::Context&); + + spi::RemoveResult remove(const spi::Bucket&, + spi::Timestamp, + const spi::DocumentId&, + spi::Context&); + + spi::RemoveResult removeIfFound(const spi::Bucket&, + spi::Timestamp, + const spi::DocumentId&, + spi::Context&); + + spi::UpdateResult update(const spi::Bucket&, + spi::Timestamp, + const document::DocumentUpdate::SP&, + spi::Context&); + + spi::GetResult get(const spi::Bucket&, + const document::FieldSet&, + const spi::DocumentId&, + spi::Context&) const; + + spi::Result flush(const spi::Bucket&, spi::Context&); + + spi::CreateIteratorResult createIterator(const spi::Bucket&, + const document::FieldSet&, + const spi::Selection&, + spi::IncludedVersions versions, + spi::Context&); + + spi::IterateResult iterate(spi::IteratorId, + uint64_t maxByteSize, spi::Context&) const; + + spi::Result destroyIterator(spi::IteratorId, spi::Context&); + + spi::Result deleteBucket(const spi::Bucket&, spi::Context&); + + spi::Result split(const spi::Bucket& source, + const spi::Bucket& target1, + const spi::Bucket& target2, + spi::Context&); + + spi::Result join(const spi::Bucket& source1, + const spi::Bucket& source2, + const spi::Bucket& target, + spi::Context&); + + spi::Result removeEntry(const spi::Bucket&, + spi::Timestamp, + spi::Context&); +}; + +} // storage + diff --git a/storage/src/tests/persistence/persistencequeuetest.cpp b/storage/src/tests/persistence/persistencequeuetest.cpp new file mode 100644 index 00000000000..06daf2a975c --- /dev/null +++ b/storage/src/tests/persistence/persistencequeuetest.cpp @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storageapi/message/bucket.h> +#include <tests/persistence/persistenceproviderwrapper.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <tests/persistence/filestorage/filestortestfixture.h> +#include <tests/persistence/filestorage/forwardingmessagesender.h> + +LOG_SETUP(".persistencequeuetest"); + +namespace storage { + +class PersistenceQueueTest : public FileStorTestFixture +{ +public: + void testFetchNextUnlockedMessageIfBucketLocked(); + + std::shared_ptr<api::StorageMessage> + createPut(uint64_t bucket, uint64_t docIdx); + + void setUp() override; + void tearDown() override; + + CPPUNIT_TEST_SUITE(PersistenceQueueTest); + CPPUNIT_TEST(testFetchNextUnlockedMessageIfBucketLocked); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(PersistenceQueueTest); + +void +PersistenceQueueTest::setUp() +{ + setupDisks(1); + _node->setPersistenceProvider( + spi::PersistenceProvider::UP( + new spi::dummy::DummyPersistence(_node->getTypeRepo(), 1))); +} + +void +PersistenceQueueTest::tearDown() +{ + _node.reset(0); +} + +std::shared_ptr<api::StorageMessage> +PersistenceQueueTest::createPut(uint64_t bucket, uint64_t docIdx) +{ + std::ostringstream id; + id << "id:foo:testdoctype1:n=" << bucket << ":" << docIdx; + document::Document::SP doc( + _node->getTestDocMan().createDocument("foobar", id.str())); + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(document::BucketId(16, bucket), doc, 1234)); + cmd->setAddress(api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 0)); + return cmd; +} + +void +PersistenceQueueTest::testFetchNextUnlockedMessageIfBucketLocked() +{ + DummyStorageLink top; + DummyStorageLink *dummyManager; + top.push_back(std::unique_ptr<StorageLink>( + dummyManager = new DummyStorageLink)); + top.open(); + ForwardingMessageSender messageSender(*dummyManager); + + documentapi::LoadTypeSet loadTypes("raw:"); + FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); + metrics.initDiskMetrics(_node->getPartitions().size(), + loadTypes.getMetricLoadTypes(), 1); + + FileStorHandler filestorHandler(messageSender, metrics, + _node->getPartitions(), + _node->getComponentRegister(), 255, 0); + + // Send 2 puts, 2 to the first bucket, 1 to the second. Calling + // getNextMessage 2 times should then return a lock on the first bucket, + // then subsequently on the second, skipping the already locked bucket. + // Puts all have same pri, so order is well defined. + filestorHandler.schedule(createPut(1234, 0), 0); + filestorHandler.schedule(createPut(1234, 1), 0); + filestorHandler.schedule(createPut(5432, 0), 0); + + auto lock0 = filestorHandler.getNextMessage(0, 255); + CPPUNIT_ASSERT(lock0.first.get()); + CPPUNIT_ASSERT_EQUAL( + document::BucketId(16, 1234), + dynamic_cast<api::PutCommand&>(*lock0.second).getBucketId()); + + auto lock1 = filestorHandler.getNextMessage(0, 255); + CPPUNIT_ASSERT(lock1.first.get()); + CPPUNIT_ASSERT_EQUAL( + document::BucketId(16, 5432), + dynamic_cast<api::PutCommand&>(*lock1.second).getBucketId()); +} + +} // namespace storage diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp new file mode 100644 index 00000000000..47ec23147f1 --- /dev/null +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -0,0 +1,412 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> + +#include <vespa/document/datatype/documenttype.h> +#include <vespa/storageapi/message/persistence.h> +#include <tests/persistence/persistencetestutils.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> + +using document::DocumentType; +using storage::framework::defaultimplementation::AllocationLogic; + +namespace storage { + +namespace { + + spi::LoadType defaultLoadType(0, "default"); + + vdstestlib::DirConfig initialize(uint32_t numDisks) { + system(vespalib::make_string("rm -rf vdsroot").c_str()); + for (uint32_t i = 0; i < numDisks; i++) { + system(vespalib::make_string("mkdir -p vdsroot/disks/d%d", i).c_str()); + } + vdstestlib::DirConfig config(getStandardConfig(true)); + return config; + } + + template<typename T> + struct ConfigReader : public T::Subscriber + { + T config; + + ConfigReader(const std::string& configId) { + T::subscribe(configId, *this); + } + void configure(const T& c) { config = c; } + }; +} + +PersistenceTestEnvironment::PersistenceTestEnvironment(DiskCount numDisks) + : _config(initialize(numDisks)), + _messageKeeper(), + _node(numDisks, NodeIndex(0), _config.getConfigId()), + _component(_node.getComponentRegister(), "persistence test env"), + _metrics(_component.getLoadTypes()->getMetricLoadTypes()) +{ + _node.setupDummyPersistence(); + _metrics.initDiskMetrics( + numDisks, _node.getLoadTypes()->getMetricLoadTypes(), 1); + _handler.reset(new FileStorHandler( + _messageKeeper, _metrics, + _node.getPersistenceProvider().getPartitionStates().getList(), + _node.getComponentRegister(), 255, 0)); + for (uint32_t i = 0; i < numDisks; i++) { + _diskEnvs.push_back( + vespalib::LinkedPtr<PersistenceUtil>( + new PersistenceUtil( + _config.getConfigId(), + _node.getComponentRegister(), + *_handler, + *_metrics.disks[i]->threads[0], + i, + 255, + _node.getPersistenceProvider()))); + } +} + +PersistenceTestUtils::PersistenceTestUtils() +{ +} + +PersistenceTestUtils::~PersistenceTestUtils() +{ +} + +std::string +PersistenceTestUtils::dumpBucket(const document::BucketId& bid, + uint16_t disk) { + return dynamic_cast<spi::dummy::DummyPersistence&>(_env->_node.getPersistenceProvider()).dumpBucket(spi::Bucket(bid, spi::PartitionId(disk))); +} + +void +PersistenceTestUtils::setupDisks(uint32_t numDisks) { + _env.reset(new PersistenceTestEnvironment(DiskCount(numDisks))); +} + +std::unique_ptr<PersistenceThread> +PersistenceTestUtils::createPersistenceThread(uint32_t disk) +{ + return std::unique_ptr<PersistenceThread>( + new PersistenceThread(_env->_node.getComponentRegister(), + _env->_config.getConfigId(), + getPersistenceProvider(), + getEnv()._fileStorHandler, + getEnv()._metrics, + disk, + 255, + false)); +} + +document::Document::SP +PersistenceTestUtils::schedulePut( + uint32_t location, + spi::Timestamp timestamp, + uint16_t disk, + uint32_t minSize, + uint32_t maxSize) +{ + document::Document::SP doc(createRandomDocumentAtLocation( + location, timestamp, minSize, maxSize)); + std::shared_ptr<api::StorageMessage> msg( + new api::PutCommand( + document::BucketId(16, location), doc, timestamp)); + fsHandler().schedule(msg, disk); + return doc; +} + +StorBucketDatabase::WrappedEntry +PersistenceTestUtils::getBucket(const document::BucketId& id) +{ + return _env->_node.getStorageBucketDatabase().get(id, "foo"); +} + +StorBucketDatabase::WrappedEntry +PersistenceTestUtils::createBucket(const document::BucketId& id) +{ + return _env->_node.getStorageBucketDatabase().get( + id, + "foo", + StorBucketDatabase::CREATE_IF_NONEXISTING); +} + +spi::PersistenceProvider& +PersistenceTestUtils::getPersistenceProvider() +{ + return _env->_node.getPersistenceProvider(); +} + +std::string +PersistenceTestUtils::getBucketStatus(const document::BucketId& id) +{ + std::ostringstream ost; + StorBucketDatabase::WrappedEntry entry( + _env->_node.getStorageBucketDatabase().get( + id, "foo")); + + ost << id << ": "; + if (!entry.exist()) { + ost << "null"; + } else { + ost << entry->getBucketInfo().getDocumentCount() << "," << entry->disk; + } + + return ost.str(); +} + +document::Document::SP +PersistenceTestUtils::doPutOnDisk( + uint16_t disk, + uint32_t location, + spi::Timestamp timestamp, + uint32_t minSize, + uint32_t maxSize) +{ + document::Document::SP doc(createRandomDocumentAtLocation( + location, timestamp, minSize, maxSize)); + spi::Bucket b(document::BucketId(16, location), spi::PartitionId(disk)); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + + getPersistenceProvider().createBucket(b, context); + + getPersistenceProvider().put(spi::Bucket(b), timestamp, doc, context); + + getPersistenceProvider().flush(b, context); + return doc; +} + +bool +PersistenceTestUtils::doRemoveOnDisk( + uint16_t disk, + const document::BucketId& bucketId, + const document::DocumentId& docId, + spi::Timestamp timestamp, + bool persistRemove) +{ + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + if (persistRemove) { + spi::RemoveResult result = getPersistenceProvider().removeIfFound( + spi::Bucket(bucketId, spi::PartitionId(disk)), + timestamp, docId, context); + return result.wasFound(); + } + spi::RemoveResult result = getPersistenceProvider().remove( + spi::Bucket(bucketId, spi::PartitionId(disk)), + timestamp, docId, context); + + return result.wasFound(); +} + +bool +PersistenceTestUtils::doUnrevertableRemoveOnDisk( + uint16_t disk, + const document::BucketId& bucketId, + const document::DocumentId& docId, + spi::Timestamp timestamp) +{ + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + spi::RemoveResult result = getPersistenceProvider().remove( + spi::Bucket(bucketId, spi::PartitionId(disk)), + timestamp, docId, context); + return result.wasFound(); +} + +spi::GetResult +PersistenceTestUtils::doGetOnDisk( + uint16_t disk, + const document::BucketId& bucketId, + const document::DocumentId& docId, + bool headerOnly) +{ + document::FieldSet::UP fieldSet(new document::AllFields()); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + if (headerOnly) { + fieldSet.reset(new document::HeaderFields()); + } + return getPersistenceProvider().get(spi::Bucket( + bucketId, spi::PartitionId(disk)), *fieldSet, docId, context); +} + +document::DocumentUpdate::SP +PersistenceTestUtils::createBodyUpdate( + const document::DocumentId& docId, + const document::FieldValue& updateValue) +{ + const DocumentType* docType(_env->_component.getTypeRepo() + ->getDocumentType("testdoctype1")); + document::DocumentUpdate::SP update( + new document::DocumentUpdate(*docType, docId)); + std::shared_ptr<document::AssignValueUpdate> assignUpdate( + new document::AssignValueUpdate(updateValue)); + document::FieldUpdate fieldUpdate(docType->getField("content")); + fieldUpdate.addUpdate(*assignUpdate); + update->addUpdate(fieldUpdate); + return update; +} + +document::DocumentUpdate::SP +PersistenceTestUtils::createHeaderUpdate( + const document::DocumentId& docId, + const document::FieldValue& updateValue) +{ + const DocumentType* docType(_env->_component.getTypeRepo() + ->getDocumentType("testdoctype1")); + document::DocumentUpdate::SP update( + new document::DocumentUpdate(*docType, docId)); + std::shared_ptr<document::AssignValueUpdate> assignUpdate( + new document::AssignValueUpdate(updateValue)); + document::FieldUpdate fieldUpdate(docType->getField("headerval")); + fieldUpdate.addUpdate(*assignUpdate); + update->addUpdate(fieldUpdate); + return update; +} + +uint16_t +PersistenceTestUtils::getDiskFromBucketDatabaseIfUnset(const document::BucketId& bucket, + uint16_t disk) +{ + if (disk == 0xffff) { + StorBucketDatabase::WrappedEntry entry( + getEnv().getBucketDatabase().get(bucket, "createTestBucket")); + if (entry.exist()) { + return entry->disk; + } else { + std::ostringstream error; + error << bucket << " not in db and disk unset"; + throw vespalib::IllegalStateException(error.str(), VESPA_STRLOC); + } + } + return disk; +} + +void +PersistenceTestUtils::doPut(const document::Document::SP& doc, + spi::Timestamp time, + uint16_t disk, + uint16_t usedBits) +{ + document::BucketId bucket( + _env->_component.getBucketIdFactory().getBucketId(doc->getId())); + bucket.setUsedBits(usedBits); + disk = getDiskFromBucketDatabaseIfUnset(bucket, disk); + + doPut(doc, bucket, time, disk); +} + +void +PersistenceTestUtils::doPut(const document::Document::SP& doc, + document::BucketId bid, + spi::Timestamp time, + uint16_t disk) +{ + spi::Bucket b(bid, spi::PartitionId(disk)); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + getPersistenceProvider().createBucket(b, context); + getPersistenceProvider().put(b, time, doc, context); +} + +spi::UpdateResult +PersistenceTestUtils::doUpdate(document::BucketId bid, + const document::DocumentUpdate::SP& update, + spi::Timestamp time, + uint16_t disk) +{ + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + return getPersistenceProvider().update( + spi::Bucket(bid, spi::PartitionId(disk)), time, update, context); +} + +void +PersistenceTestUtils::doRemove(const document::DocumentId& id, spi::Timestamp time, + uint16_t disk, bool unrevertableRemove, + uint16_t usedBits) +{ + document::BucketId bucket( + _env->_component.getBucketIdFactory().getBucketId(id)); + bucket.setUsedBits(usedBits); + disk = getDiskFromBucketDatabaseIfUnset(bucket, disk); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + if (unrevertableRemove) { + getPersistenceProvider().remove( + spi::Bucket(bucket, spi::PartitionId(disk)), time, id, context); + } else { + spi::RemoveResult result = getPersistenceProvider().removeIfFound( + spi::Bucket(bucket, spi::PartitionId(disk)), time, id, context); + if (!result.wasFound()) { + throw vespalib::IllegalStateException( + "Attempted to remove non-existing doc " + id.toString(), + VESPA_STRLOC); + } + } +} + +void +PersistenceTestUtils::clearBody(document::Document& doc) +{ + // FIXME(vekterli): temporary solution while we don't have + // fieldset pruning functionality in Document. + //doc->getBody().clear(); + vespalib::nbostream stream; + doc.serializeHeader(stream); + doc.deserialize(*_env->_component.getTypeRepo(), stream); +} + +document::Document::UP +PersistenceTestUtils::createRandomDocumentAtLocation( + uint64_t location, uint32_t seed, + uint32_t minDocSize, uint32_t maxDocSize) +{ + return _env->_testDocMan.createRandomDocumentAtLocation( + location, seed, minDocSize, maxDocSize); +} + +void +PersistenceTestUtils::createTestBucket(const document::BucketId& bucket, + uint16_t disk) +{ + + uint32_t opsPerType = 2; + uint32_t numberOfLocations = 2; + uint32_t minDocSize = 0; + uint32_t maxDocSize = 128; + for (uint32_t useHeaderOnly = 0; useHeaderOnly < 2; ++useHeaderOnly) { + bool headerOnly = (useHeaderOnly == 1); + for (uint32_t optype=0; optype < 4; ++optype) { + for (uint32_t i=0; i<opsPerType; ++i) { + uint32_t seed = useHeaderOnly * 10000 + optype * 1000 + i + 1; + uint64_t location = (seed % numberOfLocations); + location <<= 32; + location += (bucket.getRawId() & 0xffffffff); + document::Document::SP doc( + createRandomDocumentAtLocation( + location, seed, minDocSize, maxDocSize)); + if (headerOnly) { + clearBody(*doc); + } + doPut(doc, spi::Timestamp(seed), disk, bucket.getUsedBits()); + if (optype == 0) { // Regular put + } else if (optype == 1) { // Overwritten later in time + document::Document::SP doc2(new document::Document(*doc)); + doc2->setValue(doc2->getField("content"), + document::StringFieldValue("overwritten")); + doPut(doc2, spi::Timestamp(seed + 500), + disk, bucket.getUsedBits()); + } else if (optype == 2) { // Removed + doRemove(doc->getId(), spi::Timestamp(seed + 500), disk, false, + bucket.getUsedBits()); + } else if (optype == 3) { // Unrevertable removed + doRemove(doc->getId(), spi::Timestamp(seed), disk, true, + bucket.getUsedBits()); + } + } + } + } +} + +} // storage diff --git a/storage/src/tests/persistence/persistencetestutils.h b/storage/src/tests/persistence/persistencetestutils.h new file mode 100644 index 00000000000..d584b4dce45 --- /dev/null +++ b/storage/src/tests/persistence/persistencetestutils.h @@ -0,0 +1,214 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/document/base/testdocman.h> +#include <vespa/storage/common/messagesender.h> +#include <vespa/storage/common/storagecomponent.h> +#include <vespa/storage/persistence/filestorage/filestorhandler.h> +#include <vespa/storage/persistence/persistenceutil.h> +#include <vespa/storageframework/defaultimplementation/memory/memorymanager.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/testhelper.h> +#include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/storage/persistence/persistencethread.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> + +namespace storage { + +struct MessageKeeper : public MessageSender { + std::vector<api::StorageMessage::SP> _msgs; + + void sendCommand(const api::StorageCommand::SP& m) { _msgs.push_back(m); } + void sendReply(const api::StorageReply::SP& m) { _msgs.push_back(m); } +}; + +struct PersistenceTestEnvironment { + PersistenceTestEnvironment( + DiskCount numDisks); + + document::TestDocMan _testDocMan; + vdstestlib::DirConfig _config; + MessageKeeper _messageKeeper; + TestServiceLayerApp _node; + StorageComponent _component; + FileStorMetrics _metrics; + std::unique_ptr<FileStorHandler> _handler; + std::vector<vespalib::LinkedPtr<PersistenceUtil> > _diskEnvs; +}; + +class PersistenceTestUtils : public CppUnit::TestFixture { +private: + std::unique_ptr<PersistenceTestEnvironment> _env; + +public: + PersistenceTestUtils(); + virtual ~PersistenceTestUtils(); + + document::Document::SP schedulePut( + uint32_t location, + spi::Timestamp timestamp, + uint16_t disk, + uint32_t minSize = 0, + uint32_t maxSize = 128); + + void setupDisks(uint32_t disks); + + void tearDown() { + _env.reset(); + } + + std::string dumpBucket(const document::BucketId& bid, uint16_t disk = 0); + + PersistenceUtil& getEnv(uint32_t disk = 0) + { return *_env->_diskEnvs[disk]; } + FileStorHandler& fsHandler() { return *_env->_handler; } + FileStorMetrics& metrics() { return _env->_metrics; } + MessageKeeper& messageKeeper() { return _env->_messageKeeper; } + document::DocumentTypeRepo::SP getTypeRepo() { return _env->_component.getTypeRepo(); } + StorageComponent& getComponent() { return _env->_component; } + TestServiceLayerApp& getNode() { return _env->_node; } + + StorBucketDatabase::WrappedEntry getBucket(const document::BucketId& id); + StorBucketDatabase::WrappedEntry createBucket(const document::BucketId& id); + + std::string getBucketStatus(const document::BucketId& id); + + spi::PersistenceProvider& getPersistenceProvider(); + + /** + Performs a put to the given disk. + Returns the document that was inserted. + */ + document::Document::SP doPutOnDisk( + uint16_t disk, + uint32_t location, + spi::Timestamp timestamp, + uint32_t minSize = 0, + uint32_t maxSize = 128); + + document::Document::SP doPut( + uint32_t location, + spi::Timestamp timestamp, + uint32_t minSize = 0, + uint32_t maxSize = 128) + { return doPutOnDisk(0, location, timestamp, minSize, maxSize); } + + /** + Performs a remove to the given disk. + Returns the new doccount if document was removed, or -1 if not found. + */ + bool doRemoveOnDisk( + uint16_t disk, + const document::BucketId& bid, + const document::DocumentId& id, + spi::Timestamp timestamp, + bool persistRemove); + + bool doRemove( + const document::BucketId& bid, + const document::DocumentId& id, + spi::Timestamp timestamp, + bool persistRemove) { + return doRemoveOnDisk(0, bid, id, timestamp, persistRemove); + } + + bool doUnrevertableRemoveOnDisk(uint16_t disk, + const document::BucketId& bid, + const document::DocumentId& id, + spi::Timestamp timestamp); + + bool doUnrevertableRemove(const document::BucketId& bid, + const document::DocumentId& id, + spi::Timestamp timestamp) + { + return doUnrevertableRemoveOnDisk(0, bid, id, timestamp); + } + + /** + * Do a remove toward storage set up in test environment. + * + * @id Document to remove. + * @disk If set, use this disk, otherwise lookup in bucket db. + * @unrevertableRemove If set, instead of adding put, turn put to remove. + * @usedBits Generate bucket to use from docid using this amount of bits. + */ + void doRemove(const document::DocumentId& id, spi::Timestamp, uint16_t disk = 0xffff, + bool unrevertableRemove = false, uint16_t usedBits = 16); + + spi::GetResult doGetOnDisk( + uint16_t disk, + const document::BucketId& bucketId, + const document::DocumentId& docId, + bool headerOnly); + + spi::GetResult doGet( + const document::BucketId& bucketId, + const document::DocumentId& docId, + bool headerOnly) + { return doGetOnDisk(0, bucketId, docId, headerOnly); } + + document::DocumentUpdate::SP createBodyUpdate( + const document::DocumentId& id, + const document::FieldValue& updateValue); + + document::DocumentUpdate::SP createHeaderUpdate( + const document::DocumentId& id, + const document::FieldValue& updateValue); + + uint16_t getDiskFromBucketDatabaseIfUnset(const document::BucketId&, + uint16_t disk = 0xffff); + + /** + * Do a put toward storage set up in test environment. + * + * @doc Document to put. Use TestDocMan to generate easily. + * @disk If set, use this disk, otherwise lookup in bucket db. + * @usedBits Generate bucket to use from docid using this amount of bits. + */ + void doPut(const document::Document::SP& doc, spi::Timestamp, + uint16_t disk = 0xffff, uint16_t usedBits = 16); + + void doPut(const document::Document::SP& doc, + document::BucketId bid, + spi::Timestamp time, + uint16_t disk = 0); + + spi::UpdateResult doUpdate(document::BucketId bid, + const document::DocumentUpdate::SP& update, + spi::Timestamp time, + uint16_t disk = 0); + + document::Document::UP createRandomDocumentAtLocation( + uint64_t location, uint32_t seed, + uint32_t minDocSize, uint32_t maxDocSize); + + /** + * Create a test bucket with various content representing most states a + * bucket can represent. (Such that tests have a nice test bucket to use + * that require operations to handle all the various bucket contents. + * + * @disk If set, use this disk, otherwise lookup in bucket db. + */ + void createTestBucket(const document::BucketId&, uint16_t disk = 0xffff); + + /** + * Create a new persistence thread. + */ + std::unique_ptr<PersistenceThread> createPersistenceThread(uint32_t disk); + + /** + * In-place modify doc so that it has no more body fields. + */ + void clearBody(document::Document& doc); +}; + +class SingleDiskPersistenceTestUtils : public PersistenceTestUtils +{ +public: + void setUp() { + setupDisks(1); + } +}; + +} // storage + diff --git a/storage/src/tests/persistence/persistencethread_splittest.cpp b/storage/src/tests/persistence/persistencethread_splittest.cpp new file mode 100644 index 00000000000..f50e62e0aeb --- /dev/null +++ b/storage/src/tests/persistence/persistencethread_splittest.cpp @@ -0,0 +1,234 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/persistence/persistencethread.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <tests/persistence/persistencetestutils.h> + +namespace storage { +namespace { + spi::LoadType defaultLoadType(0, "default"); +} + +struct PersistenceThread_SplitTest : public SingleDiskPersistenceTestUtils +{ + enum SplitCase { + TOO_MANY_DOCS_SPLIT_ONCE, // Only one split needed to divide + TOO_MANY_DOCS_SPLIT_MULTIPLE_BITS, // Multiple bits needed to divide + TOO_MANY_DOCS_ACTUALLY_NOT, // Other copy is too big but not this one + // Multi bits needed, but dont do it. + TOO_LARGE_DOCS_SPLIT_ONCE, + TOO_LARGE_DOCS_SPLIT_MULTIPLE_BITS, + TOO_LARGE_DOCS_SINGLE_DOC, // Cannot split single doc even if too large + TOO_LARGE_DOCS_ACTUALLY_NOT, // Other copy is too large, not this one + // Need to split to X bits to get in line with other copy or distr. + SPLIT_TOO_LITTLE_SINGLE_SPLIT, // Split all to one target + SPLIT_TOO_LITTLE_JUST_RIGHT, // Just manage to split in two at that lvl + SPLIT_TOO_LITTLE_SPLIT_TOWARDS_ENOUGH, // Has to split shorter + SPLIT_INCONSISTENT_1_DOC, + SPLIT_INCONSISTENT_ALL_DOCS_SAME_GID, + }; + + void doTest(SplitCase); + + void testTooManyDocsSplitOnce() + { doTest(TOO_MANY_DOCS_SPLIT_ONCE); } + void testTooManyDocsSplitMulti() + { doTest(TOO_MANY_DOCS_SPLIT_MULTIPLE_BITS); } + void testTooManyDocsActuallyNot() + { doTest(TOO_MANY_DOCS_ACTUALLY_NOT); } + void testTooLargeDocsSplitOnce() + { doTest(TOO_LARGE_DOCS_SPLIT_ONCE); } + void testTooLargeDocsSplitMulti() + { doTest(TOO_LARGE_DOCS_SPLIT_MULTIPLE_BITS); } + void testTooLargeDocsSingleDoc() + { doTest(TOO_LARGE_DOCS_SINGLE_DOC); } + void testTooLargeDocsActuallyNot() + { doTest(TOO_LARGE_DOCS_ACTUALLY_NOT); } + void testSplitTooLittleSingleSplit() + { doTest(SPLIT_TOO_LITTLE_SINGLE_SPLIT); } + void testSplitTooLittleJustRight() + { doTest(SPLIT_TOO_LITTLE_JUST_RIGHT); } + void testSplitTooLittleSplitTowardsEnough() + { doTest(SPLIT_TOO_LITTLE_SPLIT_TOWARDS_ENOUGH); } + void testInconsistentSplitHasOneBitFallbackWhen1Doc() { + doTest(SPLIT_INCONSISTENT_1_DOC); + } + void testInconsistentSplitHasOneBitFallbackWhenAllDocsHaveSameGid() { + doTest(SPLIT_INCONSISTENT_ALL_DOCS_SAME_GID); + } + + CPPUNIT_TEST_SUITE(PersistenceThread_SplitTest); + CPPUNIT_TEST(testTooManyDocsSplitOnce); + CPPUNIT_TEST(testTooManyDocsSplitMulti); + CPPUNIT_TEST(testTooManyDocsActuallyNot); + CPPUNIT_TEST(testTooLargeDocsSplitOnce); + CPPUNIT_TEST(testTooLargeDocsSplitMulti); + CPPUNIT_TEST(testTooLargeDocsSingleDoc); + CPPUNIT_TEST(testTooLargeDocsActuallyNot); + CPPUNIT_TEST(testSplitTooLittleSingleSplit); + CPPUNIT_TEST(testSplitTooLittleJustRight); + CPPUNIT_TEST(testSplitTooLittleSplitTowardsEnough); + CPPUNIT_TEST(testInconsistentSplitHasOneBitFallbackWhen1Doc); + CPPUNIT_TEST(testInconsistentSplitHasOneBitFallbackWhenAllDocsHaveSameGid); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(PersistenceThread_SplitTest); + +void +PersistenceThread_SplitTest::doTest(SplitCase splitCase) +{ + uint32_t maxCount = 4; + uint32_t maxSize = 1000 * 1000; + uint32_t maxBits = 58; + uint32_t minBits = 1; + uint32_t docCount = 8; + uint32_t docSize = 100 * 1000; + uint32_t currentSplitLevel = 1; + uint32_t splitLevelToDivide = 2; + uint32_t resultSplitLevel = 2; + size_t resultBuckets = 2; + bool simulateGidCollision = false; + api::ReturnCode error(api::ReturnCode::OK); + switch (splitCase) { + case TOO_MANY_DOCS_SPLIT_ONCE: + break; // Default. Do nothing + case TOO_MANY_DOCS_SPLIT_MULTIPLE_BITS: + splitLevelToDivide = 3; + resultSplitLevel = 3; + break; + case TOO_MANY_DOCS_ACTUALLY_NOT: + splitLevelToDivide = 3; + docCount = 2; + resultBuckets = 1; + break; + case TOO_LARGE_DOCS_SPLIT_ONCE: + maxCount = 100; + docSize = 400 * 1000; + break; + case TOO_LARGE_DOCS_SPLIT_MULTIPLE_BITS: + maxCount = 100; + docSize = 400 * 1000; + splitLevelToDivide = 3; + resultSplitLevel = 3; + break; + case TOO_LARGE_DOCS_SINGLE_DOC: + // It is possible for bucket to be inconsistent being big enough + // to split in other copy but this copy has only 1 too big doc. + docCount = 1; + docSize = 3000 * 1000; + splitLevelToDivide = 3; + resultBuckets = 1; + break; + case TOO_LARGE_DOCS_ACTUALLY_NOT: + maxCount = 100; + splitLevelToDivide = 3; + resultSplitLevel = 2; + resultBuckets = 1; + break; + case SPLIT_TOO_LITTLE_SINGLE_SPLIT: + maxBits = 5; + maxSize = 0; + maxCount = 0; + splitLevelToDivide = 16; + resultSplitLevel = 5; + resultBuckets = 1; + break; + case SPLIT_TOO_LITTLE_JUST_RIGHT: + maxBits = 5; + maxSize = 0; + maxCount = 0; + splitLevelToDivide = 5; + resultSplitLevel = 5; + break; + case SPLIT_TOO_LITTLE_SPLIT_TOWARDS_ENOUGH: + maxBits = 8; + maxSize = 0; + maxCount = 0; + splitLevelToDivide = 5; + resultSplitLevel = 5; + break; + case SPLIT_INCONSISTENT_1_DOC: + docCount = 1; + maxSize = 0; + maxCount = 0; + currentSplitLevel = 16; + resultSplitLevel = 17; + resultBuckets = 1; + break; + case SPLIT_INCONSISTENT_ALL_DOCS_SAME_GID: + docCount = 2; + maxSize = 0; + maxCount = 0; + currentSplitLevel = 16; + resultSplitLevel = 17; + resultBuckets = 1; + simulateGidCollision = true; + break; + default: + assert(false); + } + + uint64_t location = 0; + uint64_t splitMask = 1 << (splitLevelToDivide - 1); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + spi::Bucket bucket(document::BucketId(currentSplitLevel, 1), + spi::PartitionId(0)); + spi::PersistenceProvider& spi(getPersistenceProvider()); + spi.deleteBucket(bucket, context); + spi.createBucket(bucket, context); + document::TestDocMan testDocMan; + for (uint32_t i=0; i<docCount; ++i) { + uint64_t docloc; + uint32_t seed; + if (!simulateGidCollision) { + docloc = location | (i % 2 == 0 ? 0 : splitMask); + seed = i; + } else { + docloc = location; + seed = 0; + } + document::Document::SP doc(testDocMan.createRandomDocumentAtLocation( + docloc, seed, docSize, docSize)); + spi.put(bucket, spi::Timestamp(1000 + i), doc, context); + } + + std::unique_ptr<PersistenceThread> thread(createPersistenceThread(0)); + getNode().getStateUpdater().setClusterState( + lib::ClusterState::CSP( + new lib::ClusterState("distributor:1 storage:1"))); + api::SplitBucketCommand cmd(document::BucketId(currentSplitLevel, 1)); + cmd.setMaxSplitBits(maxBits); + cmd.setMinSplitBits(minBits); + cmd.setMinByteSize(maxSize); + cmd.setMinDocCount(maxCount); + cmd.setSourceIndex(0); + MessageTracker::UP result(thread->handleSplitBucket(cmd)); + api::ReturnCode code(result->getResult()); + CPPUNIT_ASSERT_EQUAL(error, code); + if (!code.success()) return; + api::SplitBucketReply& reply( + dynamic_cast<api::SplitBucketReply&>(*result->getReply())); + std::set<std::string> expected; + for (uint32_t i=0; i<resultBuckets; ++i) { + document::BucketId b(resultSplitLevel, + location | (i == 0 ? 0 : splitMask)); + std::ostringstream ost; + ost << b << " - " << b.getUsedBits(); + expected.insert(ost.str()); + } + std::set<std::string> actual; + for (uint32_t i=0; i<reply.getSplitInfo().size(); ++i) { + std::ostringstream ost; + document::BucketId b(reply.getSplitInfo()[i].first); + ost << b << " - " << b.getUsedBits(); + actual.insert(ost.str()); + } + CPPUNIT_ASSERT_EQUAL(expected, actual); +} + +} // storage + diff --git a/storage/src/tests/persistence/processalltest.cpp b/storage/src/tests/persistence/processalltest.cpp new file mode 100644 index 00000000000..db75725db6f --- /dev/null +++ b/storage/src/tests/persistence/processalltest.cpp @@ -0,0 +1,262 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/document/base/testdocman.h> +#include <vespa/storage/persistence/processallhandler.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/persistence/messages.h> +#include <vespa/documentapi/loadtypes/loadtype.h> +#include <tests/persistence/persistencetestutils.h> + +namespace storage { + +class ProcessAllHandlerTest : public SingleDiskPersistenceTestUtils +{ + CPPUNIT_TEST_SUITE(ProcessAllHandlerTest); + CPPUNIT_TEST(testRemoveLocation); + CPPUNIT_TEST(testRemoveLocationDocumentSubset); + CPPUNIT_TEST(testRemoveLocationUnknownDocType); + CPPUNIT_TEST(testRemoveLocationBogusSelection); + CPPUNIT_TEST(testStat); + CPPUNIT_TEST(testStatWithRemove); + CPPUNIT_TEST(testStatWholeBucket); + CPPUNIT_TEST_SUITE_END(); + +public: + void testRemoveLocation(); + void testRemoveLocationDocumentSubset(); + void testRemoveLocationUnknownDocType(); + void testRemoveLocationEmptySelection(); + void testRemoveLocationBogusSelection(); + void testStat(); + void testStatWithRemove(); + void testStatWholeBucket(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(ProcessAllHandlerTest); + +void +ProcessAllHandlerTest::testRemoveLocation() +{ + document::BucketId bucketId(16, 4); + doPut(4, spi::Timestamp(1234)); + doPut(4, spi::Timestamp(2345)); + + api::RemoveLocationCommand removeLocation("id.user == 4", bucketId); + ProcessAllHandler handler(getEnv(), getPersistenceProvider()); + spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); + handler.handleRemoveLocation(removeLocation, context); + + CPPUNIT_ASSERT_EQUAL( + std::string( + "DocEntry(1234, 1, id:mail:testdoctype1:n=4:3619.html)\n" + "DocEntry(2345, 1, id:mail:testdoctype1:n=4:4008.html)\n"), + dumpBucket(bucketId)); +} + +void +ProcessAllHandlerTest::testRemoveLocationDocumentSubset() +{ + document::BucketId bucketId(16, 4); + ProcessAllHandler handler(getEnv(), getPersistenceProvider()); + + document::TestDocMan docMan; + for (int i = 0; i < 10; ++i) { + document::Document::SP doc(docMan.createRandomDocumentAtLocation(4, 1234 + i)); + doc->setValue(doc->getField("headerval"), document::IntFieldValue(i)); + doPut(doc, bucketId, spi::Timestamp(100 + i), 0); + } + + api::RemoveLocationCommand + removeLocation("testdoctype1.headerval % 2 == 0", bucketId); + spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); + handler.handleRemoveLocation(removeLocation, context); + + CPPUNIT_ASSERT_EQUAL( + std::string("DocEntry(100, 1, id:mail:testdoctype1:n=4:3619.html)\n" + "DocEntry(101, 0, Doc(id:mail:testdoctype1:n=4:33113.html))\n" + "DocEntry(102, 1, id:mail:testdoctype1:n=4:62608.html)\n" + "DocEntry(103, 0, Doc(id:mail:testdoctype1:n=4:26566.html))\n" + "DocEntry(104, 1, id:mail:testdoctype1:n=4:56061.html)\n" + "DocEntry(105, 0, Doc(id:mail:testdoctype1:n=4:20019.html))\n" + "DocEntry(106, 1, id:mail:testdoctype1:n=4:49514.html)\n" + "DocEntry(107, 0, Doc(id:mail:testdoctype1:n=4:13472.html))\n" + "DocEntry(108, 1, id:mail:testdoctype1:n=4:42967.html)\n" + "DocEntry(109, 0, Doc(id:mail:testdoctype1:n=4:6925.html))\n"), + dumpBucket(bucketId)); +} + +void +ProcessAllHandlerTest::testRemoveLocationUnknownDocType() +{ + document::BucketId bucketId(16, 4); + doPut(4, spi::Timestamp(1234)); + + api::RemoveLocationCommand + removeLocation("unknowndoctype.headerval % 2 == 0", bucketId); + + bool gotException = false; + try { + ProcessAllHandler handler(getEnv(), getPersistenceProvider()); + spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); + handler.handleRemoveLocation(removeLocation, context); + } catch (...) { + gotException = true; + } + CPPUNIT_ASSERT(gotException); + + CPPUNIT_ASSERT_EQUAL( + std::string("DocEntry(1234, 0, Doc(id:mail:testdoctype1:n=4:3619.html))\n"), + dumpBucket(bucketId)); +} + +void +ProcessAllHandlerTest::testRemoveLocationBogusSelection() +{ + document::BucketId bucketId(16, 4); + doPut(4, spi::Timestamp(1234)); + + api::RemoveLocationCommand removeLocation("id.bogus != badgers", bucketId); + + bool gotException = false; + try { + ProcessAllHandler handler(getEnv(), getPersistenceProvider()); + spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); + handler.handleRemoveLocation(removeLocation, context); + } catch (...) { + gotException = true; + } + CPPUNIT_ASSERT(gotException); + + CPPUNIT_ASSERT_EQUAL( + std::string("DocEntry(1234, 0, Doc(id:mail:testdoctype1:n=4:3619.html))\n"), + dumpBucket(bucketId)); +} + +void +ProcessAllHandlerTest::testStat() +{ + document::BucketId bucketId(16, 4); + ProcessAllHandler handler(getEnv(), getPersistenceProvider()); + + document::TestDocMan docMan; + for (int i = 0; i < 10; ++i) { + document::Document::SP doc(docMan.createRandomDocumentAtLocation(4, 1234 + i)); + doc->setValue(doc->getField("headerval"), document::IntFieldValue(i)); + doPut(doc, bucketId, spi::Timestamp(100 + i), 0); + } + + api::StatBucketCommand statBucket(bucketId, + "testdoctype1.headerval % 2 == 0"); + spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); + MessageTracker::UP tracker = handler.handleStatBucket(statBucket, context); + + CPPUNIT_ASSERT(tracker->getReply().get()); + api::StatBucketReply& reply = + dynamic_cast<api::StatBucketReply&>(*tracker->getReply().get()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::OK, reply.getResult().getResult()); + + vespalib::string expected = + "Persistence bucket BucketId(0x4000000000000004), partition 0\n" + " Timestamp: 100, Doc(id:mail:testdoctype1:n=4:3619.html), gid(0x0400000092bb8d298934253a), size: 169\n" + " Timestamp: 102, Doc(id:mail:testdoctype1:n=4:62608.html), gid(0x04000000ce878d2488413bc4), size: 147\n" + " Timestamp: 104, Doc(id:mail:testdoctype1:n=4:56061.html), gid(0x040000002b8f80f0160f6c5c), size: 124\n" + " Timestamp: 106, Doc(id:mail:testdoctype1:n=4:49514.html), gid(0x04000000d45ca9abb47567f0), size: 101\n" + " Timestamp: 108, Doc(id:mail:testdoctype1:n=4:42967.html), gid(0x04000000f19ece1668e6de48), size: 206\n"; + + + CPPUNIT_ASSERT_EQUAL(expected, reply.getResults()); +} + +void +ProcessAllHandlerTest::testStatWithRemove() +{ + document::BucketId bucketId(16, 4); + ProcessAllHandler handler(getEnv(), getPersistenceProvider()); + + document::TestDocMan docMan; + for (int i = 0; i < 10; ++i) { + document::Document::SP doc(docMan.createRandomDocumentAtLocation(4, 1234 + i)); + doc->setValue(doc->getField("headerval"), document::IntFieldValue(i)); + doPut(doc, bucketId, spi::Timestamp(100 + i), 0); + doRemove(bucketId, + doc->getId(), + spi::Timestamp(200 + i), + true); + } + + api::StatBucketCommand statBucket(bucketId, "true"); + spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); + MessageTracker::UP tracker = handler.handleStatBucket(statBucket, context); + + CPPUNIT_ASSERT(tracker->getReply().get()); + api::StatBucketReply& reply = + dynamic_cast<api::StatBucketReply&>(*tracker->getReply().get()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::OK, reply.getResult().getResult()); + + vespalib::string expected = + "Persistence bucket BucketId(0x4000000000000004), partition 0\n" + " Timestamp: 100, Doc(id:mail:testdoctype1:n=4:3619.html), gid(0x0400000092bb8d298934253a), size: 169\n" + " Timestamp: 101, Doc(id:mail:testdoctype1:n=4:33113.html), gid(0x04000000b121a632741db368), size: 95\n" + " Timestamp: 102, Doc(id:mail:testdoctype1:n=4:62608.html), gid(0x04000000ce878d2488413bc4), size: 147\n" + " Timestamp: 103, Doc(id:mail:testdoctype1:n=4:26566.html), gid(0x04000000177f8240bdd2bef0), size: 200\n" + " Timestamp: 104, Doc(id:mail:testdoctype1:n=4:56061.html), gid(0x040000002b8f80f0160f6c5c), size: 124\n" + " Timestamp: 105, Doc(id:mail:testdoctype1:n=4:20019.html), gid(0x040000001550c67f28ea7b03), size: 177\n" + " Timestamp: 106, Doc(id:mail:testdoctype1:n=4:49514.html), gid(0x04000000d45ca9abb47567f0), size: 101\n" + " Timestamp: 107, Doc(id:mail:testdoctype1:n=4:13472.html), gid(0x040000005d01f3fd960f8098), size: 154\n" + " Timestamp: 108, Doc(id:mail:testdoctype1:n=4:42967.html), gid(0x04000000f19ece1668e6de48), size: 206\n" + " Timestamp: 109, Doc(id:mail:testdoctype1:n=4:6925.html), gid(0x04000000667c0b3cada830be), size: 130\n" + " Timestamp: 200, id:mail:testdoctype1:n=4:3619.html, gid(0x0400000092bb8d298934253a) (remove)\n" + " Timestamp: 201, id:mail:testdoctype1:n=4:33113.html, gid(0x04000000b121a632741db368) (remove)\n" + " Timestamp: 202, id:mail:testdoctype1:n=4:62608.html, gid(0x04000000ce878d2488413bc4) (remove)\n" + " Timestamp: 203, id:mail:testdoctype1:n=4:26566.html, gid(0x04000000177f8240bdd2bef0) (remove)\n" + " Timestamp: 204, id:mail:testdoctype1:n=4:56061.html, gid(0x040000002b8f80f0160f6c5c) (remove)\n" + " Timestamp: 205, id:mail:testdoctype1:n=4:20019.html, gid(0x040000001550c67f28ea7b03) (remove)\n" + " Timestamp: 206, id:mail:testdoctype1:n=4:49514.html, gid(0x04000000d45ca9abb47567f0) (remove)\n" + " Timestamp: 207, id:mail:testdoctype1:n=4:13472.html, gid(0x040000005d01f3fd960f8098) (remove)\n" + " Timestamp: 208, id:mail:testdoctype1:n=4:42967.html, gid(0x04000000f19ece1668e6de48) (remove)\n" + " Timestamp: 209, id:mail:testdoctype1:n=4:6925.html, gid(0x04000000667c0b3cada830be) (remove)\n"; + + CPPUNIT_ASSERT_EQUAL(expected, reply.getResults()); +} + + +void +ProcessAllHandlerTest::testStatWholeBucket() +{ + document::BucketId bucketId(16, 4); + ProcessAllHandler handler(getEnv(), getPersistenceProvider()); + + document::TestDocMan docMan; + for (int i = 0; i < 10; ++i) { + document::Document::SP doc(docMan.createRandomDocumentAtLocation(4, 1234 + i)); + doc->setValue(doc->getField("headerval"), document::IntFieldValue(i)); + doPut(doc, bucketId, spi::Timestamp(100 + i), 0); + } + + api::StatBucketCommand statBucket(bucketId, "true"); + spi::Context context(documentapi::LoadType::DEFAULT, 0, 0); + MessageTracker::UP tracker = handler.handleStatBucket(statBucket, context); + + CPPUNIT_ASSERT(tracker->getReply().get()); + api::StatBucketReply& reply = + dynamic_cast<api::StatBucketReply&>(*tracker->getReply().get()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::OK, reply.getResult().getResult()); + + vespalib::string expected = + "Persistence bucket BucketId(0x4000000000000004), partition 0\n" + " Timestamp: 100, Doc(id:mail:testdoctype1:n=4:3619.html), gid(0x0400000092bb8d298934253a), size: 169\n" + " Timestamp: 101, Doc(id:mail:testdoctype1:n=4:33113.html), gid(0x04000000b121a632741db368), size: 95\n" + " Timestamp: 102, Doc(id:mail:testdoctype1:n=4:62608.html), gid(0x04000000ce878d2488413bc4), size: 147\n" + " Timestamp: 103, Doc(id:mail:testdoctype1:n=4:26566.html), gid(0x04000000177f8240bdd2bef0), size: 200\n" + " Timestamp: 104, Doc(id:mail:testdoctype1:n=4:56061.html), gid(0x040000002b8f80f0160f6c5c), size: 124\n" + " Timestamp: 105, Doc(id:mail:testdoctype1:n=4:20019.html), gid(0x040000001550c67f28ea7b03), size: 177\n" + " Timestamp: 106, Doc(id:mail:testdoctype1:n=4:49514.html), gid(0x04000000d45ca9abb47567f0), size: 101\n" + " Timestamp: 107, Doc(id:mail:testdoctype1:n=4:13472.html), gid(0x040000005d01f3fd960f8098), size: 154\n" + " Timestamp: 108, Doc(id:mail:testdoctype1:n=4:42967.html), gid(0x04000000f19ece1668e6de48), size: 206\n" + " Timestamp: 109, Doc(id:mail:testdoctype1:n=4:6925.html), gid(0x04000000667c0b3cada830be), size: 130\n"; + + CPPUNIT_ASSERT_EQUAL(expected, reply.getResults()); +} + +} diff --git a/storage/src/tests/persistence/providershutdownwrappertest.cpp b/storage/src/tests/persistence/providershutdownwrappertest.cpp new file mode 100644 index 00000000000..0731dcb155a --- /dev/null +++ b/storage/src/tests/persistence/providershutdownwrappertest.cpp @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <tests/persistence/persistencetestutils.h> +#include <tests/persistence/persistenceproviderwrapper.h> +#include <vespa/storage/persistence/providershutdownwrapper.h> + +namespace storage { + +class ProviderShutdownWrapperTest : public SingleDiskPersistenceTestUtils +{ +public: + CPPUNIT_TEST_SUITE(ProviderShutdownWrapperTest); + CPPUNIT_TEST(testShutdownOnFatalError); + CPPUNIT_TEST_SUITE_END(); + + void testShutdownOnFatalError(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(ProviderShutdownWrapperTest); + +namespace { + +class TestShutdownListener + : public framework::defaultimplementation::ShutdownListener +{ +public: + TestShutdownListener() : _reason() {} + + void requestShutdown(vespalib::stringref reason) { + _reason = reason; + } + + bool shutdownRequested() const { return !_reason.empty(); } + const vespalib::string& getReason() const { return _reason; } +private: + vespalib::string _reason; +}; + +} + +void +ProviderShutdownWrapperTest::testShutdownOnFatalError() +{ + // We wrap the wrapper. It's turtles all the way down! + PersistenceProviderWrapper providerWrapper( + getPersistenceProvider()); + TestServiceLayerApp app; + ServiceLayerComponent component(app.getComponentRegister(), "dummy"); + + ProviderShutdownWrapper shutdownWrapper(providerWrapper, component); + + TestShutdownListener shutdownListener; + + app.getComponentRegister().registerShutdownListener(shutdownListener); + + providerWrapper.setResult( + spi::Result(spi::Result::FATAL_ERROR, "eject! eject!")); + providerWrapper.setFailureMask( + PersistenceProviderWrapper::FAIL_ALL_OPERATIONS); + + CPPUNIT_ASSERT(!shutdownListener.shutdownRequested()); + // This should cause the node to implicitly be shut down + shutdownWrapper.getBucketInfo( + spi::Bucket(document::BucketId(16, 1234), + spi::PartitionId(0))); + + CPPUNIT_ASSERT(shutdownListener.shutdownRequested()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("eject! eject!"), + shutdownListener.getReason()); + + // Triggering a new error should not cause shutdown to be requested twice. + providerWrapper.setResult( + spi::Result(spi::Result::FATAL_ERROR, "boom!")); + + shutdownWrapper.getBucketInfo( + spi::Bucket(document::BucketId(16, 1234), + spi::PartitionId(0))); + + CPPUNIT_ASSERT_EQUAL(vespalib::string("eject! eject!"), + shutdownListener.getReason()); +} + +} // ns storage + + diff --git a/storage/src/tests/persistence/splitbitdetectortest.cpp b/storage/src/tests/persistence/splitbitdetectortest.cpp new file mode 100644 index 00000000000..5cc9c5da721 --- /dev/null +++ b/storage/src/tests/persistence/splitbitdetectortest.cpp @@ -0,0 +1,363 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <fstream> +#include <vespa/storage/persistence/splitbitdetector.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/persistence/dummyimpl/dummypersistence.h> +#include <vespa/document/base/testdocman.h> + + +namespace storage { + +namespace { + spi::LoadType defaultLoadType(0, "default"); +} + +struct SplitBitDetectorTest : public CppUnit::TestFixture { + void testSingleUser(); + void testTwoUsers(); + void testMaxBits(); + void testMaxBitsOneBelowMax(); + void testUnsplittable(); + void testUnsplittableMinCount(); + void testEmpty(); + void testZeroDocLimitFallbacksToOneBitIncreaseWith1Doc(); + void testZeroDocLimitFallbacksToOneBitIncreaseOnGidCollision(); + void findBucketCollisionIds(); + + spi::DocEntry::LP + generateDocEntry(uint32_t userId, + uint32_t docNum, + spi::Timestamp timestamp) + { + std::ostringstream ost; + ost << "id:storage_test:testdoctype1:n=" << userId << ":" << docNum; + return spi::DocEntry::LP(new spi::DocEntry( + timestamp, 0, document::DocumentId(ost.str()))); + }; + + CPPUNIT_TEST_SUITE(SplitBitDetectorTest); + CPPUNIT_TEST(testSingleUser); + CPPUNIT_TEST(testTwoUsers); + CPPUNIT_TEST(testMaxBits); + CPPUNIT_TEST(testMaxBitsOneBelowMax); + CPPUNIT_TEST(testUnsplittable); + CPPUNIT_TEST(testUnsplittableMinCount); + CPPUNIT_TEST(testEmpty); + CPPUNIT_TEST(testZeroDocLimitFallbacksToOneBitIncreaseWith1Doc); + CPPUNIT_TEST(testZeroDocLimitFallbacksToOneBitIncreaseOnGidCollision); + CPPUNIT_TEST_DISABLED(findBucketCollisionIds); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(SplitBitDetectorTest); + +void +SplitBitDetectorTest::testTwoUsers() +{ + document::TestDocMan testDocMan; + spi::dummy::DummyPersistence provider(testDocMan.getTypeRepoSP(), 1); + provider.getPartitionStates(); + spi::Bucket bucket(document::BucketId(1, 1), + spi::PartitionId(0)); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + + provider.createBucket(bucket, context); + + std::vector<spi::DocEntry::LP> entries; + for (uint32_t i = 0; i < 5; ++i) { + document::Document::SP doc( + testDocMan.createRandomDocumentAtLocation(1, i, 1, 1)); + provider.put(bucket, spi::Timestamp(1000 + i), doc, context); + } + + for (uint32_t i = 5; i < 10; ++i) { + document::Document::SP doc( + testDocMan.createRandomDocumentAtLocation(3, i, 1, 1)); + provider.put(bucket, spi::Timestamp(1000 + i), doc, context); + } + + SplitBitDetector::Result result( + SplitBitDetector::detectSplit(provider, bucket, 58, context)); + CPPUNIT_ASSERT_EQUAL( + std::string("SplitTargets(2: BucketId(0x0800000000000001), " + "BucketId(0x0800000000000003))"), + result.toString()); +} + +void +SplitBitDetectorTest::testSingleUser() +{ + document::TestDocMan testDocMan; + spi::dummy::DummyPersistence provider(testDocMan.getTypeRepoSP(), 1); + provider.getPartitionStates(); + spi::Bucket bucket(document::BucketId(1, 1), + spi::PartitionId(0)); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + + provider.createBucket(bucket, context); + + std::vector<spi::DocEntry::LP> entries; + for (uint32_t i = 0; i < 10; ++i) { + document::Document::SP doc( + testDocMan.createRandomDocumentAtLocation(1, i, 1, 1)); + provider.put(bucket, spi::Timestamp(1000 + i), doc, context); + } + + SplitBitDetector::Result result( + SplitBitDetector::detectSplit(provider, bucket, 58, context)); + CPPUNIT_ASSERT_EQUAL( + std::string("SplitTargets(33: BucketId(0x8400000000000001), " + "BucketId(0x8400000100000001))"), + result.toString()); +} + +void +SplitBitDetectorTest::testMaxBits() +{ + document::TestDocMan testDocMan; + spi::dummy::DummyPersistence provider(testDocMan.getTypeRepoSP(), 1); + provider.getPartitionStates(); + spi::Bucket bucket(document::BucketId(1, 1), + spi::PartitionId(0)); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + int minContentSize = 1, maxContentSize = 1; + + provider.createBucket(bucket, context); + + std::vector<spi::DocEntry::LP> entries; + for (uint32_t seed = 0; seed < 10; ++seed) { + int location = 1; + document::Document::SP doc(testDocMan.createRandomDocumentAtLocation( + location, seed, minContentSize, maxContentSize)); + provider.put(bucket, spi::Timestamp(1000 + seed), doc, context); + } + + SplitBitDetector::Result result( + SplitBitDetector::detectSplit(provider, bucket, 3, context)); + CPPUNIT_ASSERT_EQUAL( + std::string("SplitTargets(3: BucketId(0x0c00000000000001), " + "[ BucketId(0x0c00000000000005) ])"), + result.toString()); +} + +void +SplitBitDetectorTest::testMaxBitsOneBelowMax() +{ + document::TestDocMan testDocMan; + spi::dummy::DummyPersistence provider(testDocMan.getTypeRepoSP(), 1); + provider.getPartitionStates(); + spi::Bucket bucket(document::BucketId(15, 1), spi::PartitionId(0)); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + int minContentSize = 1, maxContentSize = 1; + + provider.createBucket(bucket, context); + + std::vector<spi::DocEntry::LP> entries; + for (uint32_t seed = 0; seed < 10; ++seed) { + int location = 1 | (seed % 2 == 0 ? 0x8000 : 0); + document::Document::SP doc(testDocMan.createRandomDocumentAtLocation( + location, seed, minContentSize, maxContentSize)); + provider.put(bucket, spi::Timestamp(1000 + seed), doc, context); + } + + //std::cerr << provider.dumpBucket(bucket) << "\n"; + + SplitBitDetector::Result result( + SplitBitDetector::detectSplit(provider, bucket, 15, context)); + CPPUNIT_ASSERT_EQUAL( + std::string("SplitTargets(error: No use in trying to split " + "Bucket(0x3c00000000000001, partition 0) when max split" + " bit is set to 15.)"), + result.toString()); + + result = SplitBitDetector::detectSplit(provider, bucket, 16, context); + CPPUNIT_ASSERT_EQUAL( + std::string("SplitTargets(16: BucketId(0x4000000000000001), " + "BucketId(0x4000000000008001))"), + result.toString()); +} + +void +SplitBitDetectorTest::testUnsplittable() +{ + document::TestDocMan testDocMan; + spi::dummy::DummyPersistence provider(testDocMan.getTypeRepoSP(), 1); + provider.getPartitionStates(); + spi::Bucket bucket(document::BucketId(1, 1), + spi::PartitionId(0)); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + + provider.createBucket(bucket, context); + + std::vector<spi::DocEntry::LP> entries; + + for (uint32_t i = 0; i < 10; ++i) { + document::Document::SP doc( + testDocMan.createRandomDocumentAtLocation(1, 1, 1, 1)); + provider.put(bucket, spi::Timestamp(1000 + i), doc, context); + } + + SplitBitDetector::Result result( + SplitBitDetector::detectSplit(provider, bucket, 58, context, 100)); + CPPUNIT_ASSERT_EQUAL( + std::string("SplitTargets(58: BucketId(0xe94c074f00000001), " + "BucketId(0xeb4c074f00000001))"), + result.toString()); +} + +void +SplitBitDetectorTest::testUnsplittableMinCount() +{ + document::TestDocMan testDocMan; + spi::dummy::DummyPersistence provider(testDocMan.getTypeRepoSP(), 1); + provider.getPartitionStates(); + spi::Bucket bucket(document::BucketId(1, 1), + spi::PartitionId(0)); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + + provider.createBucket(bucket, context); + + std::vector<spi::DocEntry::LP> entries; + + for (uint32_t i = 0; i < 10; ++i) { + document::Document::SP doc( + testDocMan.createRandomDocumentAtLocation(1, 1, 1, 1)); + provider.put(bucket, spi::Timestamp(1000 + i), doc, context); + } + + SplitBitDetector::Result result( + SplitBitDetector::detectSplit(provider, bucket, 58, context, 5, 0)); + // Still no other choice than split out to 58 bits regardless of minCount. + CPPUNIT_ASSERT_EQUAL( + std::string("SplitTargets(58: BucketId(0xe94c074f00000001), " + "BucketId(0xeb4c074f00000001))"), + result.toString()); +} + + +void +SplitBitDetectorTest::testEmpty() +{ + document::TestDocMan testDocMan; + spi::dummy::DummyPersistence provider(testDocMan.getTypeRepoSP(), 1); + provider.getPartitionStates(); + spi::Bucket bucket(document::BucketId(1, 1), + spi::PartitionId(0)); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + + provider.createBucket(bucket, context); + + SplitBitDetector::Result result( + SplitBitDetector::detectSplit(provider, bucket, 58, context)); + CPPUNIT_ASSERT_EQUAL( + std::string("SplitTargets(source empty)"), + result.toString()); +} + +void +SplitBitDetectorTest::testZeroDocLimitFallbacksToOneBitIncreaseWith1Doc() +{ + document::TestDocMan testDocMan; + spi::dummy::DummyPersistence provider(testDocMan.getTypeRepoSP(), 1); + provider.getPartitionStates(); + spi::Bucket bucket(document::BucketId(1, 1), + spi::PartitionId(0)); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + + provider.createBucket(bucket, context); + document::Document::SP doc( + testDocMan.createRandomDocumentAtLocation(1, 0, 1, 1)); + provider.put(bucket, spi::Timestamp(1000), doc, context); + + SplitBitDetector::Result result( + SplitBitDetector::detectSplit(provider, bucket, 58, context, 0, 0)); + CPPUNIT_ASSERT_EQUAL( + std::string("SplitTargets(2: BucketId(0x0800000000000001), " + "BucketId(0x0800000000000003))"), + result.toString()); +} + +void +SplitBitDetectorTest::testZeroDocLimitFallbacksToOneBitIncreaseOnGidCollision() +{ + document::TestDocMan testDocMan; + spi::dummy::DummyPersistence provider(testDocMan.getTypeRepoSP(), 1); + provider.getPartitionStates(); + spi::Bucket bucket(document::BucketId(1, 1), + spi::PartitionId(0)); + spi::Context context(defaultLoadType, spi::Priority(0), + spi::Trace::TraceLevel(0)); + + provider.createBucket(bucket, context); + document::Document::SP doc( + testDocMan.createRandomDocumentAtLocation(1, 0, 1, 1)); + provider.put(bucket, spi::Timestamp(1000), doc, context); + provider.put(bucket, spi::Timestamp(2000), doc, context); + + SplitBitDetector::Result result( + SplitBitDetector::detectSplit(provider, bucket, 58, context, 0, 0)); + CPPUNIT_ASSERT_EQUAL( + std::string("SplitTargets(2: BucketId(0x0800000000000001), " + "BucketId(0x0800000000000003))"), + result.toString()); +} + +/** + * Not a regular unit test in itself, but more of an utility to find non-unique + * document IDs that map to the same 58-bit bucket ID. Disabled by default since + * it costs CPU to do this and is not necessary during normal testing. + */ +void +SplitBitDetectorTest::findBucketCollisionIds() +{ + using document::DocumentId; + using document::BucketId; + + document::BucketIdFactory factory; + + DocumentId targetId("id:foo:music:n=123456:ABCDEFGHIJKLMN"); + BucketId targetBucket(factory.getBucketId(targetId)); + char candidateSuffix[] = "ABCDEFGHIJKLMN"; + + size_t iterations = 0; + constexpr size_t maxIterations = 100000000; + while (std::next_permutation(std::begin(candidateSuffix), + std::end(candidateSuffix) - 1)) + { + ++iterations; + + DocumentId candidateId( + vespalib::make_string("id:foo:music:n=123456:%s", + candidateSuffix)); + BucketId candidateBucket(factory.getBucketId(candidateId)); + if (targetBucket == candidateBucket) { + std::cerr << "\nFound a collision after " << iterations + << " iterations!\n" + << "target: " << targetId << " -> " << targetBucket + << "\ncollision: " << candidateId << " -> " + << candidateBucket << "\n"; + return; + } + + if (iterations == maxIterations) { + std::cerr << "\nNo collision found after " << iterations + << " iterations :[\n"; + return; + } + } + std::cerr << "\nRan out of permutations after " << iterations + << " iterations!\n"; +} + +} diff --git a/storage/src/tests/persistence/testandsettest.cpp b/storage/src/tests/persistence/testandsettest.cpp new file mode 100644 index 00000000000..984e06dc6e3 --- /dev/null +++ b/storage/src/tests/persistence/testandsettest.cpp @@ -0,0 +1,331 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// @author Vegard Sjonfjell +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/persistence/persistencethread.h> +#include <tests/persistence/persistencetestutils.h> +#include <vespa/documentapi/messagebus/messages/testandsetcondition.h> +#include <vespa/document/fieldvalue/fieldvalues.h> +#include <functional> + +using std::unique_ptr; +using std::shared_ptr; + +using namespace std::string_literals; + +namespace storage { + +class TestAndSetTest : public SingleDiskPersistenceTestUtils +{ + static constexpr int MIN_DOCUMENT_SIZE = 0; + static constexpr int MAX_DOCUMENT_SIZE = 128; + static constexpr int RANDOM_SEED = 1234; + + const document::BucketId BUCKET_ID{16, 4}; + const document::StringFieldValue MISMATCHING_HEADER{"Definitely nothing about loud canines"}; + const document::StringFieldValue MATCHING_HEADER{"Some string with woofy dog as a substring"}; + const document::StringFieldValue OLD_CONTENT{"Some old content"}; + const document::StringFieldValue NEW_CONTENT{"Freshly pressed and squeezed content"}; + + unique_ptr<PersistenceThread> thread; + shared_ptr<document::Document> testDoc; + document::DocumentId testDocId; + +public: + void setUp() override { + SingleDiskPersistenceTestUtils::setUp(); + + spi::Context context( + spi::LoadType(0, "default"), + spi::Priority(0), + spi::Trace::TraceLevel(0)); + + createBucket(BUCKET_ID); + getPersistenceProvider().createBucket( + spi::Bucket(BUCKET_ID, spi::PartitionId(0)), + context); + + thread = createPersistenceThread(0); + testDoc = createTestDocument(); + testDocId = testDoc->getId(); + } + + void tearDown() override { + thread.reset(nullptr); + SingleDiskPersistenceTestUtils::tearDown(); + } + + void conditional_put_not_executed_on_condition_mismatch(); + void conditional_put_executed_on_condition_match(); + void conditional_remove_not_executed_on_condition_mismatch(); + void conditional_remove_executed_on_condition_match(); + void conditional_update_not_executed_on_condition_mismatch(); + void conditional_update_executed_on_condition_match(); + void invalid_document_selection_should_fail(); + void non_existing_document_should_fail(); + void document_with_no_type_should_fail(); + + CPPUNIT_TEST_SUITE(TestAndSetTest); + CPPUNIT_TEST(conditional_put_not_executed_on_condition_mismatch); + CPPUNIT_TEST(conditional_put_executed_on_condition_match); + CPPUNIT_TEST(conditional_remove_not_executed_on_condition_mismatch); + CPPUNIT_TEST(conditional_remove_executed_on_condition_match); + CPPUNIT_TEST(conditional_update_not_executed_on_condition_mismatch); + CPPUNIT_TEST(conditional_update_executed_on_condition_match); + CPPUNIT_TEST(invalid_document_selection_should_fail); + CPPUNIT_TEST(non_existing_document_should_fail); + CPPUNIT_TEST(document_with_no_type_should_fail); + CPPUNIT_TEST_SUITE_END(); + +protected: + std::unique_ptr<api::UpdateCommand> conditional_update_test( + bool matchingHeader, + api::Timestamp timestampOne, + api::Timestamp timestampTwo); + + document::Document::SP createTestDocument(); + document::Document::SP retrieveTestDocument(); + void setTestCondition(api::TestAndSetCommand & command); + void putTestDocument(bool matchingHeader, api::Timestamp timestamp); + void assertTestDocumentFoundAndMatchesContent(const document::FieldValue & value); + + static std::string expectedDocEntryString( + api::Timestamp timestamp, + const document::DocumentId & testDocId, + spi::DocumentMetaFlags removeFlag = spi::NONE); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(TestAndSetTest); + +void TestAndSetTest::conditional_put_not_executed_on_condition_mismatch() +{ + // Put document with mismatching header + api::Timestamp timestampOne = 0; + putTestDocument(false, timestampOne); + + CPPUNIT_ASSERT_EQUAL(expectedDocEntryString(timestampOne, testDocId), dumpBucket(BUCKET_ID)); + + // Conditionally replace document, but fail due to lack of woofy dog + api::Timestamp timestampTwo = 1; + api::PutCommand putTwo(BUCKET_ID, testDoc, timestampTwo); + setTestCondition(putTwo); + + CPPUNIT_ASSERT(thread->handlePut(putTwo)->getResult() == api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); + CPPUNIT_ASSERT_EQUAL(expectedDocEntryString(timestampOne, testDocId), dumpBucket(BUCKET_ID)); +} + +void TestAndSetTest::conditional_put_executed_on_condition_match() +{ + // Put document with matching header + api::Timestamp timestampOne = 0; + putTestDocument(true, timestampOne); + + CPPUNIT_ASSERT_EQUAL(expectedDocEntryString(timestampOne, testDocId), dumpBucket(BUCKET_ID)); + + // Update content of document + testDoc->setValue(testDoc->getField("content"), NEW_CONTENT); + + // Conditionally replace document with updated version, succeed in doing so + api::Timestamp timestampTwo = 1; + api::PutCommand putTwo(BUCKET_ID, testDoc, timestampTwo); + setTestCondition(putTwo); + + CPPUNIT_ASSERT(thread->handlePut(putTwo)->getResult() == api::ReturnCode::Result::OK); + CPPUNIT_ASSERT_EQUAL(expectedDocEntryString(timestampOne, testDocId) + + expectedDocEntryString(timestampTwo, testDocId), + dumpBucket(BUCKET_ID)); + + assertTestDocumentFoundAndMatchesContent(NEW_CONTENT); +} + +void TestAndSetTest::conditional_remove_not_executed_on_condition_mismatch() +{ + // Put document with mismatching header + api::Timestamp timestampOne = 0; + putTestDocument(false, timestampOne); + + CPPUNIT_ASSERT_EQUAL(expectedDocEntryString(timestampOne, testDocId), dumpBucket(BUCKET_ID)); + + // Conditionally remove document, fail in doing so + api::Timestamp timestampTwo = 1; + api::RemoveCommand remove(BUCKET_ID, testDocId, timestampTwo); + setTestCondition(remove); + + CPPUNIT_ASSERT(thread->handleRemove(remove)->getResult() == api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); + CPPUNIT_ASSERT_EQUAL(expectedDocEntryString(timestampOne, testDocId), dumpBucket(BUCKET_ID)); + + // Assert that the document is still there + retrieveTestDocument(); +} + +void TestAndSetTest::conditional_remove_executed_on_condition_match() +{ + // Put document with matching header + api::Timestamp timestampOne = 0; + putTestDocument(true, timestampOne); + + CPPUNIT_ASSERT_EQUAL(expectedDocEntryString(timestampOne, testDocId), dumpBucket(BUCKET_ID)); + + // Conditionally remove document, succeed in doing so + api::Timestamp timestampTwo = 1; + api::RemoveCommand remove(BUCKET_ID, testDocId, timestampTwo); + setTestCondition(remove); + + CPPUNIT_ASSERT(thread->handleRemove(remove)->getResult() == api::ReturnCode::Result::OK); + CPPUNIT_ASSERT_EQUAL(expectedDocEntryString(timestampOne, testDocId) + + expectedDocEntryString(timestampTwo, testDocId, spi::REMOVE_ENTRY), + dumpBucket(BUCKET_ID)); +} + +std::unique_ptr<api::UpdateCommand> TestAndSetTest::conditional_update_test( + bool matchingHeader, + api::Timestamp timestampOne, + api::Timestamp timestampTwo) +{ + putTestDocument(matchingHeader, timestampOne); + + auto docUpdate = std::make_shared<document::DocumentUpdate>(testDoc->getType(), testDocId); + auto fieldUpdate = document::FieldUpdate(testDoc->getField("content")); + fieldUpdate.addUpdate(document::AssignValueUpdate(NEW_CONTENT)); + docUpdate->addUpdate(fieldUpdate); + + auto updateUp = std::make_unique<api::UpdateCommand>(BUCKET_ID, docUpdate, timestampTwo); + setTestCondition(*updateUp); + return updateUp; +} + +void TestAndSetTest::conditional_update_not_executed_on_condition_mismatch() +{ + api::Timestamp timestampOne = 0; + api::Timestamp timestampTwo = 1; + auto updateUp = conditional_update_test(false, timestampOne, timestampTwo); + + CPPUNIT_ASSERT(thread->handleUpdate(*updateUp)->getResult() == api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); + CPPUNIT_ASSERT_EQUAL(expectedDocEntryString(timestampOne, testDocId), + dumpBucket(BUCKET_ID)); + + assertTestDocumentFoundAndMatchesContent(OLD_CONTENT); +} + +void TestAndSetTest::conditional_update_executed_on_condition_match() +{ + api::Timestamp timestampOne = 0; + api::Timestamp timestampTwo = 1; + auto updateUp = conditional_update_test(true, timestampOne, timestampTwo); + + CPPUNIT_ASSERT(thread->handleUpdate(*updateUp)->getResult() == api::ReturnCode::Result::OK); + CPPUNIT_ASSERT_EQUAL(expectedDocEntryString(timestampOne, testDocId) + + expectedDocEntryString(timestampTwo, testDocId), + dumpBucket(BUCKET_ID)); + + assertTestDocumentFoundAndMatchesContent(NEW_CONTENT); +} + +void TestAndSetTest::invalid_document_selection_should_fail() +{ + // Conditionally replace nonexisting document + // Fail early since document selection is invalid + api::Timestamp timestamp = 0; + api::PutCommand put(BUCKET_ID, testDoc, timestamp); + put.setCondition(documentapi::TestAndSetCondition("bjarne")); + + CPPUNIT_ASSERT(thread->handlePut(put)->getResult() == api::ReturnCode::Result::ILLEGAL_PARAMETERS); + CPPUNIT_ASSERT_EQUAL(""s, dumpBucket(BUCKET_ID)); +} + +void TestAndSetTest::non_existing_document_should_fail() +{ + // Conditionally replace nonexisting document + // Fail since no document exists to match with test and set + api::Timestamp timestamp = 0; + api::PutCommand put(BUCKET_ID, testDoc, timestamp); + setTestCondition(put); + thread->handlePut(put); + + CPPUNIT_ASSERT(thread->handlePut(put)->getResult() == api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); + CPPUNIT_ASSERT_EQUAL(""s, dumpBucket(BUCKET_ID)); +} + +void TestAndSetTest::document_with_no_type_should_fail() +{ + // Conditionally replace nonexisting document + // Fail since no document exists to match with test and set + api::Timestamp timestamp = 0; + document::DocumentId legacyDocId("doc:mail:3619.html"); + api::RemoveCommand remove(BUCKET_ID, legacyDocId, timestamp); + setTestCondition(remove); + + auto code = thread->handleRemove(remove)->getResult(); + CPPUNIT_ASSERT(code == api::ReturnCode::Result::ILLEGAL_PARAMETERS); + CPPUNIT_ASSERT(code.getMessage() == "Document id has no doctype"); + CPPUNIT_ASSERT_EQUAL(""s, dumpBucket(BUCKET_ID)); +} + +document::Document::SP +TestAndSetTest::createTestDocument() +{ + auto doc = document::Document::SP( + createRandomDocumentAtLocation( + BUCKET_ID.getId(), + RANDOM_SEED, + MIN_DOCUMENT_SIZE, + MAX_DOCUMENT_SIZE)); + + doc->setValue(doc->getField("content"), OLD_CONTENT); + doc->setValue(doc->getField("hstringval"), MISMATCHING_HEADER); + + return doc; +} + +document::Document::SP TestAndSetTest::retrieveTestDocument() +{ + api::GetCommand get(BUCKET_ID, testDocId, "[all]"); + auto tracker = thread->handleGet(get); + CPPUNIT_ASSERT(tracker->getResult() == api::ReturnCode::Result::OK); + + auto & reply = static_cast<api::GetReply &>(*tracker->getReply()); + CPPUNIT_ASSERT(reply.wasFound()); + + return reply.getDocument(); +} + +void TestAndSetTest::setTestCondition(api::TestAndSetCommand & command) +{ + command.setCondition(documentapi::TestAndSetCondition("testdoctype1.hstringval=\"*woofy dog*\"")); +} + +void TestAndSetTest::putTestDocument(bool matchingHeader, api::Timestamp timestamp) { + if (matchingHeader) { + testDoc->setValue(testDoc->getField("hstringval"), MATCHING_HEADER); + } + + api::PutCommand put(BUCKET_ID, testDoc, timestamp); + thread->handlePut(put); +} + +void TestAndSetTest::assertTestDocumentFoundAndMatchesContent(const document::FieldValue & value) +{ + auto doc = retrieveTestDocument(); + auto & field = doc->getField("content"); + + CPPUNIT_ASSERT_EQUAL(*doc->getValue(field), value); +} + +std::string TestAndSetTest::expectedDocEntryString( + api::Timestamp timestamp, + const document::DocumentId & docId, + spi::DocumentMetaFlags removeFlag) +{ + std::stringstream ss; + + ss << "DocEntry(" << timestamp << ", " << removeFlag << ", "; + if (removeFlag == spi::REMOVE_ENTRY) { + ss << docId.toString() << ")\n"; + } else { + ss << "Doc(" << docId.toString() << "))\n"; + } + + return ss.str(); +} + +} // storage diff --git a/storage/src/tests/pstack_testrunner b/storage/src/tests/pstack_testrunner new file mode 100755 index 00000000000..320d47f7e35 --- /dev/null +++ b/storage/src/tests/pstack_testrunner @@ -0,0 +1,14 @@ +#!/usr/bin/perl -w + +use strict; + +my @pids = `ps auxww | grep "./testrunner" | grep -v grep`; +foreach (@pids) { + s/^\S+\s+(\d+)\s+.*$/$1/; + chomp; +} + +foreach my $pid (@pids) { + my $cmd = "pstack $pid"; + system($cmd) == 0 or die "Failed to run '$cmd'"; +} diff --git a/storage/src/tests/serverapp/.gitignore b/storage/src/tests/serverapp/.gitignore new file mode 100644 index 00000000000..333f254ba10 --- /dev/null +++ b/storage/src/tests/serverapp/.gitignore @@ -0,0 +1,8 @@ +*.So +*.lo +.*.swp +.depend +.depend.NEW +.deps +.libs +Makefile diff --git a/storage/src/tests/storageserver/.gitignore b/storage/src/tests/storageserver/.gitignore new file mode 100644 index 00000000000..c4098089f09 --- /dev/null +++ b/storage/src/tests/storageserver/.gitignore @@ -0,0 +1,13 @@ +*.So +*.lo +*.o +.*.swp +.config.log +.depend +.depend.NEW +.deps +.libs +Makefile +filestorage +testrunner +testrunner.core diff --git a/storage/src/tests/storageserver/CMakeLists.txt b/storage/src/tests/storageserver/CMakeLists.txt new file mode 100644 index 00000000000..2e327089b4c --- /dev/null +++ b/storage/src/tests/storageserver/CMakeLists.txt @@ -0,0 +1,17 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storage_teststorageserver + SOURCES + communicationmanagertest.cpp + statemanagertest.cpp + documentapiconvertertest.cpp + mergethrottlertest.cpp + testvisitormessagesession.cpp + bouncertest.cpp + bucketintegritycheckertest.cpp + priorityconvertertest.cpp + statereportertest.cpp + changedbucketownershiphandlertest.cpp + DEPENDS + AFTER + storage_storageconfig +) diff --git a/storage/src/tests/storageserver/bouncertest.cpp b/storage/src/tests/storageserver/bouncertest.cpp new file mode 100644 index 00000000000..f00e4b19c31 --- /dev/null +++ b/storage/src/tests/storageserver/bouncertest.cpp @@ -0,0 +1,285 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <boost/pointer_cast.hpp> +#include <cppunit/extensions/HelperMacros.h> +#include <iostream> +#include <string> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/storageapi/message/stat.h> +#include <vespa/vdslib/state/nodestate.h> +#include <vespa/storage/storageserver/bouncer.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/testhelper.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/storageapi/message/persistence.h> + +namespace storage { + +struct BouncerTest : public CppUnit::TestFixture { + std::unique_ptr<TestServiceLayerApp> _node; + std::unique_ptr<DummyStorageLink> _upper; + Bouncer* _manager; + DummyStorageLink* _lower; + + BouncerTest(); + + void setUp(); + void tearDown(); + + void testFutureTimestamp(); + void testAllowNotifyBucketChangeEvenWhenDistributorDown(); + void rejectLowerPrioritizedFeedMessagesWhenConfigured(); + void doNotRejectHigherPrioritizedFeedMessagesThanConfigured(); + void rejectionThresholdIsExclusive(); + void onlyRejectFeedMessagesWhenConfigured(); + void rejectionIsDisabledByDefaultInConfig(); + void readOnlyOperationsAreNotRejected(); + void internalOperationsAreNotRejected(); + void outOfBoundsConfigValuesThrowException(); + + CPPUNIT_TEST_SUITE(BouncerTest); + CPPUNIT_TEST(testFutureTimestamp); + CPPUNIT_TEST(testAllowNotifyBucketChangeEvenWhenDistributorDown); + CPPUNIT_TEST(rejectLowerPrioritizedFeedMessagesWhenConfigured); + CPPUNIT_TEST(doNotRejectHigherPrioritizedFeedMessagesThanConfigured); + CPPUNIT_TEST(rejectionThresholdIsExclusive); + CPPUNIT_TEST(onlyRejectFeedMessagesWhenConfigured); + CPPUNIT_TEST(rejectionIsDisabledByDefaultInConfig); + CPPUNIT_TEST(readOnlyOperationsAreNotRejected); + CPPUNIT_TEST(internalOperationsAreNotRejected); + CPPUNIT_TEST(outOfBoundsConfigValuesThrowException); + CPPUNIT_TEST_SUITE_END(); + + using Priority = api::StorageMessage::Priority; + + static constexpr int RejectionDisabledConfigValue = -1; + + // Note: newThreshold is intentionally int (rather than Priority) in order + // to be able to test out of bounds values. + void configureRejectionThreshold(int newThreshold); + + std::shared_ptr<api::StorageCommand> createDummyFeedMessage( + api::Timestamp timestamp, + Priority priority = 0); + + void assertMessageBouncedWithRejection(); + void assertMessageNotBounced(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(BouncerTest); + +BouncerTest::BouncerTest() + : _node(), + _upper(), + _manager(0), + _lower(0) +{ +} + +void +BouncerTest::setUp() { + try{ + vdstestlib::DirConfig config(getStandardConfig(true)); + _node.reset(new TestServiceLayerApp( + DiskCount(1), NodeIndex(2), config.getConfigId())); + _upper.reset(new DummyStorageLink()); + _manager = new Bouncer(_node->getComponentRegister(), + config.getConfigId()); + _lower = new DummyStorageLink(); + _upper->push_back(std::unique_ptr<StorageLink>(_manager)); + _upper->push_back(std::unique_ptr<StorageLink>(_lower)); + _upper->open(); + } catch (std::exception& e) { + std::cerr << "Failed to static initialize objects: " << e.what() + << "\n"; + } + _node->getClock().setAbsoluteTimeInSeconds(10); +} + +void +BouncerTest::tearDown() { + _manager = 0; + _lower = 0; + _upper->close(); + _upper->flush(); + _upper.reset(0); + _node.reset(0); +} + +std::shared_ptr<api::StorageCommand> +BouncerTest::createDummyFeedMessage(api::Timestamp timestamp, + api::StorageMessage::Priority priority) +{ + auto cmd = std::make_shared<api::RemoveCommand>( + document::BucketId(0), + document::DocumentId("doc:foo:bar"), + timestamp); + cmd->setPriority(priority); + return cmd; +} + +void +BouncerTest::testFutureTimestamp() +{ + + // Fail when future timestamps (more than 5 seconds) are received. + { + _upper->sendDown(createDummyFeedMessage(16 * 1000000)); + + CPPUNIT_ASSERT_EQUAL(1, (int)_upper->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(0, (int)_upper->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ABORTED, + static_cast<api::RemoveReply&>(*_upper->getReply(0)). + getResult().getResult()); + _upper->reset(); + } + + // Verify that 1 second clock skew is OK + { + _upper->sendDown(createDummyFeedMessage(11 * 1000000)); + + CPPUNIT_ASSERT_EQUAL(0, (int)_upper->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(1, (int)_lower->getNumCommands()); + _lower->reset(); + } + + // Verify that past is OK + { + _upper->sendDown(createDummyFeedMessage(5 * 1000000)); + + CPPUNIT_ASSERT_EQUAL(1, (int)_lower->getNumCommands()); + } + + +} + +void +BouncerTest::testAllowNotifyBucketChangeEvenWhenDistributorDown() +{ + lib::NodeState state(lib::NodeType::DISTRIBUTOR, lib::State::DOWN); + _node->getNodeStateUpdater().setReportedNodeState(state); + // Trigger Bouncer state update + auto clusterState = std::make_shared<lib::ClusterState>( + "distributor:3 storage:3"); + _node->getNodeStateUpdater().setClusterState(clusterState); + + + document::BucketId bucket(16, 1234); + api::BucketInfo info(0x1, 0x2, 0x3); + auto cmd = std::make_shared<api::NotifyBucketChangeCommand>(bucket, info); + _upper->sendDown(cmd); + + CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(size_t(1), _lower->getNumCommands()); +} + +void +BouncerTest::assertMessageBouncedWithRejection() +{ + CPPUNIT_ASSERT_EQUAL(size_t(1), _upper->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::REJECTED, + static_cast<api::RemoveReply&>(*_upper->getReply(0)). + getResult().getResult()); + CPPUNIT_ASSERT_EQUAL(size_t(0), _lower->getNumCommands()); +} + +void +BouncerTest::assertMessageNotBounced() +{ + CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(size_t(1), _lower->getNumCommands()); +} + +void +BouncerTest::configureRejectionThreshold(int newThreshold) +{ + using Builder = vespa::config::content::core::StorBouncerConfigBuilder; + auto config = std::make_unique<Builder>(); + config->feedRejectionPriorityThreshold = newThreshold; + _manager->configure(std::move(config)); +} + +void +BouncerTest::rejectLowerPrioritizedFeedMessagesWhenConfigured() +{ + configureRejectionThreshold(Priority(120)); + _upper->sendDown(createDummyFeedMessage(11 * 1000000, Priority(121))); + assertMessageBouncedWithRejection(); +} + +void +BouncerTest::doNotRejectHigherPrioritizedFeedMessagesThanConfigured() +{ + configureRejectionThreshold(Priority(120)); + _upper->sendDown(createDummyFeedMessage(11 * 1000000, Priority(119))); + assertMessageNotBounced(); +} + +void +BouncerTest::rejectionThresholdIsExclusive() +{ + configureRejectionThreshold(Priority(120)); + _upper->sendDown(createDummyFeedMessage(11 * 1000000, Priority(120))); + assertMessageNotBounced(); +} + +void +BouncerTest::onlyRejectFeedMessagesWhenConfigured() +{ + configureRejectionThreshold(RejectionDisabledConfigValue); + // A message with even the lowest priority should not be rejected. + _upper->sendDown(createDummyFeedMessage(11 * 1000000, Priority(255))); + assertMessageNotBounced(); +} + +void +BouncerTest::rejectionIsDisabledByDefaultInConfig() +{ + _upper->sendDown(createDummyFeedMessage(11 * 1000000, Priority(255))); + assertMessageNotBounced(); +} + +void +BouncerTest::readOnlyOperationsAreNotRejected() +{ + configureRejectionThreshold(Priority(1)); + // StatBucket is an external operation, but it's not a mutating operation + // and should therefore not be blocked. + auto cmd = std::make_shared<api::StatBucketCommand>( + document::BucketId(16, 5), ""); + cmd->setPriority(Priority(2)); + _upper->sendDown(cmd); + assertMessageNotBounced(); +} + +void +BouncerTest::internalOperationsAreNotRejected() +{ + configureRejectionThreshold(Priority(1)); + document::BucketId bucket(16, 1234); + api::BucketInfo info(0x1, 0x2, 0x3); + auto cmd = std::make_shared<api::NotifyBucketChangeCommand>(bucket, info); + cmd->setPriority(Priority(2)); + _upper->sendDown(cmd); + assertMessageNotBounced(); +} + +void +BouncerTest::outOfBoundsConfigValuesThrowException() +{ + try { + configureRejectionThreshold(256); + CPPUNIT_FAIL("Upper bound violation not caught"); + } catch (config::InvalidConfigException) {} + + try { + configureRejectionThreshold(-2); + CPPUNIT_FAIL("Lower bound violation not caught"); + } catch (config::InvalidConfigException) {} +} + +} // storage + diff --git a/storage/src/tests/storageserver/bucketintegritycheckertest.cpp b/storage/src/tests/storageserver/bucketintegritycheckertest.cpp new file mode 100644 index 00000000000..88a5546b174 --- /dev/null +++ b/storage/src/tests/storageserver/bucketintegritycheckertest.cpp @@ -0,0 +1,302 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <boost/lexical_cast.hpp> +#include <cppunit/extensions/HelperMacros.h> +#include <vespa/log/log.h> +#include <vespa/storage/bucketdb/bucketmanager.h> +#include <vespa/storage/bucketdb/storbucketdb.h> +#include <vespa/storage/persistence/filestorage/filestormanager.h> +#include <vespa/storage/storageserver/bucketintegritychecker.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/persistence.h> +#include <tests/common/testhelper.h> +#include <tests/common/storagelinktest.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/vespalib/io/fileutil.h> +#include <tests/common/teststorageapp.h> + +LOG_SETUP(".test.bucketintegritychecker"); + +namespace storage { + +struct BucketIntegrityCheckerTest : public CppUnit::TestFixture { + std::unique_ptr<vdstestlib::DirConfig> _config; + std::unique_ptr<TestServiceLayerApp> _node; + int _timeout; // Timeout in seconds before aborting + + void setUp() { + _timeout = 60*2; + _config.reset(new vdstestlib::DirConfig(getStandardConfig(true))); + _node.reset(new TestServiceLayerApp(DiskCount(256), + NodeIndex(0), + _config->getConfigId())); + } + + void tearDown() { + LOG(info, "Finished test"); + } + + void testConfig(); + void testBasicFunctionality(); + void testTiming(); + + CPPUNIT_TEST_SUITE(BucketIntegrityCheckerTest); + CPPUNIT_TEST(testConfig); + CPPUNIT_TEST(testBasicFunctionality); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(BucketIntegrityCheckerTest); + +void BucketIntegrityCheckerTest::testConfig() +{ + + // Verify that config is read correctly. Given config should not use + // any default values. + vdstestlib::DirConfig::Config& config( + _config->getConfig("stor-integritychecker")); + config.set("dailycyclestart", "60"); + config.set("dailycyclestop", "360"); + config.set("weeklycycle", "crRc-rc"); + config.set("maxpending", "2"); + config.set("mincycletime", "120"); + config.set("requestdelay", "5"); + + BucketIntegrityChecker checker(_config->getConfigId(), + _node->getComponentRegister()); + checker.setMaxThreadWaitTime(framework::MilliSecTime(10)); + SchedulingOptions& opt(checker.getSchedulingOptions()); + CPPUNIT_ASSERT_EQUAL(60u, opt._dailyCycleStart); + CPPUNIT_ASSERT_EQUAL(360u, opt._dailyCycleStop); + CPPUNIT_ASSERT_EQUAL(SchedulingOptions::CONTINUE, opt._dailyStates[0]); + CPPUNIT_ASSERT_EQUAL(SchedulingOptions::RUN_CHEAP, opt._dailyStates[1]); + CPPUNIT_ASSERT_EQUAL(SchedulingOptions::RUN_FULL, opt._dailyStates[2]); + CPPUNIT_ASSERT_EQUAL(SchedulingOptions::CONTINUE, opt._dailyStates[3]); + CPPUNIT_ASSERT_EQUAL(SchedulingOptions::DONT_RUN, opt._dailyStates[4]); + CPPUNIT_ASSERT_EQUAL(SchedulingOptions::RUN_CHEAP, opt._dailyStates[5]); + CPPUNIT_ASSERT_EQUAL(SchedulingOptions::CONTINUE, opt._dailyStates[6]); + CPPUNIT_ASSERT_EQUAL(2u, opt._maxPendingCount); + CPPUNIT_ASSERT_EQUAL(framework::SecondTime(7200), opt._minCycleTime); + CPPUNIT_ASSERT_EQUAL(framework::SecondTime(5), opt._requestDelay); +} + +namespace { + /** + * Calculate a date based on the following format: + * week<#> <day> <hh>:<mm>:<ss> + * Examples: "week3 mon 00:30:00" + * "week3 tue 04:20:00" + * "week9 thi 14:00:24" + */ + time_t getDate(const std::string& datestring) { + vespalib::string rest(datestring); + int spacePos = rest.find(' '); + uint32_t week = strtoul(rest.substr(4, spacePos-4).c_str(), NULL, 0); + rest = rest.substr(spacePos+1); + vespalib::string wday(rest.substr(0,3)); + rest = rest.substr(4); + uint32_t hours = strtoul(rest.substr(0, 2).c_str(), NULL, 0); + uint32_t minutes = strtoul(rest.substr(3, 2).c_str(), NULL, 0); + uint32_t seconds = strtoul(rest.substr(6, 2).c_str(), NULL, 0); + uint32_t day(0); + if (wday == "mon") { day = 1; } + else if (wday == "tue") { day = 2; } + else if (wday == "wed") { day = 3; } + else if (wday == "thi") { day = 4; } + else if (wday == "fri") { day = 5; } + else if (wday == "sat") { day = 6; } + else if (wday == "sun") { day = 0; } + else { assert(false); } + // Create a start time that points to the start of some week. + // A random sunday 00:00:00, which we will use as start of time + struct tm mytime; + memset(&mytime, 0, sizeof(mytime)); + mytime.tm_year = 2008 - 1900; + mytime.tm_mon = 0; + mytime.tm_mday = 1; + mytime.tm_hour = 0; + mytime.tm_min = 0; + mytime.tm_sec = 0; + time_t startTime = timegm(&mytime); + CPPUNIT_ASSERT(gmtime_r(&startTime, &mytime)); + while (mytime.tm_wday != 0) { + ++mytime.tm_mday; + startTime = timegm(&mytime); + CPPUNIT_ASSERT(gmtime_r(&startTime, &mytime)); + } + // Add the wanted values to the start time + time_t resultTime = startTime; + resultTime += week * 7 * 24 * 60 * 60 + + day * 24 * 60 * 60 + + hours * 60 * 60 + + minutes * 60 + + seconds; + // std::cerr << "Time requested " << datestring << ". Got time " + // << framework::SecondTime(resultTime).toString() << "\n"; + return resultTime; + } + + void addBucketToDatabase(TestServiceLayerApp& server, + const document::BucketId& id, uint8_t disk, + uint32_t numDocs, uint32_t crc, uint32_t totalSize) + { + bucketdb::StorageBucketInfo info; + info.setBucketInfo(api::BucketInfo(crc, numDocs, totalSize)); + info.disk = disk; + server.getStorageBucketDatabase().insert(id, info, "foo"); + } + + + /** + * In tests wanting to only have one pending, only add buckets for one disk + * as pending is per disk. If so set singleDisk true. + */ + void addBucketsToDatabase(TestServiceLayerApp& server, bool singleDisk) { + addBucketToDatabase(server, document::BucketId(16, 0x123), 0, + 14, 0x123, 1024); + addBucketToDatabase(server, document::BucketId(16, 0x234), 0, + 18, 0x234, 1024); + addBucketToDatabase(server, document::BucketId(16, 0x345), 0, + 11, 0x345, 2048); + addBucketToDatabase(server, document::BucketId(16, 0x456), 0, + 13, 0x456, 1280); + if (!singleDisk) { + addBucketToDatabase(server, document::BucketId(16, 0x567), 1, + 20, 0x567, 4096); + addBucketToDatabase(server, document::BucketId(16, 0x987), 254, + 8, 0x987, 65536); + } + } +} + +void BucketIntegrityCheckerTest::testBasicFunctionality() +{ + _node->getClock().setAbsoluteTimeInSeconds(getDate("week1 sun 00:00:00")); + addBucketsToDatabase(*_node, false); + DummyStorageLink* dummyLink = 0; + { + std::unique_ptr<BucketIntegrityChecker> midLink( + new BucketIntegrityChecker("", _node->getComponentRegister())); + BucketIntegrityChecker& checker(*midLink); + checker.setMaxThreadWaitTime(framework::MilliSecTime(10)); + // Setup and start checker + DummyStorageLink topLink; + topLink.push_back(StorageLink::UP(midLink.release())); + checker.push_back(std::unique_ptr<StorageLink>( + dummyLink = new DummyStorageLink())); + checker.getSchedulingOptions()._maxPendingCount = 2; + checker.getSchedulingOptions()._minCycleTime = framework::SecondTime(60 * 60); + topLink.open(); + // Waiting for system to be initialized + FastOS_Thread::Sleep(10); // Give next message chance to come + ASSERT_COMMAND_COUNT(0, *dummyLink); + topLink.doneInit(); + checker.bump(); + // Should have started new run with 2 pending per disk + dummyLink->waitForMessages(4, _timeout); + FastOS_Thread::Sleep(10); // Give 5th message chance to come + ASSERT_COMMAND_COUNT(4, *dummyLink); + RepairBucketCommand *cmd1 = dynamic_cast<RepairBucketCommand*>( + dummyLink->getCommand(0).get()); + CPPUNIT_ASSERT_EQUAL(230, (int)cmd1->getPriority()); + CPPUNIT_ASSERT(cmd1); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x234), + cmd1->getBucketId()); + RepairBucketCommand *cmd2 = dynamic_cast<RepairBucketCommand*>( + dummyLink->getCommand(1).get()); + CPPUNIT_ASSERT(cmd2); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x456), + cmd2->getBucketId()); + RepairBucketCommand *cmd3 = dynamic_cast<RepairBucketCommand*>( + dummyLink->getCommand(2).get()); + CPPUNIT_ASSERT(cmd3); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x567), + cmd3->getBucketId()); + RepairBucketCommand *cmd4 = dynamic_cast<RepairBucketCommand*>( + dummyLink->getCommand(3).get()); + CPPUNIT_ASSERT(cmd4); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x987), + cmd4->getBucketId()); + + // Answering a message on disk with no more buckets does not trigger new + std::shared_ptr<RepairBucketReply> reply1( + new RepairBucketReply(*cmd3)); + CPPUNIT_ASSERT(StorageLinkTest::callOnUp(checker, reply1)); + FastOS_Thread::Sleep(10); // Give next message chance to come + ASSERT_COMMAND_COUNT(4, *dummyLink); + // Answering a message on disk with more buckets trigger new repair + std::shared_ptr<RepairBucketReply> reply2( + new RepairBucketReply(*cmd2)); + CPPUNIT_ASSERT(StorageLinkTest::callOnUp(checker, reply2)); + dummyLink->waitForMessages(5, _timeout); + FastOS_Thread::Sleep(10); // Give 6th message chance to come + ASSERT_COMMAND_COUNT(5, *dummyLink); + RepairBucketCommand *cmd5 = dynamic_cast<RepairBucketCommand*>( + dummyLink->getCommand(4).get()); + CPPUNIT_ASSERT(cmd5); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x345), + cmd5->getBucketId()); + // Fail a repair, causing it to be resent later, but first continue + // with other bucket. + std::shared_ptr<RepairBucketReply> reply3( + new RepairBucketReply(*cmd1)); + reply3->setResult(api::ReturnCode(api::ReturnCode::IGNORED)); + CPPUNIT_ASSERT(StorageLinkTest::callOnUp(checker, reply3)); + dummyLink->waitForMessages(6, _timeout); + FastOS_Thread::Sleep(10); // Give 7th message chance to come + ASSERT_COMMAND_COUNT(6, *dummyLink); + RepairBucketCommand *cmd6 = dynamic_cast<RepairBucketCommand*>( + dummyLink->getCommand(5).get()); + CPPUNIT_ASSERT(cmd6); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x123), + cmd6->getBucketId()); + // Fail a repair with not found. That is an acceptable return code. + // (No more requests as this was last for that disk) + std::shared_ptr<RepairBucketReply> reply4( + new RepairBucketReply(*cmd4)); + reply3->setResult(api::ReturnCode(api::ReturnCode::BUCKET_NOT_FOUND)); + CPPUNIT_ASSERT(StorageLinkTest::callOnUp(checker, reply4)); + FastOS_Thread::Sleep(10); // Give 7th message chance to come + ASSERT_COMMAND_COUNT(6, *dummyLink); + + // Send a repair reply that actually have corrected the bucket. + api::BucketInfo newInfo(0x3456, 4, 8192); + std::shared_ptr<RepairBucketReply> reply5( + new RepairBucketReply(*cmd5, newInfo)); + reply5->setAltered(true); + CPPUNIT_ASSERT(StorageLinkTest::callOnUp(checker, reply5)); + + // Finish run. New iteration should not start yet as min + // cycle time has not passed + std::shared_ptr<RepairBucketReply> reply6( + new RepairBucketReply(*cmd6)); + CPPUNIT_ASSERT(StorageLinkTest::callOnUp(checker, reply6)); + dummyLink->waitForMessages(7, _timeout); + ASSERT_COMMAND_COUNT(7, *dummyLink); + RepairBucketCommand *cmd7 = dynamic_cast<RepairBucketCommand*>( + dummyLink->getCommand(6).get()); + CPPUNIT_ASSERT(cmd7); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x234), + cmd7->getBucketId()); + std::shared_ptr<RepairBucketReply> reply7( + new RepairBucketReply(*cmd7)); + CPPUNIT_ASSERT(StorageLinkTest::callOnUp(checker, reply7)); + FastOS_Thread::Sleep(10); // Give 8th message chance to come + ASSERT_COMMAND_COUNT(7, *dummyLink); + + // Still not time for next iteration + dummyLink->reset(); + _node->getClock().setAbsoluteTimeInSeconds(getDate("week1 sun 00:59:59")); + FastOS_Thread::Sleep(10); // Give new run chance to start + ASSERT_COMMAND_COUNT(0, *dummyLink); + + // Pass time until next cycle should start + dummyLink->reset(); + _node->getClock().setAbsoluteTimeInSeconds(getDate("week1 sun 01:00:00")); + dummyLink->waitForMessages(4, _timeout); + ASSERT_COMMAND_COUNT(4, *dummyLink); + } +} + +} // storage diff --git a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp new file mode 100644 index 00000000000..3b83d71d8f3 --- /dev/null +++ b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp @@ -0,0 +1,648 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/document/base/testdocman.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/bucketdb/storbucketdb.h> +#include <vespa/storage/persistence/messages.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storageapi/message/removelocation.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/multioperation.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/testhelper.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/storage/storageserver/changedbucketownershiphandler.h> +#include <memory> + +namespace storage { + +class ChangedBucketOwnershipHandlerTest : public CppUnit::TestFixture +{ + std::unique_ptr<TestServiceLayerApp> _app; + std::unique_ptr<DummyStorageLink> _top; + ChangedBucketOwnershipHandler* _handler; + DummyStorageLink* _bottom; + document::TestDocMan _testDocRepo; + + CPPUNIT_TEST_SUITE(ChangedBucketOwnershipHandlerTest); + CPPUNIT_TEST(testEnumerateBucketsBelongingOnChangedNodes); + CPPUNIT_TEST(testNoPreExistingClusterState); + CPPUNIT_TEST(testNoAvailableDistributorsInCurrentState); + CPPUNIT_TEST(testNoAvailableDistributorsInCurrentAndNewState); + CPPUNIT_TEST(testDownEdgeToNoAvailableDistributors); + CPPUNIT_TEST(testOwnershipChangedOnDistributorUpEdge); + CPPUNIT_TEST(testDistributionConfigChangeUpdatesOwnership); + CPPUNIT_TEST(testAbortOpsWhenNoClusterStateSet); + CPPUNIT_TEST(testAbortOutdatedSplit); + CPPUNIT_TEST(testAbortOutdatedJoin); + CPPUNIT_TEST(testAbortOutdatedSetBucketState); + CPPUNIT_TEST(testAbortOutdatedCreateBucket); + CPPUNIT_TEST(testAbortOutdatedDeleteBucket); + CPPUNIT_TEST(testAbortOutdatedMergeBucket); + CPPUNIT_TEST(testAbortOutdatedRemoveLocation); + CPPUNIT_TEST(testIdealStateAbortsAreConfigurable); + CPPUNIT_TEST(testAbortOutdatedPutOperation); + CPPUNIT_TEST(testAbortOutdatedMultiOperation); + CPPUNIT_TEST(testAbortOutdatedUpdateCommand); + CPPUNIT_TEST(testAbortOutdatedRemoveCommand); + CPPUNIT_TEST(testAbortOutdatedRevertCommand); + CPPUNIT_TEST(testIdealStateAbortUpdatesMetric); + CPPUNIT_TEST(testExternalLoadOpAbortUpdatesMetric); + CPPUNIT_TEST(testExternalLoadOpAbortsAreConfigurable); + CPPUNIT_TEST_SUITE_END(); + + // TODO test: down edge triggered on cluster state with cluster down? + + std::vector<document::BucketId> insertBuckets( + uint32_t numBuckets, + uint16_t wantedOwner, + const lib::ClusterState& state); + + std::shared_ptr<api::SetSystemStateCommand> createStateCmd( + const lib::ClusterState& state) const + { + return std::make_shared<api::SetSystemStateCommand>(state); + } + + std::shared_ptr<api::SetSystemStateCommand> createStateCmd( + const std::string& stateStr) const + { + return createStateCmd(lib::ClusterState(stateStr)); + } + + void applyDistribution(Redundancy, NodeCount); + void applyClusterState(const lib::ClusterState&); + + document::BucketId nextOwnedBucket( + uint16_t wantedOwner, + const lib::ClusterState& state, + const document::BucketId& lastId) const; + + document::BucketId getBucketToAbort() const; + document::BucketId getBucketToAllow() const; + + void sendAndExpectAbortedCreateBucket(uint16_t fromDistributorIndex); + + template <typename MsgType, typename... MsgParams> + bool changeAbortsMessage(MsgParams&&... params); + + lib::ClusterState getDefaultTestClusterState() const { + return lib::ClusterState("distributor:4 storage:1"); + } + +public: + void testEnumerateBucketsBelongingOnChangedNodes(); + void testNoPreExistingClusterState(); + void testNoAvailableDistributorsInCurrentState(); + void testNoAvailableDistributorsInCurrentAndNewState(); + void testDownEdgeToNoAvailableDistributors(); + void testOwnershipChangedOnDistributorUpEdge(); + void testDistributionConfigChangeUpdatesOwnership(); + void testAbortOpsWhenNoClusterStateSet(); + void testAbortOutdatedSplit(); + void testAbortOutdatedJoin(); + void testAbortOutdatedSetBucketState(); + void testAbortOutdatedCreateBucket(); + void testAbortOutdatedDeleteBucket(); + void testAbortOutdatedMergeBucket(); + void testAbortOutdatedRemoveLocation(); + void testIdealStateAbortsAreConfigurable(); + void testAbortOutdatedPutOperation(); + void testAbortOutdatedMultiOperation(); + void testAbortOutdatedUpdateCommand(); + void testAbortOutdatedRemoveCommand(); + void testAbortOutdatedRevertCommand(); + void testIdealStateAbortUpdatesMetric(); + void testExternalLoadOpAbortUpdatesMetric(); + void testExternalLoadOpAbortsAreConfigurable(); + + void setUp(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(ChangedBucketOwnershipHandlerTest); + +document::BucketId +ChangedBucketOwnershipHandlerTest::nextOwnedBucket( + uint16_t wantedOwner, + const lib::ClusterState& state, + const document::BucketId& lastId) const +{ + uint32_t idx(lastId.getId() + 1); + while (true) { + document::BucketId candidate(16, idx); + uint16_t owner(_app->getDistribution()->getIdealDistributorNode( + state, candidate)); + if (owner == wantedOwner) { + return candidate; + } + ++idx; + } + assert(!"should never get here"); +} + +std::vector<document::BucketId> +ChangedBucketOwnershipHandlerTest::insertBuckets(uint32_t numBuckets, + uint16_t wantedOwner, + const lib::ClusterState& state) +{ + std::vector<document::BucketId> inserted; + document::BucketId bucket; + while (inserted.size() < numBuckets) { + bucket = nextOwnedBucket(wantedOwner, state, bucket); + + bucketdb::StorageBucketInfo sbi; + sbi.setBucketInfo(api::BucketInfo(1, 2, 3)); + sbi.disk = 0; + _app->getStorageBucketDatabase().insert(bucket, sbi, "test"); + inserted.push_back(bucket); + } + return inserted; +} + +void +ChangedBucketOwnershipHandlerTest::setUp() +{ + vdstestlib::DirConfig config(getStandardConfig(true)); + + _app.reset(new TestServiceLayerApp); + _top.reset(new DummyStorageLink); + _handler = new ChangedBucketOwnershipHandler(config.getConfigId(), + _app->getComponentRegister()); + _top->push_back(std::unique_ptr<StorageLink>(_handler)); + _bottom = new DummyStorageLink; + _handler->push_back(std::unique_ptr<StorageLink>(_bottom)); + _top->open(); + + // Ensure we're not dependent on config schema default values. + std::unique_ptr<vespa::config::content::PersistenceConfigBuilder> pconfig( + new vespa::config::content::PersistenceConfigBuilder); + pconfig->abortOutdatedMutatingIdealStateOps = true; + pconfig->abortOutdatedMutatingExternalLoadOps = true; + _handler->configure(std::move(pconfig)); +} + +namespace { + +template <typename Set, typename K> +bool has(const Set& s, const K& key) { + return s.find(key) != s.end(); +} + +template <typename Vec> +bool +hasAbortedAllOf(const AbortBucketOperationsCommand::SP& cmd, const Vec& v) +{ + for (auto& b : v) { + if (!cmd->shouldAbort(b)) { + return false; + } + } + return true; +} + +template <typename Vec> +bool +hasAbortedNoneOf(const AbortBucketOperationsCommand::SP& cmd, const Vec& v) +{ + for (auto& b : v) { + if (cmd->shouldAbort(b)) { + return false; + } + } + return true; +} + +bool +hasOnlySetSystemStateCmdQueued(DummyStorageLink& link) { + if (link.getNumCommands() != 1) { + std::cerr << "expected 1 command, found" + << link.getNumCommands() << "\n"; + } + api::SetSystemStateCommand::SP cmd( + std::dynamic_pointer_cast<api::SetSystemStateCommand>( + link.getCommand(0))); + return (cmd.get() != 0); +} + +} + +void +ChangedBucketOwnershipHandlerTest::applyDistribution( + Redundancy redundancy, NodeCount nodeCount) +{ + _app->setDistribution(redundancy, nodeCount); + _handler->storageDistributionChanged(); +} + +void +ChangedBucketOwnershipHandlerTest::applyClusterState( + const lib::ClusterState& state) +{ + _app->setClusterState(state); + _handler->reloadClusterState(); +} + +void +ChangedBucketOwnershipHandlerTest::testEnumerateBucketsBelongingOnChangedNodes() +{ + lib::ClusterState stateBefore("distributor:4 storage:1"); + applyDistribution(Redundancy(1), NodeCount(4)); + applyClusterState(stateBefore); + auto node1Buckets(insertBuckets(2, 1, stateBefore)); + auto node3Buckets(insertBuckets(2, 3, stateBefore)); + // Add some buckets that will not be part of the change set + auto node0Buckets(insertBuckets(3, 0, stateBefore)); + auto node2Buckets(insertBuckets(2, 2, stateBefore)); + + _top->sendDown(createStateCmd("distributor:4 .1.s:d .3.s:d storage:1")); + // TODO: refactor into own function + CPPUNIT_ASSERT_EQUAL(size_t(2), _bottom->getNumCommands()); + AbortBucketOperationsCommand::SP cmd( + std::dynamic_pointer_cast<AbortBucketOperationsCommand>( + _bottom->getCommand(0))); + CPPUNIT_ASSERT(cmd.get() != 0); + + CPPUNIT_ASSERT(hasAbortedAllOf(cmd, node1Buckets)); + CPPUNIT_ASSERT(hasAbortedAllOf(cmd, node3Buckets)); + CPPUNIT_ASSERT(hasAbortedNoneOf(cmd, node0Buckets)); + CPPUNIT_ASSERT(hasAbortedNoneOf(cmd, node2Buckets)); + + // Handler must swallow abort replies + _bottom->sendUp(api::StorageMessage::SP(cmd->makeReply().release())); + CPPUNIT_ASSERT_EQUAL(size_t(0), _top->getNumReplies()); +} + +void +ChangedBucketOwnershipHandlerTest::testNoPreExistingClusterState() +{ + applyDistribution(Redundancy(1), NodeCount(4)); + lib::ClusterState stateBefore("distributor:4 storage:1"); + insertBuckets(2, 1, stateBefore); + insertBuckets(3, 0, stateBefore); + insertBuckets(2, 2, stateBefore); + + _top->sendDown(createStateCmd("distributor:4 .1.s:d .3.s:d storage:1")); + CPPUNIT_ASSERT(hasOnlySetSystemStateCmdQueued(*_bottom)); +} + +/** + * When current state has no distributors and we receive a state with one or + * more distributors, we do not send any abort messages since this should + * already have been done on the down-edge. + */ +void +ChangedBucketOwnershipHandlerTest::testNoAvailableDistributorsInCurrentState() +{ + applyDistribution(Redundancy(1), NodeCount(3)); + lib::ClusterState insertedState("distributor:3 storage:1"); + insertBuckets(2, 0, insertedState); + insertBuckets(2, 1, insertedState); + insertBuckets(2, 2, insertedState); + lib::ClusterState downState("distributor:3 .0.s:d .1.s:d .2.s:d storage:1"); + _app->setClusterState(downState); + + _top->sendDown(createStateCmd("distributor:3 .1.s:d storage:1")); + CPPUNIT_ASSERT(hasOnlySetSystemStateCmdQueued(*_bottom)); +} + +void +ChangedBucketOwnershipHandlerTest::testNoAvailableDistributorsInCurrentAndNewState() +{ + applyDistribution(Redundancy(1), NodeCount(3)); + lib::ClusterState insertedState("distributor:3 storage:1"); + insertBuckets(2, 0, insertedState); + insertBuckets(2, 1, insertedState); + insertBuckets(2, 2, insertedState); + lib::ClusterState stateBefore("distributor:3 .0.s:s .1.s:s .2.s:d storage:1"); + applyClusterState(stateBefore); + lib::ClusterState downState("distributor:3 .0.s:d .1.s:d .2.s:d storage:1"); + + _top->sendDown(createStateCmd(downState)); + CPPUNIT_ASSERT(hasOnlySetSystemStateCmdQueued(*_bottom)); +} + +void +ChangedBucketOwnershipHandlerTest::testDownEdgeToNoAvailableDistributors() +{ + lib::ClusterState insertedState("distributor:3 storage:1"); + applyDistribution(Redundancy(1), NodeCount(3)); + applyClusterState(insertedState); + auto node0Buckets(insertBuckets(2, 0, insertedState)); + auto node1Buckets(insertBuckets(2, 1, insertedState)); + auto node2Buckets(insertBuckets(2, 2, insertedState)); + lib::ClusterState downState("distributor:3 .0.s:d .1.s:s .2.s:s storage:1"); + + _top->sendDown(createStateCmd(downState)); + // TODO: refactor into own function + CPPUNIT_ASSERT_EQUAL(size_t(2), _bottom->getNumCommands()); + AbortBucketOperationsCommand::SP cmd( + std::dynamic_pointer_cast<AbortBucketOperationsCommand>( + _bottom->getCommand(0))); + CPPUNIT_ASSERT(cmd.get() != 0); + + CPPUNIT_ASSERT(hasAbortedAllOf(cmd, node0Buckets)); + CPPUNIT_ASSERT(hasAbortedAllOf(cmd, node1Buckets)); + CPPUNIT_ASSERT(hasAbortedAllOf(cmd, node2Buckets)); +} + +void +ChangedBucketOwnershipHandlerTest::testOwnershipChangedOnDistributorUpEdge() +{ + lib::ClusterState stateBefore( + "version:10 distributor:4 .1.s:d storage:4 .1.s:d"); + lib::ClusterState stateAfter( + "version:11 distributor:4 .1.t:1369990247 storage:4 .1.s:d"); + applyDistribution(Redundancy(1), NodeCount(4)); + applyClusterState(stateBefore); + // Add buckets that will belong to distributor 1 after it has come back up + auto node1Buckets(insertBuckets(2, 1, stateAfter)); + // Add some buckets that will not be part of the change set + auto node0Buckets(insertBuckets(3, 0, stateAfter)); + auto node2Buckets(insertBuckets(2, 2, stateAfter)); + + _top->sendDown(createStateCmd(stateAfter)); + // TODO: refactor into own function + CPPUNIT_ASSERT_EQUAL(size_t(2), _bottom->getNumCommands()); + AbortBucketOperationsCommand::SP cmd( + std::dynamic_pointer_cast<AbortBucketOperationsCommand>( + _bottom->getCommand(0))); + CPPUNIT_ASSERT(cmd.get() != 0); + + CPPUNIT_ASSERT(hasAbortedAllOf(cmd, node1Buckets)); + CPPUNIT_ASSERT(hasAbortedNoneOf(cmd, node0Buckets)); + CPPUNIT_ASSERT(hasAbortedNoneOf(cmd, node2Buckets)); + + // Handler must swallow abort replies + _bottom->sendUp(api::StorageMessage::SP(cmd->makeReply().release())); + CPPUNIT_ASSERT_EQUAL(size_t(0), _top->getNumReplies()); +} + +void +ChangedBucketOwnershipHandlerTest::sendAndExpectAbortedCreateBucket( + uint16_t fromDistributorIndex) +{ + document::BucketId bucket(16, 6786); + auto msg = std::make_shared<api::CreateBucketCommand>(bucket); + msg->setSourceIndex(fromDistributorIndex); + + _top->sendDown(msg); + std::vector<api::StorageMessage::SP> replies(_top->getRepliesOnce()); + CPPUNIT_ASSERT_EQUAL(size_t(1), replies.size()); + api::StorageReply& reply(dynamic_cast<api::StorageReply&>(*replies[0])); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ABORTED, + reply.getResult().getResult()); +} + +void +ChangedBucketOwnershipHandlerTest::testAbortOpsWhenNoClusterStateSet() +{ + sendAndExpectAbortedCreateBucket(1); +} + +void +ChangedBucketOwnershipHandlerTest::testDistributionConfigChangeUpdatesOwnership() +{ + lib::ClusterState insertedState("distributor:3 storage:1"); + applyClusterState(insertedState); + applyDistribution(Redundancy(1), NodeCount(3)); + + // Apply new distribution config containing only 1 distributor, meaning + // any messages sent from >1 must be aborted. + applyDistribution(Redundancy(1), NodeCount(1)); + sendAndExpectAbortedCreateBucket(2); +} + +/** + * Generate and dispatch a message of the given type with the provided + * aruments as if that message was sent from distributor 1. Messages will + * be checked as if the state contains 4 distributors in Up state. This + * means that it suffices to send in a message with a bucket that is not + * owned by distributor 1 in this state to trigger an abort. + */ +template <typename MsgType, typename... MsgParams> +bool +ChangedBucketOwnershipHandlerTest::changeAbortsMessage(MsgParams&&... params) +{ + auto msg = std::make_shared<MsgType>(std::forward<MsgParams>(params)...); + msg->setSourceIndex(1); + + applyDistribution(Redundancy(1), NodeCount(4)); + applyClusterState(getDefaultTestClusterState()); + + _top->sendDown(msg); + std::vector<api::StorageMessage::SP> replies(_top->getRepliesOnce()); + // Test is single-threaded, no need to do any waiting. + if (replies.empty()) { + return false; + } else { + CPPUNIT_ASSERT_EQUAL(size_t(1), replies.size()); + // Make sure the message was actually aborted and not bounced with + // some other arbitrary failure code. + api::StorageReply& reply(dynamic_cast<api::StorageReply&>(*replies[0])); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ABORTED, + reply.getResult().getResult()); + return true; + } +} + +/** + * Returns a bucket that is not owned by the sending distributor (1). More + * specifically, it returns a bucket that is owned by distributor 2. + */ +document::BucketId +ChangedBucketOwnershipHandlerTest::getBucketToAbort() const +{ + lib::ClusterState state(getDefaultTestClusterState()); + return nextOwnedBucket(2, state, document::BucketId()); +} + +/** + * Returns a bucket that _is_ owned by distributor 1 and should thus be + * allowed through. + */ +document::BucketId +ChangedBucketOwnershipHandlerTest::getBucketToAllow() const +{ + lib::ClusterState state(getDefaultTestClusterState()); + return nextOwnedBucket(1, state, document::BucketId()); +} + +void +ChangedBucketOwnershipHandlerTest::testAbortOutdatedSplit() +{ + CPPUNIT_ASSERT(changeAbortsMessage<api::SplitBucketCommand>( + getBucketToAbort())); + CPPUNIT_ASSERT(!changeAbortsMessage<api::SplitBucketCommand>( + getBucketToAllow())); +} + +void +ChangedBucketOwnershipHandlerTest::testAbortOutdatedJoin() +{ + CPPUNIT_ASSERT(changeAbortsMessage<api::JoinBucketsCommand>( + getBucketToAbort())); + CPPUNIT_ASSERT(!changeAbortsMessage<api::JoinBucketsCommand>( + getBucketToAllow())); +} + +void +ChangedBucketOwnershipHandlerTest::testAbortOutdatedSetBucketState() +{ + CPPUNIT_ASSERT(changeAbortsMessage<api::SetBucketStateCommand>( + getBucketToAbort(), api::SetBucketStateCommand::ACTIVE)); + CPPUNIT_ASSERT(!changeAbortsMessage<api::SetBucketStateCommand>( + getBucketToAllow(), api::SetBucketStateCommand::ACTIVE)); +} + +void +ChangedBucketOwnershipHandlerTest::testAbortOutdatedCreateBucket() +{ + CPPUNIT_ASSERT(changeAbortsMessage<api::CreateBucketCommand>( + getBucketToAbort())); + CPPUNIT_ASSERT(!changeAbortsMessage<api::CreateBucketCommand>( + getBucketToAllow())); +} + +void +ChangedBucketOwnershipHandlerTest::testAbortOutdatedDeleteBucket() +{ + CPPUNIT_ASSERT(changeAbortsMessage<api::DeleteBucketCommand>( + getBucketToAbort())); + CPPUNIT_ASSERT(!changeAbortsMessage<api::DeleteBucketCommand>( + getBucketToAllow())); +} + +void +ChangedBucketOwnershipHandlerTest::testAbortOutdatedMergeBucket() +{ + std::vector<api::MergeBucketCommand::Node> nodes; + CPPUNIT_ASSERT(changeAbortsMessage<api::MergeBucketCommand>( + getBucketToAbort(), nodes, 0)); + CPPUNIT_ASSERT(!changeAbortsMessage<api::MergeBucketCommand>( + getBucketToAllow(), nodes, 0)); +} + +/** + * RemoveLocation is technically an external load class, but since it's also + * used as the backing operation for GC we have to treat it as if it were an + * ideal state operation class. + */ +void +ChangedBucketOwnershipHandlerTest::testAbortOutdatedRemoveLocation() +{ + std::vector<api::MergeBucketCommand::Node> nodes; + CPPUNIT_ASSERT(changeAbortsMessage<api::RemoveLocationCommand>( + "foo", getBucketToAbort())); + CPPUNIT_ASSERT(!changeAbortsMessage<api::RemoveLocationCommand>( + "foo", getBucketToAllow())); +} + +void +ChangedBucketOwnershipHandlerTest::testIdealStateAbortsAreConfigurable() +{ + std::unique_ptr<vespa::config::content::PersistenceConfigBuilder> config( + new vespa::config::content::PersistenceConfigBuilder); + config->abortOutdatedMutatingIdealStateOps = false; + _handler->configure(std::move(config)); + // Should not abort operation, even when ownership has changed. + CPPUNIT_ASSERT(!changeAbortsMessage<api::CreateBucketCommand>( + getBucketToAbort())); +} + +void +ChangedBucketOwnershipHandlerTest::testAbortOutdatedPutOperation() +{ + document::Document::SP doc(_testDocRepo.createRandomDocumentAtLocation(1)); + CPPUNIT_ASSERT(changeAbortsMessage<api::PutCommand>( + getBucketToAbort(), doc, api::Timestamp(1234))); + CPPUNIT_ASSERT(!changeAbortsMessage<api::PutCommand>( + getBucketToAllow(), doc, api::Timestamp(1234))); +} + +void +ChangedBucketOwnershipHandlerTest::testAbortOutdatedMultiOperation() +{ + CPPUNIT_ASSERT(changeAbortsMessage<api::MultiOperationCommand>( + _testDocRepo.getTypeRepoSP(), getBucketToAbort(), 1024)); + CPPUNIT_ASSERT(!changeAbortsMessage<api::MultiOperationCommand>( + _testDocRepo.getTypeRepoSP(), getBucketToAllow(), 1024)); +} + +void +ChangedBucketOwnershipHandlerTest::testAbortOutdatedUpdateCommand() +{ + const document::DocumentType* docType(_testDocRepo.getTypeRepo() + .getDocumentType("testdoctype1")); + document::DocumentId docId("id:foo:testdoctype1::bar"); + document::DocumentUpdate::SP update( + std::make_shared<document::DocumentUpdate>(*docType, docId)); + CPPUNIT_ASSERT(changeAbortsMessage<api::UpdateCommand>( + getBucketToAbort(), update, api::Timestamp(1234))); + CPPUNIT_ASSERT(!changeAbortsMessage<api::UpdateCommand>( + getBucketToAllow(), update, api::Timestamp(1234))); +} + +void +ChangedBucketOwnershipHandlerTest::testAbortOutdatedRemoveCommand() +{ + document::DocumentId docId("id:foo:testdoctype1::bar"); + CPPUNIT_ASSERT(changeAbortsMessage<api::RemoveCommand>( + getBucketToAbort(), docId, api::Timestamp(1234))); + CPPUNIT_ASSERT(!changeAbortsMessage<api::RemoveCommand>( + getBucketToAllow(), docId, api::Timestamp(1234))); +} + +void +ChangedBucketOwnershipHandlerTest::testAbortOutdatedRevertCommand() +{ + std::vector<api::Timestamp> timestamps; + CPPUNIT_ASSERT(changeAbortsMessage<api::RevertCommand>( + getBucketToAbort(), timestamps)); + CPPUNIT_ASSERT(!changeAbortsMessage<api::RevertCommand>( + getBucketToAllow(), timestamps)); +} + +void +ChangedBucketOwnershipHandlerTest::testIdealStateAbortUpdatesMetric() +{ + CPPUNIT_ASSERT(changeAbortsMessage<api::SplitBucketCommand>( + getBucketToAbort())); + CPPUNIT_ASSERT_EQUAL( + uint64_t(1), + _handler->getMetrics().idealStateOpsAborted.getValue()); + CPPUNIT_ASSERT_EQUAL( + uint64_t(0), + _handler->getMetrics().externalLoadOpsAborted.getValue()); +} + +void +ChangedBucketOwnershipHandlerTest::testExternalLoadOpAbortUpdatesMetric() +{ + document::DocumentId docId("id:foo:testdoctype1::bar"); + CPPUNIT_ASSERT(changeAbortsMessage<api::RemoveCommand>( + getBucketToAbort(), docId, api::Timestamp(1234))); + CPPUNIT_ASSERT_EQUAL( + uint64_t(0), + _handler->getMetrics().idealStateOpsAborted.getValue()); + CPPUNIT_ASSERT_EQUAL( + uint64_t(1), + _handler->getMetrics().externalLoadOpsAborted.getValue()); +} + +void +ChangedBucketOwnershipHandlerTest::testExternalLoadOpAbortsAreConfigurable() +{ + std::unique_ptr<vespa::config::content::PersistenceConfigBuilder> config( + new vespa::config::content::PersistenceConfigBuilder); + config->abortOutdatedMutatingExternalLoadOps = false; + _handler->configure(std::move(config)); + // Should not abort operation, even when ownership has changed. + document::DocumentId docId("id:foo:testdoctype1::bar"); + CPPUNIT_ASSERT(!changeAbortsMessage<api::RemoveCommand>( + getBucketToAbort(), docId, api::Timestamp(1234))); +} + +} // storage diff --git a/storage/src/tests/storageserver/communicationmanagertest.cpp b/storage/src/tests/storageserver/communicationmanagertest.cpp new file mode 100644 index 00000000000..fe062a9ee30 --- /dev/null +++ b/storage/src/tests/storageserver/communicationmanagertest.cpp @@ -0,0 +1,235 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storage/storageserver/communicationmanager.h> + +#include <vespa/messagebus/testlib/slobrok.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> +#include <vespa/storageframework/defaultimplementation/memory/nomemorymanager.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/dummystoragelink.h> +#include <tests/common/testhelper.h> +#include <vespa/vdstestlib/cppunit/macros.h> + +namespace storage { + +struct CommunicationManagerTest : public CppUnit::TestFixture { + void testSimple(); + void testDistPendingLimitConfigsArePropagatedToMessageBus(); + void testStorPendingLimitConfigsArePropagatedToMessageBus(); + void testCommandsAreDequeuedInPriorityOrder(); + void testRepliesAreDequeuedInFifoOrder(); + + static constexpr uint32_t MESSAGE_WAIT_TIME_SEC = 60; + + void doTestConfigPropagation(bool isContentNode); + + std::shared_ptr<api::StorageCommand> createDummyCommand( + api::StorageMessage::Priority priority) + { + auto cmd = std::make_shared<api::GetCommand>( + document::BucketId(0), + document::DocumentId("doc::mydoc"), + "[all]"); + cmd->setAddress(api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 1)); + cmd->setPriority(priority); + return cmd; + } + + CPPUNIT_TEST_SUITE(CommunicationManagerTest); + CPPUNIT_TEST(testSimple); + CPPUNIT_TEST(testDistPendingLimitConfigsArePropagatedToMessageBus); + CPPUNIT_TEST(testStorPendingLimitConfigsArePropagatedToMessageBus); + CPPUNIT_TEST(testCommandsAreDequeuedInPriorityOrder); + CPPUNIT_TEST(testRepliesAreDequeuedInFifoOrder); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(CommunicationManagerTest); + +void CommunicationManagerTest::testSimple() +{ + mbus::Slobrok slobrok; + vdstestlib::DirConfig distConfig(getStandardConfig(false)); + vdstestlib::DirConfig storConfig(getStandardConfig(true)); + distConfig.getConfig("stor-server").set("node_index", "1"); + storConfig.getConfig("stor-server").set("node_index", "1"); + addSlobrokConfig(distConfig, slobrok); + addSlobrokConfig(storConfig, slobrok); + + // Set up a "distributor" and a "storage" node with communication + // managers and a dummy storage link below we can use for testing. + TestServiceLayerApp storNode(storConfig.getConfigId()); + TestDistributorApp distNode(distConfig.getConfigId()); + + CommunicationManager distributor(distNode.getComponentRegister(), + distConfig.getConfigId()); + CommunicationManager storage(storNode.getComponentRegister(), + storConfig.getConfigId()); + DummyStorageLink *distributorLink = new DummyStorageLink(); + DummyStorageLink *storageLink = new DummyStorageLink(); + distributor.push_back(std::unique_ptr<StorageLink>(distributorLink)); + storage.push_back(std::unique_ptr<StorageLink>(storageLink)); + distributor.open(); + storage.open(); + + FastOS_Thread::Sleep(1000); + + // Send a message through from distributor to storage + std::shared_ptr<api::StorageCommand> cmd( + new api::GetCommand( + document::BucketId(0), document::DocumentId("doc::mydoc"), "[all]")); + cmd->setAddress(api::StorageMessageAddress( + "storage", lib::NodeType::STORAGE, 1)); + distributorLink->sendUp(cmd); + storageLink->waitForMessages(1, MESSAGE_WAIT_TIME_SEC); + CPPUNIT_ASSERT(storageLink->getNumCommands() > 0); + std::shared_ptr<api::StorageCommand> cmd2( + std::dynamic_pointer_cast<api::StorageCommand>( + storageLink->getCommand(0))); + CPPUNIT_ASSERT_EQUAL( + vespalib::string("doc::mydoc"), + static_cast<api::GetCommand&>(*cmd2).getDocumentId().toString()); + // Reply to the message + std::shared_ptr<api::StorageReply> reply(cmd2->makeReply().release()); + storageLink->sendUp(reply); + storageLink->sendUp(reply); + distributorLink->waitForMessages(1, MESSAGE_WAIT_TIME_SEC); + CPPUNIT_ASSERT(distributorLink->getNumCommands() > 0); + std::shared_ptr<api::GetReply> reply2( + std::dynamic_pointer_cast<api::GetReply>( + distributorLink->getCommand(0))); + CPPUNIT_ASSERT_EQUAL(false, reply2->wasFound()); +} + +void +CommunicationManagerTest::doTestConfigPropagation(bool isContentNode) +{ + mbus::Slobrok slobrok; + vdstestlib::DirConfig config(getStandardConfig(isContentNode)); + config.getConfig("stor-server").set("node_index", "1"); + auto& cfg = config.getConfig("stor-communicationmanager"); + cfg.set("mbus_content_node_max_pending_count", "12345"); + cfg.set("mbus_content_node_max_pending_size", "555666"); + cfg.set("mbus_distributor_node_max_pending_count", "6789"); + cfg.set("mbus_distributor_node_max_pending_size", "777888"); + addSlobrokConfig(config, slobrok); + + std::unique_ptr<TestStorageApp> node; + if (isContentNode) { + node = std::make_unique<TestServiceLayerApp>(config.getConfigId()); + } else { + node = std::make_unique<TestDistributorApp>(config.getConfigId()); + } + + CommunicationManager commMgr(node->getComponentRegister(), + config.getConfigId()); + DummyStorageLink *storageLink = new DummyStorageLink(); + commMgr.push_back(std::unique_ptr<StorageLink>(storageLink)); + commMgr.open(); + + // Outer type is RPCMessageBus, which wraps regular MessageBus. + auto& mbus = commMgr.getMessageBus().getMessageBus(); + if (isContentNode) { + CPPUNIT_ASSERT_EQUAL(uint32_t(12345), mbus.getMaxPendingCount()); + CPPUNIT_ASSERT_EQUAL(uint32_t(555666), mbus.getMaxPendingSize()); + } else { + CPPUNIT_ASSERT_EQUAL(uint32_t(6789), mbus.getMaxPendingCount()); + CPPUNIT_ASSERT_EQUAL(uint32_t(777888), mbus.getMaxPendingSize()); + } + + // Test live reconfig of limits. + using ConfigBuilder + = vespa::config::content::core::StorCommunicationmanagerConfigBuilder; + auto liveCfg = std::make_unique<ConfigBuilder>(); + liveCfg->mbusContentNodeMaxPendingCount = 777777; + liveCfg->mbusDistributorNodeMaxPendingCount = 999999; + + commMgr.configure(std::move(liveCfg)); + if (isContentNode) { + CPPUNIT_ASSERT_EQUAL(uint32_t(777777), mbus.getMaxPendingCount()); + } else { + CPPUNIT_ASSERT_EQUAL(uint32_t(999999), mbus.getMaxPendingCount()); + } +} + +void +CommunicationManagerTest::testDistPendingLimitConfigsArePropagatedToMessageBus() +{ + doTestConfigPropagation(false); +} + +void +CommunicationManagerTest::testStorPendingLimitConfigsArePropagatedToMessageBus() +{ + doTestConfigPropagation(true); +} + +void +CommunicationManagerTest::testCommandsAreDequeuedInPriorityOrder() +{ + mbus::Slobrok slobrok; + vdstestlib::DirConfig storConfig(getStandardConfig(true)); + storConfig.getConfig("stor-server").set("node_index", "1"); + addSlobrokConfig(storConfig, slobrok); + TestServiceLayerApp storNode(storConfig.getConfigId()); + + CommunicationManager storage(storNode.getComponentRegister(), + storConfig.getConfigId()); + DummyStorageLink *storageLink = new DummyStorageLink(); + storage.push_back(std::unique_ptr<StorageLink>(storageLink)); + + // Message dequeing does not start before we invoke `open` on the storage + // link chain, so we enqueue messages in randomized priority order before + // doing so. After starting the thread, we should then get messages down + // the chain in a deterministic, prioritized order. + // Lower number == higher priority. + std::vector<api::StorageMessage::Priority> pris{200, 0, 255, 128}; + for (auto pri : pris) { + storage.enqueue(createDummyCommand(pri)); + } + storage.open(); + storageLink->waitForMessages(pris.size(), MESSAGE_WAIT_TIME_SEC); + + std::sort(pris.begin(), pris.end()); + for (size_t i = 0; i < pris.size(); ++i) { + // Casting is just to avoid getting mismatched values printed to the + // output verbatim as chars. + CPPUNIT_ASSERT_EQUAL( + uint32_t(pris[i]), + uint32_t(storageLink->getCommand(i)->getPriority())); + } +} + +void +CommunicationManagerTest::testRepliesAreDequeuedInFifoOrder() +{ + mbus::Slobrok slobrok; + vdstestlib::DirConfig storConfig(getStandardConfig(true)); + storConfig.getConfig("stor-server").set("node_index", "1"); + addSlobrokConfig(storConfig, slobrok); + TestServiceLayerApp storNode(storConfig.getConfigId()); + + CommunicationManager storage(storNode.getComponentRegister(), + storConfig.getConfigId()); + DummyStorageLink *storageLink = new DummyStorageLink(); + storage.push_back(std::unique_ptr<StorageLink>(storageLink)); + + std::vector<api::StorageMessage::Priority> pris{200, 0, 255, 128}; + for (auto pri : pris) { + storage.enqueue(createDummyCommand(pri)->makeReply()); + } + storage.open(); + storageLink->waitForMessages(pris.size(), MESSAGE_WAIT_TIME_SEC); + + // Want FIFO order for replies, not priority-sorted order. + for (size_t i = 0; i < pris.size(); ++i) { + CPPUNIT_ASSERT_EQUAL( + uint32_t(pris[i]), + uint32_t(storageLink->getCommand(i)->getPriority())); + } +} + +} // storage diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp new file mode 100644 index 00000000000..69083352c4a --- /dev/null +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -0,0 +1,529 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <vespa/document/config/config-documenttypes.h> +#include <vespa/document/datatype/datatype.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/messagebus/emptyreply.h> +#include <vespa/storage/storageserver/documentapiconverter.h> +#include <vespa/storageapi/message/batch.h> +#include <vespa/storageapi/message/datagram.h> +#include <vespa/storageapi/message/multioperation.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/visitor.h> +#include <vespa/vdslib/container/writabledocumentlist.h> + +using document::DataType; +using document::DocIdString; +using document::Document; +using document::DocumentId; +using document::DocumentTypeRepo; +using document::readDocumenttypesConfig; + +namespace storage { + +struct DocumentApiConverterTest : public CppUnit::TestFixture +{ + std::unique_ptr<DocumentApiConverter> _converter; + const DocumentTypeRepo::SP _repo; + const DataType& _html_type; + + DocumentApiConverterTest() + : _repo(new DocumentTypeRepo(readDocumenttypesConfig( + "config-doctypes.cfg"))), + _html_type(*_repo->getDocumentType("text/html")) + { + } + + void setUp() { + _converter.reset(new DocumentApiConverter("raw:")); + }; + + void testPut(); + void testForwardedPut(); + void testRemove(); + void testGet(); + void testCreateVisitor(); + void testCreateVisitorHighTimeout(); + void testCreateVisitorReplyNotReady(); + void testCreateVisitorReplyLastBucket(); + void testDestroyVisitor(); + void testVisitorInfo(); + void testDocBlock(); + void testDocBlockWithKeepTimeStamps(); + void testMultiOperation(); + void testBatchDocumentUpdate(); + + CPPUNIT_TEST_SUITE(DocumentApiConverterTest); + CPPUNIT_TEST(testPut); + CPPUNIT_TEST(testForwardedPut); + CPPUNIT_TEST(testRemove); + CPPUNIT_TEST(testGet); + CPPUNIT_TEST(testCreateVisitor); + CPPUNIT_TEST(testCreateVisitorHighTimeout); + CPPUNIT_TEST(testCreateVisitorReplyNotReady); + CPPUNIT_TEST(testCreateVisitorReplyLastBucket); + CPPUNIT_TEST(testDestroyVisitor); + CPPUNIT_TEST(testVisitorInfo); + CPPUNIT_TEST(testDocBlock); + CPPUNIT_TEST(testDocBlockWithKeepTimeStamps); + CPPUNIT_TEST(testMultiOperation); + CPPUNIT_TEST(testBatchDocumentUpdate); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(DocumentApiConverterTest); + +void DocumentApiConverterTest::testPut() +{ + Document::SP + doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); + + documentapi::PutDocumentMessage putmsg(doc); + putmsg.setTimestamp(1234); + + std::unique_ptr<storage::api::StorageCommand> cmd = + _converter->toStorageAPI(putmsg, _repo); + + api::PutCommand* pc = dynamic_cast<api::PutCommand*>(cmd.get()); + + CPPUNIT_ASSERT(pc); + CPPUNIT_ASSERT(pc->getDocument().get() == doc.get()); + + std::unique_ptr<mbus::Reply> reply = putmsg.createReply(); + CPPUNIT_ASSERT(reply.get()); + + std::unique_ptr<storage::api::StorageReply> rep = _converter->toStorageAPI( + static_cast<documentapi::DocumentReply&>(*reply), *cmd); + api::PutReply* pr = dynamic_cast<api::PutReply*>(rep.get()); + CPPUNIT_ASSERT(pr); + + std::unique_ptr<mbus::Message> mbusmsg = + _converter->toDocumentAPI(*pc, _repo); + + documentapi::PutDocumentMessage* mbusput = dynamic_cast<documentapi::PutDocumentMessage*>(mbusmsg.get()); + CPPUNIT_ASSERT(mbusput); + CPPUNIT_ASSERT(mbusput->getDocument().get() == doc.get()); + CPPUNIT_ASSERT(mbusput->getTimestamp() == 1234); +}; + +void DocumentApiConverterTest::testForwardedPut() +{ + Document::SP + doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); + + documentapi::PutDocumentMessage* putmsg = new documentapi::PutDocumentMessage(doc); + std::unique_ptr<mbus::Reply> reply(((documentapi::DocumentMessage*)putmsg)->createReply()); + reply->setMessage(std::unique_ptr<mbus::Message>(putmsg)); + + std::unique_ptr<storage::api::StorageCommand> cmd = + _converter->toStorageAPI(*putmsg, _repo); + ((storage::api::PutCommand*)cmd.get())->setTimestamp(1234); + + std::unique_ptr<storage::api::StorageReply> rep = cmd->makeReply(); + api::PutReply* pr = dynamic_cast<api::PutReply*>(rep.get()); + CPPUNIT_ASSERT(pr); + + _converter->transferReplyState(*pr, *reply); +} + +void DocumentApiConverterTest::testRemove() +{ + documentapi::RemoveDocumentMessage removemsg(document::DocumentId(document::DocIdString("test", "test"))); + std::unique_ptr<storage::api::StorageCommand> cmd = + _converter->toStorageAPI(removemsg, _repo); + + api::RemoveCommand* rc = dynamic_cast<api::RemoveCommand*>(cmd.get()); + + CPPUNIT_ASSERT(rc); + CPPUNIT_ASSERT_EQUAL(document::DocumentId(document::DocIdString("test", "test")), rc->getDocumentId()); + + std::unique_ptr<mbus::Reply> reply = removemsg.createReply(); + CPPUNIT_ASSERT(reply.get()); + + std::unique_ptr<storage::api::StorageReply> rep = _converter->toStorageAPI( + static_cast<documentapi::DocumentReply&>(*reply), *cmd); + api::RemoveReply* pr = dynamic_cast<api::RemoveReply*>(rep.get()); + CPPUNIT_ASSERT(pr); + + std::unique_ptr<mbus::Message> mbusmsg = + _converter->toDocumentAPI(*rc, _repo); + + documentapi::RemoveDocumentMessage* mbusremove = dynamic_cast<documentapi::RemoveDocumentMessage*>(mbusmsg.get()); + CPPUNIT_ASSERT(mbusremove); + CPPUNIT_ASSERT_EQUAL(document::DocumentId(document::DocIdString("test", "test")), mbusremove->getDocumentId()); +}; + +void DocumentApiConverterTest::testGet() +{ + documentapi::GetDocumentMessage getmsg( + document::DocumentId(document::DocIdString("test", "test")), + "foo bar"); + + std::unique_ptr<storage::api::StorageCommand> cmd = + _converter->toStorageAPI(getmsg, _repo); + + api::GetCommand* rc = dynamic_cast<api::GetCommand*>(cmd.get()); + + CPPUNIT_ASSERT(rc); + CPPUNIT_ASSERT_EQUAL(document::DocumentId(document::DocIdString("test", "test")), rc->getDocumentId()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("foo bar"), rc->getFieldSet()); +}; + +void DocumentApiConverterTest::testCreateVisitor() +{ + documentapi::CreateVisitorMessage cv( + "mylib", + "myinstance", + "control-dest", + "data-dest"); + + cv.setTimeRemaining(123456); + + std::unique_ptr<storage::api::StorageCommand> cmd = + _converter->toStorageAPI(cv, _repo); + + api::CreateVisitorCommand* pc = dynamic_cast<api::CreateVisitorCommand*>(cmd.get()); + + CPPUNIT_ASSERT(pc); + CPPUNIT_ASSERT_EQUAL(vespalib::string("mylib"), pc->getLibraryName()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("myinstance"), pc->getInstanceId()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("control-dest"), pc->getControlDestination()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("data-dest"), pc->getDataDestination()); + CPPUNIT_ASSERT_EQUAL(123456u, pc->getTimeout()); +} + +void DocumentApiConverterTest::testCreateVisitorHighTimeout() +{ + documentapi::CreateVisitorMessage cv( + "mylib", + "myinstance", + "control-dest", + "data-dest"); + + cv.setTimeRemaining((uint64_t)std::numeric_limits<uint32_t>::max() + 1); // Will be INT_MAX + + std::unique_ptr<storage::api::StorageCommand> cmd = + _converter->toStorageAPI(cv, _repo); + + api::CreateVisitorCommand* pc = dynamic_cast<api::CreateVisitorCommand*>(cmd.get()); + + CPPUNIT_ASSERT(pc); + CPPUNIT_ASSERT_EQUAL(vespalib::string("mylib"), pc->getLibraryName()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("myinstance"), pc->getInstanceId()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("control-dest"), pc->getControlDestination()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("data-dest"), pc->getDataDestination()); + CPPUNIT_ASSERT_EQUAL((uint32_t) std::numeric_limits<int32_t>::max(), + pc->getTimeout()); +} + +void DocumentApiConverterTest::testCreateVisitorReplyNotReady() +{ + documentapi::CreateVisitorMessage cv( + "mylib", + "myinstance", + "control-dest", + "data-dest"); + + std::unique_ptr<storage::api::StorageCommand> cmd = + _converter->toStorageAPI(cv, _repo); + CPPUNIT_ASSERT(cmd.get()); + api::CreateVisitorCommand& cvc = dynamic_cast<api::CreateVisitorCommand&>(*cmd); + + api::CreateVisitorReply cvr(cvc); + cvr.setResult(api::ReturnCode(api::ReturnCode::NOT_READY, "not ready")); + + std::unique_ptr<documentapi::CreateVisitorReply> reply( + dynamic_cast<documentapi::CreateVisitorReply*>( + cv.createReply().release())); + CPPUNIT_ASSERT(reply.get()); + + _converter->transferReplyState(cvr, *reply); + + CPPUNIT_ASSERT_EQUAL((uint32_t)documentapi::DocumentProtocol::ERROR_NODE_NOT_READY, reply->getError(0).getCode()); + + CPPUNIT_ASSERT_EQUAL(document::BucketId(INT_MAX), reply->getLastBucket()); +} + + +void DocumentApiConverterTest::testCreateVisitorReplyLastBucket() +{ + documentapi::CreateVisitorMessage cv( + "mylib", + "myinstance", + "control-dest", + "data-dest"); + + std::unique_ptr<storage::api::StorageCommand> cmd = + _converter->toStorageAPI(cv, _repo); + CPPUNIT_ASSERT(cmd.get()); + api::CreateVisitorCommand& cvc = dynamic_cast<api::CreateVisitorCommand&>(*cmd); + + + api::CreateVisitorReply cvr(cvc); + cvr.setLastBucket(document::BucketId(123)); + + + std::unique_ptr<documentapi::CreateVisitorReply> reply( + dynamic_cast<documentapi::CreateVisitorReply*>( + cv.createReply().release())); + + CPPUNIT_ASSERT(reply.get()); + + _converter->transferReplyState(cvr, *reply); + + CPPUNIT_ASSERT_EQUAL(document::BucketId(123), reply->getLastBucket()); +} + + +void DocumentApiConverterTest::testDestroyVisitor() +{ + documentapi::DestroyVisitorMessage cv("myinstance"); + + std::unique_ptr<storage::api::StorageCommand> cmd = + _converter->toStorageAPI(cv, _repo); + + api::DestroyVisitorCommand* pc = dynamic_cast<api::DestroyVisitorCommand*>(cmd.get()); + + CPPUNIT_ASSERT(pc); + CPPUNIT_ASSERT_EQUAL(vespalib::string("myinstance"), pc->getInstanceId()); +} + +void +DocumentApiConverterTest::testVisitorInfo() +{ + api::VisitorInfoCommand vicmd; + std::vector<api::VisitorInfoCommand::BucketTimestampPair> bucketsCompleted; + bucketsCompleted.push_back(api::VisitorInfoCommand::BucketTimestampPair(document::BucketId(16, 1), 0)); + bucketsCompleted.push_back(api::VisitorInfoCommand::BucketTimestampPair(document::BucketId(16, 2), 0)); + bucketsCompleted.push_back(api::VisitorInfoCommand::BucketTimestampPair(document::BucketId(16, 4), 0)); + + vicmd.setBucketsCompleted(bucketsCompleted); + + std::unique_ptr<mbus::Message> mbusmsg = + _converter->toDocumentAPI(vicmd, _repo); + + documentapi::VisitorInfoMessage* mbusvi = dynamic_cast<documentapi::VisitorInfoMessage*>(mbusmsg.get()); + CPPUNIT_ASSERT(mbusvi); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1), mbusvi->getFinishedBuckets()[0]); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 2), mbusvi->getFinishedBuckets()[1]); + CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 4), mbusvi->getFinishedBuckets()[2]); + + std::unique_ptr<mbus::Reply> reply = mbusvi->createReply(); + CPPUNIT_ASSERT(reply.get()); + + std::unique_ptr<storage::api::StorageReply> rep = _converter->toStorageAPI( + static_cast<documentapi::DocumentReply&>(*reply), vicmd); + api::VisitorInfoReply* pr = dynamic_cast<api::VisitorInfoReply*>(rep.get()); + CPPUNIT_ASSERT(pr); +} + +void +DocumentApiConverterTest::testDocBlock() +{ + Document::SP + doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); + + char buffer[10000]; + vdslib::WritableDocumentList docBlock(_repo, buffer, sizeof(buffer)); + docBlock.addPut(*doc, 100); + + document::BucketIdFactory fac; + document::BucketId bucketId = fac.getBucketId(doc->getId()); + bucketId.setUsedBits(32); + + api::DocBlockCommand dbcmd(bucketId, docBlock, std::shared_ptr<void>()); + + dbcmd.setTimeout(123456); + + std::unique_ptr<mbus::Message> mbusmsg = + _converter->toDocumentAPI(dbcmd, _repo); + + documentapi::MultiOperationMessage* mbusdb = dynamic_cast<documentapi::MultiOperationMessage*>(mbusmsg.get()); + CPPUNIT_ASSERT(mbusdb); + + CPPUNIT_ASSERT_EQUAL((uint64_t)123456, mbusdb->getTimeRemaining()); + + const vdslib::DocumentList& list = mbusdb->getOperations(); + CPPUNIT_ASSERT_EQUAL((uint32_t)1, list.size()); + CPPUNIT_ASSERT_EQUAL(*doc, *dynamic_cast<document::Document*>(list.begin()->getDocument().get())); + + std::unique_ptr<mbus::Reply> reply = mbusdb->createReply(); + CPPUNIT_ASSERT(reply.get()); + + std::unique_ptr<storage::api::StorageReply> rep = + _converter->toStorageAPI(static_cast<documentapi::DocumentReply&>(*reply), dbcmd); + api::DocBlockReply* pr = dynamic_cast<api::DocBlockReply*>(rep.get()); + CPPUNIT_ASSERT(pr); +} + + +void +DocumentApiConverterTest::testDocBlockWithKeepTimeStamps() +{ + char buffer[10000]; + vdslib::WritableDocumentList docBlock(_repo, buffer, sizeof(buffer)); + api::DocBlockCommand dbcmd(document::BucketId(0), docBlock, std::shared_ptr<void>()); + + { + CPPUNIT_ASSERT_EQUAL(dbcmd.keepTimeStamps(), false); + + std::unique_ptr<mbus::Message> mbusmsg = + _converter->toDocumentAPI(dbcmd, _repo); + + documentapi::MultiOperationMessage* mbusdb = dynamic_cast<documentapi::MultiOperationMessage*>(mbusmsg.get()); + CPPUNIT_ASSERT(mbusdb); + + CPPUNIT_ASSERT_EQUAL(mbusdb->keepTimeStamps(), false); + } + + { + dbcmd.keepTimeStamps(true); + CPPUNIT_ASSERT_EQUAL(dbcmd.keepTimeStamps(), true); + + std::unique_ptr<mbus::Message> mbusmsg = + _converter->toDocumentAPI(dbcmd, _repo); + + documentapi::MultiOperationMessage* mbusdb = dynamic_cast<documentapi::MultiOperationMessage*>(mbusmsg.get()); + CPPUNIT_ASSERT(mbusdb); + + CPPUNIT_ASSERT_EQUAL(mbusdb->keepTimeStamps(), true); + } + +} + + +void +DocumentApiConverterTest::testMultiOperation() +{ + //create a document + Document::SP + doc(new Document(_html_type, DocumentId(DocIdString("test", "test")))); + + document::BucketIdFactory fac; + document::BucketId bucketId = fac.getBucketId(doc->getId()); + bucketId.setUsedBits(32); + + { + documentapi::MultiOperationMessage momsg(_repo, bucketId, 10000); + + vdslib::WritableDocumentList operations(_repo, &(momsg.getBuffer()[0]), + momsg.getBuffer().size()); + operations.addPut(*doc, 100); + + momsg.setOperations(operations); + + CPPUNIT_ASSERT(momsg.getBuffer().size() > 0); + + // Convert it to Storage API + std::unique_ptr<api::StorageCommand> stcmd = + _converter->toStorageAPI(momsg, _repo); + + api::MultiOperationCommand* mocmd = dynamic_cast<api::MultiOperationCommand*>(stcmd.get()); + CPPUNIT_ASSERT(mocmd); + CPPUNIT_ASSERT(mocmd->getBuffer().size() > 0); + + // Get operations from Storage API message and check document + const vdslib::DocumentList& list = mocmd->getOperations(); + CPPUNIT_ASSERT_EQUAL((uint32_t)1, list.size()); + CPPUNIT_ASSERT_EQUAL(*doc, *dynamic_cast<document::Document*>(list.begin()->getDocument().get())); + + // Create Storage API Reply + std::unique_ptr<api::MultiOperationReply> moreply = std::unique_ptr<api::MultiOperationReply>(new api::MultiOperationReply(*mocmd)); + CPPUNIT_ASSERT(moreply.get()); + + // convert storage api reply to mbus reply..... + // ... + } + + { + api::MultiOperationCommand mocmd(_repo, bucketId, 10000, false); + mocmd.getOperations().addPut(*doc, 100); + + // Convert it to documentapi + std::unique_ptr<mbus::Message> mbmsg = + _converter->toDocumentAPI(mocmd, _repo); + documentapi::MultiOperationMessage* momsg = dynamic_cast<documentapi::MultiOperationMessage*>(mbmsg.get()); + CPPUNIT_ASSERT(momsg); + + // Get operations from Document API msg and check document + const vdslib::DocumentList& list = momsg->getOperations(); + CPPUNIT_ASSERT_EQUAL((uint32_t)1, list.size()); + CPPUNIT_ASSERT_EQUAL(*doc, *dynamic_cast<document::Document*>(list.begin()->getDocument().get())); + + // Create Document API reply + mbus::Reply::UP moreply = momsg->createReply(); + CPPUNIT_ASSERT(moreply.get()); + + //Convert DocumentAPI reply to storageapi reply + std::unique_ptr<api::StorageReply> streply = + _converter->toStorageAPI(static_cast<documentapi::DocumentReply&>(*moreply), mocmd); + api::MultiOperationReply* mostreply = dynamic_cast<api::MultiOperationReply*>(streply.get()); + CPPUNIT_ASSERT(mostreply); + + } +} + +void +DocumentApiConverterTest::testBatchDocumentUpdate() +{ + std::vector<document::DocumentUpdate::SP > updates; + + { + document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test1")); + document::DocumentUpdate::SP update( + new document::DocumentUpdate(_html_type, docId)); + updates.push_back(update); + } + + { + document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test2")); + document::DocumentUpdate::SP update( + new document::DocumentUpdate(_html_type, docId)); + updates.push_back(update); + } + + { + document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test3")); + document::DocumentUpdate::SP update( + new document::DocumentUpdate(_html_type, docId)); + updates.push_back(update); + } + + std::shared_ptr<documentapi::BatchDocumentUpdateMessage> msg( + new documentapi::BatchDocumentUpdateMessage(1234)); + for (std::size_t i = 0; i < updates.size(); ++i) { + msg->addUpdate(updates[i]); + } + + std::unique_ptr<storage::api::StorageCommand> cmd = + _converter->toStorageAPI(*msg, _repo); + api::BatchDocumentUpdateCommand* batchCmd = dynamic_cast<api::BatchDocumentUpdateCommand*>(cmd.get()); + CPPUNIT_ASSERT(batchCmd); + CPPUNIT_ASSERT_EQUAL(updates.size(), batchCmd->getUpdates().size()); + for (std::size_t i = 0; i < updates.size(); ++i) { + CPPUNIT_ASSERT_EQUAL(*updates[i], *batchCmd->getUpdates()[i]); + } + + api::BatchDocumentUpdateReply batchReply(*batchCmd); + batchReply.getDocumentsNotFound().resize(3); + batchReply.getDocumentsNotFound()[0] = true; + batchReply.getDocumentsNotFound()[2] = true; + + std::unique_ptr<mbus::Reply> mbusReply = msg->createReply(); + documentapi::BatchDocumentUpdateReply* mbusBatchReply( + dynamic_cast<documentapi::BatchDocumentUpdateReply*>(mbusReply.get())); + CPPUNIT_ASSERT(mbusBatchReply != 0); + + _converter->transferReplyState(batchReply, *mbusReply); + + CPPUNIT_ASSERT_EQUAL(std::size_t(3), mbusBatchReply->getDocumentsNotFound().size()); + CPPUNIT_ASSERT(mbusBatchReply->getDocumentsNotFound()[0] == true); + CPPUNIT_ASSERT(mbusBatchReply->getDocumentsNotFound()[1] == false); + CPPUNIT_ASSERT(mbusBatchReply->getDocumentsNotFound()[2] == true); +} + +} diff --git a/storage/src/tests/storageserver/dummystoragelink.cpp b/storage/src/tests/storageserver/dummystoragelink.cpp new file mode 100644 index 00000000000..7194f1fba3d --- /dev/null +++ b/storage/src/tests/storageserver/dummystoragelink.cpp @@ -0,0 +1,182 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/storageframework/defaultimplementation/clock/realclock.h> +#include <tests/common/dummystoragelink.h> +#include <sys/time.h> + +namespace storage { + +DummyStorageLink* DummyStorageLink::_last(0); + +DummyStorageLink::DummyStorageLink() + : StorageLink("Dummy storage link"), + _commands(), + _replies(), + _injected(), + _autoReply(false), + _useDispatch(false), + _ignore(false), + _waitMonitor() +{ + _last = this; +} + +DummyStorageLink::~DummyStorageLink() +{ + // Often a chain with dummy link on top is deleted in unit tests. + // If they haven't been closed already, close them for a cleaner + // shutdown + if (getState() == OPENED) { + close(); + flush(); + } + closeNextLink(); + reset(); +} + +bool DummyStorageLink::onDown(const api::StorageMessage::SP& cmd) +{ + if (_ignore) { + return false; + } + if (_injected.size() > 0) { + vespalib::LockGuard guard(_lock); + sendUp(*_injected.begin()); + _injected.pop_front(); + } else if (_autoReply) { + if (!cmd->getType().isReply()) { + std::shared_ptr<api::StorageReply> reply( + std::dynamic_pointer_cast<api::StorageCommand>(cmd) + ->makeReply().release()); + reply->setResult(api::ReturnCode( + api::ReturnCode::OK, "Automatically generated reply")); + sendUp(reply); + } + } + if (isBottom()) { + vespalib::MonitorGuard lock(_waitMonitor); + { + vespalib::LockGuard guard(_lock); + _commands.push_back(cmd); + } + lock.broadcast(); + return true; + } + return StorageLink::onDown(cmd); +} + +bool DummyStorageLink::onUp(const api::StorageMessage::SP& reply) { + if (isTop()) { + vespalib::MonitorGuard lock(_waitMonitor); + { + vespalib::LockGuard guard(_lock); + _replies.push_back(reply); + } + lock.broadcast(); + return true; + } + return StorageLink::onUp(reply); + +} + +void DummyStorageLink::injectReply(api::StorageReply* reply) +{ + assert(reply); + vespalib::LockGuard guard(_lock); + _injected.push_back(std::shared_ptr<api::StorageReply>(reply)); +} + +void DummyStorageLink::reset() { + vespalib::MonitorGuard lock(_waitMonitor); + vespalib::LockGuard guard(_lock); + _commands.clear(); + _replies.clear(); + _injected.clear(); +} + +void DummyStorageLink::waitForMessages(unsigned int msgCount, int timeout) +{ + framework::defaultimplementation::RealClock clock; + framework::MilliSecTime endTime( + clock.getTimeInMillis() + framework::MilliSecTime(timeout * 1000)); + vespalib::MonitorGuard lock(_waitMonitor); + while (_commands.size() + _replies.size() < msgCount) { + if (timeout != 0 && clock.getTimeInMillis() > endTime) { + std::ostringstream ost; + ost << "Timed out waiting for " << msgCount << " messages to " + << "arrive in dummy storage link. Only " + << (_commands.size() + _replies.size()) << " messages seen " + << "after timout of " << timeout << " seconds was reached."; + throw vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); + } + if (timeout >= 0) { + lock.wait((endTime - clock.getTimeInMillis()).getTime()); + } else { + lock.wait(); + } + } +} + +void DummyStorageLink::waitForMessage(const api::MessageType& type, int timeout) +{ + framework::defaultimplementation::RealClock clock; + framework::MilliSecTime endTime( + clock.getTimeInMillis() + framework::MilliSecTime(timeout * 1000)); + vespalib::MonitorGuard lock(_waitMonitor); + while (true) { + for (uint32_t i=0; i<_commands.size(); ++i) { + if (_commands[i]->getType() == type) return; + } + for (uint32_t i=0; i<_replies.size(); ++i) { + if (_replies[i]->getType() == type) return; + } + if (timeout != 0 && clock.getTimeInMillis() > endTime) { + std::ostringstream ost; + ost << "Timed out waiting for " << type << " message to " + << "arrive in dummy storage link. Only " + << (_commands.size() + _replies.size()) << " messages seen " + << "after timout of " << timeout << " seconds was reached."; + if (_commands.size() == 1) { + ost << " Found command of type " << _commands[0]->getType(); + } + if (_replies.size() == 1) { + ost << " Found command of type " << _replies[0]->getType(); + } + throw vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); + } + if (timeout >= 0) { + lock.wait((endTime - clock.getTimeInMillis()).getTime()); + } else { + lock.wait(); + } + } +} + +api::StorageMessage::SP +DummyStorageLink::getAndRemoveMessage(const api::MessageType& type) +{ + vespalib::MonitorGuard lock(_waitMonitor); + for (std::vector<api::StorageMessage::SP>::iterator it = _commands.begin(); + it != _commands.end(); ++it) + { + if ((*it)->getType() == type) { + api::StorageMessage::SP result(*it); + _commands.erase(it); + return result; + } + } + for (std::vector<api::StorageMessage::SP>::iterator it = _replies.begin(); + it != _replies.end(); ++it) + { + if ((*it)->getType() == type) { + api::StorageMessage::SP result(*it); + _replies.erase(it); + return result; + } + } + std::ostringstream ost; + ost << "No message of type " << type << " found."; + throw vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); +} + +} // storage diff --git a/storage/src/tests/storageserver/dummystoragelink.h b/storage/src/tests/storageserver/dummystoragelink.h new file mode 100644 index 00000000000..cb9df8c5642 --- /dev/null +++ b/storage/src/tests/storageserver/dummystoragelink.h @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/sync.h> +#include <list> +#include <sstream> +#include <vespa/storageapi/messageapi/storagecommand.h> +#include <string> +#include <vector> +#include <vespa/storage/common/storagelink.h> +#include <vespa/storage/common/bucketmessages.h> +#include <vespa/storageapi/message/internal.h> + +class FastOS_ThreadPool; + +namespace storage { + +class DummyStorageLink : public StorageLink { + + mutable vespalib::Lock _lock; // to protect below containers: + std::vector<api::StorageMessage::SP> _commands; + std::vector<api::StorageMessage::SP> _replies; + std::list<api::StorageMessage::SP> _injected; + + bool _autoReply; + bool _useDispatch; + bool _ignore; + static DummyStorageLink* _last; + vespalib::Monitor _waitMonitor; + +public: + DummyStorageLink(); + ~DummyStorageLink(); + + bool onDown(const api::StorageMessage::SP&); + bool onUp(const api::StorageMessage::SP&); + + void addOnTopOfChain(StorageLink& link) { + link.addTestLinkOnTop(this); + } + + void print(std::ostream& ost, bool verbose, const std::string& indent) const + { + (void) verbose; + ost << indent << "DummyStorageLink(" + << "autoreply = " << (_autoReply ? "on" : "off") + << ", dispatch = " << (_useDispatch ? "on" : "off") + << ", " << _commands.size() << " commands" + << ", " << _replies.size() << " replies"; + if (_injected.size() > 0) + ost << ", " << _injected.size() << " injected"; + ost << ")"; + } + + void injectReply(api::StorageReply* reply); + void reset(); + void setAutoreply(bool autoReply) { _autoReply = autoReply; } + void setIgnore(bool ignore) { _ignore = ignore; } + // Timeout is given in seconds + void waitForMessages(unsigned int msgCount = 1, int timeout = -1); + // Wait for a single message of a given type + void waitForMessage(const api::MessageType&, int timeout = -1); + + api::StorageMessage::SP getCommand(size_t i) const { + vespalib::LockGuard guard(_lock); + api::StorageMessage::SP ret = _commands[i]; + return ret; + } + api::StorageMessage::SP getReply(size_t i) const { + vespalib::LockGuard guard(_lock); + api::StorageMessage::SP ret = _replies[i]; + return ret; + } + size_t getNumCommands() const { + vespalib::LockGuard guard(_lock); + return _commands.size(); + } + size_t getNumReplies() const { + vespalib::LockGuard guard(_lock); + return _replies.size(); + } + + const std::vector<api::StorageMessage::SP>& getCommands() const + { return _commands; } + const std::vector<api::StorageMessage::SP>& getReplies() const + { return _replies; } + + std::vector<api::StorageMessage::SP> getCommandsOnce() { + vespalib::MonitorGuard lock(_waitMonitor); + std::vector<api::StorageMessage::SP> retval; + { + vespalib::LockGuard guard(_lock); + retval.swap(_commands); + } + return retval; + } + + std::vector<api::StorageMessage::SP> getRepliesOnce() { + vespalib::MonitorGuard lock(_waitMonitor); + std::vector<api::StorageMessage::SP> retval; + { + vespalib::LockGuard guard(_lock); + retval.swap(_replies); + } + return retval; + } + + api::StorageMessage::SP getAndRemoveMessage(const api::MessageType&); + + static DummyStorageLink* getLast() { return _last; } +}; + +} + diff --git a/storage/src/tests/storageserver/mergethrottlertest.cpp b/storage/src/tests/storageserver/mergethrottlertest.cpp new file mode 100644 index 00000000000..e705db80788 --- /dev/null +++ b/storage/src/tests/storageserver/mergethrottlertest.cpp @@ -0,0 +1,1566 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <memory> +#include <iterator> +#include <vector> +#include <algorithm> +#include <ctime> +#include <vespa/vespalib/util/document_runnable.h> +#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> +#include <tests/common/testhelper.h> +#include <tests/common/storagelinktest.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/storage/storageserver/mergethrottler.h> +#include <vespa/storage/persistence/messages.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/state.h> + +using namespace document; +using namespace storage::api; + +namespace storage { + +namespace { + +struct MergeBuilder +{ + document::BucketId _bucket; + api::Timestamp _maxTimestamp; + std::vector<uint16_t> _nodes; + std::vector<uint16_t> _chain; + uint64_t _clusterStateVersion; + + MergeBuilder(const document::BucketId& bucket) + : _bucket(bucket), + _maxTimestamp(1234), + _chain(), + _clusterStateVersion(1) + { + nodes(0, 1, 2); + } + + MergeBuilder& nodes(uint16_t n0) { + _nodes.push_back(n0); + return *this; + } + MergeBuilder& nodes(uint16_t n0, uint16_t n1) { + _nodes.push_back(n0); + _nodes.push_back(n1); + return *this; + } + MergeBuilder& nodes(uint16_t n0, uint16_t n1, uint16_t n2) { + _nodes.push_back(n0); + _nodes.push_back(n1); + _nodes.push_back(n2); + return *this; + } + MergeBuilder& maxTimestamp(api::Timestamp maxTs) { + _maxTimestamp = maxTs; + return *this; + } + MergeBuilder& clusterStateVersion(uint64_t csv) { + _clusterStateVersion = csv; + return *this; + } + MergeBuilder& chain(uint16_t n0) { + _chain.clear(); + _chain.push_back(n0); + return *this; + } + MergeBuilder& chain(uint16_t n0, uint16_t n1) { + _chain.clear(); + _chain.push_back(n0); + _chain.push_back(n1); + return *this; + } + MergeBuilder& chain(uint16_t n0, uint16_t n1, uint16_t n2) { + _chain.clear(); + _chain.push_back(n0); + _chain.push_back(n1); + _chain.push_back(n2); + return *this; + } + + api::MergeBucketCommand::SP create() const { + std::vector<api::MergeBucketCommand::Node> n; + for (uint32_t i = 0; i < _nodes.size(); ++i) { + n.push_back(_nodes[i]); + } + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(_bucket, n, _maxTimestamp, + _clusterStateVersion, _chain)); + StorageMessageAddress address("storage", lib::NodeType::STORAGE, _nodes[0]); + cmd->setAddress(address); + return cmd; + } +}; + +std::shared_ptr<api::SetSystemStateCommand> +makeSystemStateCmd(const std::string& state) +{ + return std::make_shared<api::SetSystemStateCommand>( + lib::ClusterState(state)); +} + +} // anon ns + +class MergeThrottlerTest : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE(MergeThrottlerTest); + CPPUNIT_TEST(testMergesConfig); + CPPUNIT_TEST(testChain); + CPPUNIT_TEST(testWithSourceOnlyNode); + CPPUNIT_TEST(test42DistributorBehavior); + CPPUNIT_TEST(test42DistributorBehaviorDoesNotTakeOwnership); + CPPUNIT_TEST(testEndOfChainExecutionDoesNotTakeOwnership); + CPPUNIT_TEST(testResendHandling); + CPPUNIT_TEST(testPriorityQueuing); + CPPUNIT_TEST(testCommandInQueueDuplicateOfKnownMerge); + CPPUNIT_TEST(testInvalidReceiverNode); + CPPUNIT_TEST(testForwardQueuedMerge); + CPPUNIT_TEST(testExecuteQueuedMerge); + CPPUNIT_TEST(testFlush); + CPPUNIT_TEST(testUnseenMergeWithNodeInChain); + CPPUNIT_TEST(testMergeWithNewerClusterStateFlushesOutdatedQueued); + CPPUNIT_TEST(testUpdatedClusterStateFlushesOutdatedQueued); + CPPUNIT_TEST(test42MergesDoNotTriggerFlush); + CPPUNIT_TEST(testOutdatedClusterStateMergesAreRejectedOnArrival); + CPPUNIT_TEST(testUnknownMergeWithSelfInChain); + CPPUNIT_TEST(testBusyReturnedOnFullQueue); + CPPUNIT_TEST(testBrokenCycle); + CPPUNIT_TEST(testGetBucketDiffCommandNotInActiveSetIsRejected); + CPPUNIT_TEST(testApplyBucketDiffCommandNotInActiveSetIsRejected); + CPPUNIT_TEST(testNewClusterStateAbortsAllOutdatedActiveMerges); + CPPUNIT_TEST_SUITE_END(); +public: + void setUp(); + void tearDown(); + + void testMergesConfig(); + void testChain(); + void testWithSourceOnlyNode(); + void test42DistributorBehavior(); + void test42DistributorBehaviorDoesNotTakeOwnership(); + void testEndOfChainExecutionDoesNotTakeOwnership(); + void testResendHandling(); + void testPriorityQueuing(); + void testCommandInQueueDuplicateOfKnownMerge(); + void testInvalidReceiverNode(); + void testForwardQueuedMerge(); + void testExecuteQueuedMerge(); + void testFlush(); + void testUnseenMergeWithNodeInChain(); + void testMergeWithNewerClusterStateFlushesOutdatedQueued(); + void testUpdatedClusterStateFlushesOutdatedQueued(); + void test42MergesDoNotTriggerFlush(); + void testOutdatedClusterStateMergesAreRejectedOnArrival(); + void testUnknownMergeWithSelfInChain(); + void testBusyReturnedOnFullQueue(); + void testBrokenCycle(); + void testGetBucketDiffCommandNotInActiveSetIsRejected(); + void testApplyBucketDiffCommandNotInActiveSetIsRejected(); + void testNewClusterStateAbortsAllOutdatedActiveMerges(); +private: + static const int _storageNodeCount = 3; + static const int _messageWaitTime = 100; + + // Using n storage node links and dummy servers + std::vector<std::shared_ptr<DummyStorageLink> > _topLinks; + std::vector<std::shared_ptr<TestServiceLayerApp> > _servers; + std::vector<MergeThrottler*> _throttlers; + std::vector<DummyStorageLink*> _bottomLinks; + + api::MergeBucketCommand::SP sendMerge(const MergeBuilder&); + + void sendAndExpectReply( + const std::shared_ptr<api::StorageMessage>& msg, + const api::MessageType& expectedReplyType, + api::ReturnCode::Result expectedResultCode); +}; + +const int MergeThrottlerTest::_storageNodeCount; +const int MergeThrottlerTest::_messageWaitTime; + +CPPUNIT_TEST_SUITE_REGISTRATION(MergeThrottlerTest); + +void +MergeThrottlerTest::setUp() +{ + vdstestlib::DirConfig config(getStandardConfig(true)); + + for (int i = 0; i < _storageNodeCount; ++i) { + std::unique_ptr<TestServiceLayerApp> server( + new TestServiceLayerApp(DiskCount(1), NodeIndex(i))); + server->setClusterState(lib::ClusterState( + "distributor:100 storage:100 version:1")); + std::unique_ptr<DummyStorageLink> top; + + top.reset(new DummyStorageLink); + MergeThrottler* throttler = new MergeThrottler(config.getConfigId(), server->getComponentRegister()); + // MergeThrottler will be sandwiched in between two dummy links + top->push_back(std::unique_ptr<StorageLink>(throttler)); + DummyStorageLink* bottom = new DummyStorageLink; + throttler->push_back(std::unique_ptr<StorageLink>(bottom)); + + _servers.push_back(std::shared_ptr<TestServiceLayerApp>(server.release())); + _throttlers.push_back(throttler); + _bottomLinks.push_back(bottom); + top->open(); + _topLinks.push_back(std::shared_ptr<DummyStorageLink>(top.release())); + } +} + +void +MergeThrottlerTest::tearDown() +{ + for (std::size_t i = 0; i < _topLinks.size(); ++i) { + if (_topLinks[i]->getState() == StorageLink::OPENED) { + _topLinks[i]->close(); + _topLinks[i]->flush(); + } + _topLinks[i] = std::shared_ptr<DummyStorageLink>(); + } + _topLinks.clear(); + _bottomLinks.clear(); + _throttlers.clear(); + _servers.clear(); +} + +namespace { + +template <typename Iterator> +bool +checkChain(const StorageMessage::SP& msg, + Iterator first, Iterator end) +{ + const MergeBucketCommand& cmd = + dynamic_cast<const MergeBucketCommand&>(*msg); + + if (cmd.getChain().size() != static_cast<std::size_t>(std::distance(first, end))) { + return false; + } + + return std::equal(cmd.getChain().begin(), cmd.getChain().end(), first); +} + +void waitUntilMergeQueueIs(MergeThrottler& throttler, std::size_t sz, int timeout) +{ + std::time_t start = std::time(0); + while (true) { + std::size_t count; + { + vespalib::LockGuard lock(throttler.getStateLock()); + count = throttler.getMergeQueue().size(); + } + if (count == sz) { + break; + } + std::time_t now = std::time(0); + if (now - start > timeout) { + std::ostringstream os; + os << "Timeout while waiting for merge queue with " << sz << " items. Had " + << count << " at timeout."; + throw vespalib::IllegalStateException(os.str(), VESPA_STRLOC); + } + FastOS_Thread::Sleep(1); + } +} + +} + +// Extremely simple test that just checks that (min|max)_merges_per_node +// under the stor-server config gets propagated to all the nodes +void +MergeThrottlerTest::testMergesConfig() +{ + for (int i = 0; i < _storageNodeCount; ++i) { + CPPUNIT_ASSERT_EQUAL(uint32_t(25), _throttlers[i]->getThrottlePolicy().getMaxPendingCount()); + CPPUNIT_ASSERT_EQUAL(std::size_t(20), _throttlers[i]->getMaxQueueSize()); + } +} + +// Test that a distributor sending a merge to the lowest-index storage +// node correctly invokes a merge forwarding chain and subsequent unwind. +void +MergeThrottlerTest::testChain() +{ + uint16_t indices[_storageNodeCount]; + for (int i = 0; i < _storageNodeCount; ++i) { + indices[i] = i; + _servers[i]->setClusterState(lib::ClusterState("distributor:100 storage:100 version:123")); + } + + BucketId bid(14, 0x1337); + + // Use different node permutations to ensure it works no matter which node is + // set as the executor. More specifically, _all_ permutations. + do { + uint16_t lastNodeIdx = _storageNodeCount - 1; + uint16_t executorNode = indices[0]; + + //std::cout << "\n----\n"; + std::vector<MergeBucketCommand::Node> nodes; + for (int i = 0; i < _storageNodeCount; ++i) { + nodes.push_back(MergeBucketCommand::Node(indices[i], (i + executorNode) % 2 == 0)); + //std::cout << indices[i] << " "; + } + //std::cout << "\n"; + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(bid, nodes, UINT_MAX, 123)); + cmd->setPriority(7); + cmd->setTimeout(54321); + StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + cmd->setAddress(address); + const uint16_t distributorIndex = 123; + cmd->setSourceIndex(distributorIndex); // Dummy distributor index that must be forwarded + + StorageMessage::SP fwd = cmd; + StorageMessage::SP fwdToExec; + + // TODO: make generic wrt. _storageNodeCount + + for (int i = 0; i < _storageNodeCount - 1; ++i) { + if (i == executorNode) { + fwdToExec = fwd; + } + CPPUNIT_ASSERT_EQUAL(uint16_t(i), _servers[i]->getIndex()); + // No matter the node order, command is always sent to node 0 -> 1 -> 2 etc + _topLinks[i]->sendDown(fwd); + _topLinks[i]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + + //std::cout << "fwd " << i << " -> " << i+1 << "\n"; + + // Forwarded merge should not be sent down. Should not be necessary + // to lock throttler here, since it should be sleeping like a champion + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[i]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[i]->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[i]->getActiveMerges().size()); + + fwd = _topLinks[i]->getAndRemoveMessage(MessageType::MERGEBUCKET); + CPPUNIT_ASSERT_EQUAL(uint16_t(i + 1), fwd->getAddress()->getIndex()); + CPPUNIT_ASSERT_EQUAL(distributorIndex, dynamic_cast<const StorageCommand&>(*fwd).getSourceIndex()); + { + //uint16_t chain[] = { 0 }; + std::vector<uint16_t> chain; + for (int j = 0; j <= i; ++j) { + chain.push_back(j); + } + CPPUNIT_ASSERT(checkChain(fwd, chain.begin(), chain.end())); + } + // Ensure priority, cluster state version and timeout is correctly forwarded + CPPUNIT_ASSERT_EQUAL(7, static_cast<int>(fwd->getPriority())); + CPPUNIT_ASSERT_EQUAL(uint32_t(123), dynamic_cast<const MergeBucketCommand&>(*fwd).getClusterStateVersion()); + CPPUNIT_ASSERT_EQUAL(uint32_t(54321), dynamic_cast<const StorageCommand&>(*fwd).getTimeout()); + } + + _topLinks[lastNodeIdx]->sendDown(fwd); + + // If node 2 is the first in the node list, it should immediately execute + // the merge. Otherwise, a cycle with the first node should be formed. + if (executorNode != lastNodeIdx) { + //std::cout << "cycle " << lastNodeIdx << " -> " << executorNode << "\n"; + _topLinks[lastNodeIdx]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + // Forwarded merge should not be sent down + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[lastNodeIdx]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[lastNodeIdx]->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[lastNodeIdx]->getActiveMerges().size()); + + fwd = _topLinks[lastNodeIdx]->getAndRemoveMessage(MessageType::MERGEBUCKET); + CPPUNIT_ASSERT_EQUAL(uint16_t(executorNode), fwd->getAddress()->getIndex()); + CPPUNIT_ASSERT_EQUAL(distributorIndex, dynamic_cast<const StorageCommand&>(*fwd).getSourceIndex()); + { + std::vector<uint16_t> chain; + for (int j = 0; j < _storageNodeCount; ++j) { + chain.push_back(j); + } + CPPUNIT_ASSERT(checkChain(fwd, chain.begin(), chain.end())); + } + CPPUNIT_ASSERT_EQUAL(7, static_cast<int>(fwd->getPriority())); + CPPUNIT_ASSERT_EQUAL(uint32_t(123), dynamic_cast<const MergeBucketCommand&>(*fwd).getClusterStateVersion()); + CPPUNIT_ASSERT_EQUAL(uint32_t(54321), dynamic_cast<const StorageCommand&>(*fwd).getTimeout()); + + _topLinks[executorNode]->sendDown(fwd); + } + + _bottomLinks[executorNode]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + + // Forwarded merge has now been sent down to persistence layer + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _bottomLinks[executorNode]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[executorNode]->getNumReplies()); // No reply sent yet + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[executorNode]->getActiveMerges().size()); // no re-registering merge + + if (executorNode != lastNodeIdx) { + // The MergeBucketCommand that is kept in the executor node should + // be the one from the node it initially got it from, NOT the one + // from the last node, since the chain has looped + CPPUNIT_ASSERT(_throttlers[executorNode]->getActiveMerges().find(bid) + != _throttlers[executorNode]->getActiveMerges().end()); + CPPUNIT_ASSERT_EQUAL(static_cast<StorageMessage*>(fwdToExec.get()), + _throttlers[executorNode]->getActiveMerges().find(bid)->second.getMergeCmd().get()); + } + + // Send reply up from persistence layer to simulate a completed + // merge operation. Chain should now unwind properly + fwd = _bottomLinks[executorNode]->getAndRemoveMessage(MessageType::MERGEBUCKET); + CPPUNIT_ASSERT_EQUAL(7, static_cast<int>(fwd->getPriority())); + CPPUNIT_ASSERT_EQUAL(uint32_t(123), dynamic_cast<const MergeBucketCommand&>(*fwd).getClusterStateVersion()); + CPPUNIT_ASSERT_EQUAL(uint32_t(54321), dynamic_cast<const StorageCommand&>(*fwd).getTimeout()); + + std::shared_ptr<MergeBucketReply> reply( + new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*fwd))); + reply->setResult(ReturnCode(ReturnCode::OK, "Great success! :D-|-<")); + _bottomLinks[executorNode]->sendUp(reply); + + _topLinks[executorNode]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + + if (executorNode != lastNodeIdx) { + // Merge should not be removed yet from executor, since it's pending an unwind + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[executorNode]->getActiveMerges().size()); + CPPUNIT_ASSERT_EQUAL(static_cast<StorageMessage*>(fwdToExec.get()), + _throttlers[executorNode]->getActiveMerges().find(bid)->second.getMergeCmd().get()); + } + // MergeBucketReply waiting to be sent back to node 2. NOTE: we don't have any + // transport context stuff set up here to perform the reply mapping, so we + // have to emulate it + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[executorNode]->getNumReplies()); + + StorageMessage::SP unwind = _topLinks[executorNode]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL(uint16_t(executorNode), unwind->getAddress()->getIndex()); + + // eg: 0 -> 2 -> 1 -> 0. Or: 2 -> 1 -> 0 if no cycle + for (int i = (executorNode != lastNodeIdx ? _storageNodeCount - 1 : _storageNodeCount - 2); i >= 0; --i) { + //std::cout << "unwind " << i << "\n"; + + _topLinks[i]->sendDown(unwind); + _topLinks[i]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[i]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[i]->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _throttlers[i]->getActiveMerges().size()); + + unwind = _topLinks[i]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL(uint16_t(i), unwind->getAddress()->getIndex()); + } + + const MergeBucketReply& mbr = dynamic_cast<const MergeBucketReply&>(*unwind); + + CPPUNIT_ASSERT_EQUAL(ReturnCode::OK, mbr.getResult().getResult()); + CPPUNIT_ASSERT_EQUAL(vespalib::string("Great success! :D-|-<"), mbr.getResult().getMessage()); + CPPUNIT_ASSERT_EQUAL(bid, mbr.getBucketId()); + + } while (std::next_permutation(indices, indices + _storageNodeCount)); + + //std::cout << "\n" << *_topLinks[0] << "\n"; +} + +void +MergeThrottlerTest::testWithSourceOnlyNode() +{ + BucketId bid(14, 0x1337); + + StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(2); + nodes.push_back(MergeBucketCommand::Node(1, true)); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(bid, nodes, UINT_MAX, 123)); + + cmd->setAddress(address); + _topLinks[0]->sendDown(cmd); + + _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + StorageMessage::SP fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + CPPUNIT_ASSERT_EQUAL(uint16_t(1), fwd->getAddress()->getIndex()); + + _topLinks[1]->sendDown(fwd); + + _topLinks[1]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + fwd = _topLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); + CPPUNIT_ASSERT_EQUAL(uint16_t(2), fwd->getAddress()->getIndex()); + + _topLinks[2]->sendDown(fwd); + + _topLinks[2]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + fwd = _topLinks[2]->getAndRemoveMessage(MessageType::MERGEBUCKET); + CPPUNIT_ASSERT_EQUAL(uint16_t(0), fwd->getAddress()->getIndex()); + + _topLinks[0]->sendDown(fwd); + _bottomLinks[0]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + _bottomLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + std::shared_ptr<MergeBucketReply> reply( + new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*fwd))); + reply->setResult(ReturnCode(ReturnCode::OK, "Great success! :D-|-<")); + _bottomLinks[0]->sendUp(reply); + + _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL(uint16_t(0), fwd->getAddress()->getIndex()); + + // Assume everything's fine from here on out +} + +// 4.2 distributors don't guarantee they'll send to lowest node +// index, so we must detect such situations and execute the merge +// immediately rather than attempt to chain it. Test that this +// is done correctly. +void +MergeThrottlerTest::test42DistributorBehavior() +{ + BucketId bid(32, 0xfeef00); + + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(bid, nodes, 1234)); + + // Send to node 1, which is not the lowest index + StorageMessageAddress address("storage", lib::NodeType::STORAGE, 1); + + cmd->setAddress(address); + _topLinks[1]->sendDown(cmd); + _bottomLinks[1]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + + // Should now have been sent to persistence layer + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _bottomLinks[1]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[1]->getNumReplies()); // No reply sent yet + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[1]->getActiveMerges().size()); + + // Send reply up from persistence layer to simulate a completed + // merge operation. Merge should be removed from state. + _bottomLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); + std::shared_ptr<MergeBucketReply> reply( + new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*cmd))); + reply->setResult(ReturnCode(ReturnCode::OK, "Tonight we dine on turtle soup!")); + _bottomLinks[1]->sendUp(reply); + _topLinks[1]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[1]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[1]->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _throttlers[1]->getActiveMerges().size()); + + CPPUNIT_ASSERT_EQUAL(uint64_t(1), _throttlers[1]->getMetrics().local.ok.getValue()); +} + +// Test that we don't take ownership of the merge command when we're +// just passing it through to the persistence layer when receiving +// a merge command that presumably comes form a 4.2 distributor +void +MergeThrottlerTest::test42DistributorBehaviorDoesNotTakeOwnership() +{ + BucketId bid(32, 0xfeef00); + + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(bid, nodes, 1234)); + + // Send to node 1, which is not the lowest index + StorageMessageAddress address("storage", lib::NodeType::STORAGE, 1); + + cmd->setAddress(address); + _topLinks[1]->sendDown(cmd); + _bottomLinks[1]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + + // Should now have been sent to persistence layer + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _bottomLinks[1]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[1]->getNumReplies()); // No reply sent yet + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[1]->getActiveMerges().size()); + + _bottomLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); + + // To ensure we don't try to deref any non-owned messages + framework::HttpUrlPath path("?xml"); + std::ostringstream ss; + _throttlers[1]->reportStatus(ss, path); + + // Flush throttler (synchronously). Should NOT generate a reply + // for the merge command, as it is not owned by the throttler + StorageLinkTest::callOnFlush(*_throttlers[1], true); + + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[1]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[1]->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _throttlers[1]->getActiveMerges().size()); + + // Send a belated reply from persistence up just to ensure the + // throttler doesn't throw a fit if it receives an unknown merge + std::shared_ptr<MergeBucketReply> reply( + new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*cmd))); + reply->setResult(ReturnCode(ReturnCode::OK, "Tonight we dine on turtle soup!")); + _bottomLinks[1]->sendUp(reply); + _topLinks[1]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[1]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[1]->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _throttlers[1]->getActiveMerges().size()); +} + +// Test that we don't take ownership of the merge command when we're +// just passing it through to the persistence layer when we're at the +// the end of the chain and also the designated executor +void +MergeThrottlerTest::testEndOfChainExecutionDoesNotTakeOwnership() +{ + BucketId bid(32, 0xfeef00); + + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(2); + nodes.push_back(1); + nodes.push_back(0); + std::vector<uint16_t> chain; + chain.push_back(0); + chain.push_back(1); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(bid, nodes, 1234, 1, chain)); + + // Send to last node, which is not the lowest index + StorageMessageAddress address("storage", lib::NodeType::STORAGE, 3); + + cmd->setAddress(address); + _topLinks[2]->sendDown(cmd); + _bottomLinks[2]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + + // Should now have been sent to persistence layer + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _bottomLinks[2]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[2]->getNumReplies()); // No reply sent yet + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _throttlers[2]->getActiveMerges().size()); + + _bottomLinks[2]->getAndRemoveMessage(MessageType::MERGEBUCKET); + + // To ensure we don't try to deref any non-owned messages + framework::HttpUrlPath path(""); + std::ostringstream ss; + _throttlers[2]->reportStatus(ss, path); + + // Flush throttler (synchronously). Should NOT generate a reply + // for the merge command, as it is not owned by the throttler + StorageLinkTest::callOnFlush(*_throttlers[2], true); + + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[2]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[2]->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _throttlers[2]->getActiveMerges().size()); + + // Send a belated reply from persistence up just to ensure the + // throttler doesn't throw a fit if it receives an unknown merge + std::shared_ptr<MergeBucketReply> reply( + new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*cmd))); + reply->setResult(ReturnCode(ReturnCode::OK, "Tonight we dine on turtle soup!")); + _bottomLinks[2]->sendUp(reply); + _topLinks[2]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[2]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[2]->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _throttlers[2]->getActiveMerges().size()); +} + +// Test that nodes resending a merge command won't lead to duplicate +// state registration/forwarding or erasing the already present state +// information. +void +MergeThrottlerTest::testResendHandling() +{ + BucketId bid(32, 0xbadbed); + + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(bid, nodes, 1234)); + + StorageMessageAddress address("storage", lib::NodeType::STORAGE, 1); + + cmd->setAddress(address); + _topLinks[0]->sendDown(cmd); + _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + + StorageMessage::SP fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + + // Resend from "distributor". Just use same message, as that won't matter here + _topLinks[0]->sendDown(cmd); + _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + + // Reply should be BUSY + StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + + CPPUNIT_ASSERT_EQUAL( + static_cast<MergeBucketReply&>(*reply).getResult().getResult(), + ReturnCode::BUSY); + + _topLinks[1]->sendDown(fwd); + _topLinks[1]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + fwd = _topLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); + + _topLinks[2]->sendDown(fwd); + _topLinks[2]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + _topLinks[2]->sendDown(fwd); + _topLinks[2]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + + // Reply should be BUSY + reply = _topLinks[2]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL( + static_cast<MergeBucketReply&>(*reply).getResult().getResult(), + ReturnCode::BUSY); + + fwd = _topLinks[2]->getAndRemoveMessage(MessageType::MERGEBUCKET); + + _topLinks[0]->sendDown(fwd); + _bottomLinks[0]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + _topLinks[0]->sendDown(fwd); + _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + + reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL( + static_cast<MergeBucketReply&>(*reply).getResult().getResult(), + ReturnCode::BUSY); +} + +void +MergeThrottlerTest::testPriorityQueuing() +{ + // Fill up all active merges + std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + CPPUNIT_ASSERT(maxPending >= 4u); + for (std::size_t i = 0; i < maxPending; ++i) { + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xf00baa00 + i), nodes, 1234)); + cmd->setPriority(100); + _topLinks[0]->sendDown(cmd); + } + + // Wait till we have maxPending replies and 0 queued + _topLinks[0]->waitForMessages(maxPending, 5); + waitUntilMergeQueueIs(*_throttlers[0], 0, _messageWaitTime); + + // Queue up some merges with different priorities + int priorities[4] = { 200, 150, 120, 240 }; + int sortedPris[4] = { 120, 150, 200, 240 }; + for (int i = 0; i < 4; ++i) { + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, i), nodes, 1234)); + cmd->setPriority(priorities[i]); + _topLinks[0]->sendDown(cmd); + } + + waitUntilMergeQueueIs(*_throttlers[0], 4, _messageWaitTime); + + // Remove all but 4 forwarded merges + for (std::size_t i = 0; i < maxPending - 4; ++i) { + _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + } + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[0]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(4), _topLinks[0]->getNumReplies()); + + // Now when we start replying to merges, queued merges should be + // processed in priority order + for (int i = 0; i < 4; ++i) { + StorageMessage::SP replyTo = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + std::shared_ptr<MergeBucketReply> reply( + new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*replyTo))); + reply->setResult(ReturnCode(ReturnCode::OK, "whee")); + _topLinks[0]->sendDown(reply); + } + + _topLinks[0]->waitForMessages(8, _messageWaitTime); // 4 merges, 4 replies + waitUntilMergeQueueIs(*_throttlers[0], 0, _messageWaitTime); + + for (int i = 0; i < 4; ++i) { + StorageMessage::SP cmd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + CPPUNIT_ASSERT_EQUAL(uint8_t(sortedPris[i]), cmd->getPriority()); + } +} + +// Test that we can detect and reject merges that due to resending +// and potential priority queue sneaking etc may end up with duplicates +// in the queue for a merge that is already known. +void +MergeThrottlerTest::testCommandInQueueDuplicateOfKnownMerge() +{ + // Fill up all active merges and 1 queued one + std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + CPPUNIT_ASSERT(maxPending < 100); + for (std::size_t i = 0; i < maxPending + 1; ++i) { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(2 + i); + nodes.push_back(5 + i); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xf00baa00 + i), nodes, 1234)); + cmd->setPriority(100 - i); + _topLinks[0]->sendDown(cmd); + } + + // Wait till we have maxPending replies and 3 queued + _topLinks[0]->waitForMessages(maxPending, _messageWaitTime); + waitUntilMergeQueueIs(*_throttlers[0], 1, _messageWaitTime); + + // Add a merge for the same bucket twice to the queue + { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(12); + nodes.push_back(123); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xf000feee), nodes, 1234)); + _topLinks[0]->sendDown(cmd); + } + { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(124); // Different node set doesn't matter + nodes.push_back(14); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xf000feee), nodes, 1234)); + _topLinks[0]->sendDown(cmd); + } + + waitUntilMergeQueueIs(*_throttlers[0], 3, _messageWaitTime); + + StorageMessage::SP fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + + // Remove and success-reply for 2 merges. This will give enough room + // for the 2 first queued merges to be processed, the last one having a + // duplicate in the queue. + for (int i = 0; i < 2; ++i) { + StorageMessage::SP fwd2 = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + std::shared_ptr<MergeBucketReply> reply( + new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*fwd2))); + reply->setResult(ReturnCode(ReturnCode::OK, "")); + _topLinks[0]->sendDown(reply); + } + + _topLinks[0]->waitForMessages(maxPending + 1, _messageWaitTime); + waitUntilMergeQueueIs(*_throttlers[0], 1, _messageWaitTime); + + // Remove all current merge commands/replies so we can work with a clean slate + _topLinks[0]->getRepliesOnce(); + // Send a success-reply for fwd, allowing the duplicate from the queue + // to have its moment to shine only to then be struck down mercilessly + std::shared_ptr<MergeBucketReply> reply( + new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*fwd))); + reply->setResult(ReturnCode(ReturnCode::OK, "")); + _topLinks[0]->sendDown(reply); + + _topLinks[0]->waitForMessages(2, _messageWaitTime); + waitUntilMergeQueueIs(*_throttlers[0], 0, _messageWaitTime); + + // First reply is the successful merge reply + StorageMessage::SP reply2 = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL( + static_cast<MergeBucketReply&>(*reply2).getResult().getResult(), + ReturnCode::OK); + + // Second reply should be the BUSY-rejected duplicate + StorageMessage::SP reply1 = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL( + static_cast<MergeBucketReply&>(*reply1).getResult().getResult(), + ReturnCode::BUSY); + CPPUNIT_ASSERT(static_cast<MergeBucketReply&>(*reply1).getResult() + .getMessage().find("out of date;") != std::string::npos); +} + +// Test that sending a merge command to a node not in the set of +// to-be-merged nodes is handled gracefully. +// This is not a scenario that should ever actually happen, but for +// the sake of robustness, include it anyway. +void +MergeThrottlerTest::testInvalidReceiverNode() +{ + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(1); + nodes.push_back(5); + nodes.push_back(9); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xf00baaaa), nodes, 1234)); + + // Send to node with index 0 + _topLinks[0]->sendDown(cmd); + _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + + StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL( + static_cast<MergeBucketReply&>(*reply).getResult().getResult(), + ReturnCode::REJECTED); + CPPUNIT_ASSERT(static_cast<MergeBucketReply&>(*reply).getResult() + .getMessage().find("which is not in its forwarding chain") != std::string::npos); +} + +// Test that the throttling policy kicks in after a certain number of +// merges are forwarded and that the rest are queued in a prioritized +// order. +void +MergeThrottlerTest::testForwardQueuedMerge() +{ + // Fill up all active merges and then 3 queued ones + std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + CPPUNIT_ASSERT(maxPending < 100); + for (std::size_t i = 0; i < maxPending + 3; ++i) { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(2 + i); + nodes.push_back(5 + i); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xf00baa00 + i), nodes, 1234)); + cmd->setPriority(100 - i); + _topLinks[0]->sendDown(cmd); + } + + // Wait till we have maxPending replies and 3 queued + _topLinks[0]->waitForMessages(maxPending, _messageWaitTime); + waitUntilMergeQueueIs(*_throttlers[0], 3, _messageWaitTime); + + // Merge queue state should not be touched by worker thread now + StorageMessage::SP nextMerge = _throttlers[0]->getMergeQueue().begin()->_msg; + + StorageMessage::SP fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + + // Remove all the rest of the active merges + while (!_topLinks[0]->getReplies().empty()) { + _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + } + + std::shared_ptr<MergeBucketReply> reply( + new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*fwd))); + reply->setResult(ReturnCode(ReturnCode::OK, "Celebrate good times come on")); + _topLinks[0]->sendDown(reply); + _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); // Success rewind reply + + // Remove reply bound for distributor + StorageMessage::SP distReply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL( + static_cast<MergeBucketReply&>(*distReply).getResult().getResult(), + ReturnCode::OK); + + waitUntilMergeQueueIs(*_throttlers[0], 2, _messageWaitTime); + _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[0]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(1), _topLinks[0]->getNumReplies()); + + // First queued merge should now have been registered and forwarded + fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + + CPPUNIT_ASSERT_EQUAL( + static_cast<const MergeBucketCommand&>(*fwd).getBucketId(), + static_cast<const MergeBucketCommand&>(*nextMerge).getBucketId()); + + CPPUNIT_ASSERT( + static_cast<const MergeBucketCommand&>(*fwd).getNodes() + == static_cast<const MergeBucketCommand&>(*nextMerge).getNodes()); + + // Ensure forwarded merge has a higher priority than the next queued one + CPPUNIT_ASSERT(fwd->getPriority() < _throttlers[0]->getMergeQueue().begin()->_msg->getPriority()); + + CPPUNIT_ASSERT_EQUAL(uint64_t(1), _throttlers[0]->getMetrics().chaining.ok.getValue()); + + /*framework::HttpUrlPath path("?xml"); + _forwarders[0]->reportStatus(std::cerr, path);*/ +} + +void +MergeThrottlerTest::testExecuteQueuedMerge() +{ + MergeThrottler& throttler(*_throttlers[1]); + DummyStorageLink& topLink(*_topLinks[1]); + DummyStorageLink& bottomLink(*_bottomLinks[1]); + + // Fill up all active merges and then 3 queued ones + std::size_t maxPending = throttler.getThrottlePolicy().getMaxPendingCount(); + CPPUNIT_ASSERT(maxPending < 100); + for (std::size_t i = 0; i < maxPending + 3; ++i) { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(1); + nodes.push_back(5 + i); + nodes.push_back(7 + i); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xf00baa00 + i), nodes, 1234, 1)); + cmd->setPriority(250 - i + 5); + topLink.sendDown(cmd); + } + + // Wait till we have maxPending replies and 3 queued + topLink.waitForMessages(maxPending, _messageWaitTime); + waitUntilMergeQueueIs(throttler, 3, _messageWaitTime); + + // Sneak in a higher priority message that is bound to be executed + // on the given node + { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(1); + nodes.push_back(0); + std::vector<uint16_t> chain; + chain.push_back(0); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0x1337), nodes, 1234, 1, chain)); + cmd->setPriority(0); + topLink.sendDown(cmd); + } + + waitUntilMergeQueueIs(throttler, 4, _messageWaitTime); + + // Merge queue state should not be touched by worker thread now + StorageMessage::SP nextMerge(throttler.getMergeQueue().begin()->_msg); + /*StorageMessage::SP nextMerge; + { + vespalib::LockGuard lock(_throttlers[0]->getStateLock()); + // Dirty: have to check internal state + nextMerge = _throttlers[0]->getMergeQueue().begin()->_msg; + }*/ + + CPPUNIT_ASSERT_EQUAL( + BucketId(32, 0x1337), + dynamic_cast<const MergeBucketCommand&>(*nextMerge).getBucketId()); + + StorageMessage::SP fwd(topLink.getAndRemoveMessage(MessageType::MERGEBUCKET)); + + // Remove all the rest of the active merges + while (!topLink.getReplies().empty()) { + topLink.getAndRemoveMessage(MessageType::MERGEBUCKET); + } + + // Free up a merge slot + std::shared_ptr<MergeBucketReply> reply( + new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*fwd))); + reply->setResult(ReturnCode(ReturnCode::OK, "Celebrate good times come on")); + topLink.sendDown(reply); + + topLink.waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + // Remove chain reply + StorageMessage::SP distReply(topLink.getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY)); + CPPUNIT_ASSERT_EQUAL( + static_cast<MergeBucketReply&>(*distReply).getResult().getResult(), + ReturnCode::OK); + + waitUntilMergeQueueIs(throttler, 3, _messageWaitTime); + bottomLink.waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + + CPPUNIT_ASSERT_EQUAL(std::size_t(0), topLink.getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), topLink.getNumReplies()); + CPPUNIT_ASSERT_EQUAL(std::size_t(1), bottomLink.getNumCommands()); + + // First queued merge should now have been registered and sent down + StorageMessage::SP cmd(bottomLink.getAndRemoveMessage(MessageType::MERGEBUCKET)); + + CPPUNIT_ASSERT_EQUAL( + static_cast<const MergeBucketCommand&>(*cmd).getBucketId(), + static_cast<const MergeBucketCommand&>(*nextMerge).getBucketId()); + + CPPUNIT_ASSERT( + static_cast<const MergeBucketCommand&>(*cmd).getNodes() + == static_cast<const MergeBucketCommand&>(*nextMerge).getNodes()); +} + +void +MergeThrottlerTest::testFlush() +{ + // Fill up all active merges and then 3 queued ones + std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + CPPUNIT_ASSERT(maxPending < 100); + for (std::size_t i = 0; i < maxPending + 3; ++i) { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xf00baa00 + i), nodes, 1234, 1)); + _topLinks[0]->sendDown(cmd); + } + + // Wait till we have maxPending replies and 3 queued + _topLinks[0]->waitForMessages(maxPending, _messageWaitTime); + waitUntilMergeQueueIs(*_throttlers[0], 3, _messageWaitTime); + + // Remove all forwarded commands + uint32_t removed = _topLinks[0]->getRepliesOnce().size(); + CPPUNIT_ASSERT(removed >= 5); + + // Flush the storage link, triggering an abort of all commands + // no matter what their current state is. + _topLinks[0]->close(); + _topLinks[0]->flush(); + _topLinks[0]->waitForMessages(maxPending + 3 - removed, _messageWaitTime); + + while (!_topLinks[0]->getReplies().empty()) { + StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL( + ReturnCode::ABORTED, + static_cast<const MergeBucketReply&>(*reply).getResult().getResult()); + } + // NOTE: merges that have been immediately executed (i.e. not cycled) + // on the node should _not_ be replied to, since they're not owned + // by the throttler at that point in time +} + +// If a node goes down and another node has a merge chained through it in +// its queue, the original node can receive a final chain hop forwarding +// it knows nothing about when it comes back up. If this is not handled +// properly, it will attempt to forward this node again with a bogus +// index. This should be implicitly handled by checking for a full node +void +MergeThrottlerTest::testUnseenMergeWithNodeInChain() +{ + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(5); + nodes.push_back(9); + std::vector<uint16_t> chain; + chain.push_back(0); + chain.push_back(5); + chain.push_back(9); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xdeadbeef), nodes, 1234, 1, chain)); + + StorageMessageAddress address("storage", lib::NodeType::STORAGE, 9); + + cmd->setAddress(address); + _topLinks[0]->sendDown(cmd); + + // First, test that we get rejected when processing merge immediately + // Should get a rejection in return + _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL( + ReturnCode::REJECTED, + dynamic_cast<const MergeBucketReply&>(*reply).getResult().getResult()); + + // Second, test that we get rejected before queueing up. This is to + // avoid a hypothetical deadlock scenario. + // Fill up all active merges + { + + std::size_t maxPending( + _throttlers[0]->getThrottlePolicy().getMaxPendingCount()); + for (std::size_t i = 0; i < maxPending; ++i) { + std::shared_ptr<MergeBucketCommand> fillCmd( + new MergeBucketCommand(BucketId(32, 0xf00baa00 + i), nodes, 1234)); + _topLinks[0]->sendDown(fillCmd); + } + } + + _topLinks[0]->sendDown(cmd); + + _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL( + ReturnCode::REJECTED, + dynamic_cast<const MergeBucketReply&>(*reply).getResult().getResult()); +} + +void +MergeThrottlerTest::testMergeWithNewerClusterStateFlushesOutdatedQueued() +{ + // Fill up all active merges and then 3 queued ones with the same + // system state + std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + CPPUNIT_ASSERT(maxPending < 100); + std::vector<api::StorageMessage::Id> ids; + for (std::size_t i = 0; i < maxPending + 3; ++i) { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xf00baa00 + i), nodes, 1234, 1)); + ids.push_back(cmd->getMsgId()); + _topLinks[0]->sendDown(cmd); + } + + // Wait till we have maxPending replies and 3 queued + _topLinks[0]->waitForMessages(maxPending, _messageWaitTime); + waitUntilMergeQueueIs(*_throttlers[0], 3, _messageWaitTime); + + // Send down merge with newer system state + { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0x12345678), nodes, 1234, 2)); + ids.push_back(cmd->getMsgId()); + _topLinks[0]->sendDown(cmd); + } + + // Queue should now be flushed with all messages being returned with + // WRONG_DISTRIBUTION + _topLinks[0]->waitForMessages(maxPending + 3, _messageWaitTime); + waitUntilMergeQueueIs(*_throttlers[0], 1, _messageWaitTime); + + for (int i = 0; i < 3; ++i) { + StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL( + static_cast<MergeBucketReply&>(*reply).getResult().getResult(), + ReturnCode::WRONG_DISTRIBUTION); + CPPUNIT_ASSERT_EQUAL(1u, static_cast<MergeBucketReply&>(*reply).getClusterStateVersion()); + CPPUNIT_ASSERT_EQUAL(ids[maxPending + i], reply->getMsgId()); + } + + CPPUNIT_ASSERT_EQUAL(uint64_t(3), _throttlers[0]->getMetrics().chaining.failures.wrongdistribution.getValue()); +} + +void +MergeThrottlerTest::testUpdatedClusterStateFlushesOutdatedQueued() +{ + // State is version 1. Send down several merges with state version 2. + std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + CPPUNIT_ASSERT(maxPending < 100); + std::vector<api::StorageMessage::Id> ids; + for (std::size_t i = 0; i < maxPending + 3; ++i) { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xf00baa00 + i), nodes, 1234, 2)); + ids.push_back(cmd->getMsgId()); + _topLinks[0]->sendDown(cmd); + } + + // Wait till we have maxPending replies and 4 queued + _topLinks[0]->waitForMessages(maxPending, _messageWaitTime); + waitUntilMergeQueueIs(*_throttlers[0], 3, _messageWaitTime); + + // Send down new system state (also set it explicitly) + _servers[0]->setClusterState(lib::ClusterState("distributor:100 storage:100 version:3")); + std::shared_ptr<api::SetSystemStateCommand> stateCmd( + new api::SetSystemStateCommand(lib::ClusterState("distributor:100 storage:100 version:3"))); + _topLinks[0]->sendDown(stateCmd); + + // Queue should now be flushed with all being replied to with WRONG_DISTRIBUTION + waitUntilMergeQueueIs(*_throttlers[0], 0, _messageWaitTime); + _topLinks[0]->waitForMessages(maxPending + 3, 5); + + for (int i = 0; i < 3; ++i) { + StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL( + static_cast<MergeBucketReply&>(*reply).getResult().getResult(), + ReturnCode::WRONG_DISTRIBUTION); + CPPUNIT_ASSERT_EQUAL(2u, static_cast<MergeBucketReply&>(*reply).getClusterStateVersion()); + CPPUNIT_ASSERT_EQUAL(ids[maxPending + i], reply->getMsgId()); + } + + CPPUNIT_ASSERT_EQUAL(uint64_t(3), _throttlers[0]->getMetrics().chaining.failures.wrongdistribution.getValue()); +} + +void +MergeThrottlerTest::test42MergesDoNotTriggerFlush() +{ + // Fill up all active merges and then 1 queued one + std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + CPPUNIT_ASSERT(maxPending < 100); + for (std::size_t i = 0; i < maxPending + 1; ++i) { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xf00baa00 + i), nodes, 1234, 1)); + _topLinks[0]->sendDown(cmd); + } + + // Wait till we have maxPending replies and 1 queued + _topLinks[0]->waitForMessages(maxPending, _messageWaitTime); + waitUntilMergeQueueIs(*_throttlers[0], 1, _messageWaitTime); + + StorageMessage::SP fwd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + + // Remove all the rest of the active merges + while (!_topLinks[0]->getReplies().empty()) { + _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + } + + // Send down a merge with a cluster state version of 0, which should + // be ignored and queued as usual + { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xbaaadbed), nodes, 1234, 0)); + _topLinks[0]->sendDown(cmd); + } + + waitUntilMergeQueueIs(*_throttlers[0], 2, _messageWaitTime); + + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[0]->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[0]->getNumReplies()); + + CPPUNIT_ASSERT_EQUAL(uint64_t(0), _throttlers[0]->getMetrics().local.failures.wrongdistribution.getValue()); +} + +// Test that a merge that arrive with a state version that is less than +// that of the node is rejected immediately +void +MergeThrottlerTest::testOutdatedClusterStateMergesAreRejectedOnArrival() +{ + _servers[0]->setClusterState(lib::ClusterState("distributor:100 storage:100 version:10")); + + // Send down a merge with a cluster state version of 9, which should + // be rejected + { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xfeef00), nodes, 1234, 9)); + _topLinks[0]->sendDown(cmd); + } + + _topLinks[0]->waitForMessages(1, _messageWaitTime); + + StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL( + static_cast<MergeBucketReply&>(*reply).getResult().getResult(), + ReturnCode::WRONG_DISTRIBUTION); + + CPPUNIT_ASSERT_EQUAL(uint64_t(1), _throttlers[0]->getMetrics().chaining.failures.wrongdistribution.getValue()); +} + +// Test erroneous case where node receives merge where the merge does +// not exist in the state, but it exists in the chain without the chain +// being full. This is something that shouldn't happen, but must still +// not crash the node +void +MergeThrottlerTest::testUnknownMergeWithSelfInChain() +{ + BucketId bid(32, 0xbadbed); + + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + std::vector<uint16_t> chain; + chain.push_back(0); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(bid, nodes, 1234, 1, chain)); + + StorageMessageAddress address("storage", lib::NodeType::STORAGE, 1); + + cmd->setAddress(address); + _topLinks[0]->sendDown(cmd); + _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + + StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + + CPPUNIT_ASSERT_EQUAL( + ReturnCode::REJECTED, + static_cast<MergeBucketReply&>(*reply).getResult().getResult()); +} + +void +MergeThrottlerTest::testBusyReturnedOnFullQueue() +{ + std::size_t maxPending = _throttlers[0]->getThrottlePolicy().getMaxPendingCount(); + std::size_t maxQueue = _throttlers[0]->getMaxQueueSize(); + CPPUNIT_ASSERT(maxPending < 100); + for (std::size_t i = 0; i < maxPending + maxQueue; ++i) { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xf00000 + i), nodes, 1234, 1)); + _topLinks[0]->sendDown(cmd); + } + + // Wait till we have maxPending replies and maxQueue queued + _topLinks[0]->waitForMessages(maxPending, _messageWaitTime); + waitUntilMergeQueueIs(*_throttlers[0], maxQueue, _messageWaitTime); + + // Clear all forwarded merges + _topLinks[0]->getRepliesOnce(); + // Send down another merge which should be immediately busy-returned + { + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(0); + nodes.push_back(1); + nodes.push_back(2); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xf000baaa), nodes, 1234, 1)); + _topLinks[0]->sendDown(cmd); + } + _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET_REPLY, _messageWaitTime); + StorageMessage::SP reply = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + + CPPUNIT_ASSERT_EQUAL( + BucketId(32, 0xf000baaa), + static_cast<MergeBucketReply&>(*reply).getBucketId()); + + CPPUNIT_ASSERT_EQUAL( + ReturnCode::BUSY, + static_cast<MergeBucketReply&>(*reply).getResult().getResult()); + + CPPUNIT_ASSERT_EQUAL(uint64_t(0), + _throttlers[0]->getMetrics().chaining + .failures.busy.getValue()); + CPPUNIT_ASSERT_EQUAL(uint64_t(1), + _throttlers[0]->getMetrics().local + .failures.busy.getValue()); +} + +void +MergeThrottlerTest::testBrokenCycle() +{ + std::vector<MergeBucketCommand::Node> nodes; + nodes.push_back(1); + nodes.push_back(0); + nodes.push_back(2); + { + std::vector<uint16_t> chain; + chain.push_back(0); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xfeef00), nodes, 1234, 1, chain)); + _topLinks[1]->sendDown(cmd); + } + + _topLinks[1]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + StorageMessage::SP fwd = _topLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); + CPPUNIT_ASSERT_EQUAL(uint16_t(2), fwd->getAddress()->getIndex()); + + // Send cycled merge which will be executed + { + std::vector<uint16_t> chain; + chain.push_back(0); + chain.push_back(1); + chain.push_back(2); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xfeef00), nodes, 1234, 1, chain)); + _topLinks[1]->sendDown(cmd); + } + + _bottomLinks[1]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + StorageMessage::SP cycled = _bottomLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); + + // Now, node 2 goes down, auto sending back a failed merge + std::shared_ptr<MergeBucketReply> nodeDownReply( + new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*fwd))); + nodeDownReply->setResult(ReturnCode(ReturnCode::NOT_CONNECTED, "Node went sightseeing")); + + _topLinks[1]->sendDown(nodeDownReply); + // Merge reply also arrives from persistence + std::shared_ptr<MergeBucketReply> persistenceReply( + new MergeBucketReply(dynamic_cast<const MergeBucketCommand&>(*cycled))); + persistenceReply->setResult(ReturnCode(ReturnCode::ABORTED, "Oh dear")); + _bottomLinks[1]->sendUp(persistenceReply); + + // Should now be two replies from node 1, one to node 2 and one to node 0 + // since we must handle broken chains + _topLinks[1]->waitForMessages(2, _messageWaitTime); + // Unwind reply shares the result of the persistence reply + for (int i = 0; i < 2; ++i) { + StorageMessage::SP reply = _topLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET_REPLY); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(ReturnCode::ABORTED, "Oh dear"), + static_cast<MergeBucketReply&>(*reply).getResult()); + } + + // Make sure it has been removed from the internal state so we can + // send new merges for the bucket + { + std::vector<uint16_t> chain; + chain.push_back(0); + std::shared_ptr<MergeBucketCommand> cmd( + new MergeBucketCommand(BucketId(32, 0xfeef00), nodes, 1234, 1, chain)); + _topLinks[1]->sendDown(cmd); + } + + _topLinks[1]->waitForMessage(MessageType::MERGEBUCKET, 5); + fwd = _topLinks[1]->getAndRemoveMessage(MessageType::MERGEBUCKET); + CPPUNIT_ASSERT_EQUAL(uint16_t(2), fwd->getAddress()->getIndex()); +} + +void +MergeThrottlerTest::sendAndExpectReply( + const std::shared_ptr<api::StorageMessage>& msg, + const api::MessageType& expectedReplyType, + api::ReturnCode::Result expectedResultCode) +{ + _topLinks[0]->sendDown(msg); + _topLinks[0]->waitForMessage(expectedReplyType, _messageWaitTime); + StorageMessage::SP reply(_topLinks[0]->getAndRemoveMessage( + expectedReplyType)); + api::StorageReply& storageReply( + dynamic_cast<api::StorageReply&>(*reply)); + CPPUNIT_ASSERT_EQUAL(expectedResultCode, + storageReply.getResult().getResult()); +} + +void +MergeThrottlerTest::testGetBucketDiffCommandNotInActiveSetIsRejected() +{ + document::BucketId bucket(16, 1234); + std::vector<api::GetBucketDiffCommand::Node> nodes; + std::shared_ptr<api::GetBucketDiffCommand> getDiffCmd( + new api::GetBucketDiffCommand(bucket, nodes, api::Timestamp(1234))); + + sendAndExpectReply(getDiffCmd, + api::MessageType::GETBUCKETDIFF_REPLY, + api::ReturnCode::ABORTED); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[0]->getNumCommands()); +} + +void +MergeThrottlerTest::testApplyBucketDiffCommandNotInActiveSetIsRejected() +{ + document::BucketId bucket(16, 1234); + std::vector<api::GetBucketDiffCommand::Node> nodes; + std::shared_ptr<api::ApplyBucketDiffCommand> applyDiffCmd( + new api::ApplyBucketDiffCommand(bucket, nodes, api::Timestamp(1234))); + + sendAndExpectReply(applyDiffCmd, + api::MessageType::APPLYBUCKETDIFF_REPLY, + api::ReturnCode::ABORTED); + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _bottomLinks[0]->getNumCommands()); +} + +api::MergeBucketCommand::SP +MergeThrottlerTest::sendMerge(const MergeBuilder& builder) +{ + api::MergeBucketCommand::SP cmd(builder.create()); + _topLinks[builder._nodes[0]]->sendDown(cmd); + return cmd; +} + +void +MergeThrottlerTest::testNewClusterStateAbortsAllOutdatedActiveMerges() +{ + document::BucketId bucket(16, 6789); + _throttlers[0]->getThrottlePolicy().setMaxPendingCount(1); + + // Merge will be forwarded (i.e. active). + sendMerge(MergeBuilder(bucket).clusterStateVersion(10)); + _topLinks[0]->waitForMessage(MessageType::MERGEBUCKET, _messageWaitTime); + StorageMessage::SP fwd(_topLinks[0]->getAndRemoveMessage( + MessageType::MERGEBUCKET)); + + _topLinks[0]->sendDown(makeSystemStateCmd( + "version:11 distributor:100 storage:100")); + // Cannot send reply until we're unwinding + CPPUNIT_ASSERT_EQUAL(std::size_t(0), _topLinks[0]->getNumReplies()); + + // Trying to diff the bucket should now fail + { + std::shared_ptr<api::GetBucketDiffCommand> getDiffCmd( + new api::GetBucketDiffCommand(bucket, {}, api::Timestamp(123))); + + sendAndExpectReply(getDiffCmd, + api::MessageType::GETBUCKETDIFF_REPLY, + api::ReturnCode::ABORTED); + } +} + +// TODO test message queue aborting (use rendezvous functionality--make guard) + +} // namespace storage diff --git a/storage/src/tests/storageserver/priorityconvertertest.cpp b/storage/src/tests/storageserver/priorityconvertertest.cpp new file mode 100644 index 00000000000..ecbdcfb6b91 --- /dev/null +++ b/storage/src/tests/storageserver/priorityconvertertest.cpp @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/storage/storageserver/priorityconverter.h> +#include <tests/common/testhelper.h> + +namespace storage { + +struct PriorityConverterTest : public CppUnit::TestFixture +{ + std::unique_ptr<PriorityConverter> _converter; + + void setUp() { + vdstestlib::DirConfig config(getStandardConfig(true)); + _converter.reset(new PriorityConverter(config.getConfigId())); + }; + + void testNormalUsage(); + void testLowestPriorityIsReturnedForUnknownCode(); + + CPPUNIT_TEST_SUITE(PriorityConverterTest); + CPPUNIT_TEST(testNormalUsage); + CPPUNIT_TEST(testLowestPriorityIsReturnedForUnknownCode); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(PriorityConverterTest); + +void PriorityConverterTest::testNormalUsage() +{ + for (int p=0; p<16; ++p) { + CPPUNIT_ASSERT_EQUAL( + (uint8_t)(50+p*10), + _converter->toStoragePriority( + static_cast<documentapi::Priority::Value>(p))); + } + for (int i=0; i<256; ++i) { + uint8_t p = i; + if (p <= 50) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_HIGHEST, + _converter->toDocumentPriority(p)); + } else if (p <= 60) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_VERY_HIGH, + _converter->toDocumentPriority(p)); + } else if (p <= 70) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_HIGH_1, + _converter->toDocumentPriority(p)); + } else if (p <= 80) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_HIGH_2, + _converter->toDocumentPriority(p)); + } else if (p <= 90) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_HIGH_3, + _converter->toDocumentPriority(p)); + } else if (p <= 100) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_NORMAL_1, + _converter->toDocumentPriority(p)); + } else if (p <= 110) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_NORMAL_2, + _converter->toDocumentPriority(p)); + } else if (p <= 120) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_NORMAL_3, + _converter->toDocumentPriority(p)); + } else if (p <= 130) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_NORMAL_4, + _converter->toDocumentPriority(p)); + } else if (p <= 140) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_NORMAL_5, + _converter->toDocumentPriority(p)); + } else if (p <= 150) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_NORMAL_6, + _converter->toDocumentPriority(p)); + } else if (p <= 160) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_LOW_1, + _converter->toDocumentPriority(p)); + } else if (p <= 170) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_LOW_2, + _converter->toDocumentPriority(p)); + } else if (p <= 180) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_LOW_3, + _converter->toDocumentPriority(p)); + } else if (p <= 190) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_VERY_LOW, + _converter->toDocumentPriority(p)); + } else if (p <= 200) { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_LOWEST, + _converter->toDocumentPriority(p)); + } else { + CPPUNIT_ASSERT_EQUAL(documentapi::Priority::PRI_LOWEST, + _converter->toDocumentPriority(p)); + } + } +} + + +void +PriorityConverterTest::testLowestPriorityIsReturnedForUnknownCode() +{ + CPPUNIT_ASSERT_EQUAL(255, + static_cast<int>(_converter->toStoragePriority( + static_cast<documentapi::Priority::Value>(123)))); +} + +} diff --git a/storage/src/tests/storageserver/statemanagertest.cpp b/storage/src/tests/storageserver/statemanagertest.cpp new file mode 100644 index 00000000000..68a35ac37d9 --- /dev/null +++ b/storage/src/tests/storageserver/statemanagertest.cpp @@ -0,0 +1,264 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <boost/pointer_cast.hpp> +#include <cppunit/extensions/HelperMacros.h> +#include <iostream> +#include <vespa/metrics/metricmanager.h> +#include <string> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/storageapi/message/state.h> +#include <vespa/vdslib/state/nodestate.h> +#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> +#include <vespa/storage/storageserver/statemanager.h> +#include <vespa/storage/common/hostreporter/hostinfo.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/testhelper.h> +#include <tests/common/dummystoragelink.h> +#include <vespa/vespalib/data/slime/type.h> + +using storage::lib::NodeState; +using storage::lib::NodeType; +using storage::lib::State; +using storage::lib::ClusterState; + +namespace storage { + +struct StateManagerTest : public CppUnit::TestFixture { + std::unique_ptr<TestServiceLayerApp> _node; + std::unique_ptr<DummyStorageLink> _upper; + std::unique_ptr<metrics::MetricManager> _metricManager; + StateManager* _manager; + DummyStorageLink* _lower; + + StateManagerTest(); + + void setUp(); + void tearDown(); + + void testSystemState(); + void testReportedNodeState(); + void testClusterStateVersion(); + + CPPUNIT_TEST_SUITE(StateManagerTest); + CPPUNIT_TEST(testSystemState); + CPPUNIT_TEST(testReportedNodeState); + CPPUNIT_TEST(testClusterStateVersion); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(StateManagerTest); + +StateManagerTest::StateManagerTest() + : _node(), + _upper(), + _manager(0), + _lower(0) +{ +} + +void +StateManagerTest::setUp() { + try{ + vdstestlib::DirConfig config(getStandardConfig(true)); + _node.reset(new TestServiceLayerApp(DiskCount(1), NodeIndex(2))); + // Clock will increase 1 sec per call. + _node->getClock().setAbsoluteTimeInSeconds(1); + _metricManager.reset(new metrics::MetricManager); + _upper.reset(new DummyStorageLink()); + _manager = new StateManager(_node->getComponentRegister(), + *_metricManager, + std::unique_ptr<HostInfo>(new HostInfo)); + _lower = new DummyStorageLink(); + _upper->push_back(StorageLink::UP(_manager)); + _upper->push_back(StorageLink::UP(_lower)); + _upper->open(); + } catch (std::exception& e) { + std::cerr << "Failed to static initialize objects: " << e.what() + << "\n"; + } +} + +void +StateManagerTest::tearDown() { + CPPUNIT_ASSERT_EQUAL(size_t(0), _lower->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(size_t(0), _lower->getNumCommands()); + CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumReplies()); + CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumCommands()); + _manager = 0; + _lower = 0; + _upper->close(); + _upper->flush(); + _upper.reset(0); + _node.reset(0); + _metricManager.reset(); +} + +#define GET_ONLY_OK_REPLY(varname) \ +{ \ + CPPUNIT_ASSERT_EQUAL(size_t(1), _upper->getNumReplies()); \ + CPPUNIT_ASSERT(_upper->getReply(0)->getType().isReply()); \ + varname = std::dynamic_pointer_cast<api::StorageReply>( \ + _upper->getReply(0)); \ + CPPUNIT_ASSERT(varname != 0); \ + _upper->reset(); \ + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), \ + varname->getResult()); \ +} + +void +StateManagerTest::testSystemState() +{ + std::shared_ptr<api::StorageReply> reply; + // Verify initial state on startup + ClusterState::CSP currentState = _manager->getSystemState(); + CPPUNIT_ASSERT_EQUAL(std::string("cluster:d"), + currentState->toString(false)); + + NodeState::CSP currentNodeState = _manager->getCurrentNodeState(); + CPPUNIT_ASSERT_EQUAL(std::string("s:d"), currentNodeState->toString(false)); + + ClusterState sendState("storage:4 .2.s:m"); + std::shared_ptr<api::SetSystemStateCommand> cmd( + new api::SetSystemStateCommand(sendState)); + _upper->sendDown(cmd); + GET_ONLY_OK_REPLY(reply); + + currentState = _manager->getSystemState(); + CPPUNIT_ASSERT_EQUAL(sendState, *currentState); + + currentNodeState = _manager->getCurrentNodeState(); + CPPUNIT_ASSERT_EQUAL(std::string("s:m"), currentNodeState->toString(false)); +} + +namespace { + struct MyStateListener : public StateListener { + const NodeStateUpdater& updater; + lib::NodeState current; + std::ostringstream ost; + + MyStateListener(const NodeStateUpdater& upd) + : updater(upd), current(*updater.getReportedNodeState()) {} + + void handleNewState() + { + ost << current << " -> "; + current = *updater.getReportedNodeState(); + ost << current << "\n"; + } + }; +} + +void +StateManagerTest::testReportedNodeState() +{ + std::shared_ptr<api::StorageReply> reply; + // Add a state listener to check that we get events. + MyStateListener stateListener(*_manager); + _manager->addStateListener(stateListener); + // Test that initial state is initializing + NodeState::CSP nodeState = _manager->getReportedNodeState(); + CPPUNIT_ASSERT_EQUAL(std::string("s:i b:58 i:0 t:1"), nodeState->toString(false)); + // Test that it works to update the state + { + NodeStateUpdater::Lock::SP lock(_manager->grabStateChangeLock()); + NodeState ns(*_manager->getReportedNodeState()); + ns.setState(State::UP); + _manager->setReportedNodeState(ns); + } + // And that we get the change both through state interface + nodeState = _manager->getReportedNodeState(); + CPPUNIT_ASSERT_EQUAL(std::string("s:u b:58 t:1"), + nodeState->toString(false)); + // And get node state command (no expected state) + std::shared_ptr<api::GetNodeStateCommand> cmd( + new api::GetNodeStateCommand(lib::NodeState::UP())); + _upper->sendDown(cmd); + GET_ONLY_OK_REPLY(reply); + CPPUNIT_ASSERT_EQUAL(api::MessageType::GETNODESTATE_REPLY, + reply->getType()); + nodeState.reset(new NodeState( + dynamic_cast<api::GetNodeStateReply&>(*reply).getNodeState())); + CPPUNIT_ASSERT_EQUAL(std::string("s:u b:58 t:1"), + nodeState->toString(false)); + // We should also get it with wrong expected state + cmd.reset(new api::GetNodeStateCommand(lib::NodeState::UP(new NodeState(NodeType::STORAGE, State::INITIALIZING)))); + _upper->sendDown(cmd); + GET_ONLY_OK_REPLY(reply); + CPPUNIT_ASSERT_EQUAL(api::MessageType::GETNODESTATE_REPLY, + reply->getType()); + nodeState.reset(new NodeState( + dynamic_cast<api::GetNodeStateReply&>(*reply).getNodeState())); + CPPUNIT_ASSERT_EQUAL(std::string("s:u b:58 t:1"), + nodeState->toString(false)); + // With correct wanted state we should not get response right away + cmd.reset(new api::GetNodeStateCommand( + lib::NodeState::UP(new NodeState("s:u b:58 t:1", &NodeType::STORAGE)))); + _upper->sendDown(cmd); + CPPUNIT_ASSERT_EQUAL(size_t(0), _upper->getNumReplies()); + // But when we update state, we get the reply + { + NodeStateUpdater::Lock::SP lock(_manager->grabStateChangeLock()); + NodeState ns(*_manager->getReportedNodeState()); + ns.setState(State::STOPPING); + ns.setDescription("Stopping node"); + _manager->setReportedNodeState(ns); + } + + GET_ONLY_OK_REPLY(reply); + CPPUNIT_ASSERT_EQUAL(api::MessageType::GETNODESTATE_REPLY, + reply->getType()); + nodeState.reset(new NodeState( + dynamic_cast<api::GetNodeStateReply&>(*reply).getNodeState())); + CPPUNIT_ASSERT_EQUAL(std::string("s:s b:58 t:1 m:Stopping\\x20node"), + nodeState->toString(false)); + + // Removing state listener, it stops getting updates + _manager->removeStateListener(stateListener); + // Do another update which listener should not get.. + { + NodeStateUpdater::Lock::SP lock(_manager->grabStateChangeLock()); + NodeState ns(*_manager->getReportedNodeState()); + ns.setState(State::UP); + _manager->setReportedNodeState(ns); + } + std::string expectedEvents = + "s:i b:58 i:0 t:1 -> s:u b:58 t:1\n" + "s:u b:58 t:1 -> s:s b:58 t:1 m:Stopping\\x20node\n"; + CPPUNIT_ASSERT_EQUAL(expectedEvents, stateListener.ost.str()); +} + +void +StateManagerTest::testClusterStateVersion() +{ + ClusterState state(*_manager->getSystemState()); + state.setVersion(123); + _manager->setClusterState(state); + + std::string nodeInfoString(_manager->getNodeInfo()); + vespalib::slime::Memory goldenMemory(nodeInfoString); + vespalib::Slime nodeInfo; + vespalib::slime::JsonFormat::decode(nodeInfoString, nodeInfo); + + vespalib::slime::Symbol lookupSymbol = + nodeInfo.lookup("cluster-state-version"); + if (lookupSymbol.undefined()) { + CPPUNIT_FAIL("No cluster-state-version was found in the node info"); + } + + auto& cursor = nodeInfo.get(); + auto& clusterStateVersionCursor = cursor["cluster-state-version"]; + if (!clusterStateVersionCursor.valid()) { + CPPUNIT_FAIL("No cluster-state-version was found in the node info"); + } + + if (clusterStateVersionCursor.type().getId() != vespalib::slime::LONG::ID) { + CPPUNIT_FAIL("No cluster-state-version was found in the node info"); + } + + int version = clusterStateVersionCursor.asLong(); + CPPUNIT_ASSERT_EQUAL(123, version); +} + +} // storage + diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp new file mode 100644 index 00000000000..ef1592bce80 --- /dev/null +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -0,0 +1,279 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <vespa/log/log.h> +#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> +#include <vespa/storage/persistence/filestorage/filestormanager.h> +#include <vespa/storage/storageserver/applicationgenerationfetcher.h> +#include <vespa/storage/storageserver/statereporter.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/testhelper.h> +#include <tests/common/dummystoragelink.h> + +LOG_SETUP(".test.statereporter"); + +namespace storage { + +class DummyApplicationGenerationFether : public ApplicationGenerationFetcher { +public: + virtual int64_t getGeneration() const { return 1; } + virtual std::string getComponentName() const { return "component"; } +}; + +struct StateReporterTest : public CppUnit::TestFixture { + FastOS_ThreadPool _threadPool; + framework::defaultimplementation::FakeClock* _clock; + std::unique_ptr<TestServiceLayerApp> _node; + std::unique_ptr<DummyStorageLink> _top; + DummyApplicationGenerationFether _generationFetcher; + std::unique_ptr<StateReporter> _stateReporter; + std::unique_ptr<vdstestlib::DirConfig> _config; + std::unique_ptr<metrics::MetricSet> _topSet; + std::unique_ptr<metrics::MetricManager> _metricManager; + std::shared_ptr<FileStorMetrics> _filestorMetrics; + + StateReporterTest(); + + void setUp(); + void tearDown(); + void runLoad(uint32_t count = 1); + + void testReportConfigGeneration(); + void testReportHealth(); + void testReportMetrics(); + + CPPUNIT_TEST_SUITE(StateReporterTest); + CPPUNIT_TEST(testReportConfigGeneration); + CPPUNIT_TEST(testReportHealth); + CPPUNIT_TEST(testReportMetrics); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(StateReporterTest); + +namespace { + struct MetricClock : public metrics::MetricManager::Timer + { + framework::Clock& _clock; + MetricClock(framework::Clock& c) : _clock(c) {} + virtual time_t getTime() const + { return _clock.getTimeInSeconds().getTime(); } + virtual time_t getTimeInMilliSecs() const + { return _clock.getTimeInMillis().getTime(); } + }; +} + +StateReporterTest::StateReporterTest() + : _threadPool(256*1024), + _clock(0), + _top(), + _stateReporter() +{ +} + +void StateReporterTest::setUp() { + assert(system("rm -rf vdsroot") == 0); + _config.reset(new vdstestlib::DirConfig(getStandardConfig(true))); + try { + _node.reset(new TestServiceLayerApp(DiskCount(4), NodeIndex(0), + _config->getConfigId())); + _node->setupDummyPersistence(); + _clock = &_node->getClock(); + _clock->setAbsoluteTimeInSeconds(1000000); + _top.reset(new DummyStorageLink); + } catch (config::InvalidConfigException& e) { + fprintf(stderr, "%s\n", e.what()); + } + _metricManager.reset(new metrics::MetricManager( + std::unique_ptr<metrics::MetricManager::Timer>( + new MetricClock(*_clock)))); + _topSet.reset(new metrics::MetricSet("vds", "", "")); + { + metrics::MetricLockGuard guard(_metricManager->getMetricLock()); + _metricManager->registerMetric(guard, *_topSet); + } + + _stateReporter.reset(new StateReporter( + _node->getComponentRegister(), + *_metricManager, + _generationFetcher, + "status")); + + uint16_t diskCount = _node->getPartitions().size(); + documentapi::LoadTypeSet::SP loadTypes(_node->getLoadTypes()); + + _filestorMetrics.reset(new FileStorMetrics( + _node->getLoadTypes()->getMetricLoadTypes())); + _filestorMetrics->initDiskMetrics( + diskCount, loadTypes->getMetricLoadTypes(), 1); + _topSet->registerMetric(*_filestorMetrics); + + _metricManager->init(_config->getConfigId(), _node->getThreadPool()); +} + +void StateReporterTest::tearDown() { + _metricManager->stop(); + _stateReporter.reset(0); + _topSet.reset(0); + _metricManager.reset(0); + _top.reset(0); + _node.reset(0); + _config.reset(0); + _filestorMetrics.reset(); +} + +#define PARSE_JSON(jsonData) \ +vespalib::Slime slime; \ +{ \ + using namespace vespalib::slime; \ + size_t parsed = JsonFormat::decode(Memory(jsonData), slime); \ + SimpleBuffer buffer; \ + JsonFormat::encode(slime, buffer, false); \ + if (jsonData.size() != parsed) { \ + std::ostringstream error; \ + error << "Failed to parse JSON: '\n" \ + << jsonData << "'\n:" << buffer.get().make_string() << "\n"; \ + CPPUNIT_ASSERT_EQUAL_MSG(error.str(), jsonData.size(), parsed); \ + } \ +} + +#define ASSERT_GENERATION(jsonData, component, generation) \ +{ \ + PARSE_JSON(jsonData); \ + CPPUNIT_ASSERT_EQUAL( \ + generation, \ + slime.get()["config"][component]["generation"].asDouble()); \ +} + +#define ASSERT_NODE_STATUS(jsonData, code, message) \ +{ \ + PARSE_JSON(jsonData); \ + CPPUNIT_ASSERT_EQUAL( \ + vespalib::string(code), \ + slime.get()["status"]["code"].asString().make_string()); \ + CPPUNIT_ASSERT_EQUAL( \ + vespalib::string(message), \ + slime.get()["status"]["message"].asString().make_string()); \ +} + +#define ASSERT_METRIC_GET_PUT(jsonData, expGetCount, expPutCount) \ +{ \ + PARSE_JSON(jsonData); \ + double getCount = -1; \ + double putCount = -1; \ + size_t metricCount = slime.get()["metrics"]["values"].children(); \ + /*std::cerr << "\nmetric count=" << metricCount << "\n";*/ \ + for (size_t j=0; j<metricCount; j++) { \ + const vespalib::string name = slime.get()["metrics"]["values"][j]["name"] \ + .asString().make_string(); \ + if (name.compare("vds.filestor.alldisks.allthreads." \ + "get.sum.count") == 0) \ + { \ + getCount = slime.get()["metrics"]["values"][j]["values"]["count"] \ + .asDouble(); \ + } else if (name.compare("vds.filestor.alldisks.allthreads." \ + "put.sum.count") == 0) \ + { \ + putCount = slime.get()["metrics"]["values"][j]["values"]["count"] \ + .asDouble(); \ + } \ + } \ + CPPUNIT_ASSERT_EQUAL(expGetCount, getCount); \ + CPPUNIT_ASSERT_EQUAL(expPutCount, putCount); \ + CPPUNIT_ASSERT(metricCount > 100); \ +} + + +void StateReporterTest::testReportConfigGeneration() { + std::ostringstream ost; + framework::HttpUrlPath path("/state/v1/config"); + _stateReporter->reportStatus(ost, path); + std::string jsonData = ost.str(); + //std::cerr << "\nConfig: " << jsonData << "\n"; + ASSERT_GENERATION(jsonData, "component", 1.0); +} + +void StateReporterTest::testReportHealth() { + const int stateCount = 7; + const lib::NodeState nodeStates[stateCount] = { + lib::NodeState(lib::NodeType::STORAGE, lib::State::UNKNOWN), + lib::NodeState(lib::NodeType::STORAGE, lib::State::MAINTENANCE), + lib::NodeState(lib::NodeType::STORAGE, lib::State::DOWN), + lib::NodeState(lib::NodeType::STORAGE, lib::State::STOPPING), + lib::NodeState(lib::NodeType::STORAGE, lib::State::INITIALIZING), + lib::NodeState(lib::NodeType::STORAGE, lib::State::RETIRED), + lib::NodeState(lib::NodeType::STORAGE, lib::State::UP) + }; + const char* codes[stateCount] = { + "down", + "down", + "down", + "down", + "down", + "down", + "up" + }; + const char* messages[stateCount] = { + "Node state: Unknown", + "Node state: Maintenance", + "Node state: Down", + "Node state: Stopping", + "Node state: Initializing, init progress 0", + "Node state: Retired", + "" + }; + + framework::HttpUrlPath path("/state/v1/health"); + for (int i=0; i<stateCount; i++) { + _node->getStateUpdater().setCurrentNodeState(nodeStates[i]); + std::ostringstream ost; + _stateReporter->reportStatus(ost, path); + std::string jsonData = ost.str(); + //std::cerr << "\nHealth " << i << ":" << jsonData << "\n"; + ASSERT_NODE_STATUS(jsonData, codes[i], messages[i]); + } +} + +void StateReporterTest::testReportMetrics() { + FileStorDiskMetrics& disk0(*_filestorMetrics->disks[0]); + FileStorThreadMetrics& thread0(*disk0.threads[0]); + + LOG(info, "Adding to get metric"); + + using documentapi::LoadType; + thread0.get[LoadType::DEFAULT].count.inc(1); + + LOG(info, "Waiting for 5 minute snapshot to be taken"); + // Wait until active metrics have been added to 5 min snapshot and reset + for (uint32_t i=0; i<6; ++i) { + _clock->addSecondsToTime(60); + _metricManager->timeChangedNotification(); + while ( + uint64_t(_metricManager->getLastProcessedTime()) + < _clock->getTimeInSeconds().getTime()) + { + FastOS_Thread::Sleep(1); + } + } + LOG(info, "5 minute snapshot should have been taken. Adding put count"); + + thread0.put[LoadType::DEFAULT].count.inc(1); + + const int pathCount = 2; + const char* paths[pathCount] = { + "/state/v1/metrics", + "/state/v1/metrics?consumer=status" + }; + + for (int i=0; i<pathCount; i++) { + framework::HttpUrlPath path(paths[i]); + std::ostringstream ost; + _stateReporter->reportStatus(ost, path); + std::string jsonData = ost.str(); + //std::cerr << "\nMetrics:" << jsonData << "\n"; + ASSERT_METRIC_GET_PUT(jsonData, 1.0, 0.0); + } + } + +} // storage diff --git a/storage/src/tests/storageserver/testvisitormessagesession.cpp b/storage/src/tests/storageserver/testvisitormessagesession.cpp new file mode 100644 index 00000000000..e814f6cf229 --- /dev/null +++ b/storage/src/tests/storageserver/testvisitormessagesession.cpp @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <tests/storageserver/testvisitormessagesession.h> +#include <vespa/storageframework/defaultimplementation/clock/realclock.h> + +namespace storage { + +TestVisitorMessageSession::~TestVisitorMessageSession() +{ +} + +TestVisitorMessageSession::TestVisitorMessageSession(VisitorThread& t, + Visitor& v, + const mbus::Error& autoReplyError, + bool autoReply) + : _autoReplyError(autoReplyError), + _autoReply(autoReply), + thread(t), + visitor(v), + pendingCount(0) +{ +} + +void +TestVisitorMessageSession::reply(mbus::Reply::UP rep) { + { + vespalib::MonitorGuard guard(_waitMonitor); + pendingCount--; + } + thread.handleMessageBusReply(std::move(rep), visitor); +} + +mbus::Result +TestVisitorMessageSession::send( + std::unique_ptr<documentapi::DocumentMessage> message) +{ + vespalib::MonitorGuard guard(_waitMonitor); + if (_autoReply) { + pendingCount++; + mbus::Reply::UP rep = message->createReply(); + rep->setMessage(mbus::Message::UP(message.release())); + if (_autoReplyError.getCode() == mbus::ErrorCode::NONE) { + reply(std::move(rep)); + return mbus::Result(); + } else { + return mbus::Result(_autoReplyError, + std::unique_ptr<mbus::Message>(message.release())); + } + } else { + pendingCount++; + sentMessages.push_back( + vespalib::LinkedPtr<documentapi::DocumentMessage>( + message.release())); + guard.broadcast(); + return mbus::Result(); + } +} + +void +TestVisitorMessageSession::waitForMessages(unsigned int msgCount) { + framework::defaultimplementation::RealClock clock; + framework::MilliSecTime endTime( + clock.getTimeInMillis() + framework::MilliSecTime(60 * 1000)); + + vespalib::MonitorGuard guard(_waitMonitor); + while (sentMessages.size() < msgCount) { + if (clock.getTimeInMillis() > endTime) { + throw vespalib::IllegalStateException( + vespalib::make_string("Timed out waiting for %u messages " + "in test visitor session", msgCount), + VESPA_STRLOC); + } + guard.wait(1000); + } +}; + +} diff --git a/storage/src/tests/storageserver/testvisitormessagesession.h b/storage/src/tests/storageserver/testvisitormessagesession.h new file mode 100644 index 00000000000..3ae6ccafb84 --- /dev/null +++ b/storage/src/tests/storageserver/testvisitormessagesession.h @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <deque> +#include <vespa/storage/visiting/visitormessagesession.h> +#include <vespa/storage/visiting/visitorthread.h> +#include <vespa/documentapi/messagebus/messages/documentmessage.h> +#include <vespa/storage/storageserver/priorityconverter.h> + +namespace storage { + +class TestVisitorMessageSession : public VisitorMessageSession +{ +private: + vespalib::Monitor _waitMonitor; + mbus::Error _autoReplyError; + bool _autoReply; + +public: + typedef std::unique_ptr<TestVisitorMessageSession> UP; + + VisitorThread& thread; + Visitor& visitor; + uint32_t pendingCount; + + ~TestVisitorMessageSession(); + + std::deque<vespalib::LinkedPtr<documentapi::DocumentMessage> > sentMessages; + + TestVisitorMessageSession(VisitorThread& t, + Visitor& v, + const mbus::Error& autoReplyError, + bool autoReply); + + void reply(mbus::Reply::UP rep); + + uint32_t pending() { return pendingCount; } + + mbus::Result send(std::unique_ptr<documentapi::DocumentMessage> message); + + void waitForMessages(unsigned int msgCount); + + vespalib::Monitor& getMonitor() { return _waitMonitor; } +}; + +struct TestVisitorMessageSessionFactory : public VisitorMessageSessionFactory +{ + vespalib::Lock _accessLock; + std::vector<TestVisitorMessageSession*> _visitorSessions; + mbus::Error _autoReplyError; + bool _createAutoReplyVisitorSessions; + PriorityConverter _priConverter; + + TestVisitorMessageSessionFactory(vespalib::stringref configId = "") + : _createAutoReplyVisitorSessions(false), + _priConverter(configId) {} + + VisitorMessageSession::UP createSession(Visitor& v, VisitorThread& vt) { + vespalib::LockGuard lock(_accessLock); + TestVisitorMessageSession::UP session( + new TestVisitorMessageSession( + vt, + v, + _autoReplyError, + _createAutoReplyVisitorSessions)); + _visitorSessions.push_back(session.get()); + return VisitorMessageSession::UP(std::move(session)); + } + + documentapi::Priority::Value toDocumentPriority(uint8_t storagePriority) const + { + return _priConverter.toDocumentPriority(storagePriority); + } + +}; + +} // storage + diff --git a/storage/src/tests/storageutil/.gitignore b/storage/src/tests/storageutil/.gitignore new file mode 100644 index 00000000000..a080232d5f3 --- /dev/null +++ b/storage/src/tests/storageutil/.gitignore @@ -0,0 +1,13 @@ +*.So +*.lo +*.o +.*.swp +.config.log +.depend +.depend.NEW +.deps +.libs +Makefile +statefile* +testrunner +testrunner.core diff --git a/storage/src/tests/storageutil/CMakeLists.txt b/storage/src/tests/storageutil/CMakeLists.txt new file mode 100644 index 00000000000..a48895352e8 --- /dev/null +++ b/storage/src/tests/storageutil/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storage_teststorageutil + SOURCES + functortest.cpp + charttest.cpp + palettetest.cpp + DEPENDS + AFTER + storage_storageconfig +) diff --git a/storage/src/tests/storageutil/charttest.cpp b/storage/src/tests/storageutil/charttest.cpp new file mode 100644 index 00000000000..d9ce3d6f1b4 --- /dev/null +++ b/storage/src/tests/storageutil/charttest.cpp @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storage/storageutil/piechart.h> + +#include <fstream> +#include <vespa/vdstestlib/cppunit/macros.h> + +namespace storage { + +struct PieChartTest : public CppUnit::TestFixture +{ + void setUp() {} + void tearDown() {} + + void testWriteHtmlFile(); + + CPPUNIT_TEST_SUITE(PieChartTest); + CPPUNIT_TEST(testWriteHtmlFile); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(PieChartTest); + +namespace { + void printHtmlFile(const std::string& filename, const PieChart& chart) { + std::ofstream out(filename.c_str()); + out << "<html>\n" + << " <head>\n" + << " "; + PieChart::printHtmlHeadAdditions(out, " "); + out << "\n <title>Pie example</title>\n" + << " </head>\n" + << " <body>\n" + << " "; + chart.printCanvas(out, 500, 400); + out << "\n "; + chart.printScript(out, " "); + out << "\n </body>\n" + << "</html>\n"; + out.close(); + } +} + +void +PieChartTest::testWriteHtmlFile() +{ + { + PieChart chart("mypie"); + chart.add(10, "put"); + chart.add(20, "get"); + chart.add(50, "free"); + + printHtmlFile("piefile.html", chart); + } + { + PieChart chart("mypie", PieChart::SCHEME_CUSTOM); + chart.add(10, "put", PieChart::RED); + chart.add(20, "get", PieChart::GREEN); + chart.add(50, "free", PieChart::BLUE); + + printHtmlFile("piefile-customcols.html", chart); + } +} + +} // storage diff --git a/storage/src/tests/storageutil/functortest.cpp b/storage/src/tests/storageutil/functortest.cpp new file mode 100644 index 00000000000..00b9f5450cb --- /dev/null +++ b/storage/src/tests/storageutil/functortest.cpp @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <list> +#include <string> +#include <algorithm> +#include <vespa/storage/storageutil/functor.h> + +class Functor_Test : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(Functor_Test); + CPPUNIT_TEST(testReplace); + CPPUNIT_TEST(testDeletePointer); + CPPUNIT_TEST_SUITE_END(); + +public: + +protected: + void testReplace(); + void testDeletePointer(); +}; + +using namespace storage; +using namespace std; + +CPPUNIT_TEST_SUITE_REGISTRATION(Functor_Test); + +void Functor_Test::testReplace() +{ + string source("this.is.a.string.with.many.dots."); + for_each(source.begin(), source.end(), Functor::Replace<char>('.', '_')); + CPPUNIT_ASSERT_EQUAL(string("this_is_a_string_with_many_dots_"), source); +} + +namespace { + + static int instanceCounter = 0; + + class TestClass { + public: + TestClass() { instanceCounter++; } + ~TestClass() { instanceCounter--; } + }; +} + +void Functor_Test::testDeletePointer() +{ + list<TestClass*> mylist; + mylist.push_back(new TestClass()); + mylist.push_back(new TestClass()); + mylist.push_back(new TestClass()); + CPPUNIT_ASSERT_EQUAL(3, instanceCounter); + for_each(mylist.begin(), mylist.end(), Functor::DeletePointer()); + CPPUNIT_ASSERT_EQUAL(0, instanceCounter); +} diff --git a/storage/src/tests/storageutil/palettetest.cpp b/storage/src/tests/storageutil/palettetest.cpp new file mode 100644 index 00000000000..ffc2dd091ee --- /dev/null +++ b/storage/src/tests/storageutil/palettetest.cpp @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/storage/storageutil/palette.h> +#include <vespa/vdstestlib/cppunit/macros.h> + +namespace storage { + +struct PaletteTest : public CppUnit::TestFixture { + void setUp() {} + void tearDown() {} + + void testNormalUsage(); + + CPPUNIT_TEST_SUITE(PaletteTest); + CPPUNIT_TEST(testNormalUsage); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(PaletteTest); + +void +PaletteTest::testNormalUsage() +{ + std::ofstream out("palette.html"); + out << "<html><body>\n"; + Palette palette(75); + palette.printHtmlTablePalette(out); + out << "</body></html>\n"; + out.close(); +} + +} // storage diff --git a/storage/src/tests/storageutil/recordflatfiletest.cpp b/storage/src/tests/storageutil/recordflatfiletest.cpp new file mode 100644 index 00000000000..e08dd88dc67 --- /dev/null +++ b/storage/src/tests/storageutil/recordflatfiletest.cpp @@ -0,0 +1,314 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <cppunit/extensions/HelperMacros.h> +#include <iostream> +#include <string> +#include <vespa/storage/storageutil/recordflatfile.h> + +using namespace document; +using namespace storage; +using namespace std; +using namespace document; + +class RecordFlatFile_Test : public CppUnit::TestFixture { + CPPUNIT_TEST_SUITE(RecordFlatFile_Test); + CPPUNIT_TEST(testAdd); + CPPUNIT_TEST(testUpdate); + CPPUNIT_TEST(testRemove); + CPPUNIT_TEST(testExists); + CPPUNIT_TEST(testGetRecord); + CPPUNIT_TEST(testClear); + CPPUNIT_TEST(testSimpleUsage); + CPPUNIT_TEST(testValid); + CPPUNIT_TEST_SUITE_END(); + + string _testFile; + unsigned int _chunkSize; + + void setupTestFile(); + +public: + void setUp(); + + RecordFlatFile_Test(void) + : _testFile(), + _chunkSize(0) + { + } + +protected: + void testAdd(); + void testUpdate(); + void testRemove(); + void testExists(); + void testGetRecord(); + void testClear(); + void testSimpleUsage(); + void testValid(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(RecordFlatFile_Test); + +namespace { + + const bool debug = false; + + class MyRecord { + private: + unsigned int _id; + unsigned int _value; + unsigned int _valid; + + public: + MyRecord(void) + : _id(0u), + _value(0u), + _valid(0u) + { + } + MyRecord(unsigned int id, unsigned int value, bool valid = true) + : _id(id), _value(value), _valid(valid ? 0 : 0xFFFFFFFF) {} + + const unsigned int& getId() const { return _id; } + unsigned int getValue() const { return _value; } + void setValue(unsigned int value) { _value = value; } + bool isValid() const { return (_valid == 0); } + + bool operator==(const MyRecord& record) const { + return (_id == record._id && _value == record._value); + } + }; + + ostream& operator<<(ostream& out, MyRecord record) { + out << "MyRecord(" << record.getId() << ", " << record.getValue() + << ")"; + return out; + } + + class BlockMessage { + private: + string _name; + static unsigned int _indent; + + public: + BlockMessage(const string& name) : _name(name) { + if (debug) { + for (unsigned int i=0; i<_indent; i++) cout << " "; + cout << "Block started: " << _name << "\n" << flush; + } + _indent++; + } + ~BlockMessage() { + _indent--; + if (debug) { + for (unsigned int i=0; i<_indent; i++) cout << " "; + cout << "Block completed: " << _name << "\n" << flush; + } + } + }; + + unsigned int BlockMessage::_indent(0); + +} + +void RecordFlatFile_Test::setUp() { + _testFile = "recordflatfile.testfile"; + _chunkSize = 4; +} + +void RecordFlatFile_Test::setupTestFile() { + BlockMessage message("setupTestFile()"); + RecordFlatFile<MyRecord, unsigned int> flatfile(_testFile, _chunkSize); + flatfile.clear(); + for (unsigned int i=1; i<=8; ++i) { + flatfile.add(MyRecord(i, 10+i)); + } + CPPUNIT_ASSERT_EQUAL(8u, flatfile.getSize()); + for (unsigned int i=1; i<=8; ++i) { + CPPUNIT_ASSERT_EQUAL(MyRecord(i, 10+i), *flatfile[i-1]); + } +} + + +void RecordFlatFile_Test::testAdd() { + BlockMessage message("testAdd()"); + setupTestFile(); + RecordFlatFile<MyRecord, unsigned int> flatfile(_testFile, _chunkSize); + flatfile.add(MyRecord(9, 19)); + CPPUNIT_ASSERT_EQUAL(9u, flatfile.getSize()); + CPPUNIT_ASSERT_EQUAL(MyRecord(1, 11), *flatfile[0]); + CPPUNIT_ASSERT_EQUAL(MyRecord(2, 12), *flatfile[1]); + CPPUNIT_ASSERT_EQUAL(MyRecord(7, 17), *flatfile[6]); + CPPUNIT_ASSERT_EQUAL(MyRecord(8, 18), *flatfile[7]); + CPPUNIT_ASSERT_EQUAL(MyRecord(9, 19), *flatfile[8]); +} + +void RecordFlatFile_Test::testUpdate() { + BlockMessage message("testUpdate()"); + setupTestFile(); + RecordFlatFile<MyRecord, unsigned int> flatfile(_testFile, _chunkSize); + CPPUNIT_ASSERT(!flatfile.update(MyRecord(0, 20))); + CPPUNIT_ASSERT(flatfile.update(MyRecord(4, 19))); + CPPUNIT_ASSERT_EQUAL(8u, flatfile.getSize()); + CPPUNIT_ASSERT_EQUAL(MyRecord(1, 11), *flatfile[0]); + CPPUNIT_ASSERT_EQUAL(MyRecord(3, 13), *flatfile[2]); + CPPUNIT_ASSERT_EQUAL(MyRecord(4, 19), *flatfile[3]); + CPPUNIT_ASSERT_EQUAL(MyRecord(5, 15), *flatfile[4]); + CPPUNIT_ASSERT_EQUAL(MyRecord(8, 18), *flatfile[7]); +} + +void RecordFlatFile_Test::testRemove() { + BlockMessage message("testRemove()"); + setupTestFile(); + RecordFlatFile<MyRecord, unsigned int> flatfile(_testFile, _chunkSize); + flatfile.remove(3); + CPPUNIT_ASSERT_EQUAL(7u, flatfile.getSize()); + CPPUNIT_ASSERT_EQUAL(MyRecord(1, 11), *flatfile[0]); + CPPUNIT_ASSERT_EQUAL(MyRecord(2, 12), *flatfile[1]); + CPPUNIT_ASSERT_EQUAL(MyRecord(8, 18), *flatfile[2]); + CPPUNIT_ASSERT_EQUAL(MyRecord(4, 14), *flatfile[3]); + CPPUNIT_ASSERT_EQUAL(MyRecord(5, 15), *flatfile[4]); + CPPUNIT_ASSERT_EQUAL(MyRecord(6, 16), *flatfile[5]); + CPPUNIT_ASSERT_EQUAL(MyRecord(7, 17), *flatfile[6]); +} + +void RecordFlatFile_Test::testExists() { + BlockMessage message("testExists()"); + setupTestFile(); + RecordFlatFile<MyRecord, unsigned int> flatfile(_testFile, _chunkSize); + CPPUNIT_ASSERT(flatfile.exists(3)); + CPPUNIT_ASSERT(flatfile.exists(1)); + CPPUNIT_ASSERT(!flatfile.exists(11)); + CPPUNIT_ASSERT(flatfile.exists(6)); + CPPUNIT_ASSERT(flatfile.exists(5)); + CPPUNIT_ASSERT(!flatfile.exists(0)); +} + +void RecordFlatFile_Test::testGetRecord() { + BlockMessage message("testGetRecord()"); + setupTestFile(); + RecordFlatFile<MyRecord, unsigned int> flatfile(_testFile, _chunkSize); + CPPUNIT_ASSERT_EQUAL(MyRecord(4, 14), *flatfile.getRecord(4)); + CPPUNIT_ASSERT(flatfile.getRecord(0).get() == 0); +} + +void RecordFlatFile_Test::testClear() { + try{ + BlockMessage message("testClear()"); + setupTestFile(); + RecordFlatFile<MyRecord, unsigned int> flatfile(_testFile, _chunkSize); + flatfile.clear(); + struct stat filestats; + CPPUNIT_ASSERT(stat(_testFile.c_str(), &filestats) == -1); + } catch (exception& e) { + cerr << "Caught exception '" << e.what() << "' in testClear()" << endl; + throw; + } +} + +void RecordFlatFile_Test::testSimpleUsage() +{ + BlockMessage message("testSimpleUsage()"); + RecordFlatFile<MyRecord, unsigned int> flatfile("recordflatfile.testfile"); + flatfile.clear(); + + CPPUNIT_ASSERT_EQUAL(false, flatfile.exists(34u)); + CPPUNIT_ASSERT_EQUAL((MyRecord*) 0, flatfile.getRecord(23u).get()); + + MyRecord record1(12, 54); + MyRecord record2(34, 62); + + flatfile.add(record1); + flatfile.add(record2); + + CPPUNIT_ASSERT_EQUAL(true, flatfile.exists(12u)); + CPPUNIT_ASSERT_EQUAL((MyRecord*) 0, flatfile.getRecord(23u).get()); + unique_ptr<MyRecord> result(flatfile.getRecord(34u)); + CPPUNIT_ASSERT(result.get() != 0); + CPPUNIT_ASSERT_EQUAL(62u, result->getValue()); + + record2.setValue(67); + flatfile.update(record2); + + unique_ptr<MyRecord> result2(flatfile.getRecord(34u)); + CPPUNIT_ASSERT(result2.get() != 0); + CPPUNIT_ASSERT_EQUAL(67u, result2->getValue()); + + flatfile.remove(12); + CPPUNIT_ASSERT_EQUAL(false, flatfile.exists(12u)); + + flatfile.clear(); + CPPUNIT_ASSERT_EQUAL(false, flatfile.exists(34u)); +} + +void RecordFlatFile_Test::testValid() +{ + BlockMessage message("testValid()"); + RecordFlatFile<MyRecord, unsigned int> flatfile("recordflatfile.testfile"); + flatfile.clear(); + + MyRecord record1(12, 54, true); + MyRecord record2(34, 62, false); + MyRecord record3(15, 69, true); + MyRecord record4(50, 93, false); + + // Test that valid entries doesn't generate errors + flatfile.add(record1); + CPPUNIT_ASSERT(!flatfile.errorsFound()); + CPPUNIT_ASSERT_EQUAL((size_t) 0, flatfile.getErrors().size()); + + // Test that invalid entries do + flatfile.add(record2); + CPPUNIT_ASSERT(flatfile.errorsFound()); + CPPUNIT_ASSERT_EQUAL((size_t) 1, flatfile.getErrors().size()); + string expected("Adding invalid record '34' to file " + "recordflatfile.testfile."); + CPPUNIT_ASSERT_EQUAL(expected, *flatfile.getErrors().begin()); + + // Checking that errors are kept if not cleared + flatfile.add(record3); + CPPUNIT_ASSERT_EQUAL((size_t) 1, flatfile.getErrors().size()); + CPPUNIT_ASSERT_EQUAL(expected, *flatfile.getErrors().begin()); + + // Checking that clearing errors work + flatfile.clearErrors(); + CPPUNIT_ASSERT_EQUAL((size_t) 0, flatfile.getErrors().size()); + + flatfile.add(record4); + flatfile.clearErrors(); + + // Checking that entries read in get method generates warning + unique_ptr<MyRecord> result(flatfile.getRecord(12)); + CPPUNIT_ASSERT_EQUAL((size_t) 0, flatfile.getErrors().size()); + result = flatfile.getRecord(15); + CPPUNIT_ASSERT_EQUAL((size_t) 1, flatfile.getErrors().size()); + expected = "Found corrupted entry in file recordflatfile.testfile"; + CPPUNIT_ASSERT_EQUAL(expected, *flatfile.getErrors().begin()); + flatfile.clearErrors(); + + // Checking that reading invalid entries generate exception + try{ + result = flatfile.getRecord(50); + CPPUNIT_FAIL("Expected exception"); + } catch (IoException& e) { + expected = "IoException(): Entry requested '50' is corrupted in file " + "recordflatfile.testfile at getRecord in"; + string actual(e.what()); + if (actual.size() > expected.size()) + actual = actual.substr(0, expected.size()); + CPPUNIT_ASSERT_EQUAL(expected, actual); + } + CPPUNIT_ASSERT_EQUAL((size_t) 1, flatfile.getErrors().size()); + expected = "Found corrupted entry in file recordflatfile.testfile"; + CPPUNIT_ASSERT_EQUAL(expected, *flatfile.getErrors().begin()); + flatfile.clearErrors(); + + // Check that you get warning when deleting if last entry is invalid + flatfile.remove(12); + CPPUNIT_ASSERT_EQUAL((size_t) 1, flatfile.getErrors().size()); + expected = "Last entry in file recordflatfile.testfile is invalid"; + CPPUNIT_ASSERT_EQUAL(expected, *flatfile.getErrors().begin()); + + flatfile.clear(); +} diff --git a/storage/src/tests/subscriptions/.gitignore b/storage/src/tests/subscriptions/.gitignore new file mode 100644 index 00000000000..04a221b8052 --- /dev/null +++ b/storage/src/tests/subscriptions/.gitignore @@ -0,0 +1,8 @@ +*.So +*.lo +.depend +.depend.NEW +.deps +.libs +Makefile +features.h diff --git a/storage/src/tests/systemtests/.gitignore b/storage/src/tests/systemtests/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/storage/src/tests/systemtests/.gitignore diff --git a/storage/src/tests/testhelper.cpp b/storage/src/tests/testhelper.cpp new file mode 100644 index 00000000000..c4074aa1ac6 --- /dev/null +++ b/storage/src/tests/testhelper.cpp @@ -0,0 +1,175 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <tests/testhelper.h> + +#include <vespa/log/log.h> +#include <vespa/vespalib/io/fileutil.h> + +LOG_SETUP(".testhelper"); + +namespace storage { + +namespace { + bool useNewStorageCore() { + if ( // Unit test directory + vespalib::fileExists("use_new_storage_core") || + // src/cpp directory + vespalib::fileExists("../use_new_storage_core") || + // Top build directory where storage-HEAD remains + vespalib::fileExists("../../../../use_new_storage_core")) + { + std::cerr << "Using new storage core for unit tests\n"; + return true; + } + return false; + } + bool newStorageCore(useNewStorageCore()); +} + +void addStorageDistributionConfig(vdstestlib::DirConfig& dc) +{ + vdstestlib::DirConfig::Config* config; + config = &dc.getConfig("stor-distribution", true); + config->clear(); + config->set("group[1]"); + config->set("group[0].name", "invalid"); + config->set("group[0].index", "invalid"); + config->set("group[0].nodes[50]"); + config->set("redundancy", "2"); + + for (uint32_t i = 0; i < 50; i++) { + std::ostringstream key; key << "group[0].nodes[" << i << "].index"; + std::ostringstream val; val << i; + config->set(key.str(), val.str()); + } +} + +vdstestlib::DirConfig getStandardConfig(bool storagenode) { + std::string clusterName("storage"); + vdstestlib::DirConfig dc; + vdstestlib::DirConfig::Config* config; + config = &dc.addConfig("fleetcontroller"); + config->set("cluster_name", clusterName); + config->set("index", "0"); + config->set("zookeeper_server", "\"\""); + config->set("total_distributor_count", "10"); + config->set("total_storage_count", "10"); + config = &dc.addConfig("upgrading"); + config = &dc.addConfig("load-type"); + config = &dc.addConfig("bucket"); + config = &dc.addConfig("messagebus"); + config = &dc.addConfig("stor-prioritymapping"); + config = &dc.addConfig("stor-bucketdbupdater"); + config = &dc.addConfig("stor-bucket-init"); + config = &dc.addConfig("metricsmanager"); + config->set("consumer[1]"); + config->set("consumer[0].name", "\"status\""); + config->set("consumer[0].addedmetrics[1]"); + config->set("consumer[0].addedmetrics[0]", "\"*\""); + config = &dc.addConfig("stor-communicationmanager"); + config->set("rpcport", "0"); + config->set("mbusport", "0"); + config = &dc.addConfig("stor-bucketdb"); + config->set("chunklevel", "0"); + config = &dc.addConfig("stor-distributormanager"); + config->set("splitcount", "1000000"); + config->set("splitsize", "1000000"); + config->set("joincount", "0"); + config->set("joinsize", "0"); + config = &dc.addConfig("stor-opslogger"); + config = &dc.addConfig("persistence"); + config->set("abort_operations_with_changed_bucket_ownership", "true"); + config = &dc.addConfig("stor-filestor"); + // Easier to see what goes wrong with only 1 thread per disk. + config->set("minimum_file_meta_slots", "2"); + config->set("minimum_file_header_block_size", "368"); + config->set("minimum_file_size", "4096"); + config->set("threads[1]"); + config->set("threads[0].lowestpri 255"); + config->set("dir_spread", "4"); + config->set("dir_levels", "0"); + config->set("use_new_core", newStorageCore ? "true" : "false"); + config->set("maximum_versions_of_single_document_stored", "0"); + //config->set("enable_slotfile_cache", "false"); + // Unit tests typically use fake low time values, so don't complain + // about them or compact/delete them by default. Override in tests testing that + // behavior + config->set("time_future_limit", "5"); + config->set("time_past_limit", "2000000000"); + config->set("keep_remove_time_period", "2000000000"); + config->set("revert_time_period", "2000000000"); + // Don't want test to call exit() + config->set("fail_disk_after_error_count", "0"); + config = &dc.addConfig("stor-bouncer"); + config = &dc.addConfig("stor-integritychecker"); + config = &dc.addConfig("stor-bucketmover"); + config = &dc.addConfig("stor-messageforwarder"); + config = &dc.addConfig("stor-server"); + config->set("cluster_name", clusterName); + config->set("enable_dead_lock_detector", "false"); + config->set("enable_dead_lock_detector_warnings", "false"); + config->set("max_merges_per_node", "25"); + config->set("max_merge_queue_size", "20"); + config->set("root_folder", + (storagenode ? "vdsroot" : "vdsroot.distributor")); + config->set("is_distributor", + (storagenode ? "false" : "true")); + config = &dc.addConfig("stor-devices"); + config->set("root_folder", + (storagenode ? "vdsroot" : "vdsroot.distributor")); + config = &dc.addConfig("stor-status"); + config->set("httpport", "0"); + config = &dc.addConfig("stor-visitor"); + config->set("defaultdocblocksize", "8192"); + // By default, need "old" behaviour of maxconcurrent + config->set("maxconcurrentvisitors_fixed", "4"); + config->set("maxconcurrentvisitors_variable", "0"); + config = &dc.addConfig("stor-visitordispatcher"); + addFileConfig(dc, "documenttypes", "config-doctypes.cfg"); + addStorageDistributionConfig(dc); + return dc; +} + +void addSlobrokConfig(vdstestlib::DirConfig& dc, + const mbus::Slobrok& slobrok) +{ + std::ostringstream ost; + ost << "tcp/localhost:" << slobrok.port(); + vdstestlib::DirConfig::Config* config; + config = &dc.getConfig("slobroks", true); + config->clear(); + config->set("slobrok[1]"); + config->set("slobrok[0].connectionspec", ost.str()); +} + +void addFileConfig(vdstestlib::DirConfig& dc, + const std::string& configDefName, + const std::string& fileName) +{ + vdstestlib::DirConfig::Config* config; + config = &dc.getConfig(configDefName, true); + config->clear(); + std::ifstream in(fileName.c_str()); + std::string line; + while (std::getline(in, line, '\n')) { + std::string::size_type pos = line.find(' '); + if (pos == std::string::npos) { + config->set(line); + } else { + config->set(line.substr(0, pos), line.substr(pos + 1)); + } + } + in.close(); +} + +TestName::TestName(const std::string& n) + : name(n) +{ + LOG(debug, "Starting test %s", name.c_str()); +} + +TestName::~TestName() { + LOG(debug, "Done with test %s", name.c_str()); +} + +} // storage diff --git a/storage/src/tests/testhelper.h b/storage/src/tests/testhelper.h new file mode 100644 index 00000000000..be2c3e7ec66 --- /dev/null +++ b/storage/src/tests/testhelper.h @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include <vespa/vdstestlib/cppunit/dirconfig.h> +#include <vespa/vdstestlib/cppunit/macros.h> + + +#include <fstream> +#include <vespa/fastos/fastos.h> +#include <vespa/messagebus/testlib/slobrok.h> +#include <sstream> + +#define ASSERT_REPLY_COUNT(count, dummylink) \ + { \ + std::ostringstream msgost; \ + if ((dummylink).getNumReplies() != count) { \ + for (uint32_t ijx=0; ijx<(dummylink).getNumReplies(); ++ijx) { \ + msgost << (dummylink).getReply(ijx)->toString(true) << "\n"; \ + } \ + } \ + CPPUNIT_ASSERT_EQUAL_MSG(msgost.str(), size_t(count), \ + (dummylink).getNumReplies()); \ + } +#define ASSERT_COMMAND_COUNT(count, dummylink) \ + { \ + std::ostringstream msgost; \ + if ((dummylink).getNumCommands() != count) { \ + for (uint32_t ijx=0; ijx<(dummylink).getNumCommands(); ++ijx) { \ + msgost << (dummylink).getCommand(ijx)->toString(true) << "\n"; \ + } \ + } \ + CPPUNIT_ASSERT_EQUAL_MSG(msgost.str(), size_t(count), \ + (dummylink).getNumCommands()); \ + } + +namespace storage { + +void addFileConfig(vdstestlib::DirConfig& dc, + const std::string& configDefName, + const std::string& fileName); + + +void addStorageDistributionConfig(vdstestlib::DirConfig& dc); + +vdstestlib::DirConfig getStandardConfig(bool storagenode); + +void addSlobrokConfig(vdstestlib::DirConfig& dc, + const mbus::Slobrok& slobrok); + +// Class used to print start and end of test. Enable debug when you want to see +// which test creates what output or where we get stuck +struct TestName { + std::string name; + TestName(const std::string& n); + ~TestName(); +}; + +} // storage + diff --git a/storage/src/tests/testrunner.cpp b/storage/src/tests/testrunner.cpp new file mode 100644 index 00000000000..5d8dc8d4c1f --- /dev/null +++ b/storage/src/tests/testrunner.cpp @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <iostream> +#include <vespa/log/log.h> +#include <vespa/vdstestlib/cppunit/cppunittestrunner.h> + +LOG_SETUP("storagecppunittests"); + +int +main(int argc, char **argv) +{ + vdstestlib::CppUnitTestRunner testRunner; + return testRunner.run(argc, argv); +} diff --git a/storage/src/tests/visiting/.gitignore b/storage/src/tests/visiting/.gitignore new file mode 100644 index 00000000000..184e5d1c936 --- /dev/null +++ b/storage/src/tests/visiting/.gitignore @@ -0,0 +1,12 @@ +*.So +*.lo +*.o +.*.swp +.config.log +.depend +.depend.NEW +.deps +.libs +Makefile +testrunner +testrunner.core diff --git a/storage/src/tests/visiting/CMakeLists.txt b/storage/src/tests/visiting/CMakeLists.txt new file mode 100644 index 00000000000..60e130c003c --- /dev/null +++ b/storage/src/tests/visiting/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storage_testvisiting + SOURCES + commandqueuetest.cpp + visitormanagertest.cpp + visitortest.cpp + memory_bounded_trace_test.cpp + DEPENDS + AFTER + storage_storageconfig +) diff --git a/storage/src/tests/visiting/commandqueuetest.cpp b/storage/src/tests/visiting/commandqueuetest.cpp new file mode 100644 index 00000000000..5d6da5f7ea5 --- /dev/null +++ b/storage/src/tests/visiting/commandqueuetest.cpp @@ -0,0 +1,223 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> +#include <vespa/storage/visiting/commandqueue.h> +#include <vespa/storageapi/message/visitor.h> +#include <vespa/vdstestlib/cppunit/macros.h> + +using vespalib::string; + +namespace storage { + +struct CommandQueueTest : public CppUnit::TestFixture +{ + void testFIFO(); + void testFIFOWithPriorities(); + void testReleaseOldest(); + void testReleaseLowestPriority(); + void testDeleteIterator(); + + CPPUNIT_TEST_SUITE(CommandQueueTest); + CPPUNIT_TEST(testFIFO); + CPPUNIT_TEST(testFIFOWithPriorities); + CPPUNIT_TEST(testReleaseOldest); + CPPUNIT_TEST(testReleaseLowestPriority); + CPPUNIT_TEST(testDeleteIterator); + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(CommandQueueTest); + +namespace { + std::shared_ptr<api::CreateVisitorCommand> getCommand( + const vespalib::stringref & name, int timeout, + uint8_t priority = 0) + { + vespalib::asciistream ost; + ost << name << " t=" << timeout << " p=" << static_cast<unsigned int>(priority); + // Piggyback name in document selection + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("", "", ost.str())); + cmd->setQueueTimeout(timeout); + cmd->setPriority(priority); + return cmd; + } + + const vespalib::string & + getCommandString(const std::shared_ptr<api::CreateVisitorCommand>& cmd) + { + return cmd->getDocumentSelection(); + } + +} + +void CommandQueueTest::testFIFO() { + framework::defaultimplementation::FakeClock clock; + CommandQueue<api::CreateVisitorCommand> queue(clock); + CPPUNIT_ASSERT(queue.empty()); + // Use all default priorities, meaning what comes out should be in the same order + // as what went in + queue.add(getCommand("first", 1)); + queue.add(getCommand("second", 10)); + queue.add(getCommand("third", 5)); + queue.add(getCommand("fourth", 0)); + queue.add(getCommand("fifth", 3)); + queue.add(getCommand("sixth", 14)); + queue.add(getCommand("seventh", 7)); + + CPPUNIT_ASSERT(!queue.empty()); + std::vector<std::shared_ptr<api::CreateVisitorCommand> > commands; + for (;;) { + std::shared_ptr<api::CreateVisitorCommand> cmd( + queue.releaseNextCommand().first); + if (cmd.get() == 0) break; + commands.push_back(cmd); + } + CPPUNIT_ASSERT_EQUAL(size_t(7), commands.size()); + CPPUNIT_ASSERT_EQUAL(string("first t=1 p=0"), getCommandString(commands[0])); + CPPUNIT_ASSERT_EQUAL(string("second t=10 p=0"), getCommandString(commands[1])); + CPPUNIT_ASSERT_EQUAL(string("third t=5 p=0"), getCommandString(commands[2])); + CPPUNIT_ASSERT_EQUAL(string("fourth t=0 p=0"), getCommandString(commands[3])); + CPPUNIT_ASSERT_EQUAL(string("fifth t=3 p=0"), getCommandString(commands[4])); + CPPUNIT_ASSERT_EQUAL(string("sixth t=14 p=0"), getCommandString(commands[5])); + CPPUNIT_ASSERT_EQUAL(string("seventh t=7 p=0"), getCommandString(commands[6])); +} + +void CommandQueueTest::testFIFOWithPriorities() { + framework::defaultimplementation::FakeClock clock; + CommandQueue<api::CreateVisitorCommand> queue(clock); + CPPUNIT_ASSERT(queue.empty()); + + queue.add(getCommand("first", 1, 10)); + CPPUNIT_ASSERT_EQUAL(string("first t=1 p=10"), getCommandString(queue.peekLowestPriorityCommand())); + queue.add(getCommand("second", 10, 22)); + queue.add(getCommand("third", 5, 9)); + CPPUNIT_ASSERT_EQUAL(string("second t=10 p=22"), getCommandString(queue.peekLowestPriorityCommand())); + queue.add(getCommand("fourth", 0, 22)); + queue.add(getCommand("fifth", 3, 22)); + CPPUNIT_ASSERT_EQUAL(string("fifth t=3 p=22"), getCommandString(queue.peekLowestPriorityCommand())); + queue.add(getCommand("sixth", 14, 50)); + queue.add(getCommand("seventh", 7, 0)); + + CPPUNIT_ASSERT_EQUAL(string("sixth t=14 p=50"), getCommandString(queue.peekLowestPriorityCommand())); + + CPPUNIT_ASSERT(!queue.empty()); + std::vector<std::shared_ptr<api::CreateVisitorCommand> > commands; + for (;;) { + std::shared_ptr<api::CreateVisitorCommand> cmdPeek(queue.peekNextCommand()); + std::shared_ptr<api::CreateVisitorCommand> cmd(queue.releaseNextCommand().first); + if (cmd.get() == 0 || cmdPeek != cmd) break; + commands.push_back(cmd); + } + CPPUNIT_ASSERT_EQUAL(size_t(7), commands.size()); + CPPUNIT_ASSERT_EQUAL(string("seventh t=7 p=0"), getCommandString(commands[0])); + CPPUNIT_ASSERT_EQUAL(string("third t=5 p=9"), getCommandString(commands[1])); + CPPUNIT_ASSERT_EQUAL(string("first t=1 p=10"), getCommandString(commands[2])); + CPPUNIT_ASSERT_EQUAL(string("second t=10 p=22"), getCommandString(commands[3])); + CPPUNIT_ASSERT_EQUAL(string("fourth t=0 p=22"), getCommandString(commands[4])); + CPPUNIT_ASSERT_EQUAL(string("fifth t=3 p=22"), getCommandString(commands[5])); + CPPUNIT_ASSERT_EQUAL(string("sixth t=14 p=50"), getCommandString(commands[6])); +} + +void CommandQueueTest::testReleaseOldest() { + framework::defaultimplementation::FakeClock clock(framework::defaultimplementation::FakeClock::FAKE_ABSOLUTE); + CommandQueue<api::CreateVisitorCommand> queue(clock); + CPPUNIT_ASSERT(queue.empty()); + queue.add(getCommand("first", 10)); + queue.add(getCommand("second", 100)); + queue.add(getCommand("third", 1000)); + queue.add(getCommand("fourth", 5)); + queue.add(getCommand("fifth", 3000)); + queue.add(getCommand("sixth", 400)); + queue.add(getCommand("seventh", 700)); + CPPUNIT_ASSERT_EQUAL(7u, queue.size()); + + typedef CommandQueue<api::CreateVisitorCommand>::CommandEntry CommandEntry; + std::list<CommandEntry> timedOut(queue.releaseTimedOut()); + CPPUNIT_ASSERT(timedOut.empty()); + clock.addMilliSecondsToTime(400 * 1000); + timedOut = queue.releaseTimedOut(); + CPPUNIT_ASSERT_EQUAL(size_t(4), timedOut.size()); + std::ostringstream ost; + for (std::list<CommandEntry>::const_iterator it = timedOut.begin(); + it != timedOut.end(); ++it) + { + ost << getCommandString(it->_command) << "\n"; + } + CPPUNIT_ASSERT_EQUAL(std::string( + "fourth t=5 p=0\n" + "first t=10 p=0\n" + "second t=100 p=0\n" + "sixth t=400 p=0\n"), ost.str()); + CPPUNIT_ASSERT_EQUAL(3u, queue.size()); +} + +void CommandQueueTest::testReleaseLowestPriority() { + framework::defaultimplementation::FakeClock clock; + CommandQueue<api::CreateVisitorCommand> queue(clock); + CPPUNIT_ASSERT(queue.empty()); + + queue.add(getCommand("first", 1, 10)); + queue.add(getCommand("second", 10, 22)); + queue.add(getCommand("third", 5, 9)); + queue.add(getCommand("fourth", 0, 22)); + queue.add(getCommand("fifth", 3, 22)); + queue.add(getCommand("sixth", 14, 50)); + queue.add(getCommand("seventh", 7, 0)); + CPPUNIT_ASSERT_EQUAL(7u, queue.size()); + + std::vector<std::shared_ptr<api::CreateVisitorCommand> > commands; + for (;;) { + std::shared_ptr<api::CreateVisitorCommand> cmdPeek(queue.peekLowestPriorityCommand()); + std::pair<std::shared_ptr<api::CreateVisitorCommand>, uint64_t> cmd( + queue.releaseLowestPriorityCommand()); + if (cmd.first.get() == 0 || cmdPeek != cmd.first) break; + commands.push_back(cmd.first); + } + CPPUNIT_ASSERT_EQUAL(size_t(7), commands.size()); + CPPUNIT_ASSERT_EQUAL(string("sixth t=14 p=50"), getCommandString(commands[0])); + CPPUNIT_ASSERT_EQUAL(string("fifth t=3 p=22"), getCommandString(commands[1])); + CPPUNIT_ASSERT_EQUAL(string("fourth t=0 p=22"), getCommandString(commands[2])); + CPPUNIT_ASSERT_EQUAL(string("second t=10 p=22"), getCommandString(commands[3])); + CPPUNIT_ASSERT_EQUAL(string("first t=1 p=10"), getCommandString(commands[4])); + CPPUNIT_ASSERT_EQUAL(string("third t=5 p=9"), getCommandString(commands[5])); + CPPUNIT_ASSERT_EQUAL(string("seventh t=7 p=0"), getCommandString(commands[6])); +} + +void CommandQueueTest::testDeleteIterator() { + framework::defaultimplementation::FakeClock clock; + CommandQueue<api::CreateVisitorCommand> queue(clock); + CPPUNIT_ASSERT(queue.empty()); + queue.add(getCommand("first", 10)); + queue.add(getCommand("second", 100)); + queue.add(getCommand("third", 1000)); + queue.add(getCommand("fourth", 5)); + queue.add(getCommand("fifth", 3000)); + queue.add(getCommand("sixth", 400)); + queue.add(getCommand("seventh", 700)); + CPPUNIT_ASSERT_EQUAL(7u, queue.size()); + + CommandQueue<api::CreateVisitorCommand>::iterator it = queue.begin(); + ++it; ++it; + queue.erase(it); + CPPUNIT_ASSERT_EQUAL(6u, queue.size()); + + std::vector<std::shared_ptr<api::CreateVisitorCommand> > cmds; + for (;;) { + std::shared_ptr<api::CreateVisitorCommand> cmd( + std::dynamic_pointer_cast<api::CreateVisitorCommand>( + queue.releaseNextCommand().first)); + if (cmd.get() == 0) break; + cmds.push_back(cmd); + } + CPPUNIT_ASSERT_EQUAL(size_t(6), cmds.size()); + CPPUNIT_ASSERT_EQUAL(string("first t=10 p=0"), getCommandString(cmds[0])); + CPPUNIT_ASSERT_EQUAL(string("second t=100 p=0"), getCommandString(cmds[1])); + CPPUNIT_ASSERT_EQUAL(string("fourth t=5 p=0"), getCommandString(cmds[2])); + CPPUNIT_ASSERT_EQUAL(string("fifth t=3000 p=0"), getCommandString(cmds[3])); + CPPUNIT_ASSERT_EQUAL(string("sixth t=400 p=0"), getCommandString(cmds[4])); + CPPUNIT_ASSERT_EQUAL(string("seventh t=700 p=0"), getCommandString(cmds[5])); +} + +} + diff --git a/storage/src/tests/visiting/memory_bounded_trace_test.cpp b/storage/src/tests/visiting/memory_bounded_trace_test.cpp new file mode 100644 index 00000000000..85eae12fc34 --- /dev/null +++ b/storage/src/tests/visiting/memory_bounded_trace_test.cpp @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storage/visiting/memory_bounded_trace.h> + +namespace storage { + +class MemoryBoundedTraceTest : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE(MemoryBoundedTraceTest); + CPPUNIT_TEST(noMemoryReportedUsedWhenEmpty); + CPPUNIT_TEST(memoryUsedIsStringLengthForLeafNode); + CPPUNIT_TEST(memoryUsedIsAccumulatedRecursivelyForNonLeafNodes); + CPPUNIT_TEST(traceNodesCanBeMovedAndImplicitlyCleared); + CPPUNIT_TEST(movedTraceTreeIsMarkedAsStrict); + CPPUNIT_TEST(canNotAddMoreNodesWhenMemoryUsedExceedsUpperBound); + CPPUNIT_TEST(movedTreeIncludesStatsNodeWhenNodesOmitted); + CPPUNIT_TEST_SUITE_END(); + +public: + void noMemoryReportedUsedWhenEmpty(); + void memoryUsedIsStringLengthForLeafNode(); + void memoryUsedIsAccumulatedRecursivelyForNonLeafNodes(); + void traceNodesCanBeMovedAndImplicitlyCleared(); + void movedTraceTreeIsMarkedAsStrict(); + void canNotAddMoreNodesWhenMemoryUsedExceedsUpperBound(); + void movedTreeIncludesStatsNodeWhenNodesOmitted(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(MemoryBoundedTraceTest); + +void +MemoryBoundedTraceTest::noMemoryReportedUsedWhenEmpty() +{ + MemoryBoundedTrace trace(100); + CPPUNIT_ASSERT_EQUAL(size_t(0), trace.getApproxMemoryUsed()); +} + +void +MemoryBoundedTraceTest::memoryUsedIsStringLengthForLeafNode() +{ + MemoryBoundedTrace trace(100); + CPPUNIT_ASSERT(trace.add(mbus::TraceNode("hello world", 0))); + CPPUNIT_ASSERT_EQUAL(size_t(11), trace.getApproxMemoryUsed()); +} + +void +MemoryBoundedTraceTest::memoryUsedIsAccumulatedRecursivelyForNonLeafNodes() +{ + MemoryBoundedTrace trace(100); + mbus::TraceNode innerNode; + innerNode.addChild("hello world"); + innerNode.addChild("goodbye moon"); + CPPUNIT_ASSERT(trace.add(innerNode)); + CPPUNIT_ASSERT_EQUAL(size_t(23), trace.getApproxMemoryUsed()); +} + +void +MemoryBoundedTraceTest::traceNodesCanBeMovedAndImplicitlyCleared() +{ + MemoryBoundedTrace trace(100); + CPPUNIT_ASSERT(trace.add(mbus::TraceNode("hello world", 0))); + mbus::TraceNode target; + trace.moveTraceTo(target); + CPPUNIT_ASSERT_EQUAL(uint32_t(1), target.getNumChildren()); + CPPUNIT_ASSERT_EQUAL(size_t(0), trace.getApproxMemoryUsed()); + + mbus::TraceNode emptinessCheck; + trace.moveTraceTo(emptinessCheck); + CPPUNIT_ASSERT_EQUAL(uint32_t(0), emptinessCheck.getNumChildren()); +} + +/** + * We want trace subtrees to be strictly ordered so that the message about + * omitted traces will remain soundly as the last ordered node. There is no + * particular performance reason for not having strict mode enabled to the + * best of my knowledge, since the internal backing data structure is an + * ordered vector anyhow. + */ +void +MemoryBoundedTraceTest::movedTraceTreeIsMarkedAsStrict() +{ + MemoryBoundedTrace trace(100); + CPPUNIT_ASSERT(trace.add(mbus::TraceNode("hello world", 0))); + mbus::TraceNode target; + trace.moveTraceTo(target); + CPPUNIT_ASSERT_EQUAL(uint32_t(1), target.getNumChildren()); + CPPUNIT_ASSERT(target.getChild(0).isStrict()); +} + +void +MemoryBoundedTraceTest::canNotAddMoreNodesWhenMemoryUsedExceedsUpperBound() +{ + // Note: we allow one complete node tree to exceed the bounds, but as soon + // as the bound is exceeded no further nodes can be added. + MemoryBoundedTrace trace(10); + CPPUNIT_ASSERT(trace.add(mbus::TraceNode("hello world", 0))); + CPPUNIT_ASSERT_EQUAL(size_t(11), trace.getApproxMemoryUsed()); + + CPPUNIT_ASSERT(!trace.add(mbus::TraceNode("the quick red fox runs across " + "the freeway", 0))); + CPPUNIT_ASSERT_EQUAL(size_t(11), trace.getApproxMemoryUsed()); + + mbus::TraceNode target; + trace.moveTraceTo(target); + // Twice nested node (root -> added trace tree -> leaf with txt). + CPPUNIT_ASSERT_EQUAL(uint32_t(1), target.getNumChildren()); + CPPUNIT_ASSERT(target.getChild(0).getNumChildren() >= 1); + CPPUNIT_ASSERT_EQUAL(vespalib::string("hello world"), + target.getChild(0).getChild(0).getNote()); +} + +void +MemoryBoundedTraceTest::movedTreeIncludesStatsNodeWhenNodesOmitted() +{ + MemoryBoundedTrace trace(5); + CPPUNIT_ASSERT(trace.add(mbus::TraceNode("abcdef", 0))); + CPPUNIT_ASSERT(!trace.add(mbus::TraceNode("ghijkjlmn", 0))); + + mbus::TraceNode target; + trace.moveTraceTo(target); + CPPUNIT_ASSERT_EQUAL(uint32_t(1), target.getNumChildren()); + CPPUNIT_ASSERT_EQUAL(uint32_t(2), target.getChild(0).getNumChildren()); + vespalib::string expected("Trace too large; omitted 1 subsequent trace " + "trees containing a total of 9 bytes"); + CPPUNIT_ASSERT_EQUAL(expected, target.getChild(0).getChild(1).getNote()); +} + +} // storage + diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp new file mode 100644 index 00000000000..d782abf7d54 --- /dev/null +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -0,0 +1,1172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/document/datatype/datatype.h> +#include <vespa/document/fieldvalue/intfieldvalue.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/document/fieldvalue/rawfieldvalue.h> +#include <vespa/log/log.h> +#include <vespa/storageapi/message/datagram.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/visitor.h> +#include <vector> +#include <vespa/storage/persistence/filestorage/filestormanager.h> +#include <vespa/storage/visiting/visitormanager.h> +#include <vespa/storageframework/defaultimplementation/clock/realclock.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/testhelper.h> +#include <tests/common/dummystoragelink.h> +#include <tests/storageserver/testvisitormessagesession.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vdslib/container/visitorordering.h> +#include <vespa/documentapi/messagebus/messages/multioperationmessage.h> +#include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> + + +LOG_SETUP(".visitormanagertest"); + +namespace storage { +namespace { + typedef std::vector<api::StorageMessage::SP> msg_ptr_vector; +} + +class VisitorManagerTest : public CppUnit::TestFixture +{ +private: + CPPUNIT_TEST_SUITE(VisitorManagerTest); + CPPUNIT_TEST(testNormalUsage); + CPPUNIT_TEST(testResending); + CPPUNIT_TEST(testVisitEmptyBucket); + CPPUNIT_TEST(testMultiBucketVisit); + CPPUNIT_TEST(testNoBuckets); + CPPUNIT_TEST(testVisitPutsAndRemoves); + CPPUNIT_TEST(testVisitWithTimeframeAndSelection); + CPPUNIT_TEST(testVisitWithTimeframeAndBogusSelection); + CPPUNIT_TEST(testVisitorCallbacks); + CPPUNIT_TEST(testVisitorCleanup); + CPPUNIT_TEST(testAbortOnFailedVisitorInfo); + CPPUNIT_TEST(testAbortOnFieldPathError); + CPPUNIT_TEST(testVisitorQueueTimeout); + CPPUNIT_TEST(testVisitorProcessingTimeout); + CPPUNIT_TEST(testPrioritizedVisitorQueing); + CPPUNIT_TEST(testPrioritizedMaxConcurrentVisitors); + CPPUNIT_TEST(testVisitorQueingZeroQueueSize); + CPPUNIT_TEST(testHitCounter); + CPPUNIT_TEST(testStatusPage); + CPPUNIT_TEST_SUITE_END(); + + static uint32_t docCount; + std::vector<document::Document::SP > _documents; + std::unique_ptr<TestVisitorMessageSessionFactory> _messageSessionFactory; + std::unique_ptr<TestServiceLayerApp> _node; + std::unique_ptr<DummyStorageLink> _top; + VisitorManager* _manager; + +public: + VisitorManagerTest() : _node() {} + + // Not using setUp since can't throw exception out of it. + void initializeTest(); + void addSomeRemoves(bool removeAll = false); + void tearDown(); + TestVisitorMessageSession& getSession(uint32_t n); + uint64_t verifyCreateVisitorReply( + api::ReturnCode::Result expectedResult, + int checkStatsDocsVisited = -1, + int checkStatsBytesVisited = -1); + void getMessagesAndReply( + int expectedCount, + TestVisitorMessageSession& session, + std::vector<document::Document::SP >& docs, + std::vector<document::DocumentId>& docIds, + api::ReturnCode::Result returnCode = api::ReturnCode::OK, + documentapi::Priority::Value priority = documentapi::Priority::PRI_NORMAL_4); + uint32_t getMatchingDocuments(std::vector<document::Document::SP >& docs); + + void testNormalUsage(); + void testResending(); + void testVisitEmptyBucket(); + void testMultiBucketVisit(); + void testNoBuckets(); + void testVisitPutsAndRemoves(); + void testVisitWithTimeframeAndSelection(); + void testVisitWithTimeframeAndBogusSelection(); + void testVisitorCallbacks(); + void testVisitorCleanup(); + void testAbortOnFailedVisitorInfo(); + void testAbortOnFieldPathError(); + void testVisitorQueueTimeout(); + void testVisitorProcessingTimeout(); + void testPrioritizedVisitorQueing(); + void testPrioritizedMaxConcurrentVisitors(); + void testVisitorQueingZeroQueueSize(); + void testHitCounter(); + void testStatusPage(); +}; + +uint32_t VisitorManagerTest::docCount = 10; + +CPPUNIT_TEST_SUITE_REGISTRATION(VisitorManagerTest); + +void +VisitorManagerTest::initializeTest() +{ + LOG(debug, "Initializing test"); + vdstestlib::DirConfig config(getStandardConfig(true)); + config.getConfig("stor-visitor").set("visitorthreads", "1"); + + try { + _messageSessionFactory.reset( + new TestVisitorMessageSessionFactory(config.getConfigId())); + _node.reset( + new TestServiceLayerApp(config.getConfigId())); + _node->setupDummyPersistence(); + _node->getStateUpdater().setClusterState( + lib::ClusterState::CSP( + new lib::ClusterState("storage:1 distributor:1"))); + _top.reset(new DummyStorageLink()); + _top->push_back(std::unique_ptr<StorageLink>(_manager + = new VisitorManager( + config.getConfigId(), _node->getComponentRegister(), + *_messageSessionFactory))); + _top->push_back(std::unique_ptr<StorageLink>(new FileStorManager( + config.getConfigId(), _node->getPartitions(), _node->getPersistenceProvider(), _node->getComponentRegister()))); + _manager->setTimeBetweenTicks(10); + _top->open(); + } catch (config::InvalidConfigException& e) { + fprintf(stderr, "%s\n", e.what()); + } + // Adding some documents so database isn't empty + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + std::string content( + "To be, or not to be: that is the question:\n" + "Whether 'tis nobler in the mind to suffer\n" + "The slings and arrows of outrageous fortune,\n" + "Or to take arms against a sea of troubles,\n" + "And by opposing end them? To die: to sleep;\n" + "No more; and by a sleep to say we end\n" + "The heart-ache and the thousand natural shocks\n" + "That flesh is heir to, 'tis a consummation\n" + "Devoutly to be wish'd. To die, to sleep;\n" + "To sleep: perchance to dream: ay, there's the rub;\n" + "For in that sleep of death what dreams may come\n" + "When we have shuffled off this mortal coil,\n" + "Must give us pause: there's the respect\n" + "That makes calamity of so long life;\n" + "For who would bear the whips and scorns of time,\n" + "The oppressor's wrong, the proud man's contumely,\n" + "The pangs of despised love, the law's delay,\n" + "The insolence of office and the spurns\n" + "That patient merit of the unworthy takes,\n" + "When he himself might his quietus make\n" + "With a bare bodkin? who would fardels bear,\n" + "To grunt and sweat under a weary life,\n" + "But that the dread of something after death,\n" + "The undiscover'd country from whose bourn\n" + "No traveller returns, puzzles the will\n" + "And makes us rather bear those ills we have\n" + "Than fly to others that we know not of?\n" + "Thus conscience does make cowards of us all;\n" + "And thus the native hue of resolution\n" + "Is sicklied o'er with the pale cast of thought,\n" + "And enterprises of great pith and moment\n" + "With this regard their currents turn awry,\n" + "And lose the name of action. - Soft you now!\n" + "The fair Ophelia! Nymph, in thy orisons\n" + "Be all my sins remember'd.\n"); + for (uint32_t i=0; i<docCount; ++i) { + std::ostringstream uri; + uri << "userdoc:test:" << i % 10 << ":http://www.ntnu.no/" + << i << ".html"; + + _documents.push_back(document::Document::SP( + _node->getTestDocMan().createDocument(content, uri.str()))); + const document::DocumentType& type(_documents.back()->getType()); + _documents.back()->setValue(type.getField("headerval"), + document::IntFieldValue(i % 4)); + } + for (uint32_t i=0; i<10; ++i) { + document::BucketId bid(16, i); + + std::shared_ptr<api::CreateBucketCommand> cmd( + new api::CreateBucketCommand(bid)); + cmd->setAddress(address); + cmd->setSourceIndex(0); + _top->sendDown(cmd); + _top->waitForMessages(1, 60); + _top->reset(); + + StorBucketDatabase::WrappedEntry entry( + _node->getStorageBucketDatabase().get(bid, "", + StorBucketDatabase::CREATE_IF_NONEXISTING)); + entry->disk = 0; + entry.write(); + } + for (uint32_t i=0; i<docCount; ++i) { + document::BucketId bid(16, i); + + std::shared_ptr<api::PutCommand> cmd( + new api::PutCommand(bid, _documents[i], i+1)); + cmd->setAddress(address); + _top->sendDown(cmd); + _top->waitForMessages(1, 60); + const msg_ptr_vector replies = _top->getRepliesOnce(); + CPPUNIT_ASSERT_EQUAL((size_t) 1, replies.size()); + std::shared_ptr<api::PutReply> reply( + std::dynamic_pointer_cast<api::PutReply>( + replies[0])); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), + reply->getResult()); + } + LOG(debug, "Done initializing test"); +} + +void +VisitorManagerTest::addSomeRemoves(bool removeAll) +{ + framework::defaultimplementation::FakeClock clock; + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + for (uint32_t i=0; i<docCount; i += (removeAll ? 1 : 4)) { + // Add it to the database + document::BucketId bid(16, i % 10); + std::shared_ptr<api::RemoveCommand> cmd( + new api::RemoveCommand( + bid, _documents[i]->getId(), clock.getTimeInMicros().getTime() + docCount + i + 1)); + cmd->setAddress(address); + _top->sendDown(cmd); + _top->waitForMessages(1, 60); + const msg_ptr_vector replies = _top->getRepliesOnce(); + CPPUNIT_ASSERT_EQUAL((size_t) 1, replies.size()); + std::shared_ptr<api::RemoveReply> reply( + std::dynamic_pointer_cast<api::RemoveReply>( + replies[0])); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), + reply->getResult()); + } +} + +void +VisitorManagerTest::tearDown() +{ + if (_top.get() != 0) { + _top->close(); + _top->flush(); + _top.reset(0); + } + _node.reset(0); + _messageSessionFactory.reset(0); + _manager = 0; +} + +TestVisitorMessageSession& +VisitorManagerTest::getSession(uint32_t n) +{ + // Wait until we have started the visitor + const std::vector<TestVisitorMessageSession*>& sessions( + _messageSessionFactory->_visitorSessions); + framework::defaultimplementation::RealClock clock; + framework::MilliSecTime endTime( + clock.getTimeInMillis() + framework::MilliSecTime(30 * 1000)); + while (true) { + { + vespalib::LockGuard lock(_messageSessionFactory->_accessLock); + if (sessions.size() > n) { + return *sessions[n]; + } + } + if (clock.getTimeInMillis() > endTime) { + throw vespalib::IllegalStateException( + "Timed out waiting for visitor session", VESPA_STRLOC); + } + FastOS_Thread::Sleep(10); + } + throw std::logic_error("unreachable"); +} + +void +VisitorManagerTest::getMessagesAndReply( + int expectedCount, + TestVisitorMessageSession& session, + std::vector<document::Document::SP >& docs, + std::vector<document::DocumentId>& docIds, + api::ReturnCode::Result result, + documentapi::Priority::Value priority) +{ + for (int i = 0; i < expectedCount; i++) { + session.waitForMessages(i + 1); + mbus::Reply::UP reply; + { + vespalib::MonitorGuard guard(session.getMonitor()); + + CPPUNIT_ASSERT_EQUAL(priority, + session.sentMessages[i]->getPriority()); + + switch (session.sentMessages[i]->getType()) { + case documentapi::DocumentProtocol::MESSAGE_PUTDOCUMENT: + docs.push_back(static_cast<documentapi::PutDocumentMessage&>( + *session.sentMessages[i]).getDocument()); + break; + case documentapi::DocumentProtocol::MESSAGE_REMOVEDOCUMENT: + docIds.push_back(static_cast<documentapi::RemoveDocumentMessage&>( + *session.sentMessages[i]).getDocumentId()); + break; + default: + break; + } + + reply = session.sentMessages[i]->createReply(); + reply->swapState(*session.sentMessages[i]); + reply->setMessage( + mbus::Message::UP(session.sentMessages[i].release())); + + if (result != api::ReturnCode::OK) { + reply->addError(mbus::Error(result, "Generic error")); + } + } + + session.reply(std::move(reply)); + } +} + +uint64_t +VisitorManagerTest::verifyCreateVisitorReply( + api::ReturnCode::Result expectedResult, + int checkStatsDocsVisited, + int checkStatsBytesVisited) +{ + _top->waitForMessages(1, 60); + const msg_ptr_vector replies = _top->getRepliesOnce(); + CPPUNIT_ASSERT_EQUAL(1, (int)replies.size()); + + std::shared_ptr<api::StorageMessage> msg(replies[0]); + + CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); + + std::shared_ptr<api::CreateVisitorReply> reply( + std::dynamic_pointer_cast<api::CreateVisitorReply>(msg)); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(expectedResult, reply->getResult().getResult()); + + if (checkStatsDocsVisited >= 0) { + CPPUNIT_ASSERT_EQUAL(checkStatsDocsVisited, + int(reply->getVisitorStatistics().getDocumentsVisited())); + } + if (checkStatsBytesVisited >= 0) { + CPPUNIT_ASSERT_EQUAL(checkStatsBytesVisited, + int(reply->getVisitorStatistics().getBytesVisited())); + } + + return reply->getMsgId(); +} + +uint32_t +VisitorManagerTest::getMatchingDocuments(std::vector<document::Document::SP >& docs) { + uint32_t equalCount = 0; + for (uint32_t i=0; i<docs.size(); ++i) { + for (uint32_t j=0; j<_documents.size(); ++j) { + if (docs[i]->getId() == _documents[j]->getId() + && *docs[i] == *_documents[j]) + + { + equalCount++; + } + } + } + + return equalCount; +} + +void +VisitorManagerTest::testHitCounter() +{ + document::OrderingSpecification spec(document::OrderingSpecification::ASCENDING, 42, 7, 2); + Visitor::HitCounter hitCounter(&spec); + + hitCounter.addHit(document::DocumentId("orderdoc(7,2):mail:1234:42:foo"), 450); + hitCounter.addHit(document::DocumentId("orderdoc(7,2):mail:1234:49:foo"), 450); + hitCounter.addHit(document::DocumentId("orderdoc(7,2):mail:1234:60:foo"), 450); + hitCounter.addHit(document::DocumentId("orderdoc(7,2):mail:1234:10:foo"), 450); + hitCounter.addHit(document::DocumentId("orderdoc(7,2):mail:1234:21:foo"), 450); + + CPPUNIT_ASSERT_EQUAL(3, (int)hitCounter.getFirstPassHits()); + CPPUNIT_ASSERT_EQUAL(1350, (int)hitCounter.getFirstPassBytes()); + CPPUNIT_ASSERT_EQUAL(2, (int)hitCounter.getSecondPassHits()); + CPPUNIT_ASSERT_EQUAL(900, (int)hitCounter.getSecondPassBytes()); +} + +namespace { + +int getTotalSerializedSize(const std::vector<document::Document::SP>& docs) +{ + int total = 0; + for (size_t i = 0; i < docs.size(); ++i) { + total += int(docs[i]->serialize()->getLength()); + } + return total; +} + +} + +void +VisitorManagerTest::testNormalUsage() +{ + initializeTest(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + cmd->addBucketToBeVisited(document::BucketId(16, 3)); + cmd->setAddress(address); + cmd->setControlDestination("foo/bar"); + _top->sendDown(cmd); + std::vector<document::Document::SP > docs; + std::vector<document::DocumentId> docIds; + + // Should receive one multioperation message (bucket 3 has one document). + getMessagesAndReply(1, getSession(0), docs, docIds); + + // All data has been replied to, expecting to get a create visitor reply + verifyCreateVisitorReply(api::ReturnCode::OK, + int(docs.size()), + getTotalSerializedSize(docs)); + + CPPUNIT_ASSERT_EQUAL(1u, getMatchingDocuments(docs)); + CPPUNIT_ASSERT(!_manager->hasPendingMessageState()); +} + +void +VisitorManagerTest::testResending() +{ + initializeTest(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + cmd->addBucketToBeVisited(document::BucketId(16, 3)); + cmd->setAddress(address); + cmd->setControlDestination("foo/bar"); + _top->sendDown(cmd); + std::vector<document::Document::SP > docs; + std::vector<document::DocumentId> docIds; + + TestVisitorMessageSession& session = getSession(0); + getMessagesAndReply(1, session, docs, docIds, api::ReturnCode::NOT_READY); + + { + session.waitForMessages(2); + + documentapi::DocumentMessage* msg = session.sentMessages[1].get(); + + mbus::Reply::UP reply = msg->createReply(); + + CPPUNIT_ASSERT_EQUAL((uint32_t)documentapi::DocumentProtocol::MESSAGE_VISITORINFO, + session.sentMessages[1]->getType()); + reply->swapState(*session.sentMessages[1]); + reply->setMessage(mbus::Message::UP(session.sentMessages[1].release())); + session.reply(std::move(reply)); + } + + _node->getClock().addSecondsToTime(1); + + { + session.waitForMessages(3); + + documentapi::DocumentMessage* msg = session.sentMessages[2].get(); + + mbus::Reply::UP reply = msg->createReply(); + + reply->swapState(*session.sentMessages[2]); + reply->setMessage(mbus::Message::UP(session.sentMessages[2].release())); + session.reply(std::move(reply)); + } + + // All data has been replied to, expecting to get a create visitor reply + verifyCreateVisitorReply(api::ReturnCode::OK); +} + +void +VisitorManagerTest::testVisitEmptyBucket() +{ + initializeTest(); + addSomeRemoves(true); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + cmd->addBucketToBeVisited(document::BucketId(16, 3)); + + cmd->setAddress(address); + _top->sendDown(cmd); + + // All data has been replied to, expecting to get a create visitor reply + verifyCreateVisitorReply(api::ReturnCode::OK); +} + +void +VisitorManagerTest::testMultiBucketVisit() +{ + initializeTest(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + for (uint32_t i=0; i<10; ++i) { + cmd->addBucketToBeVisited(document::BucketId(16, i)); + } + cmd->setAddress(address); + cmd->setDataDestination("fooclient.0"); + _top->sendDown(cmd); + std::vector<document::Document::SP > docs; + std::vector<document::DocumentId> docIds; + + // Should receive one multioperation message for each bucket + getMessagesAndReply(10, getSession(0), docs, docIds); + + // All data has been replied to, expecting to get a create visitor reply + verifyCreateVisitorReply(api::ReturnCode::OK); + + CPPUNIT_ASSERT_EQUAL(docCount, getMatchingDocuments(docs)); +} + +void +VisitorManagerTest::testNoBuckets() +{ + initializeTest(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + + cmd->setAddress(address); + _top->sendDown(cmd); + + // Should get one reply; a CreateVisitorReply with error since no + // buckets where specified in the CreateVisitorCommand + _top->waitForMessages(1, 60); + const msg_ptr_vector replies = _top->getRepliesOnce(); + CPPUNIT_ASSERT_EQUAL((size_t) 1, replies.size()); + std::shared_ptr<api::CreateVisitorReply> reply( + std::dynamic_pointer_cast<api::CreateVisitorReply>( + replies[0])); + // Verify that cast went ok => it was a CreateVisitorReply message + CPPUNIT_ASSERT(reply.get()); + api::ReturnCode ret(api::ReturnCode::ILLEGAL_PARAMETERS, + "No buckets specified"); + CPPUNIT_ASSERT_EQUAL(ret, reply->getResult()); +} + +void VisitorManagerTest::testVisitPutsAndRemoves() +{ + initializeTest(); + addSomeRemoves(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + cmd->setAddress(address); + cmd->setVisitRemoves(); + for (uint32_t i=0; i<10; ++i) { + cmd->addBucketToBeVisited(document::BucketId(16, i)); + } + _top->sendDown(cmd); + std::vector<document::Document::SP > docs; + std::vector<document::DocumentId> docIds; + + getMessagesAndReply(10, getSession(0), docs, docIds); + + verifyCreateVisitorReply(api::ReturnCode::OK); + + CPPUNIT_ASSERT_EQUAL( + docCount - (docCount + 3) / 4, + getMatchingDocuments(docs)); + + CPPUNIT_ASSERT_EQUAL( + (size_t) (docCount + 3) / 4, + docIds.size()); +} + +void VisitorManagerTest::testVisitWithTimeframeAndSelection() +{ + initializeTest(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", "testvis", + "testdoctype1.headerval < 2")); + cmd->setFromTime(3); + cmd->setToTime(8); + for (uint32_t i=0; i<10; ++i) { + cmd->addBucketToBeVisited(document::BucketId(16, i)); + } + cmd->setAddress(address); + _top->sendDown(cmd); + std::vector<document::Document::SP > docs; + std::vector<document::DocumentId> docIds; + + getMessagesAndReply(2, getSession(0), docs, docIds); + + verifyCreateVisitorReply(api::ReturnCode::OK); + + CPPUNIT_ASSERT_EQUAL((size_t) 2, docs.size()); + std::set<std::string> expected; + expected.insert("userdoc:test:4:http://www.ntnu.no/4.html"); + expected.insert("userdoc:test:5:http://www.ntnu.no/5.html"); + std::set<std::string> actual; + for (uint32_t i=0; i<docs.size(); ++i) { + actual.insert(docs[i]->getId().toString()); + } + CPPUNIT_ASSERT_EQUAL(expected, actual); +} + +void VisitorManagerTest::testVisitWithTimeframeAndBogusSelection() +{ + initializeTest(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", "testvis", + "DocType(testdoctype1---///---) XXX BAD Field(headerval) < 2")); + cmd->setFromTime(3); + cmd->setToTime(8); + for (uint32_t i=0; i<10; ++i) { + cmd->addBucketToBeVisited(document::BucketId(16, i)); + } + cmd->setAddress(address); + + _top->sendDown(cmd); + _top->waitForMessages(1, 60); + const msg_ptr_vector replies = _top->getRepliesOnce(); + CPPUNIT_ASSERT_EQUAL((size_t) 1, replies.size()); + + api::StorageReply* reply = dynamic_cast<api::StorageReply*>( + replies.front().get()); + CPPUNIT_ASSERT(reply); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ILLEGAL_PARAMETERS, + reply->getResult().getResult()); +} + +void +VisitorManagerTest::testVisitorCallbacks() +{ + initializeTest(); + std::ostringstream replydata; + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("TestVisitor", "testvis", "")); + cmd->addBucketToBeVisited(document::BucketId(16, 3)); + cmd->addBucketToBeVisited(document::BucketId(16, 5)); + cmd->setAddress(address); + _top->sendDown(cmd); + + // Wait until we have started the visitor + TestVisitorMessageSession& session = getSession(0); + + for (uint32_t i = 0; i < 6; i++) { + session.waitForMessages(i + 1); + mbus::Reply::UP reply; + { + vespalib::MonitorGuard guard(session.getMonitor()); + + CPPUNIT_ASSERT_EQUAL((uint32_t)documentapi::DocumentProtocol::MESSAGE_MAPVISITOR, session.sentMessages[i]->getType()); + + documentapi::MapVisitorMessage* mapvisitormsg( + static_cast<documentapi::MapVisitorMessage*>(session.sentMessages[i].get())); + + replydata << mapvisitormsg->getData().get("msg"); + + reply = mapvisitormsg->createReply(); + reply->swapState(*session.sentMessages[i]); + reply->setMessage(mbus::Message::UP(session.sentMessages[i].release())); + } + session.reply(std::move(reply)); + } + + // All data has been replied to, expecting to get a create visitor reply + verifyCreateVisitorReply(api::ReturnCode::OK); + + CPPUNIT_ASSERT_SUBSTRING_COUNT(replydata.str(), 1, "Starting visitor"); + CPPUNIT_ASSERT_SUBSTRING_COUNT(replydata.str(), 2, "Handling block of 1 documents"); + CPPUNIT_ASSERT_SUBSTRING_COUNT(replydata.str(), 2, "completedBucket"); + CPPUNIT_ASSERT_SUBSTRING_COUNT(replydata.str(), 1, "completedVisiting"); +} + +void +VisitorManagerTest::testVisitorCleanup() +{ + initializeTest(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + + // Start a bunch of invalid visitors + for (uint32_t i=0; i<10; ++i) { + std::ostringstream ost; + ost << "testvis" << i; + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("InvalidVisitor", ost.str(), "")); + cmd->addBucketToBeVisited(document::BucketId(16, 3)); + cmd->setAddress(address); + cmd->setQueueTimeout(0); + _top->sendDown(cmd); + _top->waitForMessages(i+1, 60); + } + + // Start a bunch of visitors + for (uint32_t i=0; i<10; ++i) { + std::ostringstream ost; + ost << "testvis" << (i + 10); + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", ost.str(), "")); + cmd->addBucketToBeVisited(document::BucketId(16, 3)); + cmd->setAddress(address); + cmd->setQueueTimeout(0); + _top->sendDown(cmd); + } + + + // Should get 14 immediate replies - 10 failures and 4 busy + { + _top->waitForMessages(14, 60); + const msg_ptr_vector replies = _top->getRepliesOnce(); + + int failures = 0; + int busy = 0; + + for (uint32_t i=0; i< 14; ++i) { + std::shared_ptr<api::StorageMessage> msg(replies[i]); + CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); + std::shared_ptr<api::CreateVisitorReply> reply( + std::dynamic_pointer_cast<api::CreateVisitorReply>(msg)); + CPPUNIT_ASSERT(reply.get()); + + if (i < 10) { + if (api::ReturnCode::ILLEGAL_PARAMETERS == reply->getResult().getResult()) { + failures++; + } else { + std::cerr << reply->getResult() << "\n"; + } + } else { + if (api::ReturnCode::BUSY == reply->getResult().getResult()) { + busy++; + } + } + } + + CPPUNIT_ASSERT_EQUAL(10, failures); + CPPUNIT_ASSERT_EQUAL(4, busy); + } + + // Finish a visitor + std::vector<document::Document::SP > docs; + std::vector<document::DocumentId> docIds; + + getMessagesAndReply(1, getSession(0), docs, docIds); + + // Should get a reply for the visitor. + verifyCreateVisitorReply(api::ReturnCode::OK); + + // Fail a visitor + getMessagesAndReply(1, getSession(1), docs, docIds, api::ReturnCode::INTERNAL_FAILURE); + + // Should get a reply for the visitor. + verifyCreateVisitorReply(api::ReturnCode::INTERNAL_FAILURE); + + while (_manager->getActiveVisitorCount() > 2) { + FastOS_Thread::Sleep(10); + } + + // Start a bunch of more visitors + for (uint32_t i=0; i<10; ++i) { + std::ostringstream ost; + ost << "testvis" << (i + 24); + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", ost.str(), "")); + cmd->addBucketToBeVisited(document::BucketId(16, 3)); + cmd->setAddress(address); + cmd->setQueueTimeout(0); + _top->sendDown(cmd); + } + + // Should now get 8 busy. + _top->waitForMessages(8, 60); + const msg_ptr_vector replies = _top->getRepliesOnce(); + CPPUNIT_ASSERT_EQUAL(8, (int)replies.size()); + + for (uint32_t i=0; i< replies.size(); ++i) { + std::shared_ptr<api::StorageMessage> msg(replies[i]); + CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); + std::shared_ptr<api::CreateVisitorReply> reply( + std::dynamic_pointer_cast<api::CreateVisitorReply>(msg)); + CPPUNIT_ASSERT(reply.get()); + + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::BUSY, reply->getResult().getResult()); + } +} + +void +VisitorManagerTest::testAbortOnFailedVisitorInfo() +{ + initializeTest(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + + { + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + cmd->addBucketToBeVisited(document::BucketId(16, 3)); + cmd->setAddress(address); + cmd->setQueueTimeout(0); + _top->sendDown(cmd); + } + + uint32_t visitorRepliesReceived = 0; + uint32_t oki = 0; + uint32_t failed = 0; + + std::vector<document::Document::SP > docs; + std::vector<document::DocumentId> docIds; + + TestVisitorMessageSession& session = getSession(0); + getMessagesAndReply(1, session, docs, docIds, api::ReturnCode::NOT_READY); + + { + session.waitForMessages(2); + + documentapi::DocumentMessage* cmd = session.sentMessages[1].get(); + + mbus::Reply::UP reply = cmd->createReply(); + + CPPUNIT_ASSERT_EQUAL((uint32_t)documentapi::DocumentProtocol::MESSAGE_VISITORINFO, session.sentMessages[1]->getType()); + reply->swapState(*session.sentMessages[1]); + reply->setMessage(mbus::Message::UP(session.sentMessages[1].release())); + reply->addError(mbus::Error(api::ReturnCode::NOT_CONNECTED, "Me no ready")); + session.reply(std::move(reply)); + } + + _top->waitForMessages(1, 60); + const msg_ptr_vector replies = _top->getRepliesOnce(); + for (uint32_t i=0; i< replies.size(); ++i) { + std::shared_ptr<api::StorageMessage> msg(replies[i]); + if (msg->getType() == api::MessageType::VISITOR_CREATE_REPLY) + { + ++visitorRepliesReceived; + std::shared_ptr<api::CreateVisitorReply> reply( + std::dynamic_pointer_cast<api::CreateVisitorReply>(msg)); + CPPUNIT_ASSERT(reply.get()); + if (reply->getResult().success()) { + ++oki; + std::cerr << "\n" << reply->toString(true) << "\n"; + } else { + ++failed; + } + } + } + + std::ostringstream errmsg; + errmsg << "oki " << oki << ", failed " << failed; + + CPPUNIT_ASSERT_EQUAL_MSG(errmsg.str(), 0u, oki); + CPPUNIT_ASSERT_EQUAL_MSG(errmsg.str(), 1u, failed); +} + +void +VisitorManagerTest::testAbortOnFieldPathError() +{ + initializeTest(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + + // Use bogus field path to force error to happen + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", + "testvis", + "testdoctype1.headerval{bogus} == 1234")); + cmd->addBucketToBeVisited(document::BucketId(16, 3)); + cmd->setAddress(address); + cmd->setQueueTimeout(0); + _top->sendDown(cmd); + + verifyCreateVisitorReply(api::ReturnCode::ILLEGAL_PARAMETERS); +} + +void +VisitorManagerTest::testVisitorQueueTimeout() +{ + initializeTest(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + _manager->enforceQueueUsage(); + + { + vespalib::MonitorGuard guard(_manager->getThread(0).getQueueMonitor()); + + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + cmd->addBucketToBeVisited(document::BucketId(16, 3)); + cmd->setAddress(address); + cmd->setQueueTimeout(1); + cmd->setTimeout(100 * 1000 * 1000); + _top->sendDown(cmd); + + _node->getClock().addSecondsToTime(1000); + } + + // Don't answer any messages. Make sure we timeout anyways. + uint32_t visitorRepliesReceived = 0; + + _top->waitForMessages(1, 60); + const msg_ptr_vector replies = _top->getRepliesOnce(); + std::shared_ptr<api::StorageMessage> msg(replies[0]); + + CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); + ++visitorRepliesReceived; + std::shared_ptr<api::CreateVisitorReply> reply( + std::dynamic_pointer_cast<api::CreateVisitorReply>(msg)); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::BUSY, + "Visitor timed out in visitor queue"), + reply->getResult()); +} + +void +VisitorManagerTest::testVisitorProcessingTimeout() +{ + initializeTest(); + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", "testvis", "")); + cmd->addBucketToBeVisited(document::BucketId(16, 3)); + cmd->setAddress(address); + cmd->setQueueTimeout(0); + cmd->setTimeout(100); + _top->sendDown(cmd); + + // Wait for Put before increasing the clock + TestVisitorMessageSession& session = getSession(0); + session.waitForMessages(1); + + _node->getClock().addSecondsToTime(1000); + + // Don't answer any messages. Make sure we timeout anyways. + uint32_t visitorRepliesReceived = 0; + + _top->waitForMessages(1, 60); + const msg_ptr_vector replies = _top->getRepliesOnce(); + std::shared_ptr<api::StorageMessage> msg(replies[0]); + + CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); + ++visitorRepliesReceived; + std::shared_ptr<api::CreateVisitorReply> reply( + std::dynamic_pointer_cast<api::CreateVisitorReply>(msg)); + CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ABORTED, + reply->getResult().getResult()); +} + +namespace { + uint32_t nextVisitor = 0; + + api::StorageMessage::Id + sendCreateVisitor(uint32_t timeout, DummyStorageLink& top, uint8_t priority = 127) { + std::ostringstream ost; + ost << "testvis" << ++nextVisitor; + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand("DumpVisitor", ost.str(), "")); + cmd->addBucketToBeVisited(document::BucketId(16, 3)); + cmd->setAddress(address); + cmd->setQueueTimeout(timeout); + cmd->setPriority(priority); + top.sendDown(cmd); + return cmd->getMsgId(); + } +} + +void +VisitorManagerTest::testPrioritizedVisitorQueing() +{ + framework::HttpUrlPath path("?verbose=true&allvisitors=true"); + initializeTest(); + + _manager->setMaxConcurrentVisitors(4); + _manager->setMaxVisitorQueueSize(4); + + api::StorageMessage::Id ids[10] = { 0 }; + + // First 4 should just start.. + for (uint32_t i = 0; i < 4; ++i) { + ids[i] = sendCreateVisitor(i, *_top, i); + } + + // Next ones should be queued - (Better not finish before we get here) + // Submit with higher priorities + for (uint32_t i = 0; i < 4; ++i) { + ids[i + 4] = sendCreateVisitor(1000, *_top, 100 - i); + } + + // Queue is now full with a pri 100 visitor at its end + // Send a lower pri visitor that will be busy-returned immediately + ids[8] = sendCreateVisitor(1000, *_top, 130); + + CPPUNIT_ASSERT_EQUAL(ids[8], verifyCreateVisitorReply(api::ReturnCode::BUSY)); + + // Send a higher pri visitor that will take the place of pri 100 visitor + ids[9] = sendCreateVisitor(1000, *_top, 60); + + CPPUNIT_ASSERT_EQUAL(ids[4], verifyCreateVisitorReply(api::ReturnCode::BUSY)); + + // Finish the first visitor + std::vector<document::Document::SP > docs; + std::vector<document::DocumentId> docIds; + getMessagesAndReply(1, getSession(0), docs, docIds, api::ReturnCode::OK, + documentapi::Priority::PRI_HIGHEST); + verifyCreateVisitorReply(api::ReturnCode::OK); + + // We should now start the highest priority visitor. + getMessagesAndReply(1, getSession(4), docs, docIds, api::ReturnCode::OK, + documentapi::Priority::PRI_VERY_HIGH); + CPPUNIT_ASSERT_EQUAL(ids[9], verifyCreateVisitorReply(api::ReturnCode::OK)); +} + +void +VisitorManagerTest::testPrioritizedMaxConcurrentVisitors() { + framework::HttpUrlPath path("?verbose=true&allvisitors=true"); + initializeTest(); + + api::StorageMessage::Id ids[17] = { 0 }; + + // Number of concurrent visitors is in [4, 8], depending on priority + // Max concurrent: + // [0, 1): 4 + // [1, 64): 3 + // [64, 128): 2 + // [128, 192): 1 + // [192, 256): 0 + _manager->setMaxConcurrentVisitors(4, 4); + _manager->setMaxVisitorQueueSize(6); + + // First 4 should just start.. + for (uint32_t i = 0; i < 4; ++i) { + ids[i] = sendCreateVisitor(i, *_top, i); + } + + // Low pri messages; get put into queue + for (uint32_t i = 0; i < 6; ++i) { + ids[i + 4] = sendCreateVisitor(1000, *_top, 203 - i); + } + + // Higher pri message: fits happily into 1 extra concurrent slot + ids[10] = sendCreateVisitor(1000, *_top, 190); + + // Should punch pri203 msg out of the queue -> busy + ids[11] = sendCreateVisitor(1000, *_top, 197); + + CPPUNIT_ASSERT_EQUAL(ids[4], verifyCreateVisitorReply(api::ReturnCode::BUSY)); + + // No concurrency slots left for this message -> busy + ids[12] = sendCreateVisitor(1000, *_top, 204); + + CPPUNIT_ASSERT_EQUAL(ids[12], verifyCreateVisitorReply(api::ReturnCode::BUSY)); + + // Gets a concurrent slot + ids[13] = sendCreateVisitor(1000, *_top, 80); + + // Kicks pri 202 out of the queue -> busy + ids[14] = sendCreateVisitor(1000, *_top, 79); + + CPPUNIT_ASSERT_EQUAL(ids[5], verifyCreateVisitorReply(api::ReturnCode::BUSY)); + + // Gets a concurrent slot + ids[15] = sendCreateVisitor(1000, *_top, 63); + + // Very Important Visitor(tm) gets a concurrent slot + ids[16] = sendCreateVisitor(1000, *_top, 0); + + std::vector<document::Document::SP > docs; + std::vector<document::DocumentId> docIds; + + std::set<uint64_t> finishedVisitors; + + // Verify that the correct visitors are running. + for (int i = 0; i < 8; i++) { + documentapi::Priority::Value priority = + documentapi::Priority::PRI_HIGHEST; // ids 0-3,16 + if (i == 4) { + priority = documentapi::Priority::PRI_VERY_LOW; // ids 10 + } else if (i == 5) { + priority = documentapi::Priority::PRI_HIGH_2; // ids 13 + } else if (i == 6) { + priority = documentapi::Priority::PRI_HIGH_1; // ids 15 + } + getMessagesAndReply(1, getSession(i), docs, docIds, api::ReturnCode::OK, + priority); + finishedVisitors.insert(verifyCreateVisitorReply(api::ReturnCode::OK)); + } + + for (int i = 0; i < 4; i++) { + CPPUNIT_ASSERT(finishedVisitors.find(ids[i]) != finishedVisitors.end()); + } + + CPPUNIT_ASSERT(finishedVisitors.find(ids[10]) != finishedVisitors.end()); + CPPUNIT_ASSERT(finishedVisitors.find(ids[13]) != finishedVisitors.end()); + CPPUNIT_ASSERT(finishedVisitors.find(ids[15]) != finishedVisitors.end()); + CPPUNIT_ASSERT(finishedVisitors.find(ids[16]) != finishedVisitors.end()); + + finishedVisitors.clear(); + + for (int i = 8; i < 14; i++) { + documentapi::Priority::Value priority = + documentapi::Priority::PRI_LOWEST; // ids 6-9,11 + if (i == 8) { + priority = documentapi::Priority::PRI_HIGH_2; // ids 14 + } + getMessagesAndReply(1, getSession(i), docs, docIds, api::ReturnCode::OK, + priority); + uint64_t msgId = verifyCreateVisitorReply(api::ReturnCode::OK); + finishedVisitors.insert(msgId); + } + + for (int i = 6; i < 10; i++) { + CPPUNIT_ASSERT(finishedVisitors.find(ids[i]) != finishedVisitors.end()); + } + + CPPUNIT_ASSERT(finishedVisitors.find(ids[11]) != finishedVisitors.end()); + CPPUNIT_ASSERT(finishedVisitors.find(ids[14]) != finishedVisitors.end()); +} + +void +VisitorManagerTest::testVisitorQueingZeroQueueSize() { + framework::HttpUrlPath path("?verbose=true&allvisitors=true"); + initializeTest(); + + _manager->setMaxConcurrentVisitors(4); + _manager->setMaxVisitorQueueSize(0); + + // First 4 should just start.. + for (uint32_t i = 0; i < 4; ++i) { + sendCreateVisitor(i, *_top, i); + } + // Queue size is zero, all visitors will be busy-returned + for (uint32_t i = 0; i < 5; ++i) { + sendCreateVisitor(1000, *_top, 100 - i); + verifyCreateVisitorReply(api::ReturnCode::BUSY); + } +} + +void +VisitorManagerTest::testStatusPage() { + framework::HttpUrlPath path("?verbose=true&allvisitors=true"); + initializeTest(); + + _manager->setMaxConcurrentVisitors(1, 1); + _manager->setMaxVisitorQueueSize(6); + // 1 running, 1 queued + sendCreateVisitor(1000000, *_top, 1); + sendCreateVisitor(1000000, *_top, 128); + + TestVisitorMessageSession& session = getSession(0); + session.waitForMessages(1); + + std::ostringstream ss; + static_cast<framework::HtmlStatusReporter&>(*_manager).reportHtmlStatus(ss, path); + + std::string str(ss.str()); + CPPUNIT_ASSERT(str.find("Currently running visitors") != std::string::npos); + // Should be propagated to visitor thread + CPPUNIT_ASSERT(str.find("Running 1 visitors") != std::string::npos); // 1 active + CPPUNIT_ASSERT(str.find("waiting visitors 1") != std::string::npos); // 1 queued + CPPUNIT_ASSERT(str.find("Visitor thread 0") != std::string::npos); + CPPUNIT_ASSERT(str.find("Disconnected visitor timeout") != std::string::npos); // verbose per thread + CPPUNIT_ASSERT(str.find("Message #1 <b>putdocumentmessage</b>") != std::string::npos); // 1 active +} + +} diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp new file mode 100644 index 00000000000..aed08a676b8 --- /dev/null +++ b/storage/src/tests/visiting/visitortest.cpp @@ -0,0 +1,1023 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/document/datatype/datatype.h> +#include <vespa/document/fieldvalue/intfieldvalue.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/document/fieldvalue/rawfieldvalue.h> +#include <vespa/log/log.h> +#include <vespa/storageapi/message/datagram.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/visitor.h> +#include <vespa/storage/persistence/filestorage/filestormanager.h> +#include <vespa/storage/visiting/visitormanager.h> +#include <tests/common/testhelper.h> +#include <tests/common/teststorageapp.h> +#include <tests/common/dummystoragelink.h> +#include <tests/storageserver/testvisitormessagesession.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vdslib/container/visitorordering.h> +#include <vespa/documentapi/messagebus/messages/multioperationmessage.h> +#include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> +#include <vector> +#include <thread> +#include <chrono> + +LOG_SETUP(".visitortest"); + +using namespace std::chrono_literals; + +namespace storage { + +namespace { + +using msg_ptr_vector = std::vector<api::StorageMessage::SP>; + +struct TestParams +{ + TestParams& iteratorsPerBucket(uint32_t n) { + _iteratorsPerBucket = n; + return *this; + } + TestParams& maxVisitorMemoryUsage(uint32_t bytes) { + _maxVisitorMemoryUsage = bytes; + return *this; + } + TestParams& parallelBuckets(uint32_t n) { + _parallelBuckets = n; + return *this; + } + TestParams& autoReplyError(const mbus::Error& error) { + _autoReplyError = error; + return *this; + } + + uint32_t _iteratorsPerBucket {1}; + uint32_t _maxVisitorMemoryUsage {UINT32_MAX}; + uint32_t _parallelBuckets {1}; + mbus::Error _autoReplyError; +}; + +} + +class VisitorTest : public CppUnit::TestFixture +{ +private: + CPPUNIT_TEST_SUITE(VisitorTest); + CPPUNIT_TEST(testNormalUsage); + CPPUNIT_TEST(testFailedCreateIterator); + CPPUNIT_TEST(testFailedGetIter); + CPPUNIT_TEST(testMultipleFailedGetIter); + CPPUNIT_TEST(testDocumentAPIClientError); + CPPUNIT_TEST(testNoDocumentAPIResendingForFailedVisitor); + CPPUNIT_TEST(testIteratorCreatedForFailedVisitor); + CPPUNIT_TEST(testFailedDocumentAPISend); + CPPUNIT_TEST(testNoVisitorNotificationForTransientFailures); + CPPUNIT_TEST(testNotificationSentIfTransientErrorRetriedManyTimes); + CPPUNIT_TEST(testNoMbusTracingIfTraceLevelIsZero); + CPPUNIT_TEST(testReplyContainsTraceIfTraceLevelAboveZero); + CPPUNIT_TEST(testNoMoreIteratorsSentWhileMemoryUsedAboveLimit); + CPPUNIT_TEST(testDumpVisitorInvokesStrongReadConsistencyIteration); + CPPUNIT_TEST(testTestVisitorInvokesWeakReadConsistencyIteration); + CPPUNIT_TEST_SUITE_END(); + + static uint32_t docCount; + std::vector<document::Document::SP > _documents; + std::unique_ptr<TestVisitorMessageSessionFactory> _messageSessionFactory; + std::unique_ptr<TestServiceLayerApp> _node; + std::unique_ptr<DummyStorageLink> _top; + DummyStorageLink* _bottom; + VisitorManager* _manager; + +public: + VisitorTest() : _node() {} + + void testNormalUsage(); + void testFailedCreateIterator(); + void testFailedGetIter(); + void testMultipleFailedGetIter(); + void testDocumentAPIClientError(); + void testNoDocumentAPIResendingForFailedVisitor(); + void testIteratorCreatedForFailedVisitor(); + void testFailedDocumentAPISend(); + void testNoVisitorNotificationForTransientFailures(); + void testNotificationSentIfTransientErrorRetriedManyTimes(); + void testNoMbusTracingIfTraceLevelIsZero(); + void testReplyContainsTraceIfTraceLevelAboveZero(); + void testNoMoreIteratorsSentWhileMemoryUsedAboveLimit(); + void testDumpVisitorInvokesStrongReadConsistencyIteration(); + void testTestVisitorInvokesWeakReadConsistencyIteration(); + // TODO: + void testVisitMultipleBuckets() {} + + // Not using setUp since can't throw exception out of it. + void initializeTest(const TestParams& params = TestParams()); + + struct VisitorOptions { + std::string visitorType{"dumpvisitor"}; + + VisitorOptions() {} + + VisitorOptions& withVisitorType(vespalib::stringref type) { + visitorType = type; + return *this; + } + }; + + std::shared_ptr<api::CreateVisitorCommand> makeCreateVisitor( + const VisitorOptions& options = VisitorOptions()); + void tearDown(); + bool waitUntilNoActiveVisitors(); + TestVisitorMessageSession& getSession(uint32_t n); + uint64_t verifyCreateVisitorReply( + api::ReturnCode::Result expectedResult, + int checkStatsDocsVisited = -1, + int checkStatsBytesVisited = -1); + void getMessagesAndReply( + int expectedCount, + TestVisitorMessageSession& session, + std::vector<document::Document::SP >& docs, + std::vector<document::DocumentId>& docIds, + std::vector<std::string>& infoMessages, + api::ReturnCode::Result returnCode = api::ReturnCode::OK); + uint32_t getMatchingDocuments(std::vector<document::Document::SP >& docs); + +private: + void doTestVisitorInstanceHasConsistencyLevel( + vespalib::stringref visitorType, + spi::ReadConsistency expectedConsistency); + + template <typename T> + std::vector<std::shared_ptr<T> > + fetchMultipleCommands(DummyStorageLink& link, size_t count); + + template <typename T> + std::shared_ptr<T> + fetchSingleCommand(DummyStorageLink& link); + + void sendGetIterReply(GetIterCommand& cmd, + const api::ReturnCode& result = + api::ReturnCode(api::ReturnCode::OK), + uint32_t maxDocuments = 0, + bool overrideCompleted = false); + void sendCreateIteratorReply(uint64_t iteratorId = 1234); + std::shared_ptr<api::CreateVisitorReply> doCompleteVisitingSession( + const std::shared_ptr<api::CreateVisitorCommand>& cmd); + + void sendInitialCreateVisitorAndGetIterRound(); + + int64_t getFailedVisitorDestinationReplyCount() const { + // There's no metric manager attached to these tests, so even if the + // test should magically freeze here for 5+ minutes, nothing should + // come in and wipe our accumulated failure metrics. + // Only 1 visitor thread running, so we know it has the metrics. + const auto& metrics = _manager->getThread(0).getMetrics(); + auto loadType = documentapi::LoadType::DEFAULT; + return metrics.visitorDestinationFailureReplies[loadType].getCount(); + } +}; + +uint32_t VisitorTest::docCount = 10; + +CPPUNIT_TEST_SUITE_REGISTRATION(VisitorTest); + +void +VisitorTest::initializeTest(const TestParams& params) +{ + LOG(debug, "Initializing test"); + vdstestlib::DirConfig config(getStandardConfig(true)); + config.getConfig("stor-visitor").set("visitorthreads", "1"); + config.getConfig("stor-visitor").set( + "iterators_per_bucket", + std::to_string(params._iteratorsPerBucket)); + config.getConfig("stor-visitor").set( + "defaultparalleliterators", + std::to_string(params._parallelBuckets)); + config.getConfig("stor-visitor").set( + "visitor_memory_usage_limit", + std::to_string(params._maxVisitorMemoryUsage)); + + system("chmod 755 vdsroot 2>/dev/null"); + system("rm -rf vdsroot* 2>/dev/null"); + assert(system("mkdir -p vdsroot/disks/d0") == 0); + assert(system("mkdir -p vdsroot/disks/d1") == 0); + + try { + _messageSessionFactory.reset( + new TestVisitorMessageSessionFactory(config.getConfigId())); + if (params._autoReplyError.getCode() != mbus::ErrorCode::NONE) { + _messageSessionFactory->_autoReplyError = params._autoReplyError; + _messageSessionFactory->_createAutoReplyVisitorSessions = true; + } + _node.reset(new TestServiceLayerApp(config.getConfigId())); + _top.reset(new DummyStorageLink()); + _top->push_back(std::unique_ptr<StorageLink>(_manager + = new VisitorManager( + config.getConfigId(), + _node->getComponentRegister(), *_messageSessionFactory))); + _bottom = new DummyStorageLink(); + _top->push_back(std::unique_ptr<StorageLink>(_bottom)); + _manager->setTimeBetweenTicks(10); + _top->open(); + } catch (config::InvalidConfigException& e) { + fprintf(stderr, "%s\n", e.what()); + } + std::string content( + "To be, or not to be: that is the question:\n" + "Whether 'tis nobler in the mind to suffer\n" + "The slings and arrows of outrageous fortune,\n" + "Or to take arms against a sea of troubles,\n" + "And by opposing end them? To die: to sleep;\n" + "No more; and by a sleep to say we end\n" + "The heart-ache and the thousand natural shocks\n" + "That flesh is heir to, 'tis a consummation\n" + "Devoutly to be wish'd. To die, to sleep;\n" + "To sleep: perchance to dream: ay, there's the rub;\n" + "For in that sleep of death what dreams may come\n" + "When we have shuffled off this mortal coil,\n" + "Must give us pause: there's the respect\n" + "That makes calamity of so long life;\n" + "For who would bear the whips and scorns of time,\n" + "The oppressor's wrong, the proud man's contumely,\n" + "The pangs of despised love, the law's delay,\n" + "The insolence of office and the spurns\n" + "That patient merit of the unworthy takes,\n" + "When he himself might his quietus make\n" + "With a bare bodkin? who would fardels bear,\n" + "To grunt and sweat under a weary life,\n" + "But that the dread of something after death,\n" + "The undiscover'd country from whose bourn\n" + "No traveller returns, puzzles the will\n" + "And makes us rather bear those ills we have\n" + "Than fly to others that we know not of?\n" + "Thus conscience does make cowards of us all;\n" + "And thus the native hue of resolution\n" + "Is sicklied o'er with the pale cast of thought,\n" + "And enterprises of great pith and moment\n" + "With this regard their currents turn awry,\n" + "And lose the name of action. - Soft you now!\n" + "The fair Ophelia! Nymph, in thy orisons\n" + "Be all my sins remember'd.\n"); + _documents.clear(); + for (uint32_t i=0; i<docCount; ++i) { + std::ostringstream uri; + uri << "userdoc:test:" << i % 10 << ":http://www.ntnu.no/" + << i << ".html"; + + _documents.push_back(document::Document::SP( + _node->getTestDocMan().createDocument(content, uri.str()))); + const document::DocumentType& type(_documents.back()->getType()); + _documents.back()->setValue(type.getField("headerval"), + document::IntFieldValue(i % 4)); + } + LOG(debug, "Done initializing test"); +} + +void +VisitorTest::tearDown() +{ + if (_top.get() != 0) { + _top->close(); + _top->flush(); + _top.reset(0); + } + _node.reset(0); + _messageSessionFactory.reset(0); + _manager = 0; +} + +bool +VisitorTest::waitUntilNoActiveVisitors() +{ + int i = 0; + for (; i < 1000; ++i) { + if (_manager->getActiveVisitorCount() == 0) { + return true; + } + std::this_thread::sleep_for(10ms); + } + return false; +} + +TestVisitorMessageSession& +VisitorTest::getSession(uint32_t n) +{ + // Wait until we have started the visitor + const std::vector<TestVisitorMessageSession*>& sessions( + _messageSessionFactory->_visitorSessions); + framework::defaultimplementation::RealClock clock; + framework::MilliSecTime endTime( + clock.getTimeInMillis() + framework::MilliSecTime(30 * 1000)); + while (true) { + { + vespalib::LockGuard lock(_messageSessionFactory->_accessLock); + if (sessions.size() > n) { + return *sessions[n]; + } + } + if (clock.getTimeInMillis() > endTime) { + throw vespalib::IllegalStateException( + "Timed out waiting for visitor session", VESPA_STRLOC); + } + std::this_thread::sleep_for(10ms); + } + throw std::logic_error("unreachable"); +} + +void +VisitorTest::getMessagesAndReply( + int expectedCount, + TestVisitorMessageSession& session, + std::vector<document::Document::SP >& docs, + std::vector<document::DocumentId>& docIds, + std::vector<std::string>& infoMessages, + api::ReturnCode::Result result) +{ + for (int i = 0; i < expectedCount; i++) { + session.waitForMessages(1); + mbus::Reply::UP reply; + { + vespalib::MonitorGuard guard(session.getMonitor()); + CPPUNIT_ASSERT(!session.sentMessages.empty()); + vespalib::LinkedPtr<documentapi::DocumentMessage> msg( + session.sentMessages.front()); + CPPUNIT_ASSERT(msg->getPriority() < 16); + + switch (msg->getType()) { + case documentapi::DocumentProtocol::MESSAGE_PUTDOCUMENT: + docs.push_back( + static_cast<documentapi::PutDocumentMessage&>(*msg) + .getDocument()); + break; + case documentapi::DocumentProtocol::MESSAGE_REMOVEDOCUMENT: + docIds.push_back( + static_cast<documentapi::RemoveDocumentMessage&>(*msg) + .getDocumentId()); + break; + case documentapi::DocumentProtocol::MESSAGE_VISITORINFO: + infoMessages.push_back( + static_cast<documentapi::VisitorInfoMessage&>(*msg) + .getErrorMessage()); + break; + default: + break; + } + + reply = msg->createReply(); + reply->swapState(*msg); + + session.sentMessages.pop_front(); // Release linked ptr ref. + reply->setMessage(mbus::Message::UP(msg.release())); + + if (result != api::ReturnCode::OK) { + reply->addError(mbus::Error(result, "Generic error")); + } + } + session.reply(std::move(reply)); + } +} + +uint64_t +VisitorTest::verifyCreateVisitorReply( + api::ReturnCode::Result expectedResult, + int checkStatsDocsVisited, + int checkStatsBytesVisited) +{ + _top->waitForMessages(1, 60); + const msg_ptr_vector replies = _top->getRepliesOnce(); + CPPUNIT_ASSERT_EQUAL(1, (int)replies.size()); + + std::shared_ptr<api::StorageMessage> msg(replies[0]); + + CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, msg->getType()); + + std::shared_ptr<api::CreateVisitorReply> reply( + std::dynamic_pointer_cast<api::CreateVisitorReply>(msg)); + CPPUNIT_ASSERT(reply.get()); + CPPUNIT_ASSERT_EQUAL(expectedResult, reply->getResult().getResult()); + + if (checkStatsDocsVisited >= 0) { + CPPUNIT_ASSERT_EQUAL(checkStatsDocsVisited, + int(reply->getVisitorStatistics().getDocumentsVisited())); + } + if (checkStatsBytesVisited >= 0) { + CPPUNIT_ASSERT_EQUAL(checkStatsBytesVisited, + int(reply->getVisitorStatistics().getBytesVisited())); + } + + return reply->getMsgId(); +} + +uint32_t +VisitorTest::getMatchingDocuments(std::vector<document::Document::SP >& docs) { + uint32_t equalCount = 0; + for (uint32_t i=0; i<docs.size(); ++i) { + for (uint32_t j=0; j<_documents.size(); ++j) { + if (*docs[i] == *_documents[j] && + docs[i]->getId() == _documents[j]->getId()) + { + equalCount++; + } + } + } + + return equalCount; +} + +void +VisitorTest::sendGetIterReply(GetIterCommand& cmd, + const api::ReturnCode& result, + uint32_t maxDocuments, + bool overrideCompleted) +{ + GetIterReply::SP reply(new GetIterReply(cmd)); + if (result.failed()) { + reply->setResult(result); + _bottom->sendUp(reply); + return; + } + assert(maxDocuments < _documents.size()); + size_t documentCount = maxDocuments != 0 ? maxDocuments : _documents.size(); + for (size_t i = 0; i < documentCount; ++i) { + reply->getEntries().push_back( + spi::DocEntry::LP( + new spi::DocEntry( + spi::Timestamp(1000 + i), + spi::NONE, + document::Document::UP(_documents[i]->clone())))); + } + if (documentCount == _documents.size() || overrideCompleted) { + reply->setCompleted(); + } + _bottom->sendUp(reply); +} + +template <typename T> +std::vector<std::shared_ptr<T> > +VisitorTest::fetchMultipleCommands(DummyStorageLink& link, size_t count) +{ + link.waitForMessages(count, 60); + std::vector<api::StorageMessage::SP> msgs(link.getCommandsOnce()); + std::vector<std::shared_ptr<T> > fetched; + if (msgs.size() != count) { + std::ostringstream oss; + oss << "Expected " + << count + << " messages, got " + << msgs.size() + << ":\n"; + for (size_t i = 0; i < msgs.size(); ++i) { + oss << i << ": " << *msgs[i] << "\n"; + } + CPPUNIT_FAIL(oss.str()); + } + for (size_t i = 0; i < count; ++i) { + std::shared_ptr<T> ret(std::dynamic_pointer_cast<T>(msgs[i])); + if (!ret) { + std::ostringstream oss; + oss << "Expected message of type " + << typeid(T).name() + << ", but got " + << msgs[0]->toString(); + CPPUNIT_FAIL(oss.str()); + } + fetched.push_back(ret); + } + return fetched; +} + +template <typename T> +std::shared_ptr<T> +VisitorTest::fetchSingleCommand(DummyStorageLink& link) +{ + std::vector<std::shared_ptr<T> > ret( + fetchMultipleCommands<T>(link, 1)); + return ret[0]; +} + +std::shared_ptr<api::CreateVisitorCommand> +VisitorTest::makeCreateVisitor(const VisitorOptions& options) +{ + api::StorageMessageAddress address("storage", lib::NodeType::STORAGE, 0); + std::shared_ptr<api::CreateVisitorCommand> cmd( + new api::CreateVisitorCommand(options.visitorType, "testvis", "")); + cmd->addBucketToBeVisited(document::BucketId(16, 3)); + cmd->setAddress(address); + cmd->setMaximumPendingReplyCount(UINT32_MAX); + cmd->setControlDestination("foo/bar"); + return cmd; +} + +void +VisitorTest::sendCreateIteratorReply(uint64_t iteratorId) +{ + CreateIteratorCommand::SP createCmd( + fetchSingleCommand<CreateIteratorCommand>(*_bottom)); + spi::IteratorId id(iteratorId); + api::StorageReply::SP reply( + new CreateIteratorReply(*createCmd, id)); + _bottom->sendUp(reply); +} + +void +VisitorTest::testNormalUsage() +{ + initializeTest(); + std::shared_ptr<api::CreateVisitorCommand> cmd( + makeCreateVisitor()); + _top->sendDown(cmd); + + CreateIteratorCommand::SP createCmd( + fetchSingleCommand<CreateIteratorCommand>(*_bottom)); + CPPUNIT_ASSERT_EQUAL(uint8_t(0), createCmd->getPriority()); // Highest pri + spi::IteratorId id(1234); + api::StorageReply::SP reply( + new CreateIteratorReply(*createCmd, id)); + _bottom->sendUp(reply); + + GetIterCommand::SP getIterCmd( + fetchSingleCommand<GetIterCommand>(*_bottom)); + CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1234), + getIterCmd->getIteratorId()); + + sendGetIterReply(*getIterCmd); + + std::vector<document::Document::SP> docs; + std::vector<document::DocumentId> docIds; + std::vector<std::string> infoMessages; + getMessagesAndReply(_documents.size(), getSession(0), docs, docIds, infoMessages); + CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + CPPUNIT_ASSERT_EQUAL(size_t(0), docIds.size()); + + DestroyIteratorCommand::SP destroyIterCmd( + fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + + verifyCreateVisitorReply(api::ReturnCode::OK); + CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); + CPPUNIT_ASSERT_EQUAL(0L, getFailedVisitorDestinationReplyCount()); +} + +void +VisitorTest::testFailedCreateIterator() +{ + initializeTest(); + std::shared_ptr<api::CreateVisitorCommand> cmd( + makeCreateVisitor()); + cmd->addBucketToBeVisited(document::BucketId(16, 4)); + _top->sendDown(cmd); + + CreateIteratorCommand::SP createCmd( + fetchSingleCommand<CreateIteratorCommand>(*_bottom)); + spi::IteratorId id(0); + api::StorageReply::SP reply( + new CreateIteratorReply(*createCmd, id)); + reply->setResult(api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE)); + _bottom->sendUp(reply); + + verifyCreateVisitorReply(api::ReturnCode::INTERNAL_FAILURE, 0, 0); + CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); +} + +void +VisitorTest::testFailedGetIter() +{ + initializeTest(); + std::shared_ptr<api::CreateVisitorCommand> cmd( + makeCreateVisitor()); + _top->sendDown(cmd); + sendCreateIteratorReply(); + + GetIterCommand::SP getIterCmd( + fetchSingleCommand<GetIterCommand>(*_bottom)); + CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1234), + getIterCmd->getIteratorId()); + + sendGetIterReply(*getIterCmd, + api::ReturnCode(api::ReturnCode::BUCKET_NOT_FOUND)); + + DestroyIteratorCommand::SP destroyIterCmd( + fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + + verifyCreateVisitorReply(api::ReturnCode::BUCKET_NOT_FOUND, 0, 0); + CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); +} + +void +VisitorTest::testMultipleFailedGetIter() +{ + initializeTest(TestParams().iteratorsPerBucket(2)); + std::shared_ptr<api::CreateVisitorCommand> cmd( + makeCreateVisitor()); + _top->sendDown(cmd); + sendCreateIteratorReply(); + + std::vector<GetIterCommand::SP> getIterCmds( + fetchMultipleCommands<GetIterCommand>(*_bottom, 2)); + + sendGetIterReply(*getIterCmds[0], + api::ReturnCode(api::ReturnCode::BUCKET_NOT_FOUND)); + + // Wait for an "appropriate" amount of time so that wrongful logic + // will send a DestroyIteratorCommand before all pending GetIters + // have been replied to. + std::this_thread::sleep_for(100ms); + + CPPUNIT_ASSERT_EQUAL(size_t(0), _bottom->getNumCommands()); + + sendGetIterReply(*getIterCmds[1], + api::ReturnCode(api::ReturnCode::BUCKET_DELETED)); + + DestroyIteratorCommand::SP destroyIterCmd( + fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + + verifyCreateVisitorReply(api::ReturnCode::BUCKET_DELETED, 0, 0); + CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); +} + +void +VisitorTest::testDocumentAPIClientError() +{ + initializeTest(); + std::shared_ptr<api::CreateVisitorCommand> cmd( + makeCreateVisitor()); + _top->sendDown(cmd); + sendCreateIteratorReply(); + + { + GetIterCommand::SP getIterCmd( + fetchSingleCommand<GetIterCommand>(*_bottom)); + CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1234), + getIterCmd->getIteratorId()); + + sendGetIterReply(*getIterCmd, api::ReturnCode(api::ReturnCode::OK), 1); + } + + std::vector<document::Document::SP> docs; + std::vector<document::DocumentId> docIds; + std::vector<std::string> infoMessages; + getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, + api::ReturnCode::INTERNAL_FAILURE); + // INTERNAL_FAILURE is critical, so no visitor info sent + CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + + std::this_thread::sleep_for(100ms); + + { + GetIterCommand::SP getIterCmd( + fetchSingleCommand<GetIterCommand>(*_bottom)); + CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1234), + getIterCmd->getIteratorId()); + + sendGetIterReply(*getIterCmd); + } + + DestroyIteratorCommand::SP destroyIterCmd( + fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + + verifyCreateVisitorReply(api::ReturnCode::INTERNAL_FAILURE); + CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); +} + +void +VisitorTest::testNoDocumentAPIResendingForFailedVisitor() +{ + initializeTest(); + std::shared_ptr<api::CreateVisitorCommand> cmd( + makeCreateVisitor()); + _top->sendDown(cmd); + sendCreateIteratorReply(); + + { + GetIterCommand::SP getIterCmd( + fetchSingleCommand<GetIterCommand>(*_bottom)); + CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1234), + getIterCmd->getIteratorId()); + + sendGetIterReply(*getIterCmd, api::ReturnCode(api::ReturnCode::OK), 2, true); + } + + std::vector<document::Document::SP> docs; + std::vector<document::DocumentId> docIds; + std::vector<std::string> infoMessages; + // Use non-critical result. Visitor info message should be received + // after we send a NOT_CONNECTED reply. Failing this message as well + // should cause the entire visitor to fail. + getMessagesAndReply(3, getSession(0), docs, docIds, infoMessages, + api::ReturnCode::NOT_CONNECTED); + CPPUNIT_ASSERT_EQUAL(size_t(1), infoMessages.size()); + CPPUNIT_ASSERT_EQUAL( + std::string("[From content node 0] NOT_CONNECTED: Generic error"), + infoMessages[0]); + + DestroyIteratorCommand::SP destroyIterCmd( + fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + + verifyCreateVisitorReply(api::ReturnCode::NOT_CONNECTED); + CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); + CPPUNIT_ASSERT_EQUAL(3L, getFailedVisitorDestinationReplyCount()); +} + +void +VisitorTest::testIteratorCreatedForFailedVisitor() +{ + initializeTest(TestParams().iteratorsPerBucket(1).parallelBuckets(2)); + std::shared_ptr<api::CreateVisitorCommand> cmd( + makeCreateVisitor()); + cmd->addBucketToBeVisited(document::BucketId(16, 4)); + _top->sendDown(cmd); + + std::vector<CreateIteratorCommand::SP> createCmds( + fetchMultipleCommands<CreateIteratorCommand>(*_bottom, 2)); + { + spi::IteratorId id(0); + api::StorageReply::SP reply( + new CreateIteratorReply(*createCmds[0], id)); + reply->setResult(api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE)); + _bottom->sendUp(reply); + } + { + spi::IteratorId id(1234); + api::StorageReply::SP reply( + new CreateIteratorReply(*createCmds[1], id)); + _bottom->sendUp(reply); + } + // Want to immediately receive destroyiterator for newly created + // iterator, since we cannot use it anyway when the visitor has failed. + DestroyIteratorCommand::SP destroyCmd( + fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + + verifyCreateVisitorReply(api::ReturnCode::INTERNAL_FAILURE, 0, 0); + CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); +} + +/** + * Test that if a visitor fails to send a document API message outright + * (i.e. a case where it will never get a reply), the session is failed + * and the visitor terminates cleanly without counting the failed message + * as pending. + */ +void +VisitorTest::testFailedDocumentAPISend() +{ + initializeTest(TestParams().autoReplyError( + mbus::Error(mbus::ErrorCode::HANDSHAKE_FAILED, + "abandon ship!"))); + std::shared_ptr<api::CreateVisitorCommand> cmd( + makeCreateVisitor()); + cmd->addBucketToBeVisited(document::BucketId(16, 4)); + _top->sendDown(cmd); + + sendCreateIteratorReply(); + GetIterCommand::SP getIterCmd( + fetchSingleCommand<GetIterCommand>(*_bottom)); + CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1234), + getIterCmd->getIteratorId()); + sendGetIterReply(*getIterCmd, + api::ReturnCode(api::ReturnCode::OK), + 2, + true); + + DestroyIteratorCommand::SP destroyIterCmd( + fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + + verifyCreateVisitorReply( + static_cast<api::ReturnCode::Result>( + mbus::ErrorCode::HANDSHAKE_FAILED), + 0, + 0); + CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); + // We currently don't count failures to send in this metric; send failures + // indicate a message bus problem and already log a warning when they happen + CPPUNIT_ASSERT_EQUAL(0L, getFailedVisitorDestinationReplyCount()); +} + +void +VisitorTest::sendInitialCreateVisitorAndGetIterRound() +{ + std::shared_ptr<api::CreateVisitorCommand> cmd( + makeCreateVisitor()); + _top->sendDown(cmd); + sendCreateIteratorReply(); + + { + GetIterCommand::SP getIterCmd( + fetchSingleCommand<GetIterCommand>(*_bottom)); + sendGetIterReply(*getIterCmd, api::ReturnCode(api::ReturnCode::OK), + 1, true); + } +} + +void +VisitorTest::testNoVisitorNotificationForTransientFailures() +{ + initializeTest(); + sendInitialCreateVisitorAndGetIterRound(); + + std::vector<document::Document::SP> docs; + std::vector<document::DocumentId> docIds; + std::vector<std::string> infoMessages; + // Have to make sure time increases in visitor thread so that resend + // times are reached. + _node->getClock().setFakeCycleMode(); + // Should not get info message for BUCKET_DELETED, but resend of Put. + getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, + api::ReturnCode::BUCKET_DELETED); + CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + // Should not get info message for BUCKET_NOT_FOUND, but resend of Put. + getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, + api::ReturnCode::BUCKET_NOT_FOUND); + CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + // MessageBus error codes guaranteed to fit in return code result. + // Should not get info message for SESSION_BUSY, but resend of Put. + getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, + static_cast<api::ReturnCode::Result>( + mbus::ErrorCode::SESSION_BUSY)); + CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + // WRONG_DISTRIBUTION should not be reported, as it will happen all the + // time when initiating remote migrations et al. + getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, + api::ReturnCode::WRONG_DISTRIBUTION); + CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + + // Complete message successfully to finish the visitor. + getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, + api::ReturnCode::OK); + CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + + fetchSingleCommand<DestroyIteratorCommand>(*_bottom); + + verifyCreateVisitorReply(api::ReturnCode::OK); + CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); +} + +void +VisitorTest::testNotificationSentIfTransientErrorRetriedManyTimes() +{ + constexpr size_t retries( + Visitor::TRANSIENT_ERROR_RETRIES_BEFORE_NOTIFY); + + initializeTest(); + sendInitialCreateVisitorAndGetIterRound(); + + std::vector<document::Document::SP> docs; + std::vector<document::DocumentId> docIds; + std::vector<std::string> infoMessages; + // Have to make sure time increases in visitor thread so that resend + // times are reached. + _node->getClock().setFakeCycleMode(); + for (size_t attempt = 0; attempt < retries; ++attempt) { + getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, + api::ReturnCode::WRONG_DISTRIBUTION); + CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); + } + // Should now have a client notification along for the ride. + // This has to be ACKed as OK or the visitor will fail. + getMessagesAndReply(2, getSession(0), docs, docIds, infoMessages, + api::ReturnCode::OK); + CPPUNIT_ASSERT_EQUAL(size_t(1), infoMessages.size()); + // TODO(vekterli) ideally we'd want to test that this happens only once + // per message, but this seems frustratingly complex to do currently. + fetchSingleCommand<DestroyIteratorCommand>(*_bottom); + + verifyCreateVisitorReply(api::ReturnCode::OK); + CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); +} + +std::shared_ptr<api::CreateVisitorReply> +VisitorTest::doCompleteVisitingSession( + const std::shared_ptr<api::CreateVisitorCommand>& cmd) +{ + initializeTest(); + _top->sendDown(cmd); + sendCreateIteratorReply(); + + GetIterCommand::SP getIterCmd( + fetchSingleCommand<GetIterCommand>(*_bottom)); + sendGetIterReply(*getIterCmd, + api::ReturnCode(api::ReturnCode::OK), + 1, + true); + + std::vector<document::Document::SP> docs; + std::vector<document::DocumentId> docIds; + std::vector<std::string> infoMessages; + getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages); + + DestroyIteratorCommand::SP destroyIterCmd( + fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + + _top->waitForMessages(1, 60); + const msg_ptr_vector replies = _top->getRepliesOnce(); + CPPUNIT_ASSERT_EQUAL(size_t(1), replies.size()); + + std::shared_ptr<api::StorageMessage> msg(replies[0]); + + CPPUNIT_ASSERT_EQUAL(api::MessageType::VISITOR_CREATE_REPLY, + msg->getType()); + return std::dynamic_pointer_cast<api::CreateVisitorReply>(msg); +} + +void +VisitorTest::testNoMbusTracingIfTraceLevelIsZero() +{ + std::shared_ptr<api::CreateVisitorCommand> cmd(makeCreateVisitor()); + cmd->getTrace().setLevel(0); + auto reply = doCompleteVisitingSession(cmd); + CPPUNIT_ASSERT(reply->getTrace().getRoot().isEmpty()); +} + +void +VisitorTest::testReplyContainsTraceIfTraceLevelAboveZero() +{ + std::shared_ptr<api::CreateVisitorCommand> cmd(makeCreateVisitor()); + cmd->getTrace().setLevel(1); + auto reply = doCompleteVisitingSession(cmd); + CPPUNIT_ASSERT(!reply->getTrace().getRoot().isEmpty()); +} + +void +VisitorTest::testNoMoreIteratorsSentWhileMemoryUsedAboveLimit() +{ + initializeTest(TestParams().maxVisitorMemoryUsage(1) + .parallelBuckets(1) + .iteratorsPerBucket(1)); + std::shared_ptr<api::CreateVisitorCommand> cmd( + makeCreateVisitor()); + _top->sendDown(cmd); + sendCreateIteratorReply(); + + GetIterCommand::SP getIterCmd( + fetchSingleCommand<GetIterCommand>(*_bottom)); + sendGetIterReply(*getIterCmd, + api::ReturnCode(api::ReturnCode::OK), + 1); + + // Pending Document API message towards client; memory usage should prevent + // visitor from sending down additional GetIter messages until the pending + // client message has been replied to and cleared from the internal state. + getSession(0).waitForMessages(1); + // Note that it's possible for this test to exhibit false negatives (but not + // false positives) since the _absence_ of a message means we don't have any + // kind of explicit barrier with which we can synchronize the test and the + // running visitor thread. + std::this_thread::sleep_for(100ms); + CPPUNIT_ASSERT_EQUAL(size_t(0), _bottom->getNumCommands()); + + std::vector<document::Document::SP> docs; + std::vector<document::DocumentId> docIds; + std::vector<std::string> infoMessages; + getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages); + + // 2nd round of GetIter now allowed. Send reply indicating completion. + getIterCmd = fetchSingleCommand<GetIterCommand>(*_bottom); + sendGetIterReply(*getIterCmd, + api::ReturnCode(api::ReturnCode::OK), + 1, + true); + + getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages); + + DestroyIteratorCommand::SP destroyIterCmd( + fetchSingleCommand<DestroyIteratorCommand>(*_bottom)); + + verifyCreateVisitorReply(api::ReturnCode::OK); + CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); +} + +void +VisitorTest::doTestVisitorInstanceHasConsistencyLevel( + vespalib::stringref visitorType, + spi::ReadConsistency expectedConsistency) +{ + initializeTest(); + std::shared_ptr<api::CreateVisitorCommand> cmd( + makeCreateVisitor(VisitorOptions().withVisitorType(visitorType))); + _top->sendDown(cmd); + + auto createCmd = fetchSingleCommand<CreateIteratorCommand>(*_bottom); + CPPUNIT_ASSERT_EQUAL(expectedConsistency, + createCmd->getReadConsistency()); +} + +void +VisitorTest::testDumpVisitorInvokesStrongReadConsistencyIteration() +{ + doTestVisitorInstanceHasConsistencyLevel( + "dumpvisitor", spi::ReadConsistency::STRONG); +} + +// NOTE: SearchVisitor cannot be tested here since it's in a separate module +// which depends on _this_ module for compilation. Instead we let TestVisitor +// use weak consistency, as this is just some internal stuff not used for/by +// any external client use cases. Our primary concern is to test that each +// visitor subclass might report its own read consistency requirement and that +// this is carried along to the CreateIteratorCommand. +void +VisitorTest::testTestVisitorInvokesWeakReadConsistencyIteration() +{ + doTestVisitorInstanceHasConsistencyLevel( + "testvisitor", spi::ReadConsistency::WEAK); +} + +} // namespace storage |