diff options
Diffstat (limited to 'storage')
101 files changed, 1094 insertions, 666 deletions
diff --git a/storage/src/tests/bucketdb/initializertest.cpp b/storage/src/tests/bucketdb/initializertest.cpp index 47e80f0c202..9aa7da92e36 100644 --- a/storage/src/tests/bucketdb/initializertest.cpp +++ b/storage/src/tests/bucketdb/initializertest.cpp @@ -423,8 +423,7 @@ struct FakePersistenceLayer : public StorageLink { } return 0; } - - bool onDown(const api::StorageMessage::SP& msg) override { + virtual bool onDown(const api::StorageMessage::SP& msg) override { fatalError = ""; if (messageCallback) { messageCallback->onMessage(*msg); diff --git a/storage/src/tests/bucketmover/bucketmovertest.cpp b/storage/src/tests/bucketmover/bucketmovertest.cpp index c233765ad27..3a844b6d1d8 100644 --- a/storage/src/tests/bucketmover/bucketmovertest.cpp +++ b/storage/src/tests/bucketmover/bucketmovertest.cpp @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/common/bucketmessages.h> #include <vespa/storage/bucketmover/bucketmover.h> diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index 15f0373e680..8dea2fad94f 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -67,8 +67,10 @@ namespace { { framework::Clock& _clock; MetricClock(framework::Clock& c) : _clock(c) {} - time_t getTime() const override { return _clock.getTimeInSeconds().getTime(); } - time_t getTimeInMilliSecs() const override { return _clock.getTimeInMillis().getTime(); } + virtual time_t getTime() const override + { return _clock.getTimeInSeconds().getTime(); } + virtual time_t getTimeInMilliSecs() const override + { return _clock.getTimeInMillis().getTime(); } }; } diff --git a/storage/src/tests/distributor/blockingoperationstartertest.cpp b/storage/src/tests/distributor/blockingoperationstartertest.cpp index 64f3dae8fbe..2ae8a819d7e 100644 --- a/storage/src/tests/distributor/blockingoperationstartertest.cpp +++ b/storage/src/tests/distributor/blockingoperationstartertest.cpp @@ -1,5 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> +#include <string> +#include <sstream> +#include <memory> #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storage/distributor/blockingoperationstarter.h> #include <vespa/storage/distributor/pendingmessagetracker.h> diff --git a/storage/src/tests/distributor/bucketdatabasetest.cpp b/storage/src/tests/distributor/bucketdatabasetest.cpp index 5d1bb17a1c6..13a8aff239c 100644 --- a/storage/src/tests/distributor/bucketdatabasetest.cpp +++ b/storage/src/tests/distributor/bucketdatabasetest.cpp @@ -1,6 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <tests/distributor/bucketdatabasetest.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> +#include <iostream> +#include <fstream> #include <iomanip> namespace storage { diff --git a/storage/src/tests/distributor/bucketdbupdatertest.cpp b/storage/src/tests/distributor/bucketdbupdatertest.cpp index e35f147d0c8..7ad083dc5a1 100644 --- a/storage/src/tests/distributor/bucketdbupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbupdatertest.cpp @@ -13,6 +13,7 @@ #include <iostream> #include <fstream> +#include <string> using namespace storage::api; using namespace storage::lib; diff --git a/storage/src/tests/distributor/bucketstateoperationtest.cpp b/storage/src/tests/distributor/bucketstateoperationtest.cpp index 58061def8ab..74a31de1541 100644 --- a/storage/src/tests/distributor/bucketstateoperationtest.cpp +++ b/storage/src/tests/distributor/bucketstateoperationtest.cpp @@ -28,10 +28,13 @@ private: void testBucketDbNotUpdatedOnFailure(); public: - void setUp() override { + void setUp() override + { createLinks(); } - void tearDown() override { + + void tearDown() override + { close(); } }; diff --git a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp index 3e4e1d6da88..e7078f08d0a 100644 --- a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp +++ b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp @@ -6,6 +6,7 @@ #include <vespa/storage/distributor/min_replica_provider.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/util/jsonstream.h> #include <vespa/vespalib/testkit/test_kit.h> #include <tests/common/hostreporter/util.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index 10b0d035770..fc09dc17c1f 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -1,5 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <iomanip> +#include <iostream> +#include <memory> #include <boost/assign/std/vector.hpp> // for 'operator+=()' #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/distributor/idealstatemetricsset.h> diff --git a/storage/src/tests/distributor/idealstatemanagertest.cpp b/storage/src/tests/distributor/idealstatemanagertest.cpp index e4fce6b9d6e..2b861d23c5e 100644 --- a/storage/src/tests/distributor/idealstatemanagertest.cpp +++ b/storage/src/tests/distributor/idealstatemanagertest.cpp @@ -11,6 +11,7 @@ #include <tests/distributor/distributortestutil.h> + namespace storage { namespace distributor { diff --git a/storage/src/tests/distributor/mapbucketdatabasetest.cpp b/storage/src/tests/distributor/mapbucketdatabasetest.cpp index 8af2a86e4c5..6892f522bba 100644 --- a/storage/src/tests/distributor/mapbucketdatabasetest.cpp +++ b/storage/src/tests/distributor/mapbucketdatabasetest.cpp @@ -1,6 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/document_runnable.h> #include <vespa/vdstestlib/cppunit/macros.h> +#include <cppunit/extensions/HelperMacros.h> #include <vespa/storage/bucketdb/mapbucketdatabase.h> +#include <vespa/storage/storageutil/utils.h> #include <tests/distributor/bucketdatabasetest.h> namespace storage { @@ -8,7 +12,8 @@ namespace distributor { struct MapBucketDatabaseTest : public BucketDatabaseTest { MapBucketDatabase _db; - BucketDatabase& db() override { return _db; }; + + virtual BucketDatabase& db() override { return _db; } CPPUNIT_TEST_SUITE(MapBucketDatabaseTest); SETUP_DATABASE_TESTS(); diff --git a/storage/src/tests/distributor/mergeoperationtest.cpp b/storage/src/tests/distributor/mergeoperationtest.cpp index 1e317e20b74..875d16a42ed 100644 --- a/storage/src/tests/distributor/mergeoperationtest.cpp +++ b/storage/src/tests/distributor/mergeoperationtest.cpp @@ -1,4 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <boost/lexical_cast.hpp> #include <cppunit/extensions/HelperMacros.h> #include <iomanip> diff --git a/storage/src/tests/distributor/messagesenderstub.h b/storage/src/tests/distributor/messagesenderstub.h index 629c1c6e346..7b552ec6883 100644 --- a/storage/src/tests/distributor/messagesenderstub.h +++ b/storage/src/tests/distributor/messagesenderstub.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/storage/distributor/distributormessagesender.h> -#include <cassert> namespace storage { @@ -21,11 +20,13 @@ struct MessageSenderStub : distributor::DistributorMessageSender replies.clear(); } - virtual void sendCommand(const std::shared_ptr<api::StorageCommand>& cmd) { + virtual void sendCommand(const std::shared_ptr<api::StorageCommand>& cmd) + { commands.push_back(cmd); } - virtual void sendReply(const std::shared_ptr<api::StorageReply>& reply) { + virtual void sendReply(const std::shared_ptr<api::StorageReply>& reply) + { replies.push_back(reply); } @@ -66,3 +67,4 @@ private: }; } + diff --git a/storage/src/tests/distributor/operationtargetresolvertest.cpp b/storage/src/tests/distributor/operationtargetresolvertest.cpp index d0dc8cedaf3..26144ac950e 100644 --- a/storage/src/tests/distributor/operationtargetresolvertest.cpp +++ b/storage/src/tests/distributor/operationtargetresolvertest.cpp @@ -1,5 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <iomanip> +#include <iostream> #include <vespa/config/helper/configgetter.h> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/repo/documenttyperepo.h> diff --git a/storage/src/tests/distributor/pendingmessagetrackertest.cpp b/storage/src/tests/distributor/pendingmessagetrackertest.cpp index 618d7980c5b..6f4539d426e 100644 --- a/storage/src/tests/distributor/pendingmessagetrackertest.cpp +++ b/storage/src/tests/distributor/pendingmessagetrackertest.cpp @@ -1,4 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/document/base/testdocman.h> #include <vespa/storage/distributor/pendingmessagetracker.h> @@ -9,6 +10,13 @@ #include <tests/common/dummystoragelink.h> #include <vespa/vdslib/state/random.h> #include <vespa/vdstestlib/cppunit/macros.h> +#include <fstream> +#include <sstream> +#include <iomanip> +#include <iostream> +#include <memory> +#include <string> +#include <iterator> namespace storage { namespace distributor { diff --git a/storage/src/tests/distributor/simplemaintenancescannertest.cpp b/storage/src/tests/distributor/simplemaintenancescannertest.cpp index 342d346b398..100d5582006 100644 --- a/storage/src/tests/distributor/simplemaintenancescannertest.cpp +++ b/storage/src/tests/distributor/simplemaintenancescannertest.cpp @@ -1,11 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/distributor/maintenance/simplemaintenancescanner.h> #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> #include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <tests/distributor/maintenancemocks.h> +#include <string> +#include <sstream> +#include <memory> +#include <algorithm> +#include <iterator> + namespace storage { namespace distributor { diff --git a/storage/src/tests/distributor/statusreporterdelegatetest.cpp b/storage/src/tests/distributor/statusreporterdelegatetest.cpp index d9f647a52a5..f136b00244a 100644 --- a/storage/src/tests/distributor/statusreporterdelegatetest.cpp +++ b/storage/src/tests/distributor/statusreporterdelegatetest.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - +#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <tests/common/testhelper.h> #include <tests/distributor/distributortestutil.h> diff --git a/storage/src/tests/distributor/throttlingoperationstartertest.cpp b/storage/src/tests/distributor/throttlingoperationstartertest.cpp index 0414ad11da4..8e7d98005c1 100644 --- a/storage/src/tests/distributor/throttlingoperationstartertest.cpp +++ b/storage/src/tests/distributor/throttlingoperationstartertest.cpp @@ -1,5 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> +#include <string> +#include <sstream> +#include <memory> #include <vespa/storage/distributor/throttlingoperationstarter.h> #include <tests/distributor/maintenancemocks.h> diff --git a/storage/src/tests/frameworkimpl/status/statustest.cpp b/storage/src/tests/frameworkimpl/status/statustest.cpp index 43d13acb28b..567f9192d34 100644 --- a/storage/src/tests/frameworkimpl/status/statustest.cpp +++ b/storage/src/tests/frameworkimpl/status/statustest.cpp @@ -1,12 +1,16 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/document/util/stringutil.h> +#include <vespa/log/log.h> +#include <sstream> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> #include <vespa/storage/frameworkimpl/status/statuswebserver.h> #include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> #include <tests/common/teststorageapp.h> #include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/document/util/stringutil.h> -#include <sstream> + +LOG_SETUP(".test.status"); namespace storage { @@ -43,13 +47,18 @@ namespace { : framework::HtmlStatusReporter(id, name), _headerAddition(headerAddition), _content(content) - {} + { + } - void reportHtmlHeaderAdditions(std::ostream& out, const framework::HttpUrlPath&) const override { + virtual void reportHtmlHeaderAdditions( + std::ostream& out, const framework::HttpUrlPath&) const override + { out << _headerAddition; } - void reportHtmlStatus(std::ostream& out, const framework::HttpUrlPath&) const override { + virtual void reportHtmlStatus( + std::ostream& out, const framework::HttpUrlPath&) const override + { out << _content; } }; @@ -57,9 +66,9 @@ namespace { struct XmlStatusReporter : public framework::XmlStatusReporter { XmlStatusReporter(const std::string& id, const std::string& name) : framework::XmlStatusReporter(id, name) {} - - vespalib::string reportXmlStatus(vespalib::xml::XmlOutputStream& xos, - const framework::HttpUrlPath&) const override + virtual vespalib::string reportXmlStatus( + vespalib::xml::XmlOutputStream& xos, + const framework::HttpUrlPath&) const override { xos << vespalib::xml::XmlTag("mytag") << vespalib::xml::XmlAttribute("foo", "bar") diff --git a/storage/src/tests/persistence/bucketownershipnotifiertest.cpp b/storage/src/tests/persistence/bucketownershipnotifiertest.cpp index 5f675c1d551..8553795db71 100644 --- a/storage/src/tests/persistence/bucketownershipnotifiertest.cpp +++ b/storage/src/tests/persistence/bucketownershipnotifiertest.cpp @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <tests/distributor/messagesenderstub.h> #include <tests/common/teststorageapp.h> diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index e2f41aa5b3e..e21d367370b 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -9,6 +9,7 @@ #include <vespa/document/select/parser.h> #include <vespa/vdslib/state/random.h> #include <vespa/vdslib/container/mutabledocumentlist.h> +#include <vespa/vdslib/container/operationlist.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/storageapi/message/bucket.h> @@ -17,17 +18,25 @@ #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/storage/bucketdb/bucketmanager.h> +#include <vespa/storage/bucketdb/storbucketdb.h> +#include <vespa/storage/common/bucketmessages.h> +#include <vespa/storageframework/storageframework.h> #include <vespa/storage/persistence/persistencethread.h> +#include <vespa/storage/persistence/messages.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> #include <tests/common/testhelper.h> #include <tests/common/storagelinktest.h> #include <tests/common/teststorageapp.h> +#include <tests/common/dummystoragelink.h> #include <tests/persistence/filestorage/forwardingmessagesender.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/storageapi/message/batch.h> #include <vespa/storage/storageserver/statemanager.h> #include <vespa/fastos/file.h> +#include <fstream> +#include <memory> +#include <atomic> #include <vespa/log/log.h> LOG_SETUP(".filestormanagertest"); @@ -2355,8 +2364,10 @@ namespace { closeNextLink(); } - void print(std::ostream& out, bool, const std::string&) const override { out << "MidLink"; } - bool onUp(const std::shared_ptr<api::StorageMessage> & msg) override { + virtual void print(std::ostream& out, bool, const std::string&) const override + { out << "MidLink"; } + + virtual bool onUp(const std::shared_ptr<api::StorageMessage> & msg) override { if (!StorageLinkTest::callOnUp(_up, msg)) _up.sendUp(msg); return true; } @@ -2384,9 +2395,10 @@ namespace { _leftAddr(leftAddr), _rightAddr(rightAddr) {} - void print(std::ostream& out, bool, const std::string&) const override { out << "BinaryStorageLink"; } + virtual void print(std::ostream& out, bool, const std::string&) const override + { out << "BinaryStorageLink"; } - bool onDown(const std::shared_ptr<api::StorageMessage> & msg) override { + virtual bool onDown(const std::shared_ptr<api::StorageMessage> & msg) override { // LOG(debug, "onDown Received msg: ->%s, %s %llu\n", msg->getAddress() ? msg->getAddress()->toString().c_str() : "(null)", msg->toString().c_str(), msg->getMsgId()); vespalib::LockGuard lock(_lock); @@ -2420,7 +2432,7 @@ namespace { return true; } - bool onUp(const std::shared_ptr<api::StorageMessage> & msg) override { + virtual bool onUp(const std::shared_ptr<api::StorageMessage> & msg) override { // LOG(debug, "onUp Received msg: ->%s, %s %llu\n", msg->getAddress() ? msg->getAddress()->toString().c_str() : "(null)", msg->toString().c_str(), msg->getMsgId()); vespalib::LockGuard lock(_lock); diff --git a/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp b/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp index 1409445b96b..d608bef3307 100644 --- a/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - +#include <vespa/fastos/fastos.h> +#include <memory> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> diff --git a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp index 58b4ae4d475..2dc60594c45 100644 --- a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp +++ b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp @@ -1,11 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <tests/common/testhelper.h> #include <tests/common/storagelinktest.h> #include <tests/common/teststorageapp.h> +#include <tests/common/dummystoragelink.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> +#include <vespa/storage/persistence/messages.h> namespace storage { diff --git a/storage/src/tests/persistence/legacyoperationhandlertest.cpp b/storage/src/tests/persistence/legacyoperationhandlertest.cpp index 353a8d24ec5..df262501299 100644 --- a/storage/src/tests/persistence/legacyoperationhandlertest.cpp +++ b/storage/src/tests/persistence/legacyoperationhandlertest.cpp @@ -1,12 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/document/base/testdocrepo.h> +#include <vespa/document/repo/documenttyperepo.h> #include <vespa/documentapi/loadtypes/loadtype.h> #include <vespa/storage/persistence/messages.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/multioperation.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <tests/persistence/persistencetestutils.h> +#include <vespa/storage/persistence/types.h> using document::DocumentTypeRepo; using document::TestDocRepo; diff --git a/storage/src/tests/persistence/mergehandlertest.cpp b/storage/src/tests/persistence/mergehandlertest.cpp index 3f0623fbef0..8147992a516 100644 --- a/storage/src/tests/persistence/mergehandlertest.cpp +++ b/storage/src/tests/persistence/mergehandlertest.cpp @@ -3,13 +3,15 @@ #include <vespa/document/base/testdocman.h> #include <vespa/storage/persistence/mergehandler.h> #include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/log/log.h> #include <tests/persistence/persistencetestutils.h> #include <tests/persistence/common/persistenceproviderwrapper.h> #include <tests/distributor/messagesenderstub.h> +#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> #include <vespa/vespalib/objects/nbostream.h> #include <cmath> -#include <vespa/log/log.h> LOG_SETUP(".test.persistence.handler.merge"); namespace storage { diff --git a/storage/src/tests/persistence/providershutdownwrappertest.cpp b/storage/src/tests/persistence/providershutdownwrappertest.cpp index c7f60a39cc8..8b1ef88aa2c 100644 --- a/storage/src/tests/persistence/providershutdownwrappertest.cpp +++ b/storage/src/tests/persistence/providershutdownwrappertest.cpp @@ -1,8 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <tests/persistence/persistencetestutils.h> #include <tests/persistence/common/persistenceproviderwrapper.h> +#include <vespa/storage/persistence/providershutdownwrapper.h> namespace storage { diff --git a/storage/src/tests/storageserver/bouncertest.cpp b/storage/src/tests/storageserver/bouncertest.cpp index 751d6b535a3..e5b9a3e6092 100644 --- a/storage/src/tests/storageserver/bouncertest.cpp +++ b/storage/src/tests/storageserver/bouncertest.cpp @@ -1,7 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <boost/pointer_cast.hpp> #include <cppunit/extensions/HelperMacros.h> +#include <iostream> +#include <string> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> #include <vespa/storageapi/message/stat.h> diff --git a/storage/src/tests/storageserver/bucketintegritycheckertest.cpp b/storage/src/tests/storageserver/bucketintegritycheckertest.cpp index f4eedc64971..d9a175f3b04 100644 --- a/storage/src/tests/storageserver/bucketintegritycheckertest.cpp +++ b/storage/src/tests/storageserver/bucketintegritycheckertest.cpp @@ -1,14 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <boost/lexical_cast.hpp> #include <cppunit/extensions/HelperMacros.h> #include <vespa/log/log.h> #include <vespa/storage/bucketdb/bucketmanager.h> +#include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/storageserver/bucketintegritychecker.h> +#include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> #include <tests/common/testhelper.h> #include <tests/common/storagelinktest.h> +#include <tests/common/dummystoragelink.h> #include <vespa/vespalib/io/fileutil.h> #include <tests/common/teststorageapp.h> diff --git a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp index 89eaff0ab6e..8eb6b74a840 100644 --- a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp +++ b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/document/base/testdocman.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/bucketdb/storbucketdb.h> @@ -14,6 +15,7 @@ #include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> #include <vespa/storage/storageserver/changedbucketownershiphandler.h> +#include <memory> namespace storage { diff --git a/storage/src/tests/storageserver/priorityconvertertest.cpp b/storage/src/tests/storageserver/priorityconvertertest.cpp index ab6c27f48d6..ecbbd25b8b1 100644 --- a/storage/src/tests/storageserver/priorityconvertertest.cpp +++ b/storage/src/tests/storageserver/priorityconvertertest.cpp @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/documentapi/documentapi.h> #include <vespa/storage/storageserver/priorityconverter.h> #include <tests/common/testhelper.h> diff --git a/storage/src/tests/storageserver/statemanagertest.cpp b/storage/src/tests/storageserver/statemanagertest.cpp index f9c5c97e6bc..4244ce0c4ce 100644 --- a/storage/src/tests/storageserver/statemanagertest.cpp +++ b/storage/src/tests/storageserver/statemanagertest.cpp @@ -1,15 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <boost/pointer_cast.hpp> #include <cppunit/extensions/HelperMacros.h> #include <vespa/metrics/metricmanager.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> +#include <vespa/vdslib/state/nodestate.h> #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storage/storageserver/statemanager.h> +#include <vespa/storage/common/hostreporter/hostinfo.h> #include <tests/common/teststorageapp.h> #include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> #include <vespa/vespalib/data/slime/slime.h> +#include <iostream> using storage::lib::NodeState; using storage::lib::NodeType; @@ -134,7 +138,8 @@ namespace { MyStateListener(const NodeStateUpdater& upd) : updater(upd), current(*updater.getReportedNodeState()) {} - void handleNewState() override { + void handleNewState() override + { ost << current << " -> "; current = *updater.getReportedNodeState(); ost << current << "\n"; @@ -254,3 +259,4 @@ StateManagerTest::testClusterStateVersion() } } // storage + diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index 8a164361c88..265b9e95d83 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -18,8 +18,8 @@ namespace storage { class DummyApplicationGenerationFether : public ApplicationGenerationFetcher { public: - int64_t getGeneration() const override { return 1; } - std::string getComponentName() const override { return "component"; } + virtual int64_t getGeneration() const override { return 1; } + virtual std::string getComponentName() const override { return "component"; } }; struct StateReporterTest : public CppUnit::TestFixture { @@ -58,8 +58,10 @@ namespace { { framework::Clock& _clock; MetricClock(framework::Clock& c) : _clock(c) {} - time_t getTime() const override { return _clock.getTimeInSeconds().getTime(); } - time_t getTimeInMilliSecs() const override { return _clock.getTimeInMillis().getTime(); } + virtual time_t getTime() const override + { return _clock.getTimeInSeconds().getTime(); } + virtual time_t getTimeInMilliSecs() const override + { return _clock.getTimeInMillis().getTime(); } }; } diff --git a/storage/src/tests/storageutil/charttest.cpp b/storage/src/tests/storageutil/charttest.cpp index c7c1c43b000..4f83a9f26f4 100644 --- a/storage/src/tests/storageutil/charttest.cpp +++ b/storage/src/tests/storageutil/charttest.cpp @@ -1,13 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/storage/storageutil/piechart.h> -#include <vespa/vdstestlib/cppunit/macros.h> + #include <fstream> +#include <vespa/vdstestlib/cppunit/macros.h> namespace storage { struct PieChartTest : public CppUnit::TestFixture { + void setUp() override {} + void tearDown() override {} + void testWriteHtmlFile(); CPPUNIT_TEST_SUITE(PieChartTest); diff --git a/storage/src/tests/storageutil/palettetest.cpp b/storage/src/tests/storageutil/palettetest.cpp index 24eaee18996..e5a4e73d58a 100644 --- a/storage/src/tests/storageutil/palettetest.cpp +++ b/storage/src/tests/storageutil/palettetest.cpp @@ -1,11 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/storage/storageutil/palette.h> #include <vespa/vdstestlib/cppunit/macros.h> namespace storage { struct PaletteTest : public CppUnit::TestFixture { + void setUp() override {} + void tearDown() override {} + void testNormalUsage(); CPPUNIT_TEST_SUITE(PaletteTest); diff --git a/storage/src/tests/testrunner.cpp b/storage/src/tests/testrunner.cpp index 9bf86e3276a..5d8dc8d4c1f 100644 --- a/storage/src/tests/testrunner.cpp +++ b/storage/src/tests/testrunner.cpp @@ -1,12 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <iostream> +#include <vespa/log/log.h> #include <vespa/vdstestlib/cppunit/cppunittestrunner.h> -#include <vespa/log/log.h> LOG_SETUP("storagecppunittests"); int -main(int argc, const char *argv[]) +main(int argc, char **argv) { vdstestlib::CppUnitTestRunner testRunner; return testRunner.run(argc, argv); diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp index 1824b976aea..8aa57b982db 100644 --- a/storage/src/tests/visiting/visitormanagertest.cpp +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -1,11 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/rawfieldvalue.h> +#include <vespa/log/log.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/visitor.h> +#include <vector> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/visiting/visitormanager.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> @@ -13,11 +17,16 @@ #include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> #include <tests/storageserver/testvisitormessagesession.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vdslib/container/visitorordering.h> #include <vespa/documentapi/messagebus/messages/multioperationmessage.h> #include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> #include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> #include <vespa/vespalib/util/exceptions.h> + +LOG_SETUP(".visitormanagertest"); + namespace storage { namespace { typedef std::vector<api::StorageMessage::SP> msg_ptr_vector; @@ -104,6 +113,7 @@ CPPUNIT_TEST_SUITE_REGISTRATION(VisitorManagerTest); void VisitorManagerTest::initializeTest() { + LOG(debug, "Initializing test"); vdstestlib::DirConfig config(getStandardConfig(true)); config.getConfig("stor-visitor").set("visitorthreads", "1"); @@ -211,6 +221,7 @@ VisitorManagerTest::initializeTest() CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), reply->getResult()); } + LOG(debug, "Done initializing test"); } void diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp index 84f1297b05f..e6a3542cbf2 100644 --- a/storage/src/tests/visiting/visitortest.cpp +++ b/storage/src/tests/visiting/visitortest.cpp @@ -6,17 +6,25 @@ #include <vespa/document/fieldvalue/rawfieldvalue.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/visitor.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/visiting/visitormanager.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <tests/common/dummystoragelink.h> #include <tests/storageserver/testvisitormessagesession.h> +#include <vespa/vdstestlib/cppunit/macros.h> +#include <vespa/vdslib/container/visitorordering.h> #include <vespa/documentapi/messagebus/messages/multioperationmessage.h> #include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> #include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> #include <vespa/vespalib/util/exceptions.h> +#include <vector> #include <thread> +#include <chrono> + +#include <vespa/log/log.h> +LOG_SETUP(".visitortest"); using namespace std::chrono_literals; @@ -177,6 +185,7 @@ CPPUNIT_TEST_SUITE_REGISTRATION(VisitorTest); void VisitorTest::initializeTest(const TestParams& params) { + LOG(debug, "Initializing test"); vdstestlib::DirConfig config(getStandardConfig(true, "visitortest")); config.getConfig("stor-visitor").set("visitorthreads", "1"); config.getConfig("stor-visitor").set( @@ -264,6 +273,7 @@ VisitorTest::initializeTest(const TestParams& params) _documents.back()->setValue(type.getField("headerval"), document::IntFieldValue(i % 4)); } + LOG(debug, "Done initializing test"); } void diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h index da8dcf1c1cd..64498386466 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.h +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h @@ -102,7 +102,8 @@ public: ~BucketManager(); void startWorkerThread(); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; + + virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; /** Dump the whole database to the given output. Use for debugging. */ void dump(std::ostream& out) const; @@ -119,11 +120,11 @@ private: vespalib::string getReportContentType(const framework::HttpUrlPath&) const override; bool reportStatus(std::ostream&, const framework::HttpUrlPath&) const override; - /** Event saying node is up and running. We can start to build cache. */ - void onOpen() override; - void onDoneInit() override { _doneInitialized = true; } - void onClose() override; - void onFlush(bool downwards) override; + /** Event saying node is up and running. We can start to build cache. */ + virtual void onOpen() override; + virtual void onDoneInit() override { _doneInitialized = true; } + virtual void onClose() override; + virtual void onFlush(bool downwards) override; void updateMetrics(bool updateDocCount); void updateMetrics(const MetricLockGuard &) override { updateMetrics(true); } diff --git a/storage/src/vespa/storage/bucketdb/judyarray.h b/storage/src/vespa/storage/bucketdb/judyarray.h index e28d89967b2..b5d8a50cb16 100644 --- a/storage/src/vespa/storage/bucketdb/judyarray.h +++ b/storage/src/vespa/storage/bucketdb/judyarray.h @@ -83,7 +83,7 @@ public: data_type& operator[](key_type key); size_type getMemoryUsage() const; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; + virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; class ConstIterator : public vespalib::Printable { @@ -101,7 +101,8 @@ public: key_type key() const { return _key; } data_type value() const { return *_data; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; + virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; + protected: // For creating end() iterator ConstIterator(const JudyArray&); @@ -226,3 +227,4 @@ JudyArray::ConstIterator::operator==(const JudyArray::ConstIterator &cp) const } } // storage + diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.h b/storage/src/vespa/storage/bucketdb/lockablemap.h index 3c40c214136..93dfb23c4db 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.h +++ b/storage/src/vespa/storage/bucketdb/lockablemap.h @@ -168,7 +168,7 @@ public: const char* clientId, uint32_t chunkSize = DEFAULT_CHUNK_SIZE); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; + virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; /** * Returns all buckets in the bucket database that can contain the given diff --git a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp index 10a6bb6bd24..a271803714a 100644 --- a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp +++ b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp @@ -1,4 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <vespa/storage/common/bucketoperationlogger.h> @@ -464,7 +465,7 @@ namespace { struct Writer : public BucketDatabase::EntryProcessor { std::ostream& _ost; Writer(std::ostream& ost) : _ost(ost) {} - bool process(const BucketDatabase::Entry& e) override { + virtual bool process(const BucketDatabase::Entry& e) override { _ost << e.toString() << "\n"; return true; } diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h index 4c87c39fdd2..6e9bef38672 100644 --- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h +++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h @@ -138,12 +138,12 @@ public: ServiceLayerComponentRegister&); ~StorageBucketDBInitializer(); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; + virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void onOpen() override; - void onClose() override; + virtual void onOpen() override; + virtual void onClose() override; - void run(framework::ThreadHandle&) override; + virtual void run(framework::ThreadHandle&) override; bool onDown(const std::shared_ptr<api::StorageMessage>&) override; bool onInternalReply(const std::shared_ptr<api::InternalReply>&) override; @@ -215,3 +215,4 @@ public: }; } // storage + diff --git a/storage/src/vespa/storage/bucketmover/run.h b/storage/src/vespa/storage/bucketmover/run.h index d05020e6ba5..a626960eec7 100644 --- a/storage/src/vespa/storage/bucketmover/run.h +++ b/storage/src/vespa/storage/bucketmover/run.h @@ -15,13 +15,12 @@ #pragma once - +#include <list> +#include <map> #include <vespa/storage/bucketmover/move.h> #include <vespa/storage/bucketmover/runstatistics.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/nodestate.h> -#include <list> -#include <map> namespace storage { @@ -92,7 +91,8 @@ public: RunStatistics& getStatistics() { return _statistics; } const RunStatistics& getStatistics() const { return _statistics; } - void print(std::ostream&, bool verbose, const std::string& indent) const override; + + virtual void print(std::ostream&, bool verbose, const std::string& indent) const override; private: void removePending(Move&); void finalize(); @@ -100,3 +100,4 @@ private: } // bucketmover } // storage + diff --git a/storage/src/vespa/storage/common/messagesender.h b/storage/src/vespa/storage/common/messagesender.h index 332103e12db..025695dff29 100644 --- a/storage/src/vespa/storage/common/messagesender.h +++ b/storage/src/vespa/storage/common/messagesender.h @@ -16,7 +16,6 @@ #pragma once -#include <memory> namespace storage { namespace api { diff --git a/storage/src/vespa/storage/common/servicelayercomponent.h b/storage/src/vespa/storage/common/servicelayercomponent.h index 5b9db275b27..e54421b1ce3 100644 --- a/storage/src/vespa/storage/common/servicelayercomponent.h +++ b/storage/src/vespa/storage/common/servicelayercomponent.h @@ -25,7 +25,7 @@ #pragma once -#include "storagecomponent.h" +#include <vespa/storage/common/storagecomponent.h> namespace storage { @@ -54,10 +54,12 @@ class ServiceLayerComponent : public StorageComponent, StorBucketDatabase* _bucketDatabase; MinimumUsedBitsTracker* _minUsedBitsTracker; - // ServiceLayerManagedComponent implementation - void setDiskCount(uint16_t count) override { _diskCount = count; } - void setBucketDatabase(StorBucketDatabase& db) override { _bucketDatabase = &db; } - void setMinUsedBitsTracker(MinimumUsedBitsTracker& tracker) override { + // ServiceLayerManagedComponent implementation + virtual void setDiskCount(uint16_t count) override { _diskCount = count; } + virtual void setBucketDatabase(StorBucketDatabase& db) override { + _bucketDatabase = &db; + } + virtual void setMinUsedBitsTracker(MinimumUsedBitsTracker& tracker) override { _minUsedBitsTracker = &tracker; } public: @@ -89,3 +91,4 @@ public: }; } // storage + diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.h b/storage/src/vespa/storage/common/statusmetricconsumer.h index 6a2c18f78bd..0027cef9cd0 100644 --- a/storage/src/vespa/storage/common/statusmetricconsumer.h +++ b/storage/src/vespa/storage/common/statusmetricconsumer.h @@ -9,11 +9,11 @@ #pragma once -#include "storagecomponent.h" #include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/sync.h> #include <map> #include <vespa/metrics/metrics.h> +#include <vespa/storage/common/storagecomponent.h> #include <vespa/storageframework/storageframework.h> namespace vespalib { diff --git a/storage/src/vespa/storage/common/storagelink.h b/storage/src/vespa/storage/common/storagelink.h index a9af5fa5622..e9ea8403321 100644 --- a/storage/src/vespa/storage/common/storagelink.h +++ b/storage/src/vespa/storage/common/storagelink.h @@ -19,11 +19,13 @@ #pragma once -#include "messagesender.h" -#include "storagecomponent.h" +#include <vespa/vespalib/util/printable.h> +#include <memory> #include <vespa/storageapi/messageapi/messagehandler.h> #include <vespa/storageapi/messageapi/storagemessage.h> -#include <vespa/vespalib/util/printable.h> +#include <string> +#include <vespa/storage/common/messagesender.h> +#include <vespa/storage/common/storagecomponent.h> namespace storage { @@ -82,10 +84,10 @@ public: void flush(); /** Send message down the storage chain. */ - void sendDown(const api::StorageMessage::SP&) override; + virtual void sendDown(const api::StorageMessage::SP&); /** Send message up the storage chain. */ - void sendUp(const api::StorageMessage::SP&) override; + virtual void sendUp(const api::StorageMessage::SP&); void printChain(std::ostream&, std::string indent = "") const; @@ -124,7 +126,8 @@ public: */ virtual bool onUp(const api::StorageMessage::SP&); - void print(std::ostream& out, bool, const std::string&) const override { + virtual void print(std::ostream& out, bool, + const std::string&) const { out << getName(); } @@ -185,3 +188,4 @@ private: std::ostream& operator<<(std::ostream& out, StorageLink& link); } + diff --git a/storage/src/vespa/storage/common/storagelinkqueued.h b/storage/src/vespa/storage/common/storagelinkqueued.h index c894b6196e8..cd7b91a7a68 100644 --- a/storage/src/vespa/storage/common/storagelinkqueued.h +++ b/storage/src/vespa/storage/common/storagelinkqueued.h @@ -14,11 +14,11 @@ #pragma once -#include "storagelink.h" -#include <vespa/storageframework/storageframework.h> #include <vespa/vespalib/util/document_runnable.h> #include <deque> #include <limits> +#include <vespa/storageframework/storageframework.h> +#include <vespa/storage/common/storagelink.h> namespace storage { @@ -40,13 +40,13 @@ public: void dispatchUp(const std::shared_ptr<api::StorageMessage>&); /** Remember to call this method if you override it. */ - void onClose() override { + virtual void onClose() override { _commandDispatcher.flush(); _closeState |= 1; } /** Remember to call this method if you override it. */ - void onFlush(bool downwards) override { + virtual void onFlush(bool downwards) override { if (downwards) { _commandDispatcher.flush(); _closeState |= 2; @@ -79,7 +79,7 @@ private: public: Dispatcher(StorageLinkQueued& parent, unsigned int maxQueueSize, bool replyDispatcher); - ~Dispatcher(); + virtual ~Dispatcher(); void start(); void run(framework::ThreadHandle&) override; @@ -106,7 +106,7 @@ private: void send(const std::shared_ptr<api::StorageMessage> & reply) override { _parent.sendUp(reply); } - ~ReplyDispatcher() { terminate(); } + virtual ~ReplyDispatcher() { terminate(); } }; class CommandDispatcher : public Dispatcher<api::StorageMessage> @@ -117,7 +117,7 @@ private: parent, std::numeric_limits<unsigned int>::max(), false) { } - ~CommandDispatcher() { terminate(); } + virtual ~CommandDispatcher() { terminate(); } void send(const std::shared_ptr<api::StorageMessage> & command) override { _parent.sendDown(command); } @@ -132,6 +132,7 @@ private: protected: ReplyDispatcher& getReplyDispatcher() { return _replyDispatcher; } + }; } diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h index a8f76c2c7a0..b87f0e5f898 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h @@ -49,14 +49,23 @@ public: const std::shared_ptr<api::RequestBucketInfoReply> & repl) override; bool onMergeBucketReply(const std::shared_ptr<api::MergeBucketReply>& reply) override; + bool onNotifyBucketChange(const std::shared_ptr<api::NotifyBucketChangeCommand>&) override; + void resendDelayedMessages(); + void storageDistributionChanged(const lib::Distribution&); - vespalib::string reportXmlStatus(vespalib::xml::XmlOutputStream&, const framework::HttpUrlPath&) const; - vespalib::string getReportContentType(const framework::HttpUrlPath&) const override; + vespalib::string reportXmlStatus(vespalib::xml::XmlOutputStream&, + const framework::HttpUrlPath&) const; + + vespalib::string getReportContentType( + const framework::HttpUrlPath&) const override; bool reportStatus(std::ostream&, const framework::HttpUrlPath&) const override; - void print(std::ostream& out, bool verbose, const std::string& indent) const; + + virtual void print(std::ostream& out, bool verbose, + const std::string& indent) const; + DistributorComponent& getDistributorComponent() { return _bucketSpaceComponent; } /** @@ -231,8 +240,12 @@ private: _upStates(upStates) {} ~NodeRemover(); - bool process(BucketDatabase::Entry& e) override; - void logRemove(const document::BucketId& bucketId, const char* msg) const; + + virtual bool process(BucketDatabase::Entry& e) override; + + void logRemove(const document::BucketId& bucketId, + const char* msg) const; + bool distributorOwnsBucket(const document::BucketId&) const; const std::vector<document::BucketId>& getBucketsToRemove() const { @@ -253,7 +266,8 @@ private: const char* _upStates; }; - std::deque<std::pair<framework::MilliSecTime, BucketRequest> > _delayedRequests; + std::deque<std::pair<framework::MilliSecTime, + BucketRequest> > _delayedRequests; std::map<uint64_t, BucketRequest> _sentMessages; std::unique_ptr<PendingClusterState> _pendingClusterState; std::list<PendingClusterState::Summary> _history; @@ -266,3 +280,4 @@ private: } } + diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h index 182988104a9..40d6ee76e07 100644 --- a/storage/src/vespa/storage/distributor/distributor.h +++ b/storage/src/vespa/storage/distributor/distributor.h @@ -2,6 +2,7 @@ #pragma once + #include "idealstatemanager.h" #include "bucketdbupdater.h" #include "pendingmessagetracker.h" @@ -21,6 +22,7 @@ #include <vespa/storageframework/generic/thread/tickingthread.h> #include <vespa/config/config.h> #include <vespa/vespalib/util/sync.h> + #include <unordered_map> namespace storage { @@ -55,12 +57,16 @@ public: ~Distributor(); void onOpen() override; + void onClose() override; + bool onDown(const std::shared_ptr<api::StorageMessage>&) override; + void sendUp(const std::shared_ptr<api::StorageMessage>&) override; + void sendDown(const std::shared_ptr<api::StorageMessage>&) override; - ChainedMessageSender& getMessageSender() override { + virtual ChainedMessageSender& getMessageSender() override { return (_messageSender == 0 ? *this : *_messageSender); } @@ -93,7 +99,8 @@ public: bool handleReply(const std::shared_ptr<api::StorageReply>& reply) override; // StatusReporter implementation - vespalib::string getReportContentType(const framework::HttpUrlPath&) const override; + vespalib::string getReportContentType( + const framework::HttpUrlPath&) const override; bool reportStatus(std::ostream&, const framework::HttpUrlPath&) const override; bool handleStatusRequest(const DelegatedStatusRequest& request) const override; @@ -110,7 +117,8 @@ public: * Checks whether a bucket needs to be split, and sends a split * if so. */ - void checkBucketForSplit(const BucketDatabase::Entry& e, uint8_t priority) override; + void checkBucketForSplit(const BucketDatabase::Entry& e, + uint8_t priority) override; const lib::Distribution& getDistribution() const override; @@ -122,9 +130,8 @@ public: * @return Returns the states in which the distributors consider * storage nodes to be up. */ - const char* getStorageNodeUpStates() const override { - return _initializingIsUp ? "uri" : "ur"; - } + const char* getStorageNodeUpStates() const override + { return _initializingIsUp ? "uri" : "ur"; } /** * Called by bucket db updater after a merge has finished, and all the @@ -147,10 +154,13 @@ public: } int getDistributorIndex() const override; + const std::string& getClusterName() const override; + const PendingMessageTracker& getPendingMessageTracker() const override; - void sendCommand(const std::shared_ptr<api::StorageCommand>&) override; - void sendReply(const std::shared_ptr<api::StorageReply>&) override; + + virtual void sendCommand(const std::shared_ptr<api::StorageCommand>&) override; + virtual void sendReply(const std::shared_ptr<api::StorageReply>&) override; const BucketGcTimeCalculator::BucketIdHasher& getBucketIdHasher() const override { diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.h b/storage/src/vespa/storage/distributor/operations/external/getoperation.h index 57a7c55fb81..ecdfb1e677a 100644 --- a/storage/src/vespa/storage/distributor/operations/external/getoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.h @@ -6,11 +6,15 @@ #include <vespa/storage/bucketdb/bucketcopy.h> #include <vespa/storageapi/messageapi/storagemessage.h> -namespace document { class Document; } +namespace document { +class Document; +} namespace storage { -namespace api { class GetCommand; } +namespace api { +class GetCommand; +} class PersistenceOperationMetricSet; @@ -26,9 +30,13 @@ public: PersistenceOperationMetricSet& metric); void onClose(DistributorMessageSender& sender) override; + void onStart(DistributorMessageSender& sender) override; + void onReceive(DistributorMessageSender& sender, const std::shared_ptr<api::StorageReply> & msg) override; + const char* getName() const override { return "get"; } + std::string getStatus() const override { return ""; } bool hasConsistentCopies() const; @@ -40,9 +48,13 @@ private: GroupId(const document::BucketId& id, uint32_t checksum, int node); bool operator<(const GroupId& other) const; + bool operator==(const GroupId& other) const; + const document::BucketId& getBucketId() const { return _id; } + int getNode() const { return _node; } + private: document::BucketId _id; uint32_t _checksum; @@ -53,8 +65,7 @@ private: public: BucketChecksumGroup(const BucketCopy& c) : copy(c), - sent(0), received(false), returnCode(api::ReturnCode::OK) - {} + sent(0), received(false), returnCode(api::ReturnCode::OK) {}; BucketCopy copy; api::StorageMessage::Id sent; @@ -94,4 +105,8 @@ private: }; } + + } + + diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.h b/storage/src/vespa/storage/distributor/operations/external/putoperation.h index 6ba9e9c3dbf..5043fed307f 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.h @@ -28,9 +28,14 @@ public: PersistenceOperationMetricSet& metric); void onStart(DistributorMessageSender& sender) override; - const char* getName() const override { return "put"; }; - std::string getStatus() const override { return ""; }; - void onReceive(DistributorMessageSender& sender, const std::shared_ptr<api::StorageReply> &) override; + + const char* getName() const override { return "put"; } + + std::string getStatus() const override { return ""; } + + void onReceive(DistributorMessageSender& sender, + const std::shared_ptr<api::StorageReply> &) override; + void onClose(DistributorMessageSender& sender) override; /** @@ -66,9 +71,11 @@ private: const uint16_t node, std::vector<PersistenceMessageTracker::ToSend>& putBatch); - bool shouldImplicitlyActivateReplica(const OperationTargetList& targets) const; + bool shouldImplicitlyActivateReplica( + const OperationTargetList& targets) const; std::shared_ptr<api::PutCommand> _msg; + DistributorComponent& _manager; }; diff --git a/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h b/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h index dce744b6648..25cc0825b7d 100644 --- a/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h @@ -7,7 +7,9 @@ namespace storage { -namespace api { class GetBucketListCommand; } +namespace api { +class GetBucketListCommand; +} namespace distributor { @@ -21,21 +23,22 @@ public: const MaintenanceOperationGenerator& generator, uint16_t distributorIndex, const std::shared_ptr<api::GetBucketListCommand>& cmd); - ~StatBucketListOperation() {} + virtual ~StatBucketListOperation() {} - const char* getName() const override { return "statBucketList"; } - std::string getStatus() const override { return ""; } + virtual const char* getName() const override { return "statBucketList"; } + virtual std::string getStatus() const override { return ""; } - void onStart(DistributorMessageSender& sender) override; - void onReceive(DistributorMessageSender&, const std::shared_ptr<api::StorageReply>&) override - { + virtual void onStart(DistributorMessageSender& sender) override; + virtual void onReceive(DistributorMessageSender&, const std::shared_ptr<api::StorageReply>&) override { // Never called. assert(false); } - void onClose(DistributorMessageSender&) override {} + void onClose(DistributorMessageSender&) override { + } private: - void getBucketStatus(const BucketDatabase::Entry& entry, std::ostream& os) const; + void getBucketStatus(const BucketDatabase::Entry& entry, + std::ostream& os) const; const BucketDatabase& _bucketDb; const MaintenanceOperationGenerator& _generator; @@ -45,3 +48,4 @@ private: } // distributor } // storage + diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 7b1ff03b8af..15f8d06965a 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -52,24 +52,24 @@ struct IntermediateMessageSender : DistributorMessageSender { DistributorMessageSender & fwd); ~IntermediateMessageSender(); - void sendCommand(const std::shared_ptr<api::StorageCommand>& cmd) override { + virtual void sendCommand(const std::shared_ptr<api::StorageCommand>& cmd) override { msgMap.insert(cmd->getMsgId(), callback); forward.sendCommand(cmd); }; - void sendReply(const std::shared_ptr<api::StorageReply>& reply) override { + virtual void sendReply(const std::shared_ptr<api::StorageReply>& reply) override { _reply = reply; } - int getDistributorIndex() const override { + virtual int getDistributorIndex() const override { return forward.getDistributorIndex(); } - const std::string& getClusterName() const override { + virtual const std::string& getClusterName() const override { return forward.getClusterName(); } - const PendingMessageTracker& getPendingMessageTracker() const override { + virtual const PendingMessageTracker& getPendingMessageTracker() const override { return forward.getPendingMessageTracker(); } }; diff --git a/storage/src/vespa/storage/distributor/operations/external/updateoperation.h b/storage/src/vespa/storage/distributor/operations/external/updateoperation.h index fef9c01efa9..0a2a9e564a0 100644 --- a/storage/src/vespa/storage/distributor/operations/external/updateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/updateoperation.h @@ -25,9 +25,14 @@ public: PersistenceOperationMetricSet& metric); void onStart(DistributorMessageSender& sender) override; - const char* getName() const override { return "update"; }; - std::string getStatus() const override { return ""; }; - void onReceive(DistributorMessageSender& sender, const std::shared_ptr<api::StorageReply> & msg) override; + + const char* getName() const override { return "update"; } + + std::string getStatus() const override { return ""; } + + void onReceive(DistributorMessageSender& sender, + const std::shared_ptr<api::StorageReply> & msg) override; + void onClose(DistributorMessageSender& sender) override; std::pair<document::BucketId, uint16_t> getNewestTimestampLocation() const { @@ -60,3 +65,4 @@ private: } } + diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h index fd133674bd1..802bb0f4bac 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h @@ -2,8 +2,8 @@ #pragma once -#include "operationtargetresolver.h" #include <vespa/storage/bucketdb/bucketdatabase.h> +#include <vespa/storage/distributor/operationtargetresolver.h> #include <vespa/vdslib/distribution/idealnodecalculator.h> namespace storage { @@ -91,20 +91,23 @@ public: _idealNodeCalculator(idealNodeCalc), _minUsedBucketBits(minUsedBucketBits), _redundancy(redundancy) - {} + { + } BucketInstanceList getAllInstances(OperationType type, const document::BucketId& id); - BucketInstanceList getInstances(OperationType type, const document::BucketId& id) { + BucketInstanceList getInstances(OperationType type, const document::BucketId& id) + { BucketInstanceList result(getAllInstances(type, id)); result.limitToRedundancyCopies(_redundancy); return result; } - OperationTargetList getTargets(OperationType type, const document::BucketId& id) override { - return getInstances(type, id).createTargets(); - } + virtual OperationTargetList getTargets(OperationType type, + const document::BucketId& id) override + { return getInstances(type, id).createTargets(); } }; } // distributor } // storage + diff --git a/storage/src/vespa/storage/distributor/statecheckers.h b/storage/src/vespa/storage/distributor/statecheckers.h index dbd9503f9f2..0113749f2fd 100644 --- a/storage/src/vespa/storage/distributor/statecheckers.h +++ b/storage/src/vespa/storage/distributor/statecheckers.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "idealstatemanager.h" +#include <vespa/storage/distributor/idealstatemanager.h> namespace storage { @@ -11,7 +11,9 @@ class SynchronizeAndMoveStateChecker : public StateChecker { public: std::string getStatusText() const override { return "Synchronization and moving"; } + Result check(Context& c) override; + const char* getName() const override { return "SynchronizeAndMove"; } }; @@ -19,7 +21,9 @@ class DeleteExtraCopiesStateChecker : public StateChecker { public: std::string getStatusText() const override { return "Delete extra copies"; } + Result check(Context& c) override; + const char* getName() const override { return "DeleteExtraCopies"; } private: @@ -45,7 +49,9 @@ class JoinBucketsStateChecker : public StateChecker { public: std::string getStatusText() const override { return "Join buckets"; } + Result check(Context& c) override; + const char* getName() const override { return "JoinBuckets"; } private: uint64_t getTotalUsedFileSize(const Context& c) const; @@ -63,9 +69,13 @@ class SplitBucketStateChecker : public StateChecker { public: std::string getStatusText() const override { return "Split buckets"; } + Result check(Context& c) override; + const char* getName() const override { return "SplitBucket"; } + private: + Result generateMinimumBucketSplitOperation(Context& c); Result generateMaxSizeExceededSplitOperation(Context& c); @@ -76,8 +86,11 @@ private: class SplitInconsistentStateChecker : public StateChecker { public: - std::string getStatusText() const override { return "Fix inconsistently split buckets"; } + std::string getStatusText() const override + { return "Fix inconsistently split buckets"; } + Result check(Context& c) override; + const char* getName() const override { return "SplitInconsistentBuckets"; } private: @@ -102,7 +115,9 @@ class BucketStateStateChecker : public StateChecker const StateChecker::Context& c) const; public: std::string getStatusText() const override { return "Set bucket copy state"; } + Result check(Context& c) override; + const char* getName() const override { return "SetBucketState"; } }; @@ -110,11 +125,15 @@ class GarbageCollectionStateChecker : public StateChecker { public: std::string getStatusText() const override { return "Garbage collection"; } + bool needsGarbageCollection(const Context& c) const; + Result check(Context& c) override; + const char* getName() const override { return "GarbageCollection"; } }; } } + diff --git a/storage/src/vespa/storage/distributor/statusreporterdelegate.h b/storage/src/vespa/storage/distributor/statusreporterdelegate.h index 93a1fe1cd48..0c836910765 100644 --- a/storage/src/vespa/storage/distributor/statusreporterdelegate.h +++ b/storage/src/vespa/storage/distributor/statusreporterdelegate.h @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "delegatedstatusrequest.h" -#include "statusdelegator.h" #include <vespa/storageframework/storageframework.h> +#include <vespa/storage/distributor/delegatedstatusrequest.h> +#include <vespa/storage/distributor/statusdelegator.h> namespace storage { namespace distributor { @@ -20,7 +20,9 @@ public: const framework::StatusReporter& target); void registerStatusPage(); - vespalib::string getReportContentType(const framework::HttpUrlPath&) const override; + + vespalib::string getReportContentType( + const framework::HttpUrlPath&) const override; bool reportStatus(std::ostream&, const framework::HttpUrlPath&) const override; }; diff --git a/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h index 90ddb71d678..2cf0255bf9f 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h @@ -7,10 +7,10 @@ */ #pragma once -#include "storagecomponentregisterimpl.h" #include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <vespa/storage/common/distributorcomponent.h> #include <vespa/storage/common/nodestateupdater.h> +#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> namespace storage { namespace lib { @@ -36,13 +36,18 @@ public: DistributorComponentRegisterImpl(); ~DistributorComponentRegisterImpl(); - void registerDistributorComponent(DistributorManagedComponent&) override; + virtual void registerDistributorComponent(DistributorManagedComponent&) override; + void setTimeCalculator(UniqueTimeCalculator& calc); void setDistributorConfig(const DistributorConfig&); void setVisitorConfig(const VisitorConfig&); + private: - void handleNewState() override; - void setNodeStateUpdater(NodeStateUpdater& updater) override; + virtual void handleNewState() override; + + virtual void setNodeStateUpdater(NodeStateUpdater& updater) override; }; } // storage + + diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h index d071ba8ba3e..7d322f027d5 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h @@ -7,10 +7,10 @@ */ #pragma once -#include "storagecomponentregisterimpl.h" #include <vespa/storage/bucketdb/minimumusedbitstracker.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/common/servicelayercomponent.h> +#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> namespace storage { @@ -35,8 +35,11 @@ public: return _minUsedBitsTracker; } - void registerServiceLayerComponent(ServiceLayerManagedComponent&) override; + virtual void registerServiceLayerComponent(ServiceLayerManagedComponent&) override; + void setDiskCount(uint16_t count); }; } // storage + + diff --git a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp index 4f2ed2bfc25..51a30d26def 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp @@ -14,9 +14,9 @@ StorageComponentRegisterImpl::StorageComponentRegisterImpl() _index(0xffff), _loadTypes(new documentapi::LoadTypeSet), _nodeStateUpdater(0) -{ } +{ -StorageComponentRegisterImpl::~StorageComponentRegisterImpl() { } +} void StorageComponentRegisterImpl::registerStorageComponent(StorageComponent& smc) diff --git a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h index 38773be167f..cf9abbb6864 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h @@ -40,7 +40,6 @@ public: typedef std::unique_ptr<StorageComponentRegisterImpl> UP; StorageComponentRegisterImpl(); - ~StorageComponentRegisterImpl(); const vespalib::string& getClusterName() const { return _clusterName; } const lib::NodeType& getNodeType() const @@ -48,12 +47,13 @@ public: uint16_t getIndex() const { return _index; } document::DocumentTypeRepo::SP getTypeRepo() { return _docTypeRepo; } documentapi::LoadTypeSet::SP getLoadTypes() { return _loadTypes; } - const document::BucketIdFactory& getBucketIdFactory() { return _bucketIdFactory; } + const document::BucketIdFactory& getBucketIdFactory() + { return _bucketIdFactory; } lib::Distribution::SP getDistribution() { return _distribution; } NodeStateUpdater& getNodeStateUpdater() { assert(_nodeStateUpdater != 0); return *_nodeStateUpdater; } - void registerStorageComponent(StorageComponent&) override; + virtual void registerStorageComponent(StorageComponent&) override; void setNodeInfo(vespalib::stringref clusterName, const lib::NodeType& nodeType, @@ -68,3 +68,5 @@ public: }; } // storage + + diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp index 18e0f2280f8..12e2c31068a 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp @@ -236,7 +236,7 @@ StatusWebServer::WebServer::onGetRequest(const string & tmpurl, const string &se namespace { class IndexPageReporter : public framework::HtmlStatusReporter { std::ostringstream ost; - void reportHtmlStatus(std::ostream& out,const framework::HttpUrlPath&) const override{ + virtual void reportHtmlStatus(std::ostream& out, const framework::HttpUrlPath&) const override { out << ost.str(); } diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h index d1a0a33da95..c89c286387b 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h @@ -10,12 +10,13 @@ #pragma once +#include <vespa/vespalib/stllike/string.h> +#include <vespa/fastlib/net/httpserver.h> +#include <list> #include <vespa/storage/config/config-stor-status.h> #include <vespa/storageframework/storageframework.h> #include <vespa/config/config.h> #include <vespa/config/helper/configfetcher.h> -#include <vespa/fastlib/net/httpserver.h> -#include <list> namespace storage { @@ -29,7 +30,7 @@ class StatusWebServer : private config::IFetcherCallback<vespa::config::content: public: WebServer(StatusWebServer&, uint16_t port); - void onGetRequest(const string & url, + virtual void onGetRequest(const string & url, const string & serverSpec, Fast_HTTPConnection& conn) override; const vespalib::string &getServerSpec() const { @@ -68,12 +69,16 @@ public: virtual ~StatusWebServer(); void handlePage(const framework::HttpUrlPath&, std::ostream& out); + static vespalib::string getServerSpec(const vespalib::string &requestSpec, const vespalib::string &serverSpec); + private: - void configure(std::unique_ptr<vespa::config::content::core::StorStatusConfig> config) override; + virtual void configure(std::unique_ptr<vespa::config::content::core::StorStatusConfig> config) override; void getPage(const char* url, Fast_HTTPConnection& conn); - void run(framework::ThreadHandle&) override; + virtual void run(framework::ThreadHandle&) override; + }; } + diff --git a/storage/src/vespa/storage/frameworkimpl/thread/appkiller.h b/storage/src/vespa/storage/frameworkimpl/thread/appkiller.h index 437927ccaf1..b785c98e505 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/appkiller.h +++ b/storage/src/vespa/storage/frameworkimpl/thread/appkiller.h @@ -22,7 +22,7 @@ struct AppKiller { }; struct RealAppKiller : public AppKiller { - void kill() override; + virtual void kill(); }; } // storage diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp index 2861ab904ff..5d58a092053 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp +++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp @@ -2,6 +2,8 @@ #include "deadlockdetector.h" #include <vespa/storage/bucketdb/storbucketdb.h> +#include <vespa/storageframework/storageframework.h> +#include <vespa/vdslib/state/nodetype.h> #include <vespa/storage/bucketmover/htmltable.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -144,10 +146,10 @@ namespace { ThreadChecker(DeadLockDetector& d, const framework::MilliSecTime& time) : _detector(d), _currentTime(time) {} - void visitThread(const vespalib::string& id, - const framework::ThreadProperties& tp, - const framework::ThreadTickData& tick, - DeadLockDetector::State& state) override + virtual void visitThread(const vespalib::string& id, + const framework::ThreadProperties& tp, + const framework::ThreadTickData& tick, + DeadLockDetector::State& state) override { // In case we just got a new tick, ignore the thread if (tick._lastTickMs > _currentTime.getTime()) return; diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h index c8cb3f35ddf..146d0ccb751 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h +++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h @@ -12,9 +12,9 @@ #pragma once -#include "appkiller.h" #include <vespa/storage/common/distributorcomponent.h> #include <vespa/storage/common/servicelayercomponent.h> +#include <vespa/storage/frameworkimpl/thread/appkiller.h> #include <vespa/storageframework/storageframework.h> #include <vespa/vespalib/util/sync.h> #include <map> @@ -88,9 +88,13 @@ private: StorageComponent* _component; framework::Thread::UP _thread; - void run(framework::ThreadHandle&) override; - void reportHtmlStatus(std::ostream& out, const framework::HttpUrlPath&) const override; + virtual void run(framework::ThreadHandle&) override; + + // Status implementation + virtual void reportHtmlStatus(std::ostream& out, + const framework::HttpUrlPath&) const override; vespalib::string getBucketLockInfo() const; }; } + diff --git a/storage/src/vespa/storage/persistence/diskthread.h b/storage/src/vespa/storage/persistence/diskthread.h index a1af83a65ef..3626bbb2c70 100644 --- a/storage/src/vespa/storage/persistence/diskthread.h +++ b/storage/src/vespa/storage/persistence/diskthread.h @@ -60,7 +60,7 @@ public: bool operator==(const OperationCount& c) const { return (count == c.count && pending == c.pending); } - void print(std::ostream& out, bool, const std::string&) const override + void print(std::ostream& out, bool, const std::string&) const { out << "OperationCount(" << count << (pending ? ", pending" : "") << ")"; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index b549aca35b6..e658599a3dc 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -13,9 +13,9 @@ #pragma once -#include "mergestatus.h" #include <vespa/document/bucket/bucketid.h> #include <ostream> +#include <vespa/storage/persistence/filestorage/mergestatus.h> #include <vespa/storage/storageutil/resumeguard.h> #include <vespa/storage/common/messagesender.h> diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index 39d9a60a950..a6942102c42 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -8,8 +8,6 @@ #pragma once -#include "filestorhandler.h" -#include "filestormetrics.h" #include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/sync.h> #include <vespa/document/bucket/bucketid.h> @@ -22,7 +20,8 @@ #include <vespa/storage/common/storagelinkqueued.h> #include <vespa/config-stor-filestor.h> #include <vespa/storage/persistence/diskthread.h> - +#include <vespa/storage/persistence/filestorage/filestorhandler.h> +#include <vespa/storage/persistence/filestorage/filestormetrics.h> #include <vespa/storage/persistence/providershutdownwrapper.h> #include <vespa/storageframework/storageframework.h> #include <vespa/storage/common/nodestateupdater.h> @@ -105,7 +104,7 @@ public: ServiceLayerComponentRegister&); ~FileStorManager(); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; + virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; // Return true if we are currently merging the given bucket. bool isMerging(const document::BucketId& bucket) const; @@ -123,7 +122,8 @@ public: private: void configure(std::unique_ptr<vespa::config::content::StorFilestorConfig> config) override; - void replyWithBucketNotFound(api::StorageMessage&, const document::BucketId&); + void replyWithBucketNotFound(api::StorageMessage&, + const document::BucketId&); void replyDroppedOperation(api::StorageMessage& msg, const document::BucketId& bucket, @@ -135,12 +135,17 @@ private: api::StorageMessage& msg, const char* callerId); - bool validateApplyDiffCommandBucket(api::StorageMessage& msg, const StorBucketDatabase::WrappedEntry&); - bool validateDiffReplyBucket(const StorBucketDatabase::WrappedEntry&, const document::BucketId&); + bool validateApplyDiffCommandBucket(api::StorageMessage& msg, + const StorBucketDatabase::WrappedEntry&); + bool validateDiffReplyBucket(const StorBucketDatabase::WrappedEntry&, + const document::BucketId&); - StorBucketDatabase::WrappedEntry mapOperationToDisk(api::StorageMessage&, const document::BucketId&); - StorBucketDatabase::WrappedEntry mapOperationToBucketAndDisk(api::BucketCommand&, const document::DocumentId*); - bool handlePersistenceMessage(const std::shared_ptr<api::StorageMessage>&, uint16_t disk); + StorBucketDatabase::WrappedEntry mapOperationToDisk( + api::StorageMessage&, const document::BucketId&); + StorBucketDatabase::WrappedEntry mapOperationToBucketAndDisk( + api::BucketCommand&, const document::DocumentId*); + bool handlePersistenceMessage(const std::shared_ptr<api::StorageMessage>&, + uint16_t disk); // Document operations bool onPut(const std::shared_ptr<api::PutCommand>&) override; @@ -170,15 +175,23 @@ private: bool onInternal(const std::shared_ptr<api::InternalCommand>&) override; bool onInternalReply(const std::shared_ptr<api::InternalReply>&) override; - void handleAbortBucketOperations(const std::shared_ptr<AbortBucketOperationsCommand>&); + void handleAbortBucketOperations( + const std::shared_ptr<AbortBucketOperationsCommand>&); + void sendCommand(const std::shared_ptr<api::StorageCommand>&) override; void sendReply(const std::shared_ptr<api::StorageReply>&) override; + void sendUp(const std::shared_ptr<api::StorageMessage>&) override; + void onClose() override; void onFlush(bool downwards) override; - void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; - void storageDistributionChanged() override; + + virtual void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; + + virtual void storageDistributionChanged() override; + void updateState(); }; } // storage + diff --git a/storage/src/vespa/storage/persistence/filestorage/mergestatus.h b/storage/src/vespa/storage/persistence/filestorage/mergestatus.h index ff7e9e5dfb4..9967fc24c7d 100644 --- a/storage/src/vespa/storage/persistence/filestorage/mergestatus.h +++ b/storage/src/vespa/storage/persistence/filestorage/mergestatus.h @@ -41,7 +41,7 @@ public: const std::vector<api::ApplyBucketDiffCommand::Entry>& part, uint16_t hasMask); void print(std::ostream& out, bool verbose, - const std::string& indent) const override; + const std::string& indent) const; bool isFirstNode() const { return (reply.get() != 0); } }; diff --git a/storage/src/vespa/storage/persistence/persistencethread.h b/storage/src/vespa/storage/persistence/persistencethread.h index d2c4dba3a5f..f4c70353fab 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.h +++ b/storage/src/vespa/storage/persistence/persistencethread.h @@ -2,15 +2,15 @@ #pragma once -#include "diskthread.h" -#include "processallhandler.h" -#include "mergehandler.h" -#include "diskmoveoperationhandler.h" -#include "persistenceutil.h" -#include "providershutdownwrapper.h" +#include <vespa/storage/common/statusmessages.h> +#include <vespa/storage/persistence/diskthread.h> +#include <vespa/storage/persistence/processallhandler.h> +#include <vespa/storage/persistence/mergehandler.h> +#include <vespa/storage/persistence/diskmoveoperationhandler.h> #include <vespa/storageframework/storageframework.h> #include <vespa/storage/common/storagecomponent.h> -#include <vespa/storage/common/statusmessages.h> +#include <vespa/storage/persistence/persistenceutil.h> +#include <vespa/storage/persistence/providershutdownwrapper.h> namespace storage { @@ -35,7 +35,7 @@ public: bool isMerging(const BucketId& bucket) const; - framework::Thread& getThread() override { return *_thread; } + virtual framework::Thread& getThread() override { return *_thread; } MessageTracker::UP handlePut(api::PutCommand& cmd); MessageTracker::UP handleRemove(api::RemoveCommand& cmd); @@ -99,8 +99,10 @@ private: void processMessages(FileStorHandler::LockedMessage & lock); // Thread main loop - void run(framework::ThreadHandle&) override; + virtual void run(framework::ThreadHandle&) override; + bool checkForError(const spi::Result& response, MessageTracker& tracker); + spi::Bucket getBucket(const DocumentId& id, const BucketId& bucket) const; void flushAllReplies(const document::BucketId& bucketId, @@ -112,3 +114,4 @@ private: }; } // storage + diff --git a/storage/src/vespa/storage/persistence/processallhandler.cpp b/storage/src/vespa/storage/persistence/processallhandler.cpp index c08b19a9e45..2095e19261c 100644 --- a/storage/src/vespa/storage/persistence/processallhandler.cpp +++ b/storage/src/vespa/storage/persistence/processallhandler.cpp @@ -2,6 +2,7 @@ #include "processallhandler.h" #include "bucketprocessor.h" +#include <vespa/storage/common/bucketmessages.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> diff --git a/storage/src/vespa/storage/persistence/splitbitdetector.cpp b/storage/src/vespa/storage/persistence/splitbitdetector.cpp index 62039f36133..8d816760815 100644 --- a/storage/src/vespa/storage/persistence/splitbitdetector.cpp +++ b/storage/src/vespa/storage/persistence/splitbitdetector.cpp @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> #include <vespa/storage/persistence/splitbitdetector.h> #include <vespa/storage/persistence/bucketprocessor.h> #include <vespa/document/bucket/bucketidfactory.h> diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 79ce0206005..17638f55b3a 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -8,7 +8,6 @@ vespa_add_library(storage_storageserver fnetlistener.cpp rpcrequestwrapper.cpp communicationmanager.cpp - communicationmanagermetrics.cpp statemanager.cpp documentapiconverter.cpp opslogger.cpp @@ -21,7 +20,6 @@ vespa_add_library(storage_storageserver distributornode.cpp servicelayernode.cpp statereporter.cpp - storagemetricsset.cpp changedbucketownershiphandler.cpp INSTALL lib64 DEPENDS diff --git a/storage/src/vespa/storage/storageserver/applicationgenerationfetcher.h b/storage/src/vespa/storage/storageserver/applicationgenerationfetcher.h index 7d5b64c1d17..0e33c8c86c6 100644 --- a/storage/src/vespa/storage/storageserver/applicationgenerationfetcher.h +++ b/storage/src/vespa/storage/storageserver/applicationgenerationfetcher.h @@ -9,14 +9,11 @@ #pragma once -#include <cstdint> -#include <string> - namespace storage { class ApplicationGenerationFetcher { public: - virtual ~ApplicationGenerationFetcher() {} + virtual ~ApplicationGenerationFetcher() {}; virtual int64_t getGeneration() const = 0; virtual std::string getComponentName() const = 0; diff --git a/storage/src/vespa/storage/storageserver/bucketintegritychecker.h b/storage/src/vespa/storage/storageserver/bucketintegritychecker.h index fd9516a85aa..6b87be804d0 100644 --- a/storage/src/vespa/storage/storageserver/bucketintegritychecker.h +++ b/storage/src/vespa/storage/storageserver/bucketintegritychecker.h @@ -112,29 +112,47 @@ public: ServiceLayerComponentRegister&); ~BucketIntegrityChecker(); - void onClose() override; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; + virtual void onClose() override; + + virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; + SchedulingOptions& getSchedulingOptions() { return _scheduleOptions; } + bool isWorkingOnCycle() const; + uint32_t getCycleCount() const; /** Give thread a bump by signalling it. */ void bump() const; - void setMaxThreadWaitTime(framework::MilliSecTime milliSecs) { _maxThreadWaitTime = milliSecs; } + void setMaxThreadWaitTime(framework::MilliSecTime milliSecs) + { _maxThreadWaitTime = milliSecs; } framework::Clock& getClock() { return _component.getClock(); } private: - void configure(std::unique_ptr<vespa::config::content::core::StorIntegritycheckerConfig>) override; + virtual void configure(std::unique_ptr<vespa::config::content::core::StorIntegritycheckerConfig>) override; + void onDoneInit() override; + bool onInternalReply(const std::shared_ptr<api::InternalReply>&) override; bool onSetSystemState(const std::shared_ptr<api::SetSystemStateCommand>&) override; - bool onNotifyBucketChangeReply(const std::shared_ptr<api::NotifyBucketChangeReply>&) override { return true; } - SchedulingOptions::RunState getCurrentRunState(framework::SecondTime time) const; - void run(framework::ThreadHandle&) override; + bool onNotifyBucketChangeReply( + const std::shared_ptr<api::NotifyBucketChangeReply>&) override + { return true; } + + SchedulingOptions::RunState getCurrentRunState( + framework::SecondTime time) const; + + virtual void run(framework::ThreadHandle&) override; + uint32_t getTotalPendingCount() const; - void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; + + // Status::Reporter implementation + virtual void reportHtmlStatus(std::ostream&, + const framework::HttpUrlPath&) const override; + }; } + diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index 3cae63d3586..31100de78d1 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** - * @class CommunicationManager + + +* @class CommunicationManager * @ingroup storageserver * * @brief Class used for sending messages over the network. @@ -10,26 +12,26 @@ #pragma once -#include <vespa/documentapi/documentapi.h> -#include "communicationmanagermetrics.h" -#include "messageallocationtypes.h" -#include "documentapiconverter.h" -#include <vespa/storage/common/storagelink.h> -#include <vespa/storage/config/config-stor-communicationmanager.h> -#include <vespa/storageframework/storageframework.h> -#include <vespa/storageframework/storageframework.h> -#include <vespa/storageapi/messageapi/storagecommand.h> -#include <vespa/storageapi/messageapi/storagemessage.h> -#include <vespa/storageapi/mbusprot/storagecommand.h> -#include <vespa/storageapi/mbusprot/storagereply.h> -#include <vespa/messagebus/rpcmessagebus.h> -#include <vespa/metrics/metrics.h> #include <vespa/vespalib/util/document_runnable.h> #include <map> #include <memory> #include <queue> #include <vector> #include <atomic> +#include <vespa/metrics/metrics.h> +#include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/storageframework/storageframework.h> +#include <vespa/storage/common/storagelink.h> +#include <vespa/storage/config/config-stor-communicationmanager.h> +#include <vespa/storageapi/messageapi/storagecommand.h> +#include <vespa/storageapi/messageapi/storagemessage.h> +#include <vespa/storageapi/mbusprot/storagecommand.h> +#include <vespa/storageapi/mbusprot/storagereply.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/storage/storageserver/communicationmanagermetrics.h> +#include <vespa/storage/storageserver/messageallocationtypes.h> +#include "documentapiconverter.h" +#include <vespa/storageframework/storageframework.h> namespace storage { @@ -187,32 +189,43 @@ private: public: CommunicationManager(StorageComponentRegister& compReg, const config::ConfigUri & configUri); - ~CommunicationManager(); + virtual ~CommunicationManager(); void enqueue(const std::shared_ptr<api::StorageMessage> & msg); + mbus::RPCMessageBus& getMessageBus() { assert(_mbus.get()); return *_mbus; } + const PriorityConverter& getPriorityConverter() const { return _docApiConverter.getPriorityConverter(); } /** * From StorageLink. Called when messages arrive from storage * modules. Will convert and dispatch messages to MessageServer */ - bool onUp(const std::shared_ptr<api::StorageMessage>&) override; + virtual bool onUp(const std::shared_ptr<api::StorageMessage>&) override; + bool sendCommand(const std::shared_ptr<api::StorageCommand>& command); + bool sendReply(const std::shared_ptr<api::StorageReply>& reply); void sendDirectRPCReply(RPCRequestWrapper& request, const std::shared_ptr<api::StorageReply>& reply); void sendMessageBusReply(StorageTransportContext& context, const std::shared_ptr<api::StorageReply>& reply); // Pump thread void run(framework::ThreadHandle&) override; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; + virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; + + /** Get messages from messagebus. */ void handleMessage(std::unique_ptr<mbus::Message> msg) override; + void sendMessageBusMessage(const std::shared_ptr<api::StorageCommand>& msg, - std::unique_ptr<mbus::Message> mbusMsg, const mbus::Route& route); + std::unique_ptr<mbus::Message> mbusMsg, const mbus::Route& route); + /** Get replies from messagebus. */ void handleReply(std::unique_ptr<mbus::Reply> msg) override; + void updateMessagebusProtocol(const document::DocumentTypeRepo::SP &repo); + }; } // storage + diff --git a/storage/src/vespa/storage/storageserver/communicationmanagermetrics.cpp b/storage/src/vespa/storage/storageserver/communicationmanagermetrics.cpp deleted file mode 100644 index d1e71b6e8a5..00000000000 --- a/storage/src/vespa/storage/storageserver/communicationmanagermetrics.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "communicationmanagermetrics.h" -#include <vespa/documentapi/loadtypes/loadtypeset.h> - -using namespace metrics; -namespace storage { - -CommunicationManagerMetrics::CommunicationManagerMetrics(const LoadTypeSet& loadTypes, MetricSet* owner) - : MetricSet("communication", "", "Metrics for the communication manager", owner), - queueSize("messagequeue", "", "Size of input message queue.", this), - messageProcessTime(loadTypes, - DoubleAverageMetric("messageprocesstime", "", - "Time transport thread uses to process a single message"), - this), - exceptionMessageProcessTime(loadTypes, - DoubleAverageMetric("exceptionmessageprocesstime", "", - "Time transport thread uses to process a single message " - "that fails with an exception thrown into communication manager"), - this), - failedDueToTooLittleMemory("toolittlememory", "", "Number of messages failed due to too little memory available", this), - convertToStorageAPIFailures("convertfailures", "", - "Number of messages that failed to get converted to storage API messages", this), - sendCommandLatency("sendcommandlatency", "", "Average ms used to send commands to MBUS", this), - sendReplyLatency("sendreplylatency", "", "Average ms used to send replies to MBUS", this) -{ -} - -CommunicationManagerMetrics::~CommunicationManagerMetrics() { } - -} - diff --git a/storage/src/vespa/storage/storageserver/communicationmanagermetrics.h b/storage/src/vespa/storage/storageserver/communicationmanagermetrics.h index 40c3646647e..4b580f79904 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanagermetrics.h +++ b/storage/src/vespa/storage/storageserver/communicationmanagermetrics.h @@ -9,6 +9,7 @@ #pragma once #include <vespa/metrics/metrics.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> namespace storage { @@ -21,8 +22,34 @@ struct CommunicationManagerMetrics : public metrics::MetricSet { metrics::DoubleAverageMetric sendCommandLatency; metrics::DoubleAverageMetric sendReplyLatency; - CommunicationManagerMetrics(const metrics::LoadTypeSet& loadTypes, metrics::MetricSet* owner = 0); - ~CommunicationManagerMetrics(); + CommunicationManagerMetrics(const metrics::LoadTypeSet& loadTypes, + metrics::MetricSet* owner = 0) + : metrics::MetricSet("communication", "", + "Metrics for the communication manager", owner), + queueSize("messagequeue", "", "Size of input message queue.", this), + messageProcessTime(loadTypes, metrics::DoubleAverageMetric( + "messageprocesstime", "", + "Time transport thread uses to process a single message"), + this), + exceptionMessageProcessTime(loadTypes, metrics::DoubleAverageMetric( + "exceptionmessageprocesstime", "", + "Time transport thread uses to process a single message " + "that fails with an exception thrown into communication " + "manager"), + this), + failedDueToTooLittleMemory("toolittlememory", "", + "Number of messages failed due to too little memory " + "available", this), + convertToStorageAPIFailures("convertfailures", "", + "Number of messages that failed to get converted to " + "storage API messages", this), + sendCommandLatency("sendcommandlatency", "", + "Average ms used to send commands to MBUS", this), + sendReplyLatency("sendreplylatency", "", + "Average ms used to send replies to MBUS", this) + { + } + }; } diff --git a/storage/src/vespa/storage/storageserver/distributornode.h b/storage/src/vespa/storage/storageserver/distributornode.h index 31fe8fe7878..03792d8fb56 100644 --- a/storage/src/vespa/storage/storageserver/distributornode.h +++ b/storage/src/vespa/storage/storageserver/distributornode.h @@ -8,9 +8,9 @@ #pragma once -#include "distributornodecontext.h" -#include "storagenode.h" #include <vespa/storage/common/distributorcomponent.h> +#include <vespa/storage/storageserver/distributornodecontext.h> +#include <vespa/storage/storageserver/storagenode.h> #include <vespa/storageframework/generic/thread/tickingthread.h> namespace storage { @@ -41,16 +41,19 @@ public: StorageLink::UP communicationManager = StorageLink::UP()); ~DistributorNode(); - const lib::NodeType& getNodeType() const override { return lib::NodeType::DISTRIBUTOR; } - ResumeGuard pause() override; + virtual const lib::NodeType& getNodeType() const override + { return lib::NodeType::DISTRIBUTOR; } + + virtual ResumeGuard pause() override; void handleConfigChange(vespa::config::content::core::StorDistributormanagerConfig&); void handleConfigChange(vespa::config::content::core::StorVisitordispatcherConfig&); private: - void initializeNodeSpecific() override; - StorageLink::UP createChain() override; - api::Timestamp getUniqueTimestamp() override; + virtual void initializeNodeSpecific() override; + virtual StorageLink::UP createChain() override; + + virtual api::Timestamp getUniqueTimestamp() override; /** * Shut down necessary distributor-specific components before shutting @@ -60,3 +63,4 @@ private: }; } // storage + diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index 577fedb58e1..ebfa10fe814 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -1,12 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "mergethrottler.h" -#include "storagemetricsset.h" +#include <vespa/fastos/fastos.h> +#include <vespa/storage/storageserver/mergethrottler.h> + #include <iostream> #include <sstream> +#include <iterator> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/persistence/messages.h> +#include <vespa/storage/storageserver/storagemetricsset.h> #include <vespa/log/log.h> LOG_SETUP(".mergethrottler"); @@ -43,71 +46,6 @@ const mbus::string DummyMbusMessage<Base>::NAME = "SkyNet"; } -MergeThrottler::ChainedMergeState::ChainedMergeState() - : _cmd(), - _cmdString(), - _clusterStateVersion(0), - _inCycle(false), - _executingLocally(false), - _unwinding(false), - _cycleBroken(false), - _aborted(false) -{ } - -MergeThrottler::ChainedMergeState::ChainedMergeState(const api::StorageMessage::SP& cmd, bool executing) - : _cmd(cmd), - _cmdString(cmd->toString()), - _clusterStateVersion(static_cast<const api::MergeBucketCommand&>(*cmd).getClusterStateVersion()), - _inCycle(false), - _executingLocally(executing), - _unwinding(false), - _cycleBroken(false), - _aborted(false) -{ } -MergeThrottler::ChainedMergeState::~ChainedMergeState() {} - -MergeThrottler::Metrics::Metrics(metrics::MetricSet* owner) - : metrics::MetricSet("mergethrottler", "", "", owner), - averageQueueWaitingTime("averagequeuewaitingtime", "", "Average time a merge spends in the throttler queue", this), - chaining("mergechains", this), - local("locallyexecutedmerges", this) -{ } -MergeThrottler::Metrics::~Metrics() {} - -MergeThrottler::MergeFailureMetrics::MergeFailureMetrics(metrics::MetricSet* owner) - : metrics::MetricSet("failures", "", "Detailed failure statistics", owner), - sum("total", "", "Sum of all failures", this), - notready("notready", "", "The number of merges discarded because distributor was not ready", this), - timeout("timeout", "", "The number of merges that failed because they timed out towards storage", this), - aborted("aborted", "", "The number of merges that failed because the storage node was (most likely) shutting down", this), - wrongdistribution("wrongdistribution", "", "The number of merges that were discarded (flushed) because they were initiated at an older cluster state than the current", this), - bucketnotfound("bucketnotfound", "", "The number of operations that failed because the bucket did not exist", this), - busy("busy", "", "The number of merges that failed because the storage node was busy", this), - exists("exists", "", "The number of merges that were rejected due to a merge operation for their bucket already being processed", this), - rejected("rejected", "", "The number of merges that were rejected", this), - other("other", "", "The number of other failures", this) -{ - sum.addMetricToSum(notready); - sum.addMetricToSum(timeout); - sum.addMetricToSum(aborted); - sum.addMetricToSum(wrongdistribution); - sum.addMetricToSum(bucketnotfound); - sum.addMetricToSum(busy); - sum.addMetricToSum(exists); - sum.addMetricToSum(rejected); - sum.addMetricToSum(other); -} -MergeThrottler::MergeFailureMetrics::~MergeFailureMetrics() { } - - -MergeThrottler::MergeOperationMetrics::MergeOperationMetrics(const std::string& name, metrics::MetricSet* owner) - : metrics::MetricSet(name, "", vespalib::make_string("Statistics for %s", name.c_str()), owner), - ok("ok", "", vespalib::make_string("The number of successful merges for '%s'", name.c_str()), this), - failures(this) -{ -} -MergeThrottler::MergeOperationMetrics::~MergeOperationMetrics() { } - MergeThrottler::MergeNodeSequence::MergeNodeSequence( const api::MergeBucketCommand& cmd, uint16_t thisIndex) diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.h b/storage/src/vespa/storage/storageserver/mergethrottler.h index 74ffe095d7c..8ee4ca7be71 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.h +++ b/storage/src/vespa/storage/storageserver/mergethrottler.h @@ -7,6 +7,11 @@ */ #pragma once +#include <map> +#include <utility> +#include <vector> +#include <set> +#include <memory> #include <vespa/vespalib/util/document_runnable.h> #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/common/storagelink.h> @@ -43,8 +48,37 @@ public: metrics::LongCountMetric rejected; metrics::LongCountMetric other; - MergeFailureMetrics(metrics::MetricSet* owner); - ~MergeFailureMetrics(); + MergeFailureMetrics(metrics::MetricSet* owner) + : metrics::MetricSet("failures", "", "Detailed failure statistics", owner), + sum("total", "", "Sum of all failures", this), + notready("notready", "", "The number of merges discarded " + "because distributor was not ready", this), + timeout("timeout", "", "The number of merges that failed because " + "they timed out towards storage", this), + aborted("aborted", "", "The number of merges that failed " + "because the storage node was (most likely) shutting down", this), + wrongdistribution("wrongdistribution", "", "The number of merges that " + "were discarded (flushed) because they were initiated at an " + "older cluster state than the current", this), + bucketnotfound("bucketnotfound", "", "The number of operations that failed " + "because the bucket did not exist", this), + busy("busy", "", "The number of merges that failed because the " + "storage node was busy", this), + exists("exists", "", "The number of merges that were rejected due to a " + "merge operation for their bucket already being processed", this), + rejected("rejected", "", "The number of merges that were rejected", this), + other("other", "", "The number of other failures", this) + { + sum.addMetricToSum(notready); + sum.addMetricToSum(timeout); + sum.addMetricToSum(aborted); + sum.addMetricToSum(wrongdistribution); + sum.addMetricToSum(bucketnotfound); + sum.addMetricToSum(busy); + sum.addMetricToSum(exists); + sum.addMetricToSum(rejected); + sum.addMetricToSum(other); + } }; class MergeOperationMetrics : public metrics::MetricSet @@ -53,8 +87,12 @@ public: metrics::LongCountMetric ok; MergeFailureMetrics failures; - MergeOperationMetrics(const std::string& name, metrics::MetricSet* owner); - ~MergeOperationMetrics(); + MergeOperationMetrics(const std::string& name, metrics::MetricSet* owner) + : metrics::MetricSet(name, "", vespalib::make_string("Statistics for %s", name.c_str()), owner), + ok("ok", "", vespalib::make_string("The number of successful merges for '%s'", name.c_str()), this), + failures(this) + { + } }; class Metrics : public metrics::MetricSet @@ -64,8 +102,15 @@ public: MergeOperationMetrics chaining; MergeOperationMetrics local; - Metrics(metrics::MetricSet* owner = 0); - ~Metrics(); + Metrics(metrics::MetricSet* owner = 0) + : metrics::MetricSet("mergethrottler", "", "", owner), + averageQueueWaitingTime( + "averagequeuewaitingtime", "", "Average time a merge spends in " + "the throttler queue", this), + chaining("mergechains", this), + local("locallyexecutedmerges", this) + { + } }; private: @@ -106,9 +151,30 @@ private: bool _cycleBroken; bool _aborted; - ChainedMergeState(); - ChainedMergeState(const api::StorageMessage::SP& cmd, bool executing = false); - ~ChainedMergeState(); + ChainedMergeState() + : _cmd(), + _cmdString(), + _clusterStateVersion(0), + _inCycle(false), + _executingLocally(false), + _unwinding(false), + _cycleBroken(false), + _aborted(false) + { + } + + ChainedMergeState(const api::StorageMessage::SP& cmd, bool executing = false) + : _cmd(cmd), + _cmdString(cmd->toString()), + _clusterStateVersion(static_cast<const api::MergeBucketCommand&>( + *cmd).getClusterStateVersion()), + _inCycle(false), + _executingLocally(executing), + _unwinding(false), + _cycleBroken(false), + _aborted(false) + { + } // Use default copy-constructor/assignment operator bool isExecutingLocally() const { return _executingLocally; } @@ -173,7 +239,9 @@ public: * windowSizeIncrement used for allowing unit tests to start out with more * than 1 as their window size. */ - MergeThrottler(const config::ConfigUri & configUri, StorageComponentRegister&); + MergeThrottler(const config::ConfigUri & configUri, + StorageComponentRegister&); + ~MergeThrottler(); /** Implements document::Runnable::run */ @@ -199,8 +267,12 @@ public: vespalib::Lock& getStateLock() { return _stateLock; } Metrics& getMetrics() { return *_metrics; } + std::size_t getMaxQueueSize() const { return _maxQueueSize; } + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + + // HtmlStatusReporter implementation void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; private: friend class ThreadRendezvousGuard; // impl in .cpp file @@ -213,7 +285,9 @@ private: std::size_t _sortedIndex; // Index of current storage node in the sorted node sequence const uint16_t _thisIndex; // Index of the current storage node - MergeNodeSequence(const api::MergeBucketCommand& cmd, uint16_t thisIndex); + MergeNodeSequence( + const api::MergeBucketCommand& cmd, + uint16_t thisIndex); std::size_t getSortedIndex() const { return _sortedIndex; } const std::vector<api::MergeBucketCommand::Node>& getSortedNodes() const { @@ -258,8 +332,13 @@ private: // NOTE: unless explicitly specified, all the below functions require // _sync lock to be held upon call (usually implicitly via MessageGuard) - void handleMessageDown(const std::shared_ptr<api::StorageMessage>& msg, MessageGuard& msgGuard); - void handleMessageUp(const std::shared_ptr<api::StorageMessage>& msg, MessageGuard& msgGuard); + void handleMessageDown( + const std::shared_ptr<api::StorageMessage>& msg, + MessageGuard& msgGuard); + + void handleMessageUp( + const std::shared_ptr<api::StorageMessage>& msg, + MessageGuard& msgGuard); /** * Handle the receival of MergeBucketReply, be it from another node @@ -293,13 +372,17 @@ private: * * Precondition: no existing merge state exists for msg's bucketid. */ - void processNewMergeCommand(const api::StorageMessage::SP& msg, MessageGuard& msgGuard); + void processNewMergeCommand( + const api::StorageMessage::SP& msg, + MessageGuard& msgGuard); /** * Precondition: an existing merge state exists for msg's bucketid. * @return true if message was handled, false otherwise (see onUp/onDown). */ - bool processCycledMergeCommand(const api::StorageMessage::SP& msg, MessageGuard& msgGuard); + bool processCycledMergeCommand( + const api::StorageMessage::SP& msg, + MessageGuard& msgGuard); /** * Forwards the given MergeBucketCommand to the storage node given @@ -320,7 +403,10 @@ private: * @return Highest priority waiting merge or null SP if queue is empty */ api::StorageMessage::SP getNextQueuedMerge(); - void enqueueMerge(const api::StorageMessage::SP& msg, MessageGuard& msgGuard); + + void enqueueMerge( + const api::StorageMessage::SP& msg, + MessageGuard& msgGuard); /** * @return true if throttle policy says at least one additional @@ -348,15 +434,25 @@ private: * Immediately reject all queued merges whose cluster state version is * less than that of rejectLessThanVersion */ - void rejectOutdatedQueuedMerges(MessageGuard& msgGuard, uint32_t rejectLessThanVersion); + void rejectOutdatedQueuedMerges(MessageGuard& msgGuard, + uint32_t rejectLessThanVersion); + bool attemptProcessNextQueuedMerge(MessageGuard& msgGuard); + bool processQueuedMerges(MessageGuard& msgGuard); + void handleRendezvous(vespalib::MonitorGuard& guard); + void rendezvousWithWorkerThread(vespalib::MonitorGuard&); + void releaseWorkerThreadRendezvous(vespalib::MonitorGuard&); + bool isDiffCommand(const api::StorageMessage& msg) const; + bool isMergeCommand(const api::StorageMessage& msg) const; + bool isMergeReply(const api::StorageMessage& msg) const; + bool bucketIsUnknownOrAborted(const document::BucketId& bucket) const; std::shared_ptr<api::StorageMessage> makeAbortReply( @@ -364,7 +460,8 @@ private: vespalib::stringref reason) const; void handleOutdatedMerges(const api::SetSystemStateCommand&); - void rejectOperationsInThreadQueue(MessageGuard&, uint32_t minimumStateVersion); + void rejectOperationsInThreadQueue(MessageGuard&, + uint32_t minimumStateVersion); void markActiveMergesAsAborted(uint32_t minimumStateVersion); // const function, but metrics are mutable @@ -374,3 +471,4 @@ private: }; } // namespace storage + diff --git a/storage/src/vespa/storage/storageserver/opslogger.h b/storage/src/vespa/storage/storageserver/opslogger.h index 905c0fd4e85..57f304a04aa 100644 --- a/storage/src/vespa/storage/storageserver/opslogger.h +++ b/storage/src/vespa/storage/storageserver/opslogger.h @@ -27,15 +27,20 @@ public: ~OpsLogger(); void onClose() override; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; + + virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; + bool onPutReply(const std::shared_ptr<api::PutReply>& msg) override; bool onUpdateReply(const std::shared_ptr<api::UpdateReply>& msg) override; bool onRemoveReply(const std::shared_ptr<api::RemoveReply>& msg) override; bool onGetReply(const std::shared_ptr<api::GetReply>& msg) override; /** Ignore all replies on the way down the storage chain. */ - bool onDown(const std::shared_ptr<api::StorageMessage>&) override { return false; }; + bool onDown(const std::shared_ptr<api::StorageMessage>&) override + { return false; }; + void configure(std::unique_ptr<vespa::config::content::core::StorOpsloggerConfig> config) override; + private: vespalib::Lock _lock; std::string _fileName; @@ -46,3 +51,4 @@ private: }; } + diff --git a/storage/src/vespa/storage/storageserver/statemanager.h b/storage/src/vespa/storage/storageserver/statemanager.h index 974b8ca2393..028168937b5 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.h +++ b/storage/src/vespa/storage/storageserver/statemanager.h @@ -72,19 +72,23 @@ public: void tick(); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; + virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; - lib::NodeState::CSP getReportedNodeState() const override; - lib::NodeState::CSP getCurrentNodeState() const override; - lib::ClusterState::CSP getSystemState() const override; + /** Implementation of HtmlStatusReporter */ + virtual void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; - void addStateListener(StateListener&) override; - void removeStateListener(StateListener&) override; + virtual lib::NodeState::CSP getReportedNodeState() const override; + virtual lib::NodeState::CSP getCurrentNodeState() const override; + virtual lib::ClusterState::CSP getSystemState() const override; + + virtual void addStateListener(StateListener&) override; + virtual void removeStateListener(StateListener&) override; + + virtual Lock::SP grabStateChangeLock() override; + virtual void setReportedNodeState(const lib::NodeState& state) override; - Lock::SP grabStateChangeLock() override; - void setReportedNodeState(const lib::NodeState& state) override; void setClusterState(const lib::ClusterState& c); + HostInfo& getHostInfo() { return *_hostInfo; } private: @@ -132,7 +136,10 @@ private: */ std::string getNodeInfo() const; - void run(framework::ThreadHandle&) override; + virtual void run(framework::ThreadHandle&) override; + }; } // storage + + diff --git a/storage/src/vespa/storage/storageserver/statereporter.h b/storage/src/vespa/storage/storageserver/statereporter.h index c46a878ef40..51a9e93a197 100644 --- a/storage/src/vespa/storage/storageserver/statereporter.h +++ b/storage/src/vespa/storage/storageserver/statereporter.h @@ -10,11 +10,11 @@ #pragma once -#include "applicationgenerationfetcher.h" -#include <vespa/storage/common/storagecomponent.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/metrics/metrics.h> #include <vespa/metrics/state_api_adapter.h> +#include <vespa/storage/common/storagecomponent.h> +#include <vespa/storage/storageserver/applicationgenerationfetcher.h> +#include <vespa/storageframework/storageframework.h> #include <vespa/vespalib/net/metrics_producer.h> #include <vespa/vespalib/net/state_api.h> @@ -37,8 +37,11 @@ public: const std::string& name = "status"); ~StateReporter(); - vespalib::string getReportContentType(const framework::HttpUrlPath&) const override; - bool reportStatus(std::ostream& out, const framework::HttpUrlPath& path) const override; + vespalib::string getReportContentType( + const framework::HttpUrlPath&) const override; + bool reportStatus(std::ostream& out, + const framework::HttpUrlPath& path) const override; + private: metrics::MetricManager &_manager; metrics::StateApiAdapter _metricsAdapter; @@ -47,10 +50,16 @@ private: ApplicationGenerationFetcher& _generationFetcher; std::string _name; - vespalib::string getMetrics(const vespalib::string &consumer) override; - vespalib::string getTotalMetrics(const vespalib::string &consumer) override; - Health getHealth() const override; - void getComponentConfig(Consumer &consumer) override; + // Implements vespalib::MetricsProducer + virtual vespalib::string getMetrics(const vespalib::string &consumer) override; + virtual vespalib::string getTotalMetrics(const vespalib::string &consumer) override; + + // Implements vespalib::HealthProducer + virtual Health getHealth() const override; + + // Implements vespalib::ComponentConfigProducer + virtual void getComponentConfig(Consumer &consumer) override; }; } // storage + diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp deleted file mode 100644 index aa57e3ebcd5..00000000000 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "storagemetricsset.h" -#include <vespa/document/fieldvalue/serializablearray.h> - -namespace storage { - -MessageMemoryUseMetricSet::MessageMemoryUseMetricSet(metrics::MetricSet* owner) - : metrics::MetricSet("message_memory_use", "memory", "Message use from storage messages", owner), - total("total", "memory", "Message use from storage messages", this), - lowpri("lowpri", "memory", "Message use from low priority storage messages", this), - normalpri("normalpri", "memory", "Message use from normal priority storage messages", this), - highpri("highpri", "memory", "Message use from high priority storage messages", this), - veryhighpri("veryhighpri", "memory", "Message use from very high priority storage messages", this) -{ } -MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() {} - -DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSet* owner) - : metrics::MetricSet("document_serialization", "docserialization", - "Counts of document serialization of various types", owner), - usedCachedSerializationCount( - "cached_serialization_count", "docserialization", - "Number of times we didn't need to serialize the document as " - "we already had serialized version cached", this), - compressedDocumentCount( - "compressed_serialization_count", "docserialization", - "Number of times we compressed document when serializing", - this), - compressionDidntHelpCount( - "compressed_didnthelp_count", "docserialization", - "Number of times we compressed document when serializing, but " - "the compressed version was bigger, so it was dumped", this), - uncompressableCount( - "uncompressable_serialization_count", "docserialization", - "Number of times we didn't attempt compression as document " - "had already been tagged uncompressable", this), - serializedUncompressed( - "uncompressed_serialization_count", "docserialization", - "Number of times we serialized a document uncompressed", this), - inputWronglySerialized( - "input_wrongly_serialized_count", "docserialization", - "Number of times we reserialized a document because the " - "compression it had in cache did not match what was configured", - this) -{ } -DocumentSerializationMetricSet::~DocumentSerializationMetricSet() { } - -StorageMetricSet::StorageMetricSet() - : metrics::MetricSet("server", "memory", - "Metrics for VDS applications"), - memoryUse("memoryusage", "memory", "", this), - memoryUse_messages(this), - memoryUse_visiting("memoryusage_visiting", "memory", - "Message use from visiting", this), - documentSerialization(this) -{ } -StorageMetricSet::~StorageMetricSet() { } - -void StorageMetricSet::updateMetrics() { - document::SerializableArray::Statistics stats( - document::SerializableArray::getStatistics()); - - documentSerialization.usedCachedSerializationCount.set( - stats._usedCachedSerializationCount); - documentSerialization.compressedDocumentCount.set( - stats._compressedDocumentCount); - documentSerialization.compressionDidntHelpCount.set( - stats._compressionDidntHelpCount); - documentSerialization.uncompressableCount.set( - stats._uncompressableCount); - documentSerialization.serializedUncompressed.set( - stats._serializedUncompressed); - documentSerialization.inputWronglySerialized.set( - stats._inputWronglySerialized); -} - -} // storage diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.h b/storage/src/vespa/storage/storageserver/storagemetricsset.h index f7083705763..18be3e21ada 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.h +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/metrics/metrics.h> +#include <vespa/document/fieldvalue/serializablearray.h> namespace storage { @@ -15,8 +16,22 @@ public: metrics::LongValueMetric highpri; metrics::LongValueMetric veryhighpri; - MessageMemoryUseMetricSet(metrics::MetricSet* owner); - ~MessageMemoryUseMetricSet(); + MessageMemoryUseMetricSet(metrics::MetricSet* owner) + : metrics::MetricSet("message_memory_use", "memory", + "Message use from storage messages", owner), + total("total", "memory", + "Message use from storage messages", this), + lowpri("lowpri", "memory", + "Message use from low priority storage messages", this), + normalpri("normalpri", "memory", + "Message use from normal priority storage messages", this), + highpri("highpri", "memory", + "Message use from high priority storage messages", this), + veryhighpri("veryhighpri", "memory", + "Message use from very high priority storage messages", this) + { + } + }; struct DocumentSerializationMetricSet : public metrics::MetricSet @@ -28,8 +43,36 @@ struct DocumentSerializationMetricSet : public metrics::MetricSet metrics::LongCountMetric serializedUncompressed; metrics::LongCountMetric inputWronglySerialized; - DocumentSerializationMetricSet(metrics::MetricSet* owner); - ~DocumentSerializationMetricSet(); + DocumentSerializationMetricSet(metrics::MetricSet* owner) + : metrics::MetricSet("document_serialization", "docserialization", + "Counts of document serialization of various types", owner), + usedCachedSerializationCount( + "cached_serialization_count", "docserialization", + "Number of times we didn't need to serialize the document as " + "we already had serialized version cached", this), + compressedDocumentCount( + "compressed_serialization_count", "docserialization", + "Number of times we compressed document when serializing", + this), + compressionDidntHelpCount( + "compressed_didnthelp_count", "docserialization", + "Number of times we compressed document when serializing, but " + "the compressed version was bigger, so it was dumped", this), + uncompressableCount( + "uncompressable_serialization_count", "docserialization", + "Number of times we didn't attempt compression as document " + "had already been tagged uncompressable", this), + serializedUncompressed( + "uncompressed_serialization_count", "docserialization", + "Number of times we serialized a document uncompressed", this), + inputWronglySerialized( + "input_wrongly_serialized_count", "docserialization", + "Number of times we reserialized a document because the " + "compression it had in cache did not match what was configured", + this) + { + } + }; struct StorageMetricSet : public metrics::MetricSet @@ -39,9 +82,34 @@ struct StorageMetricSet : public metrics::MetricSet metrics::LongValueMetric memoryUse_visiting; DocumentSerializationMetricSet documentSerialization; - StorageMetricSet(); - ~StorageMetricSet(); - void updateMetrics(); + StorageMetricSet() + : metrics::MetricSet("server", "memory", + "Metrics for VDS applications"), + memoryUse("memoryusage", "memory", "", this), + memoryUse_messages(this), + memoryUse_visiting("memoryusage_visiting", "memory", + "Message use from visiting", this), + documentSerialization(this) + { + } + + void updateMetrics() { + document::SerializableArray::Statistics stats( + document::SerializableArray::getStatistics()); + + documentSerialization.usedCachedSerializationCount.set( + stats._usedCachedSerializationCount); + documentSerialization.compressedDocumentCount.set( + stats._compressedDocumentCount); + documentSerialization.compressionDidntHelpCount.set( + stats._compressionDidntHelpCount); + documentSerialization.uncompressableCount.set( + stats._uncompressableCount); + documentSerialization.serializedUncompressed.set( + stats._serializedUncompressed); + documentSerialization.inputWronglySerialized.set( + stats._inputWronglySerialized); + } }; } // storage diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index 5eea62a17ad..5df29191489 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -12,10 +12,9 @@ #pragma once -#include "storagemetricsset.h" -#include "storagenodecontext.h" -#include "applicationgenerationfetcher.h" #include <vespa/document/bucket/bucketidfactory.h> +#include <memory> +#include <string> #include <vespa/storage/config/config-stor-server.h> #include <vespa/config/helper/legacysubscriber.h> @@ -35,6 +34,9 @@ #include <vespa/storageframework/defaultimplementation/memory/memorymanager.h> #include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> #include <vespa/storage/frameworkimpl/memory/memorystatusviewer.h> +#include <vespa/storage/storageserver/applicationgenerationfetcher.h> +#include <vespa/storage/storageserver/storagenodecontext.h> +#include <vespa/storage/storageserver/storagemetricsset.h> #include <vespa/storage/visiting/visitormessagesessionfactory.h> #include <vespa/storageframework/storageframework.h> #include <vespa/storage/storageutil/resumeguard.h> @@ -76,9 +78,12 @@ public: virtual ~StorageNode(); virtual const lib::NodeType& getNodeType() const = 0; + bool attemptedStopped() const; - void notifyDoneInitializing() override; + + virtual void notifyDoneInitializing() override; void waitUntilInitialized(uint32_t timeoutSeconds = 15); + void updateMetrics(const MetricLockGuard & guard) override; /** Updates the document type repo. */ @@ -89,12 +94,17 @@ public: * is alive, no calls will be made towards the persistence provider. */ virtual ResumeGuard pause() = 0; + void requestShutdown(vespalib::stringref reason) override; - void notifyPartitionDown(int partId, vespalib::stringref reason); + + void + notifyPartitionDown(int partId, vespalib::stringref reason); + DoneInitializeHandler& getDoneInitializeHandler() { return *this; } - // For testing + // For testing StorageLink* getChain() { return _chain.get(); } + virtual void initializeStatusWebServer(); private: @@ -130,10 +140,10 @@ private: std::unique_ptr<StorageLink> _chain; /** Implementation of config callbacks. */ - void configure(std::unique_ptr<vespa::config::content::core::StorServerConfig> config) override; - void configure(std::unique_ptr<vespa::config::content::UpgradingConfig> config) override; - void configure(std::unique_ptr<vespa::config::content::StorDistributionConfig> config) override; - void configure(std::unique_ptr<vespa::config::content::core::StorPrioritymappingConfig>) override; + virtual void configure(std::unique_ptr<vespa::config::content::core::StorServerConfig> config) override; + virtual void configure(std::unique_ptr<vespa::config::content::UpgradingConfig> config) override; + virtual void configure(std::unique_ptr<vespa::config::content::StorDistributionConfig> config) override; + virtual void configure(std::unique_ptr<vespa::config::content::core::StorPrioritymappingConfig>) override; virtual void configure(std::unique_ptr<document::DocumenttypesConfig> config, bool hasChanged, int64_t generation); void updateUpgradeFlag(const vespa::config::content::UpgradingConfig&); @@ -174,6 +184,8 @@ protected: virtual void handleLiveConfigUpdate(); void shutdown(); virtual void removeConfigSubscriptions(); + }; } // storage + diff --git a/storage/src/vespa/storage/storageutil/palette.h b/storage/src/vespa/storage/storageutil/palette.h index 338d8a3add7..fe42eae0d31 100644 --- a/storage/src/vespa/storage/storageutil/palette.h +++ b/storage/src/vespa/storage/storageutil/palette.h @@ -11,8 +11,6 @@ #pragma once #include <vector> -#include <cstdint> -#include <iosfwd> namespace storage { diff --git a/storage/src/vespa/storage/tools/throttlingsim.h b/storage/src/vespa/storage/tools/throttlingsim.h index a7384d0f362..b583a7d2533 100644 --- a/storage/src/vespa/storage/tools/throttlingsim.h +++ b/storage/src/vespa/storage/tools/throttlingsim.h @@ -75,7 +75,7 @@ public: double max_diff; virtual void returnMessage(const Message& m) = 0; - void run() override; + virtual void run() override; virtual void print(double timenow); Client(Messaging& msgng, double windowSize, int to); @@ -91,15 +91,17 @@ class FixedClient : public Client { public: FixedClient(Messaging& msgng, int winsize, int to) : Client(msgng, winsize, to) {}; - void returnMessage(const Message& m) override; + + virtual void returnMessage(const Message& m) override; }; class LoadBalancingClient : public Client { public: LoadBalancingClient(Messaging& msgng, int winsize, int to); - void returnMessage(const Message& m) override; - void run() override; - void print(double timenow) override; + + virtual void returnMessage(const Message& m) override; + virtual void run() override; + virtual void print(double timenow) override; std::vector<double> weights; }; @@ -107,9 +109,10 @@ public: class BusyCounterBalancingClient : public Client { public: BusyCounterBalancingClient(Messaging& msgng, int winsize, int to); - void returnMessage(const Message& m) override; - void run() override; - void print(double timenow) override; + + virtual void returnMessage(const Message& m) override; + virtual void run() override; + virtual void print(double timenow) override; std::vector<int> busyCount; }; @@ -123,7 +126,8 @@ public: DynamicClient(Messaging& msgng, int maxWinSize, double to) : Client(msgng, 1, static_cast<int>(to)), maxwinsize(maxWinSize), threshold(maxWinSize / 2), lastFailTimestamp(0) {}; - void returnMessage(const Message& m) override; + + virtual void returnMessage(const Message& m) override; }; class LatencyControlClient : public Client { @@ -133,8 +137,10 @@ public: LatencyControlClient(Messaging& msgng, double to) : Client(msgng, 1, static_cast<int>(to)), count(0){}; - void returnMessage(const Message& m) override; - void print(double timenow) override; + + virtual void returnMessage(const Message& m) override; + + virtual void print(double timenow) override; }; @@ -148,4 +154,5 @@ public: ThrottlingApp() {}; int Main() override; + }; diff --git a/storage/src/vespa/storage/visiting/commandqueue.h b/storage/src/vespa/storage/visiting/commandqueue.h index ba0141add45..07677fdcd38 100644 --- a/storage/src/vespa/storage/visiting/commandqueue.h +++ b/storage/src/vespa/storage/visiting/commandqueue.h @@ -22,189 +22,229 @@ namespace storage { -template<class Command> -class CommandQueue : public vespalib::Printable -{ -public: - struct CommandEntry { - typedef typename Command::Priority PriorityType; - std::shared_ptr<Command> _command; - uint64_t _time; - uint64_t _sequenceId; - PriorityType _priority; - - CommandEntry(const std::shared_ptr<Command>& cmd, - uint64_t time, - uint64_t sequenceId, - PriorityType priority) - : _command(cmd), _time(time), _sequenceId(sequenceId), _priority(priority) - {} - - // Sort on both priority and sequence ID - bool operator<(const CommandEntry& entry) const { - if (_priority != entry._priority) { - return (_priority < entry._priority); + template<class Command> + class CommandQueue : public vespalib::Printable + { + public: + struct CommandEntry { + typedef typename Command::Priority PriorityType; + std::shared_ptr<Command> _command; + uint64_t _time; + uint64_t _sequenceId; + PriorityType _priority; + + CommandEntry(const std::shared_ptr<Command>& cmd, + uint64_t time, + uint64_t sequenceId, + PriorityType priority) + : _command(cmd), _time(time), _sequenceId(sequenceId), _priority(priority) + {} + + // Sort on both priority and sequence ID + bool operator<(const CommandEntry& entry) const { + if (_priority != entry._priority) { + return (_priority < entry._priority); + } + return (_sequenceId < entry._sequenceId); } - return (_sequenceId < entry._sequenceId); + }; + + private: + typedef boost::multi_index::multi_index_container< + CommandEntry, + boost::multi_index::indexed_by< + boost::multi_index::ordered_unique< + boost::multi_index::identity<CommandEntry> + >, + boost::multi_index::ordered_non_unique< + boost::multi_index::member<CommandEntry, uint64_t, &CommandEntry::_time> + > + > + > CommandList; + typedef typename boost::multi_index + ::nth_index<CommandList, 1>::type timelist; + + framework::Clock& _clock; + mutable CommandList _commands; + uint64_t _sequenceId; + + public: + typedef typename CommandList::iterator iterator; + typedef typename CommandList::reverse_iterator reverse_iterator; + typedef typename CommandList::const_iterator const_iterator; + typedef typename CommandList::const_reverse_iterator const_reverse_iterator; + typedef typename timelist::const_iterator const_titerator; + + CommandQueue(framework::Clock& clock) + : _clock(clock), + _sequenceId(0) {} + + const framework::Clock& getTimer() const { return _clock; } + + iterator begin() { return _commands.begin(); } + iterator end() { return _commands.end(); } + + const_iterator begin() const { return _commands.begin(); } + const_iterator end() const { return _commands.end(); } + + const_titerator tbegin() const { + timelist& tl = boost::multi_index::get<1>(_commands); + return tl.begin(); + } + const_titerator tend() const { + timelist& tl = boost::multi_index::get<1>(_commands); + return tl.end(); } - }; -private: - typedef boost::multi_index::multi_index_container< - CommandEntry, - boost::multi_index::indexed_by< - boost::multi_index::ordered_unique< - boost::multi_index::identity<CommandEntry> - >, - boost::multi_index::ordered_non_unique< - boost::multi_index::member<CommandEntry, uint64_t, &CommandEntry::_time> - > - > - > CommandList; - typedef typename boost::multi_index - ::nth_index<CommandList, 1>::type timelist; + bool empty() const; + + uint32_t size() const; + + std::pair<std::shared_ptr<Command>, time_t> releaseNextCommand(); - framework::Clock& _clock; - mutable CommandList _commands; - uint64_t _sequenceId; + std::shared_ptr<Command> peekNextCommand() const; -public: - typedef typename CommandList::iterator iterator; - typedef typename CommandList::reverse_iterator reverse_iterator; - typedef typename CommandList::const_iterator const_iterator; - typedef typename CommandList::const_reverse_iterator const_reverse_iterator; - typedef typename timelist::const_iterator const_titerator; + void add(const std::shared_ptr<Command>& msg); - CommandQueue(framework::Clock& clock) - : _clock(clock), - _sequenceId(0) {} + void erase(iterator it); - const framework::Clock& getTimer() const { return _clock; } + std::list<CommandEntry> releaseTimedOut(); - iterator begin() { return _commands.begin(); } - iterator end() { return _commands.end(); } + std::pair<std::shared_ptr<Command>, time_t> + releaseLowestPriorityCommand(); - const_iterator begin() const { return _commands.begin(); } - const_iterator end() const { return _commands.end(); } + std::shared_ptr<Command> peekLowestPriorityCommand() const; - const_titerator tbegin() const { - timelist& tl = boost::multi_index::get<1>(_commands); - return tl.begin(); + void clear(); + + void print(std::ostream& out, bool verbose, + const std::string& indent) const; + }; + + template<class Command> + inline bool + CommandQueue<Command>::empty() const + { + return _commands.empty(); } - const_titerator tend() const { - timelist& tl = boost::multi_index::get<1>(_commands); - return tl.end(); + + template<class Command> + inline uint32_t + CommandQueue<Command>::size() const + { + return _commands.size(); } - bool empty() const { return _commands.empty(); } - uint32_t size() const { return _commands.size(); } - std::pair<std::shared_ptr<Command>, time_t> releaseNextCommand(); - std::shared_ptr<Command> peekNextCommand() const; - void add(const std::shared_ptr<Command>& msg); - void erase(iterator it) { _commands.erase(it); } - std::list<CommandEntry> releaseTimedOut(); - std::pair<std::shared_ptr<Command>, time_t> releaseLowestPriorityCommand(); - - std::shared_ptr<Command> peekLowestPriorityCommand() const; - void clear() { return _commands.clear(); } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - - -template<class Command> -std::pair<std::shared_ptr<Command>, time_t> -CommandQueue<Command>::releaseNextCommand() -{ - std::pair<std::shared_ptr<Command>, time_t> retVal( - std::shared_ptr<Command>(), 0); - if (!_commands.empty()) { - iterator first = _commands.begin(); - retVal.first = first->_command; - retVal.second = first->_time; - _commands.erase(first); + template<class Command> + inline std::pair<std::shared_ptr<Command>, time_t> + CommandQueue<Command>::releaseNextCommand() + { + std::pair<std::shared_ptr<Command>, time_t> retVal( + std::shared_ptr<Command>(), 0); + if (!_commands.empty()) { + iterator first = _commands.begin(); + retVal.first = first->_command; + retVal.second = first->_time; + _commands.erase(first); + } + return retVal; } - return retVal; -} - -template<class Command> -std::shared_ptr<Command> -CommandQueue<Command>::peekNextCommand() const -{ - if (!_commands.empty()) { - const_iterator first = _commands.begin(); - return first->_command; - } else { - return std::shared_ptr<Command>(); + + template<class Command> + inline std::shared_ptr<Command> + CommandQueue<Command>::peekNextCommand() const + { + if (!_commands.empty()) { + const_iterator first = _commands.begin(); + return first->_command; + } else { + return std::shared_ptr<Command>(); + } + } + + template<class Command> + inline void + CommandQueue<Command>::add( + const std::shared_ptr<Command>& cmd) + { + framework::MicroSecTime time(_clock.getTimeInMicros() + + framework::MicroSecTime(cmd->getQueueTimeout() * 1000000)); + _commands.insert(CommandEntry(cmd, time.getTime(), ++_sequenceId, cmd->getPriority())); } -} - -template<class Command> -void -CommandQueue<Command>::add( - const std::shared_ptr<Command>& cmd) -{ - framework::MicroSecTime time(_clock.getTimeInMicros() - + framework::MicroSecTime(cmd->getQueueTimeout() * 1000000)); - _commands.insert(CommandEntry(cmd, time.getTime(), ++_sequenceId, cmd->getPriority())); -} - -template<class Command> -std::list<typename CommandQueue<Command>::CommandEntry> -CommandQueue<Command>::releaseTimedOut() -{ - std::list<CommandEntry> mylist; - framework::MicroSecTime time(_clock.getTimeInMicros()); - while (!empty() && tbegin()->_time <= time.getTime()) { - mylist.push_back(*tbegin()); - timelist& tl = boost::multi_index::get<1>(_commands); - tl.erase(tbegin()); + + template<class Command> + inline void + CommandQueue<Command>::erase(iterator it) + { + _commands.erase(it); + } + + template<class Command> + inline std::list<typename CommandQueue<Command>::CommandEntry> + CommandQueue<Command>::releaseTimedOut() + { + std::list<CommandEntry> mylist; + framework::MicroSecTime time(_clock.getTimeInMicros()); + while (!empty() && tbegin()->_time <= time.getTime()) { + mylist.push_back(*tbegin()); + timelist& tl = boost::multi_index::get<1>(_commands); + tl.erase(tbegin()); + } + return mylist; } - return mylist; -} - -template <class Command> -std::pair<std::shared_ptr<Command>, time_t> -CommandQueue<Command>::releaseLowestPriorityCommand() -{ - if (!_commands.empty()) { - iterator last = (++_commands.rbegin()).base(); - time_t time = last->_time; - std::shared_ptr<Command> cmd(last->_command); - _commands.erase(last); - return std::pair<std::shared_ptr<Command>, time_t>(cmd, time); - } else { - return std::pair<std::shared_ptr<Command>, time_t>( - std::shared_ptr<Command>(), 0); + + template <class Command> + inline std::pair<std::shared_ptr<Command>, time_t> + CommandQueue<Command>::releaseLowestPriorityCommand() + { + if (!_commands.empty()) { + iterator last = (++_commands.rbegin()).base(); + time_t time = last->_time; + std::shared_ptr<Command> cmd(last->_command); + _commands.erase(last); + return std::pair<std::shared_ptr<Command>, time_t>(cmd, time); + } else { + return std::pair<std::shared_ptr<Command>, time_t>( + std::shared_ptr<Command>(), 0); + } } -} - -template <class Command> -std::shared_ptr<Command> -CommandQueue<Command>::peekLowestPriorityCommand() const -{ - if (!_commands.empty()) { - const_reverse_iterator last = _commands.rbegin(); - return last->_command; - } else { - return std::shared_ptr<Command>(); + + template <class Command> + inline std::shared_ptr<Command> + CommandQueue<Command>::peekLowestPriorityCommand() const + { + if (!_commands.empty()) { + const_reverse_iterator last = _commands.rbegin(); + return last->_command; + } else { + return std::shared_ptr<Command>(); + } } -} - -template<class Command> -void -CommandQueue<Command>::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - (void) verbose; - out << "Insert order:\n"; - for (const_iterator it = begin(); it != end(); ++it) { - out << indent << *it->_command << ", priority " << it->_priority - << ", time " << it->_time << "\n"; + + template<class Command> + inline void + CommandQueue<Command>::clear() + { + _commands.clear(); } - out << indent << "Time order:"; - for (const_titerator it = tbegin(); it != tend(); ++it) { - out << "\n" << indent << *it->_command << ", priority " << it->_priority - << ", time " << it->_time; + + template<class Command> + inline void + CommandQueue<Command>::print(std::ostream& out, bool verbose, + const std::string& indent) const + { + (void) verbose; + out << "Insert order:\n"; + for (const_iterator it = begin(); it != end(); ++it) { + out << indent << *it->_command << ", priority " << it->_priority + << ", time " << it->_time << "\n"; + } + out << indent << "Time order:"; + for (const_titerator it = tbegin(); it != tend(); ++it) { + out << "\n" << indent << *it->_command << ", priority " << it->_priority + << ", time " << it->_time; + } } -} } // storage + diff --git a/storage/src/vespa/storage/visiting/countvisitor.h b/storage/src/vespa/storage/visiting/countvisitor.h index 814cba9501b..ecf51035a03 100644 --- a/storage/src/vespa/storage/visiting/countvisitor.h +++ b/storage/src/vespa/storage/visiting/countvisitor.h @@ -9,7 +9,7 @@ */ #pragma once -#include "visitor.h" +#include <vespa/storage/visiting/visitor.h> namespace storage { diff --git a/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp b/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp index acafa30ef9d..a36ace1d427 100644 --- a/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp +++ b/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp @@ -3,10 +3,12 @@ #include "dumpvisitorsingle.h" #include <vespa/documentapi/messagebus/messages/multioperationmessage.h> #include <vespa/document/update/documentupdate.h> +#include <vespa/log/log.h> +#include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> #include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> +#include <memory> -#include <vespa/log/log.h> LOG_SETUP(".visitor.instance.dumpvisitorsingle"); namespace storage { diff --git a/storage/src/vespa/storage/visiting/dumpvisitorsingle.h b/storage/src/vespa/storage/visiting/dumpvisitorsingle.h index b0fd4b2b9f0..d7f7b63d851 100644 --- a/storage/src/vespa/storage/visiting/dumpvisitorsingle.h +++ b/storage/src/vespa/storage/visiting/dumpvisitorsingle.h @@ -9,7 +9,7 @@ */ #pragma once -#include "visitor.h" +#include <vespa/storage/visiting/visitor.h> namespace storage { @@ -19,7 +19,8 @@ public: const vdslib::Parameters& params); private: - void handleDocuments(const document::BucketId&, std::vector<spi::DocEntry::UP>&, HitCounter&) override; + void handleDocuments(const document::BucketId&, + std::vector<spi::DocEntry::UP>&, HitCounter&) override; }; struct DumpVisitorSingleFactory : public VisitorFactory { @@ -30,10 +31,14 @@ struct DumpVisitorSingleFactory : public VisitorFactory { }; Visitor* - - makeVisitor(StorageComponent& c, VisitorEnvironment&, const vdslib::Parameters& params) override { + makeVisitor(StorageComponent& c, + VisitorEnvironment&, const vdslib::Parameters& params) override + { return new DumpVisitorSingle(c, params); } }; } + + + diff --git a/storage/src/vespa/storage/visiting/messages.h b/storage/src/vespa/storage/visiting/messages.h index b79fae0575f..d646355895f 100644 --- a/storage/src/vespa/storage/visiting/messages.h +++ b/storage/src/vespa/storage/visiting/messages.h @@ -29,9 +29,11 @@ public: _config(config) {} std::unique_ptr<api::StorageReply> makeReply() override; + const vespa::config::content::core::StorVisitorConfig& getConfig() const { return _config; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override { + virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override + { out << "PropagateVisitorConfig()"; if (verbose) { @@ -51,8 +53,12 @@ public: PropagateVisitorConfigReply(const PropagateVisitorConfig& cmd) : api::InternalReply(ID, cmd) - {} - void print(std::ostream& out, bool verbose, const std::string& indent) const override { + { + } + + + virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override + { out << "PropagateVisitorConfigReply()"; if (verbose) { @@ -70,3 +76,4 @@ PropagateVisitorConfig::makeReply() } } // storage + diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.h b/storage/src/vespa/storage/visiting/recoveryvisitor.h index b9f54a5d0ea..dac6996028b 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.h +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.h @@ -9,7 +9,7 @@ */ #pragma once -#include "visitor.h" +#include <vespa/storage/visiting/visitor.h> #include <vespa/storageapi/message/datagram.h> namespace documentapi { diff --git a/storage/src/vespa/storage/visiting/testvisitor.h b/storage/src/vespa/storage/visiting/testvisitor.h index db4e1d60939..c2740b11d56 100644 --- a/storage/src/vespa/storage/visiting/testvisitor.h +++ b/storage/src/vespa/storage/visiting/testvisitor.h @@ -8,7 +8,7 @@ */ #pragma once -#include "visitor.h" +#include <vespa/storage/visiting/visitor.h> namespace storage { @@ -30,9 +30,10 @@ private: } void completedVisiting(HitCounter& hitCounter) override; + void abortedVisiting() override; - // Send datagram with message back to client + // Send datagram with message back to client void report(const std::string& message); std::string _params; @@ -54,3 +55,6 @@ struct TestVisitorFactory : public VisitorFactory { }; } + + + diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index 918d2d0e339..3d5e9fc2c53 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -174,7 +174,8 @@ private: bool hasPendingIterators() const { return _pendingIterators > 0; } - void print(std::ostream& out, bool, const std::string& ) const override { + void print(std::ostream& out, bool, const std::string& /*indent*/) const override + { out << "BucketIterationState(" << _bucketId << ", pending GetIters: " << _pendingIterators diff --git a/storage/src/vespa/storage/visiting/visitormanager.h b/storage/src/vespa/storage/visiting/visitormanager.h index b966526ad7a..9b750e3af4f 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.h +++ b/storage/src/vespa/storage/visiting/visitormanager.h @@ -19,18 +19,19 @@ #pragma once -#include "commandqueue.h" -#include "visitor.h" -#include "visitormetrics.h" -#include "visitorthread.h" -#include "config-stor-visitor.h" -#include <vespa/storageframework/storageframework.h> -#include <vespa/storage/common/storagelink.h> +#include <vespa/vespalib/util/document_runnable.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/internal.h> #include <vespa/storageapi/message/visitor.h> +#include <vespa/storage/common/storagelink.h> +#include <vespa/storage/visiting/commandqueue.h> +#include <vespa/storage/visiting/config-stor-visitor.h> +#include <vespa/storage/visiting/visitor.h> +#include <vespa/storage/visiting/visitormetrics.h> +#include <vespa/storage/visiting/visitorthread.h> +#include <vespa/storageframework/storageframework.h> +#include <vespa/storageframework/storageframework.h> #include <vespa/config/config.h> -#include <vespa/vespalib/util/document_runnable.h> namespace storage { namespace api { @@ -89,11 +90,14 @@ public: VisitorManager(const config::ConfigUri & configUri, StorageComponentRegister&, VisitorMessageSessionFactory&, const VisitorFactory::Map& external = VisitorFactory::Map()); - ~VisitorManager(); + virtual ~VisitorManager(); + + virtual void onClose() override; + + virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void onClose() override; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; uint32_t getActiveVisitorCount() const; + void setTimeBetweenTicks(uint32_t time); void setMaxConcurrentVisitors(uint32_t count) { // Used in unit testing @@ -122,7 +126,7 @@ public: private: void configure(std::unique_ptr<vespa::config::content::core::StorVisitorConfig>) override; - void run(framework::ThreadHandle&) override; + virtual void run(framework::ThreadHandle&) override; /** * Schedules a visitor for running. onCreateVisitor will typically call @@ -153,12 +157,13 @@ private: */ bool attemptScheduleQueuedVisitor(vespalib::MonitorGuard& visitorLock); - // VisitorMessageHandler implementation + // VisitorMessageHandler implementation void send(const std::shared_ptr<api::StorageCommand>& cmd, Visitor& visitor) override; void send(const std::shared_ptr<api::StorageReply>& reply) override; void closed(api::VisitorId id) override; - void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; + // Status::Reporter implementation + virtual void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; /** * The maximum amount of concurrent visitors for a priority is given @@ -174,3 +179,4 @@ private: }; } + diff --git a/storage/src/vespa/storage/visiting/visitorthread.h b/storage/src/vespa/storage/visiting/visitorthread.h index 5f9d8897d9f..2f0a125d6ce 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.h +++ b/storage/src/vespa/storage/visiting/visitorthread.h @@ -13,17 +13,17 @@ #pragma once -#include "visitor.h" -#include "visitormetrics.h" -#include "visitormessagesessionfactory.h" -#include <vespa/storageframework/storageframework.h> -#include <vespa/storage/persistence/messages.h> -#include <vespa/storage/common/storagecomponent.h> -#include <vespa/storageapi/messageapi/messagehandler.h> -#include <vespa/metrics/metrictimer.h> +#include <deque> #include <vespa/vespalib/util/document_runnable.h> +#include <vespa/metrics/metrictimer.h> +#include <vespa/storageapi/messageapi/messagehandler.h> #include <vespa/vespalib/util/sync.h> -#include <deque> +#include <vespa/storage/common/storagecomponent.h> +#include <vespa/storage/persistence/messages.h> +#include <vespa/storage/visiting/visitor.h> +#include <vespa/storage/visiting/visitormetrics.h> +#include <vespa/storage/visiting/visitormessagesessionfactory.h> +#include <vespa/storageframework/storageframework.h> namespace storage { @@ -36,7 +36,8 @@ class VisitorThread : public framework::Runnable, typedef std::map<api::VisitorId, std::shared_ptr<Visitor> > VisitorMap; VisitorMap _visitors; - std::deque<std::pair<api::VisitorId, framework::SecondTime> > _recentlyCompleted; + std::deque<std::pair<api::VisitorId, + framework::SecondTime> > _recentlyCompleted; struct Event { enum Type { @@ -99,9 +100,13 @@ public: VisitorMessageHandler& sender); ~VisitorThread(); - void processMessage(api::VisitorId visitorId, const std::shared_ptr<api::StorageMessage>& msg); + void processMessage(api::VisitorId visitorId, + const std::shared_ptr<api::StorageMessage>& msg); + void shutdown(); + void setTimeBetweenTicks(uint32_t time) { _timeBetweenTicks = time; } + void handleMessageBusReply(std::unique_ptr<mbus::Reply> reply, Visitor& visitor); /** For unit tests needing to pause thread. */ @@ -112,7 +117,7 @@ public: } private: - void run(framework::ThreadHandle&) override; + virtual void run(framework::ThreadHandle&) override; /** * Attempt to fetch an event from the visitor thread's queue. If an event * was available, pop it from the queue and return it. If not, return @@ -122,11 +127,12 @@ private: Event popNextQueuedEventIfAvailable(); void tick(); void trimRecentlyCompletedList(framework::SecondTime currentTime); - void handleNonExistingVisitorCall(const Event& entry, api::ReturnCode& code); + void handleNonExistingVisitorCall(const Event& entry, + api::ReturnCode& code); std::shared_ptr<Visitor> createVisitor(const vespalib::stringref & libName, - const vdslib::Parameters& params, - vespalib::asciistream & error); + const vdslib::Parameters& params, + vespalib::asciistream & error); bool onCreateVisitor(const std::shared_ptr<api::CreateVisitorCommand>&) override; @@ -136,9 +142,12 @@ private: /** Deletes a visitor instance. */ void close(); - void getStatus(vespalib::asciistream & out, const framework::HttpUrlPath& path) const; + void getStatus(vespalib::asciistream & out, + const framework::HttpUrlPath& path) const; + void updateMetrics(const MetricLockGuard &) override; }; } // storage + |