diff options
98 files changed, 612 insertions, 936 deletions
diff --git a/storage/src/tests/bucketdb/initializertest.cpp b/storage/src/tests/bucketdb/initializertest.cpp index 9aa7da92e36..47e80f0c202 100644 --- a/storage/src/tests/bucketdb/initializertest.cpp +++ b/storage/src/tests/bucketdb/initializertest.cpp @@ -423,7 +423,8 @@ struct FakePersistenceLayer : public StorageLink { } return 0; } - virtual bool onDown(const api::StorageMessage::SP& msg) override { + + bool onDown(const api::StorageMessage::SP& msg) override { fatalError = ""; if (messageCallback) { messageCallback->onMessage(*msg); diff --git a/storage/src/tests/bucketmover/bucketmovertest.cpp b/storage/src/tests/bucketmover/bucketmovertest.cpp index 3a844b6d1d8..c233765ad27 100644 --- a/storage/src/tests/bucketmover/bucketmovertest.cpp +++ b/storage/src/tests/bucketmover/bucketmovertest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/common/bucketmessages.h> #include <vespa/storage/bucketmover/bucketmover.h> diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index 8dea2fad94f..15f0373e680 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -67,10 +67,8 @@ namespace { { framework::Clock& _clock; MetricClock(framework::Clock& c) : _clock(c) {} - virtual time_t getTime() const override - { return _clock.getTimeInSeconds().getTime(); } - virtual time_t getTimeInMilliSecs() const override - { return _clock.getTimeInMillis().getTime(); } + time_t getTime() const override { return _clock.getTimeInSeconds().getTime(); } + time_t getTimeInMilliSecs() const override { return _clock.getTimeInMillis().getTime(); } }; } diff --git a/storage/src/tests/distributor/blockingoperationstartertest.cpp b/storage/src/tests/distributor/blockingoperationstartertest.cpp index 2ae8a819d7e..64f3dae8fbe 100644 --- a/storage/src/tests/distributor/blockingoperationstartertest.cpp +++ b/storage/src/tests/distributor/blockingoperationstartertest.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> -#include <string> -#include <sstream> -#include <memory> #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storage/distributor/blockingoperationstarter.h> #include <vespa/storage/distributor/pendingmessagetracker.h> diff --git a/storage/src/tests/distributor/bucketdatabasetest.cpp b/storage/src/tests/distributor/bucketdatabasetest.cpp index 13a8aff239c..5d1bb17a1c6 100644 --- a/storage/src/tests/distributor/bucketdatabasetest.cpp +++ b/storage/src/tests/distributor/bucketdatabasetest.cpp @@ -1,9 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <tests/distributor/bucketdatabasetest.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> -#include <iostream> -#include <fstream> #include <iomanip> namespace storage { diff --git a/storage/src/tests/distributor/bucketdbupdatertest.cpp b/storage/src/tests/distributor/bucketdbupdatertest.cpp index 7ad083dc5a1..e35f147d0c8 100644 --- a/storage/src/tests/distributor/bucketdbupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbupdatertest.cpp @@ -13,7 +13,6 @@ #include <iostream> #include <fstream> -#include <string> using namespace storage::api; using namespace storage::lib; diff --git a/storage/src/tests/distributor/bucketstateoperationtest.cpp b/storage/src/tests/distributor/bucketstateoperationtest.cpp index 74a31de1541..58061def8ab 100644 --- a/storage/src/tests/distributor/bucketstateoperationtest.cpp +++ b/storage/src/tests/distributor/bucketstateoperationtest.cpp @@ -28,13 +28,10 @@ private: void testBucketDbNotUpdatedOnFailure(); public: - void setUp() override - { + void setUp() override { createLinks(); } - - void tearDown() override - { + void tearDown() override { close(); } }; diff --git a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp index e7078f08d0a..3e4e1d6da88 100644 --- a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp +++ b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp @@ -6,7 +6,6 @@ #include <vespa/storage/distributor/min_replica_provider.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/util/jsonstream.h> #include <vespa/vespalib/testkit/test_kit.h> #include <tests/common/hostreporter/util.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index fc09dc17c1f..10b0d035770 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -1,8 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <iomanip> -#include <iostream> -#include <memory> #include <boost/assign/std/vector.hpp> // for 'operator+=()' #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/distributor/idealstatemetricsset.h> diff --git a/storage/src/tests/distributor/idealstatemanagertest.cpp b/storage/src/tests/distributor/idealstatemanagertest.cpp index 2b861d23c5e..e4fce6b9d6e 100644 --- a/storage/src/tests/distributor/idealstatemanagertest.cpp +++ b/storage/src/tests/distributor/idealstatemanagertest.cpp @@ -11,7 +11,6 @@ #include <tests/distributor/distributortestutil.h> - namespace storage { namespace distributor { diff --git a/storage/src/tests/distributor/mapbucketdatabasetest.cpp b/storage/src/tests/distributor/mapbucketdatabasetest.cpp index 6892f522bba..8af2a86e4c5 100644 --- a/storage/src/tests/distributor/mapbucketdatabasetest.cpp +++ b/storage/src/tests/distributor/mapbucketdatabasetest.cpp @@ -1,10 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/vespalib/util/document_runnable.h> #include <vespa/vdstestlib/cppunit/macros.h> -#include <cppunit/extensions/HelperMacros.h> #include <vespa/storage/bucketdb/mapbucketdatabase.h> -#include <vespa/storage/storageutil/utils.h> #include <tests/distributor/bucketdatabasetest.h> namespace storage { @@ -12,8 +8,7 @@ namespace distributor { struct MapBucketDatabaseTest : public BucketDatabaseTest { MapBucketDatabase _db; - - virtual BucketDatabase& db() override { return _db; } + BucketDatabase& db() override { return _db; }; CPPUNIT_TEST_SUITE(MapBucketDatabaseTest); SETUP_DATABASE_TESTS(); diff --git a/storage/src/tests/distributor/mergeoperationtest.cpp b/storage/src/tests/distributor/mergeoperationtest.cpp index 875d16a42ed..1e317e20b74 100644 --- a/storage/src/tests/distributor/mergeoperationtest.cpp +++ b/storage/src/tests/distributor/mergeoperationtest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <boost/lexical_cast.hpp> #include <cppunit/extensions/HelperMacros.h> #include <iomanip> diff --git a/storage/src/tests/distributor/messagesenderstub.h b/storage/src/tests/distributor/messagesenderstub.h index 7b552ec6883..629c1c6e346 100644 --- a/storage/src/tests/distributor/messagesenderstub.h +++ b/storage/src/tests/distributor/messagesenderstub.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/storage/distributor/distributormessagesender.h> +#include <cassert> namespace storage { @@ -20,13 +21,11 @@ struct MessageSenderStub : distributor::DistributorMessageSender replies.clear(); } - virtual void sendCommand(const std::shared_ptr<api::StorageCommand>& cmd) - { + virtual void sendCommand(const std::shared_ptr<api::StorageCommand>& cmd) { commands.push_back(cmd); } - virtual void sendReply(const std::shared_ptr<api::StorageReply>& reply) - { + virtual void sendReply(const std::shared_ptr<api::StorageReply>& reply) { replies.push_back(reply); } @@ -67,4 +66,3 @@ private: }; } - diff --git a/storage/src/tests/distributor/operationtargetresolvertest.cpp b/storage/src/tests/distributor/operationtargetresolvertest.cpp index 26144ac950e..d0dc8cedaf3 100644 --- a/storage/src/tests/distributor/operationtargetresolvertest.cpp +++ b/storage/src/tests/distributor/operationtargetresolvertest.cpp @@ -1,7 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <iomanip> -#include <iostream> #include <vespa/config/helper/configgetter.h> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/repo/documenttyperepo.h> diff --git a/storage/src/tests/distributor/pendingmessagetrackertest.cpp b/storage/src/tests/distributor/pendingmessagetrackertest.cpp index 6f4539d426e..618d7980c5b 100644 --- a/storage/src/tests/distributor/pendingmessagetrackertest.cpp +++ b/storage/src/tests/distributor/pendingmessagetrackertest.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/base/testdocman.h> #include <vespa/storage/distributor/pendingmessagetracker.h> @@ -10,13 +9,6 @@ #include <tests/common/dummystoragelink.h> #include <vespa/vdslib/state/random.h> #include <vespa/vdstestlib/cppunit/macros.h> -#include <fstream> -#include <sstream> -#include <iomanip> -#include <iostream> -#include <memory> -#include <string> -#include <iterator> namespace storage { namespace distributor { diff --git a/storage/src/tests/distributor/simplemaintenancescannertest.cpp b/storage/src/tests/distributor/simplemaintenancescannertest.cpp index 100d5582006..342d346b398 100644 --- a/storage/src/tests/distributor/simplemaintenancescannertest.cpp +++ b/storage/src/tests/distributor/simplemaintenancescannertest.cpp @@ -1,18 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/distributor/maintenance/simplemaintenancescanner.h> #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> #include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <tests/distributor/maintenancemocks.h> -#include <string> -#include <sstream> -#include <memory> -#include <algorithm> -#include <iterator> - namespace storage { namespace distributor { diff --git a/storage/src/tests/distributor/statusreporterdelegatetest.cpp b/storage/src/tests/distributor/statusreporterdelegatetest.cpp index f136b00244a..d9f647a52a5 100644 --- a/storage/src/tests/distributor/statusreporterdelegatetest.cpp +++ b/storage/src/tests/distributor/statusreporterdelegatetest.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include <vespa/vdstestlib/cppunit/macros.h> #include <tests/common/testhelper.h> #include <tests/distributor/distributortestutil.h> diff --git a/storage/src/tests/distributor/throttlingoperationstartertest.cpp b/storage/src/tests/distributor/throttlingoperationstartertest.cpp index 8e7d98005c1..0414ad11da4 100644 --- a/storage/src/tests/distributor/throttlingoperationstartertest.cpp +++ b/storage/src/tests/distributor/throttlingoperationstartertest.cpp @@ -1,9 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> -#include <string> -#include <sstream> -#include <memory> #include <vespa/storage/distributor/throttlingoperationstarter.h> #include <tests/distributor/maintenancemocks.h> diff --git a/storage/src/tests/frameworkimpl/status/statustest.cpp b/storage/src/tests/frameworkimpl/status/statustest.cpp index 567f9192d34..43d13acb28b 100644 --- a/storage/src/tests/frameworkimpl/status/statustest.cpp +++ b/storage/src/tests/frameworkimpl/status/statustest.cpp @@ -1,16 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/document/util/stringutil.h> -#include <vespa/log/log.h> -#include <sstream> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> #include <vespa/storage/frameworkimpl/status/statuswebserver.h> #include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> #include <tests/common/teststorageapp.h> #include <vespa/vdstestlib/cppunit/macros.h> - -LOG_SETUP(".test.status"); +#include <vespa/document/util/stringutil.h> +#include <sstream> namespace storage { @@ -47,18 +43,13 @@ namespace { : framework::HtmlStatusReporter(id, name), _headerAddition(headerAddition), _content(content) - { - } + {} - virtual void reportHtmlHeaderAdditions( - std::ostream& out, const framework::HttpUrlPath&) const override - { + void reportHtmlHeaderAdditions(std::ostream& out, const framework::HttpUrlPath&) const override { out << _headerAddition; } - virtual void reportHtmlStatus( - std::ostream& out, const framework::HttpUrlPath&) const override - { + void reportHtmlStatus(std::ostream& out, const framework::HttpUrlPath&) const override { out << _content; } }; @@ -66,9 +57,9 @@ namespace { struct XmlStatusReporter : public framework::XmlStatusReporter { XmlStatusReporter(const std::string& id, const std::string& name) : framework::XmlStatusReporter(id, name) {} - virtual vespalib::string reportXmlStatus( - vespalib::xml::XmlOutputStream& xos, - const framework::HttpUrlPath&) const override + + vespalib::string reportXmlStatus(vespalib::xml::XmlOutputStream& xos, + const framework::HttpUrlPath&) const override { xos << vespalib::xml::XmlTag("mytag") << vespalib::xml::XmlAttribute("foo", "bar") diff --git a/storage/src/tests/persistence/bucketownershipnotifiertest.cpp b/storage/src/tests/persistence/bucketownershipnotifiertest.cpp index 8553795db71..5f675c1d551 100644 --- a/storage/src/tests/persistence/bucketownershipnotifiertest.cpp +++ b/storage/src/tests/persistence/bucketownershipnotifiertest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <tests/distributor/messagesenderstub.h> #include <tests/common/teststorageapp.h> diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index e21d367370b..e2f41aa5b3e 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -9,7 +9,6 @@ #include <vespa/document/select/parser.h> #include <vespa/vdslib/state/random.h> #include <vespa/vdslib/container/mutabledocumentlist.h> -#include <vespa/vdslib/container/operationlist.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/storageapi/message/bucket.h> @@ -18,25 +17,17 @@ #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/storage/bucketdb/bucketmanager.h> -#include <vespa/storage/bucketdb/storbucketdb.h> -#include <vespa/storage/common/bucketmessages.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/storage/persistence/persistencethread.h> -#include <vespa/storage/persistence/messages.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> #include <tests/common/testhelper.h> #include <tests/common/storagelinktest.h> #include <tests/common/teststorageapp.h> -#include <tests/common/dummystoragelink.h> #include <tests/persistence/filestorage/forwardingmessagesender.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/storageapi/message/batch.h> #include <vespa/storage/storageserver/statemanager.h> #include <vespa/fastos/file.h> -#include <fstream> -#include <memory> -#include <atomic> #include <vespa/log/log.h> LOG_SETUP(".filestormanagertest"); @@ -2364,10 +2355,8 @@ namespace { closeNextLink(); } - virtual void print(std::ostream& out, bool, const std::string&) const override - { out << "MidLink"; } - - virtual bool onUp(const std::shared_ptr<api::StorageMessage> & msg) override { + void print(std::ostream& out, bool, const std::string&) const override { out << "MidLink"; } + bool onUp(const std::shared_ptr<api::StorageMessage> & msg) override { if (!StorageLinkTest::callOnUp(_up, msg)) _up.sendUp(msg); return true; } @@ -2395,10 +2384,9 @@ namespace { _leftAddr(leftAddr), _rightAddr(rightAddr) {} - virtual void print(std::ostream& out, bool, const std::string&) const override - { out << "BinaryStorageLink"; } + void print(std::ostream& out, bool, const std::string&) const override { out << "BinaryStorageLink"; } - virtual bool onDown(const std::shared_ptr<api::StorageMessage> & msg) override { + bool onDown(const std::shared_ptr<api::StorageMessage> & msg) override { // LOG(debug, "onDown Received msg: ->%s, %s %llu\n", msg->getAddress() ? msg->getAddress()->toString().c_str() : "(null)", msg->toString().c_str(), msg->getMsgId()); vespalib::LockGuard lock(_lock); @@ -2432,7 +2420,7 @@ namespace { return true; } - virtual bool onUp(const std::shared_ptr<api::StorageMessage> & msg) override { + bool onUp(const std::shared_ptr<api::StorageMessage> & msg) override { // LOG(debug, "onUp Received msg: ->%s, %s %llu\n", msg->getAddress() ? msg->getAddress()->toString().c_str() : "(null)", msg->toString().c_str(), msg->getMsgId()); vespalib::LockGuard lock(_lock); diff --git a/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp b/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp index d608bef3307..1409445b96b 100644 --- a/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <memory> + #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> diff --git a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp index 2dc60594c45..58b4ae4d475 100644 --- a/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp +++ b/storage/src/tests/persistence/filestorage/modifiedbucketcheckertest.cpp @@ -1,14 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <tests/common/testhelper.h> #include <tests/common/storagelinktest.h> #include <tests/common/teststorageapp.h> -#include <tests/common/dummystoragelink.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> -#include <vespa/storage/persistence/messages.h> namespace storage { diff --git a/storage/src/tests/persistence/legacyoperationhandlertest.cpp b/storage/src/tests/persistence/legacyoperationhandlertest.cpp index df262501299..353a8d24ec5 100644 --- a/storage/src/tests/persistence/legacyoperationhandlertest.cpp +++ b/storage/src/tests/persistence/legacyoperationhandlertest.cpp @@ -1,15 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/base/testdocrepo.h> -#include <vespa/document/repo/documenttyperepo.h> #include <vespa/documentapi/loadtypes/loadtype.h> #include <vespa/storage/persistence/messages.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/multioperation.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <tests/persistence/persistencetestutils.h> -#include <vespa/storage/persistence/types.h> using document::DocumentTypeRepo; using document::TestDocRepo; diff --git a/storage/src/tests/persistence/mergehandlertest.cpp b/storage/src/tests/persistence/mergehandlertest.cpp index 8147992a516..3f0623fbef0 100644 --- a/storage/src/tests/persistence/mergehandlertest.cpp +++ b/storage/src/tests/persistence/mergehandlertest.cpp @@ -3,15 +3,13 @@ #include <vespa/document/base/testdocman.h> #include <vespa/storage/persistence/mergehandler.h> #include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/storageapi/message/bucket.h> -#include <vespa/log/log.h> #include <tests/persistence/persistencetestutils.h> #include <tests/persistence/common/persistenceproviderwrapper.h> #include <tests/distributor/messagesenderstub.h> -#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> #include <vespa/vespalib/objects/nbostream.h> #include <cmath> +#include <vespa/log/log.h> LOG_SETUP(".test.persistence.handler.merge"); namespace storage { diff --git a/storage/src/tests/persistence/providershutdownwrappertest.cpp b/storage/src/tests/persistence/providershutdownwrappertest.cpp index 8b1ef88aa2c..c7f60a39cc8 100644 --- a/storage/src/tests/persistence/providershutdownwrappertest.cpp +++ b/storage/src/tests/persistence/providershutdownwrappertest.cpp @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <tests/persistence/persistencetestutils.h> #include <tests/persistence/common/persistenceproviderwrapper.h> -#include <vespa/storage/persistence/providershutdownwrapper.h> namespace storage { diff --git a/storage/src/tests/storageserver/bouncertest.cpp b/storage/src/tests/storageserver/bouncertest.cpp index e5b9a3e6092..751d6b535a3 100644 --- a/storage/src/tests/storageserver/bouncertest.cpp +++ b/storage/src/tests/storageserver/bouncertest.cpp @@ -1,10 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <boost/pointer_cast.hpp> #include <cppunit/extensions/HelperMacros.h> -#include <iostream> -#include <string> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> #include <vespa/storageapi/message/stat.h> diff --git a/storage/src/tests/storageserver/bucketintegritycheckertest.cpp b/storage/src/tests/storageserver/bucketintegritycheckertest.cpp index d9a175f3b04..f4eedc64971 100644 --- a/storage/src/tests/storageserver/bucketintegritycheckertest.cpp +++ b/storage/src/tests/storageserver/bucketintegritycheckertest.cpp @@ -1,18 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <boost/lexical_cast.hpp> #include <cppunit/extensions/HelperMacros.h> #include <vespa/log/log.h> #include <vespa/storage/bucketdb/bucketmanager.h> -#include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/storageserver/bucketintegritychecker.h> -#include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> #include <tests/common/testhelper.h> #include <tests/common/storagelinktest.h> -#include <tests/common/dummystoragelink.h> #include <vespa/vespalib/io/fileutil.h> #include <tests/common/teststorageapp.h> diff --git a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp index 8eb6b74a840..89eaff0ab6e 100644 --- a/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp +++ b/storage/src/tests/storageserver/changedbucketownershiphandlertest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/base/testdocman.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/bucketdb/storbucketdb.h> @@ -15,7 +14,6 @@ #include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> #include <vespa/storage/storageserver/changedbucketownershiphandler.h> -#include <memory> namespace storage { diff --git a/storage/src/tests/storageserver/priorityconvertertest.cpp b/storage/src/tests/storageserver/priorityconvertertest.cpp index ecbbd25b8b1..ab6c27f48d6 100644 --- a/storage/src/tests/storageserver/priorityconvertertest.cpp +++ b/storage/src/tests/storageserver/priorityconvertertest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/documentapi/documentapi.h> #include <vespa/storage/storageserver/priorityconverter.h> #include <tests/common/testhelper.h> diff --git a/storage/src/tests/storageserver/statemanagertest.cpp b/storage/src/tests/storageserver/statemanagertest.cpp index 4244ce0c4ce..f9c5c97e6bc 100644 --- a/storage/src/tests/storageserver/statemanagertest.cpp +++ b/storage/src/tests/storageserver/statemanagertest.cpp @@ -1,19 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <boost/pointer_cast.hpp> #include <cppunit/extensions/HelperMacros.h> #include <vespa/metrics/metricmanager.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> -#include <vespa/vdslib/state/nodestate.h> #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storage/storageserver/statemanager.h> -#include <vespa/storage/common/hostreporter/hostinfo.h> #include <tests/common/teststorageapp.h> #include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> #include <vespa/vespalib/data/slime/slime.h> -#include <iostream> using storage::lib::NodeState; using storage::lib::NodeType; @@ -138,8 +134,7 @@ namespace { MyStateListener(const NodeStateUpdater& upd) : updater(upd), current(*updater.getReportedNodeState()) {} - void handleNewState() override - { + void handleNewState() override { ost << current << " -> "; current = *updater.getReportedNodeState(); ost << current << "\n"; @@ -259,4 +254,3 @@ StateManagerTest::testClusterStateVersion() } } // storage - diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index 265b9e95d83..8a164361c88 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -18,8 +18,8 @@ namespace storage { class DummyApplicationGenerationFether : public ApplicationGenerationFetcher { public: - virtual int64_t getGeneration() const override { return 1; } - virtual std::string getComponentName() const override { return "component"; } + int64_t getGeneration() const override { return 1; } + std::string getComponentName() const override { return "component"; } }; struct StateReporterTest : public CppUnit::TestFixture { @@ -58,10 +58,8 @@ namespace { { framework::Clock& _clock; MetricClock(framework::Clock& c) : _clock(c) {} - virtual time_t getTime() const override - { return _clock.getTimeInSeconds().getTime(); } - virtual time_t getTimeInMilliSecs() const override - { return _clock.getTimeInMillis().getTime(); } + time_t getTime() const override { return _clock.getTimeInSeconds().getTime(); } + time_t getTimeInMilliSecs() const override { return _clock.getTimeInMillis().getTime(); } }; } diff --git a/storage/src/tests/storageutil/charttest.cpp b/storage/src/tests/storageutil/charttest.cpp index 4f83a9f26f4..c7c1c43b000 100644 --- a/storage/src/tests/storageutil/charttest.cpp +++ b/storage/src/tests/storageutil/charttest.cpp @@ -1,18 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/storage/storageutil/piechart.h> - -#include <fstream> #include <vespa/vdstestlib/cppunit/macros.h> +#include <fstream> namespace storage { struct PieChartTest : public CppUnit::TestFixture { - void setUp() override {} - void tearDown() override {} - void testWriteHtmlFile(); CPPUNIT_TEST_SUITE(PieChartTest); diff --git a/storage/src/tests/storageutil/palettetest.cpp b/storage/src/tests/storageutil/palettetest.cpp index e5a4e73d58a..24eaee18996 100644 --- a/storage/src/tests/storageutil/palettetest.cpp +++ b/storage/src/tests/storageutil/palettetest.cpp @@ -1,15 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/storage/storageutil/palette.h> #include <vespa/vdstestlib/cppunit/macros.h> namespace storage { struct PaletteTest : public CppUnit::TestFixture { - void setUp() override {} - void tearDown() override {} - void testNormalUsage(); CPPUNIT_TEST_SUITE(PaletteTest); diff --git a/storage/src/tests/testrunner.cpp b/storage/src/tests/testrunner.cpp index 5d8dc8d4c1f..9bf86e3276a 100644 --- a/storage/src/tests/testrunner.cpp +++ b/storage/src/tests/testrunner.cpp @@ -1,14 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <iostream> -#include <vespa/log/log.h> #include <vespa/vdstestlib/cppunit/cppunittestrunner.h> +#include <vespa/log/log.h> LOG_SETUP("storagecppunittests"); int -main(int argc, char **argv) +main(int argc, const char *argv[]) { vdstestlib::CppUnitTestRunner testRunner; return testRunner.run(argc, argv); diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp index 8aa57b982db..1824b976aea 100644 --- a/storage/src/tests/visiting/visitormanagertest.cpp +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -1,15 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/intfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/rawfieldvalue.h> -#include <vespa/log/log.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/persistence.h> -#include <vespa/storageapi/message/visitor.h> -#include <vector> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/visiting/visitormanager.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> @@ -17,16 +13,11 @@ #include <tests/common/testhelper.h> #include <tests/common/dummystoragelink.h> #include <tests/storageserver/testvisitormessagesession.h> -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/vdslib/container/visitorordering.h> #include <vespa/documentapi/messagebus/messages/multioperationmessage.h> #include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> #include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> #include <vespa/vespalib/util/exceptions.h> - -LOG_SETUP(".visitormanagertest"); - namespace storage { namespace { typedef std::vector<api::StorageMessage::SP> msg_ptr_vector; @@ -113,7 +104,6 @@ CPPUNIT_TEST_SUITE_REGISTRATION(VisitorManagerTest); void VisitorManagerTest::initializeTest() { - LOG(debug, "Initializing test"); vdstestlib::DirConfig config(getStandardConfig(true)); config.getConfig("stor-visitor").set("visitorthreads", "1"); @@ -221,7 +211,6 @@ VisitorManagerTest::initializeTest() CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), reply->getResult()); } - LOG(debug, "Done initializing test"); } void diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp index e6a3542cbf2..84f1297b05f 100644 --- a/storage/src/tests/visiting/visitortest.cpp +++ b/storage/src/tests/visiting/visitortest.cpp @@ -6,25 +6,17 @@ #include <vespa/document/fieldvalue/rawfieldvalue.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/persistence.h> -#include <vespa/storageapi/message/visitor.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/visiting/visitormanager.h> #include <tests/common/testhelper.h> #include <tests/common/teststorageapp.h> #include <tests/common/dummystoragelink.h> #include <tests/storageserver/testvisitormessagesession.h> -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/vdslib/container/visitorordering.h> #include <vespa/documentapi/messagebus/messages/multioperationmessage.h> #include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> #include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> #include <vespa/vespalib/util/exceptions.h> -#include <vector> #include <thread> -#include <chrono> - -#include <vespa/log/log.h> -LOG_SETUP(".visitortest"); using namespace std::chrono_literals; @@ -185,7 +177,6 @@ CPPUNIT_TEST_SUITE_REGISTRATION(VisitorTest); void VisitorTest::initializeTest(const TestParams& params) { - LOG(debug, "Initializing test"); vdstestlib::DirConfig config(getStandardConfig(true, "visitortest")); config.getConfig("stor-visitor").set("visitorthreads", "1"); config.getConfig("stor-visitor").set( @@ -273,7 +264,6 @@ VisitorTest::initializeTest(const TestParams& params) _documents.back()->setValue(type.getField("headerval"), document::IntFieldValue(i % 4)); } - LOG(debug, "Done initializing test"); } void diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h index 64498386466..da8dcf1c1cd 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.h +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h @@ -102,8 +102,7 @@ public: ~BucketManager(); void startWorkerThread(); - - virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; /** Dump the whole database to the given output. Use for debugging. */ void dump(std::ostream& out) const; @@ -120,11 +119,11 @@ private: vespalib::string getReportContentType(const framework::HttpUrlPath&) const override; bool reportStatus(std::ostream&, const framework::HttpUrlPath&) const override; - /** Event saying node is up and running. We can start to build cache. */ - virtual void onOpen() override; - virtual void onDoneInit() override { _doneInitialized = true; } - virtual void onClose() override; - virtual void onFlush(bool downwards) override; + /** Event saying node is up and running. We can start to build cache. */ + void onOpen() override; + void onDoneInit() override { _doneInitialized = true; } + void onClose() override; + void onFlush(bool downwards) override; void updateMetrics(bool updateDocCount); void updateMetrics(const MetricLockGuard &) override { updateMetrics(true); } diff --git a/storage/src/vespa/storage/bucketdb/judyarray.h b/storage/src/vespa/storage/bucketdb/judyarray.h index b5d8a50cb16..e28d89967b2 100644 --- a/storage/src/vespa/storage/bucketdb/judyarray.h +++ b/storage/src/vespa/storage/bucketdb/judyarray.h @@ -83,7 +83,7 @@ public: data_type& operator[](key_type key); size_type getMemoryUsage() const; - virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; class ConstIterator : public vespalib::Printable { @@ -101,8 +101,7 @@ public: key_type key() const { return _key; } data_type value() const { return *_data; } - virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; - + void print(std::ostream& out, bool verbose, const std::string& indent) const override; protected: // For creating end() iterator ConstIterator(const JudyArray&); @@ -227,4 +226,3 @@ JudyArray::ConstIterator::operator==(const JudyArray::ConstIterator &cp) const } } // storage - diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.h b/storage/src/vespa/storage/bucketdb/lockablemap.h index 93dfb23c4db..3c40c214136 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.h +++ b/storage/src/vespa/storage/bucketdb/lockablemap.h @@ -168,7 +168,7 @@ public: const char* clientId, uint32_t chunkSize = DEFAULT_CHUNK_SIZE); - virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; /** * Returns all buckets in the bucket database that can contain the given diff --git a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp index a271803714a..10a6bb6bd24 100644 --- a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp +++ b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <vespa/storage/common/bucketoperationlogger.h> @@ -465,7 +464,7 @@ namespace { struct Writer : public BucketDatabase::EntryProcessor { std::ostream& _ost; Writer(std::ostream& ost) : _ost(ost) {} - virtual bool process(const BucketDatabase::Entry& e) override { + bool process(const BucketDatabase::Entry& e) override { _ost << e.toString() << "\n"; return true; } diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h index 6e9bef38672..4c87c39fdd2 100644 --- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h +++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.h @@ -138,12 +138,12 @@ public: ServiceLayerComponentRegister&); ~StorageBucketDBInitializer(); - virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; - virtual void onOpen() override; - virtual void onClose() override; + void onOpen() override; + void onClose() override; - virtual void run(framework::ThreadHandle&) override; + void run(framework::ThreadHandle&) override; bool onDown(const std::shared_ptr<api::StorageMessage>&) override; bool onInternalReply(const std::shared_ptr<api::InternalReply>&) override; @@ -215,4 +215,3 @@ public: }; } // storage - diff --git a/storage/src/vespa/storage/bucketdb/storbucketdb.cpp b/storage/src/vespa/storage/bucketdb/storbucketdb.cpp index 4cc535be9de..02a87e1a381 100644 --- a/storage/src/vespa/storage/bucketdb/storbucketdb.cpp +++ b/storage/src/vespa/storage/bucketdb/storbucketdb.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> + #include "storbucketdb.h" -#include <vespa/storage/common/bucketoperationlogger.h> #include "judymultimap.hpp" namespace storage { diff --git a/storage/src/vespa/storage/bucketmover/CMakeLists.txt b/storage/src/vespa/storage/bucketmover/CMakeLists.txt index 637934cb592..f444d667e21 100644 --- a/storage/src/vespa/storage/bucketmover/CMakeLists.txt +++ b/storage/src/vespa/storage/bucketmover/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(storage_bucketmover OBJECT SOURCES + htmltable.cpp move.cpp runstatistics.cpp run.cpp diff --git a/storage/src/vespa/storage/bucketmover/htmltable.cpp b/storage/src/vespa/storage/bucketmover/htmltable.cpp new file mode 100644 index 00000000000..0fae9e326ba --- /dev/null +++ b/storage/src/vespa/storage/bucketmover/htmltable.cpp @@ -0,0 +1,176 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "htmltable.h" + +namespace storage { + +Column::Column(const std::string& colName, HtmlTable* table) + : _colName(colName), _alignment(RIGHT) +{ + if (table != 0) table->addColumn(*this); +} + +Column::~Column() {} + +HtmlTable::HtmlTable(const std::string& rowId) + : _rowId(rowId), _columns(), _rows() +{} + +HtmlTable::~HtmlTable() {} + +void HtmlTable::print(std::ostream& out) +{ + out << "<table border=\"1\" cellpadding=\"2\" cellspacing=\"0\">\n<tr><th"; + if (!_colHeaders.empty()) out << " rowspan=\"2\""; + out << ">" << _rowId << "</th>"; + if (!_colHeaders.empty()) { + for (uint32_t i=0; i<_colHeaders.size(); ++i) { + out << "<th colspan=\"" << _colHeaders[i]._span << "\">" + << _colHeaders[i]._name << "</th>"; + } + out << "</tr>\n"; + } + for (uint32_t i=0; i<_columns.size(); ++i) { + _columns[i]->finalize(); + out << "<th>" << _columns[i]->_colName << "</th>"; + } + out << "</tr>\n"; + for (uint32_t i=0; i<_rows.size(); ++i) { + out << "<tr><td"; + Column::printTdColor(out, _rows[i]._backgroundColor); + out << ">" << _rows[i]._name << "</td>"; + for (uint32_t j=0; j<_columns.size(); ++j) { + _columns[j]->printElement(out, i); + } + out << "</tr>\n"; + } + if (_totalRow.get()) { + out << "<tr><td>" << *_totalRow << "</td>"; + for (uint32_t j=0; j<_columns.size(); ++j) { + _columns[j]->printElement(out, Column::TOTAL); + } + out << "</tr>\n"; + } + out << "</table>\n"; +} + +void +PercentageColumn::finalize() { + uint64_t total = _total; + if (total == 0) { + for (const auto & entry : _values) { + total += entry.second; + } + } + for (const auto & entry : _values) { + ValueColumn<double>::_values[entry.first] = 100.0 * entry.second / total; + } + ValueColumn<double>::finalize(); +} + + +void +ByteSizeColumn::finalize() { + uint64_t max = 0; + for (const auto & entry : _values) { + max = std::max(max, entry.second); + } + uint64_t oldMax = max; + const char* type = "B"; + if (max > 10 * 1024) { max /= 1024; type = "kB"; } + if (max > 10 * 1024) { max /= 1024; type = "MB"; } + if (max > 10 * 1024) { max /= 1024; type = "GB"; } + if (max > 10 * 1024) { max /= 1024; type = "TB"; } + _denomination = std::pair<const char*, uint64_t>(type, max == 0 ? 1 : oldMax / max); + ValueColumn<uint64_t>::finalize(); +} + +void +ByteSizeColumn::printValue(std::ostream& out, uint16_t row) { + uint64_t value; + if (row == TOTAL) { + value = getTotalValue(); + } else { + std::map<uint16_t, uint64_t>::iterator val(_values.find(row)); + if (val == _values.end()) { + Column::printValue(out, row); + return; + } + value = val->second; + } + out << (value / _denomination.second) << ' ' << _denomination.first; +} + +template<typename T> +ValueColumn<T>::ValueColumn(const std::string& colName, + const std::string& denomination, + HtmlTable* table) + : Column(colName, table), _values(), _denomination(denomination), + _colorLimits(), _totalIsAvg(false) +{ + _valuePrinter << std::fixed << std::setprecision(2); +} + +template<typename T> +ValueColumn<T>::~ValueColumn() {} + +template<typename T> +void +ValueColumn<T>::finalize() { + for (const auto & val : _values) { + Color c = DEFAULT_COLOR; + for (const auto & colorLimit : _colorLimits) { + if (val.second <= colorLimit.first) { + c = colorLimit.second; + break; + } + } + _colors[val.first] = c; + } + // Set color for total too. + T total = getTotalValue(); + Color c = DEFAULT_COLOR; + for (const auto & colorLimit : _colorLimits) { + if (total <= colorLimit.first) { + c = colorLimit.second; + break; + } + } + _colors[TOTAL] = c; +} + +template<typename T> +T +ValueColumn<T>::getTotalValue() { + T value = 0; + for (const auto & val : _values) { + value += val.second; + } + if (_totalIsAvg) value /= _values.size(); + return value; +} + +template<typename T> +void +ValueColumn<T>::printValue(std::ostream& out, uint16_t row) { + T value; + if (row == TOTAL) { + value = getTotalValue(); + } else { + typename std::map<uint16_t, T>::iterator val = _values.find(row); + if (val == _values.end()) { + Column::printValue(out, row); + return; + } + value = val->second; + } + _valuePrinter.str(""); + _valuePrinter << value << _denomination; + out << _valuePrinter.str(); +} + +template class ValueColumn<int64_t>; +template class ValueColumn<uint64_t>; +template class ValueColumn<double>; + +} diff --git a/storage/src/vespa/storage/bucketmover/htmltable.h b/storage/src/vespa/storage/bucketmover/htmltable.h index ae9f3d7635b..bcb66602091 100644 --- a/storage/src/vespa/storage/bucketmover/htmltable.h +++ b/storage/src/vespa/storage/bucketmover/htmltable.h @@ -22,7 +22,7 @@ struct Column { enum { TOTAL = 0xffff }; Column(const std::string& colName, HtmlTable* table = 0); - virtual ~Column() {} + virtual ~Column(); virtual void finalize() {} // Called before print is issued @@ -87,8 +87,8 @@ class HtmlTable { std::unique_ptr<std::string> _totalRow; public: - HtmlTable(const std::string& rowId) - : _rowId(rowId), _columns(), _rows() {} + HtmlTable(const std::string& rowId); + ~HtmlTable(); void addTotalRow(const std::string& name) { _totalRow.reset(new std::string(name)); } @@ -102,15 +102,9 @@ public: { _rows.back()._backgroundColor = c; } uint32_t getRowCount() const { return _rows.size(); } - inline void print(std::ostream& out); + void print(std::ostream& out); }; -inline Column::Column(const std::string& colName, HtmlTable* table) - : _colName(colName), _alignment(RIGHT) -{ - if (table != 0) table->addColumn(*this); -} - /** Writes content just as you supply it. */ template<typename T> struct ValueColumn : public Column { @@ -123,80 +117,27 @@ struct ValueColumn : public Column { ValueColumn(const std::string& colName, const std::string& denomination = "", - HtmlTable* table = 0) - : Column(colName, table), _values(), _denomination(denomination), - _colorLimits(), _totalIsAvg(false) - { - _valuePrinter << std::fixed << std::setprecision(2); - } + HtmlTable* table = 0); + ~ValueColumn(); T& operator[](uint16_t row) { return _values[row]; } - ValueColumn<T>& setPrecision(int precision) - { _valuePrinter << std::setprecision(precision); return *this; } - ValueColumn<T>& setTotalAsAverage(bool setAsAvg = true) - { _totalIsAvg = setAsAvg; return *this; } - - void addColorLimit(T limit, Color c) { - _colorLimits[limit] = c; + ValueColumn<T>& setPrecision(int precision) { + _valuePrinter << std::setprecision(precision); + return *this; } - - virtual void finalize() { - for (typename std::map<uint16_t, T>::iterator val = _values.begin(); - val != _values.end(); ++val) - { - Color c = DEFAULT_COLOR; - for (typename std::map<T, Color>::iterator it - = _colorLimits.begin(); it != _colorLimits.end(); ++it) - { - if (val->second <= it->first) { - c = it->second; - break; - } - } - _colors[val->first] = c; - } - // Set color for total too. - T total = getTotalValue(); - Color c = DEFAULT_COLOR; - for (typename std::map<T, Color>::iterator it - = _colorLimits.begin(); it != _colorLimits.end(); ++it) - { - if (total <= it->first) { - c = it->second; - break; - } - } - _colors[TOTAL] = c; + ValueColumn<T>& setTotalAsAverage(bool setAsAvg = true) { + _totalIsAvg = setAsAvg; + return *this; } - virtual T getTotalValue() { - T value = 0; - for (typename std::map<uint16_t, T>::iterator val = _values.begin(); - val != _values.end(); ++val) - { - value += val->second; - } - if (_totalIsAvg) value /= _values.size(); - return value; + void addColorLimit(T limit, Color c) { + _colorLimits[limit] = c; } - virtual void printValue(std::ostream& out, uint16_t row) { - T value; - if (row == TOTAL) { - value = getTotalValue(); - } else { - typename std::map<uint16_t, T>::iterator val = _values.find(row); - if (val == _values.end()) { - Column::printValue(out, row); - return; - } - value = val->second; - } - _valuePrinter.str(""); - _valuePrinter << value << _denomination; - out << _valuePrinter.str(); - } + void finalize() override; + virtual T getTotalValue(); + void printValue(std::ostream& out, uint16_t row) override; }; /** Writes content as percentage of a total */ @@ -212,23 +153,7 @@ struct PercentageColumn : public ValueColumn<double> { if (total != 0) _totalIsAvg = true; } - virtual void finalize() { - uint64_t total = _total; - if (total == 0) { - for (std::map<uint16_t, uint64_t>::iterator it = _values.begin(); - it != _values.end(); ++it) - { - total += it->second; - } - } - for (std::map<uint16_t, uint64_t>::iterator it = _values.begin(); - it != _values.end(); ++it) - { - ValueColumn<double>::_values[it->first] - = 100.0 * it->second / total; - } - ValueColumn<double>::finalize(); - } + void finalize() override; uint64_t& operator[](uint16_t row) { return _values[row]; } }; @@ -241,78 +166,11 @@ struct ByteSizeColumn : public ValueColumn<uint64_t> { : ValueColumn<uint64_t>(colName, "", table) {} uint64_t& operator[](uint16_t row) { return _values[row]; } - - virtual void finalize() { - uint64_t max = 0; - for (std::map<uint16_t, uint64_t>::iterator it = _values.begin(); - it != _values.end(); ++it) - { - max = std::max(max, it->second); - } - uint64_t oldMax = max; - const char* type = "B"; - if (max > 10 * 1024) { max /= 1024; type = "kB"; } - if (max > 10 * 1024) { max /= 1024; type = "MB"; } - if (max > 10 * 1024) { max /= 1024; type = "GB"; } - if (max > 10 * 1024) { max /= 1024; type = "TB"; } - _denomination = std::pair<const char*, uint64_t>( - type, max == 0 ? 1 : oldMax / max); - ValueColumn<uint64_t>::finalize(); - } - - virtual void printValue(std::ostream& out, uint16_t row) { - uint64_t value; - if (row == TOTAL) { - value = getTotalValue(); - } else { - std::map<uint16_t, uint64_t>::iterator val(_values.find(row)); - if (val == _values.end()) { - Column::printValue(out, row); - return; - } - value = val->second; - } - out << (value / _denomination.second) << ' ' << _denomination.first; - } + void finalize() override; + void printValue(std::ostream& out, uint16_t row) override; }; typedef ValueColumn<int64_t> LongColumn; typedef ValueColumn<double> DoubleColumn; -inline void HtmlTable::print(std::ostream& out) { - out << "<table border=\"1\" cellpadding=\"2\" cellspacing=\"0\">\n<tr><th"; - if (!_colHeaders.empty()) out << " rowspan=\"2\""; - out << ">" << _rowId << "</th>"; - if (!_colHeaders.empty()) { - for (uint32_t i=0; i<_colHeaders.size(); ++i) { - out << "<th colspan=\"" << _colHeaders[i]._span << "\">" - << _colHeaders[i]._name << "</th>"; - } - out << "</tr>\n"; - } - for (uint32_t i=0; i<_columns.size(); ++i) { - _columns[i]->finalize(); - out << "<th>" << _columns[i]->_colName << "</th>"; - } - out << "</tr>\n"; - for (uint32_t i=0; i<_rows.size(); ++i) { - out << "<tr><td"; - Column::printTdColor(out, _rows[i]._backgroundColor); - out << ">" << _rows[i]._name << "</td>"; - for (uint32_t j=0; j<_columns.size(); ++j) { - _columns[j]->printElement(out, i); - } - out << "</tr>\n"; - } - if (_totalRow.get()) { - out << "<tr><td>" << *_totalRow << "</td>"; - for (uint32_t j=0; j<_columns.size(); ++j) { - _columns[j]->printElement(out, Column::TOTAL); - } - out << "</tr>\n"; - } - out << "</table>\n"; -} - } // storage - diff --git a/storage/src/vespa/storage/bucketmover/move.cpp b/storage/src/vespa/storage/bucketmover/move.cpp index d570eb48c5e..4619327a5d3 100644 --- a/storage/src/vespa/storage/bucketmover/move.cpp +++ b/storage/src/vespa/storage/bucketmover/move.cpp @@ -1,11 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/bucketmover/move.h> - -#include <vespa/log/log.h> - -LOG_SETUP(".bucketmover.move"); +#include "move.h" +#include <ostream> namespace storage { namespace bucketmover { diff --git a/storage/src/vespa/storage/bucketmover/move.h b/storage/src/vespa/storage/bucketmover/move.h index b032c5f3c51..a8018bb2323 100644 --- a/storage/src/vespa/storage/bucketmover/move.h +++ b/storage/src/vespa/storage/bucketmover/move.h @@ -36,7 +36,7 @@ public: uint8_t getPriority() const { return _priority; } uint32_t getTotalDocSize() const { return _totalDocSize; } - void print(std::ostream& out, bool verbose, const std::string& indent) const; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; }; } // bucketmover diff --git a/storage/src/vespa/storage/bucketmover/run.h b/storage/src/vespa/storage/bucketmover/run.h index a626960eec7..d05020e6ba5 100644 --- a/storage/src/vespa/storage/bucketmover/run.h +++ b/storage/src/vespa/storage/bucketmover/run.h @@ -15,12 +15,13 @@ #pragma once -#include <list> -#include <map> + #include <vespa/storage/bucketmover/move.h> #include <vespa/storage/bucketmover/runstatistics.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/nodestate.h> +#include <list> +#include <map> namespace storage { @@ -91,8 +92,7 @@ public: RunStatistics& getStatistics() { return _statistics; } const RunStatistics& getStatistics() const { return _statistics; } - - virtual void print(std::ostream&, bool verbose, const std::string& indent) const override; + void print(std::ostream&, bool verbose, const std::string& indent) const override; private: void removePending(Move&); void finalize(); @@ -100,4 +100,3 @@ private: } // bucketmover } // storage - diff --git a/storage/src/vespa/storage/bucketmover/runstatistics.cpp b/storage/src/vespa/storage/bucketmover/runstatistics.cpp index 6ba0dfd463a..093d5290203 100644 --- a/storage/src/vespa/storage/bucketmover/runstatistics.cpp +++ b/storage/src/vespa/storage/bucketmover/runstatistics.cpp @@ -1,13 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/bucketmover/runstatistics.h> - -#include <iomanip> -#include <vespa/log/log.h> -#include <vespa/storage/bucketmover/htmltable.h> - -LOG_SETUP(".bucketmover.run.statistics"); +#include "runstatistics.h" +#include "htmltable.h" namespace storage { namespace bucketmover { diff --git a/storage/src/vespa/storage/bucketmover/runstatistics.h b/storage/src/vespa/storage/bucketmover/runstatistics.h index 625d7f5fac4..8a12d286d16 100644 --- a/storage/src/vespa/storage/bucketmover/runstatistics.h +++ b/storage/src/vespa/storage/bucketmover/runstatistics.h @@ -92,10 +92,8 @@ struct RunStatistics : public document::Printable { double getProgress() const; uint64_t getBucketCount(uint16_t disk, bool includeWrongLocation) const; - void print(std::ostream& out, bool verbose, - const std::string& indent) const; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; }; } // bucketmover } // storage - diff --git a/storage/src/vespa/storage/common/messagesender.h b/storage/src/vespa/storage/common/messagesender.h index 025695dff29..332103e12db 100644 --- a/storage/src/vespa/storage/common/messagesender.h +++ b/storage/src/vespa/storage/common/messagesender.h @@ -16,6 +16,7 @@ #pragma once +#include <memory> namespace storage { namespace api { diff --git a/storage/src/vespa/storage/common/servicelayercomponent.h b/storage/src/vespa/storage/common/servicelayercomponent.h index e54421b1ce3..5b9db275b27 100644 --- a/storage/src/vespa/storage/common/servicelayercomponent.h +++ b/storage/src/vespa/storage/common/servicelayercomponent.h @@ -25,7 +25,7 @@ #pragma once -#include <vespa/storage/common/storagecomponent.h> +#include "storagecomponent.h" namespace storage { @@ -54,12 +54,10 @@ class ServiceLayerComponent : public StorageComponent, StorBucketDatabase* _bucketDatabase; MinimumUsedBitsTracker* _minUsedBitsTracker; - // ServiceLayerManagedComponent implementation - virtual void setDiskCount(uint16_t count) override { _diskCount = count; } - virtual void setBucketDatabase(StorBucketDatabase& db) override { - _bucketDatabase = &db; - } - virtual void setMinUsedBitsTracker(MinimumUsedBitsTracker& tracker) override { + // ServiceLayerManagedComponent implementation + void setDiskCount(uint16_t count) override { _diskCount = count; } + void setBucketDatabase(StorBucketDatabase& db) override { _bucketDatabase = &db; } + void setMinUsedBitsTracker(MinimumUsedBitsTracker& tracker) override { _minUsedBitsTracker = &tracker; } public: @@ -91,4 +89,3 @@ public: }; } // storage - diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.h b/storage/src/vespa/storage/common/statusmetricconsumer.h index 0027cef9cd0..6a2c18f78bd 100644 --- a/storage/src/vespa/storage/common/statusmetricconsumer.h +++ b/storage/src/vespa/storage/common/statusmetricconsumer.h @@ -9,11 +9,11 @@ #pragma once +#include "storagecomponent.h" #include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/sync.h> #include <map> #include <vespa/metrics/metrics.h> -#include <vespa/storage/common/storagecomponent.h> #include <vespa/storageframework/storageframework.h> namespace vespalib { diff --git a/storage/src/vespa/storage/common/storagelink.h b/storage/src/vespa/storage/common/storagelink.h index e9ea8403321..a9af5fa5622 100644 --- a/storage/src/vespa/storage/common/storagelink.h +++ b/storage/src/vespa/storage/common/storagelink.h @@ -19,13 +19,11 @@ #pragma once -#include <vespa/vespalib/util/printable.h> -#include <memory> +#include "messagesender.h" +#include "storagecomponent.h" #include <vespa/storageapi/messageapi/messagehandler.h> #include <vespa/storageapi/messageapi/storagemessage.h> -#include <string> -#include <vespa/storage/common/messagesender.h> -#include <vespa/storage/common/storagecomponent.h> +#include <vespa/vespalib/util/printable.h> namespace storage { @@ -84,10 +82,10 @@ public: void flush(); /** Send message down the storage chain. */ - virtual void sendDown(const api::StorageMessage::SP&); + void sendDown(const api::StorageMessage::SP&) override; /** Send message up the storage chain. */ - virtual void sendUp(const api::StorageMessage::SP&); + void sendUp(const api::StorageMessage::SP&) override; void printChain(std::ostream&, std::string indent = "") const; @@ -126,8 +124,7 @@ public: */ virtual bool onUp(const api::StorageMessage::SP&); - virtual void print(std::ostream& out, bool, - const std::string&) const { + void print(std::ostream& out, bool, const std::string&) const override { out << getName(); } @@ -188,4 +185,3 @@ private: std::ostream& operator<<(std::ostream& out, StorageLink& link); } - diff --git a/storage/src/vespa/storage/common/storagelinkqueued.h b/storage/src/vespa/storage/common/storagelinkqueued.h index cd7b91a7a68..c894b6196e8 100644 --- a/storage/src/vespa/storage/common/storagelinkqueued.h +++ b/storage/src/vespa/storage/common/storagelinkqueued.h @@ -14,11 +14,11 @@ #pragma once +#include "storagelink.h" +#include <vespa/storageframework/storageframework.h> #include <vespa/vespalib/util/document_runnable.h> #include <deque> #include <limits> -#include <vespa/storageframework/storageframework.h> -#include <vespa/storage/common/storagelink.h> namespace storage { @@ -40,13 +40,13 @@ public: void dispatchUp(const std::shared_ptr<api::StorageMessage>&); /** Remember to call this method if you override it. */ - virtual void onClose() override { + void onClose() override { _commandDispatcher.flush(); _closeState |= 1; } /** Remember to call this method if you override it. */ - virtual void onFlush(bool downwards) override { + void onFlush(bool downwards) override { if (downwards) { _commandDispatcher.flush(); _closeState |= 2; @@ -79,7 +79,7 @@ private: public: Dispatcher(StorageLinkQueued& parent, unsigned int maxQueueSize, bool replyDispatcher); - virtual ~Dispatcher(); + ~Dispatcher(); void start(); void run(framework::ThreadHandle&) override; @@ -106,7 +106,7 @@ private: void send(const std::shared_ptr<api::StorageMessage> & reply) override { _parent.sendUp(reply); } - virtual ~ReplyDispatcher() { terminate(); } + ~ReplyDispatcher() { terminate(); } }; class CommandDispatcher : public Dispatcher<api::StorageMessage> @@ -117,7 +117,7 @@ private: parent, std::numeric_limits<unsigned int>::max(), false) { } - virtual ~CommandDispatcher() { terminate(); } + ~CommandDispatcher() { terminate(); } void send(const std::shared_ptr<api::StorageMessage> & command) override { _parent.sendDown(command); } @@ -132,7 +132,6 @@ private: protected: ReplyDispatcher& getReplyDispatcher() { return _replyDispatcher; } - }; } diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h index b87f0e5f898..a8f76c2c7a0 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h @@ -49,23 +49,14 @@ public: const std::shared_ptr<api::RequestBucketInfoReply> & repl) override; bool onMergeBucketReply(const std::shared_ptr<api::MergeBucketReply>& reply) override; - bool onNotifyBucketChange(const std::shared_ptr<api::NotifyBucketChangeCommand>&) override; - void resendDelayedMessages(); - void storageDistributionChanged(const lib::Distribution&); - vespalib::string reportXmlStatus(vespalib::xml::XmlOutputStream&, - const framework::HttpUrlPath&) const; - - vespalib::string getReportContentType( - const framework::HttpUrlPath&) const override; + vespalib::string reportXmlStatus(vespalib::xml::XmlOutputStream&, const framework::HttpUrlPath&) const; + vespalib::string getReportContentType(const framework::HttpUrlPath&) const override; bool reportStatus(std::ostream&, const framework::HttpUrlPath&) const override; - - virtual void print(std::ostream& out, bool verbose, - const std::string& indent) const; - + void print(std::ostream& out, bool verbose, const std::string& indent) const; DistributorComponent& getDistributorComponent() { return _bucketSpaceComponent; } /** @@ -240,12 +231,8 @@ private: _upStates(upStates) {} ~NodeRemover(); - - virtual bool process(BucketDatabase::Entry& e) override; - - void logRemove(const document::BucketId& bucketId, - const char* msg) const; - + bool process(BucketDatabase::Entry& e) override; + void logRemove(const document::BucketId& bucketId, const char* msg) const; bool distributorOwnsBucket(const document::BucketId&) const; const std::vector<document::BucketId>& getBucketsToRemove() const { @@ -266,8 +253,7 @@ private: const char* _upStates; }; - std::deque<std::pair<framework::MilliSecTime, - BucketRequest> > _delayedRequests; + std::deque<std::pair<framework::MilliSecTime, BucketRequest> > _delayedRequests; std::map<uint64_t, BucketRequest> _sentMessages; std::unique_ptr<PendingClusterState> _pendingClusterState; std::list<PendingClusterState::Summary> _history; @@ -280,4 +266,3 @@ private: } } - diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h index 40d6ee76e07..182988104a9 100644 --- a/storage/src/vespa/storage/distributor/distributor.h +++ b/storage/src/vespa/storage/distributor/distributor.h @@ -2,7 +2,6 @@ #pragma once - #include "idealstatemanager.h" #include "bucketdbupdater.h" #include "pendingmessagetracker.h" @@ -22,7 +21,6 @@ #include <vespa/storageframework/generic/thread/tickingthread.h> #include <vespa/config/config.h> #include <vespa/vespalib/util/sync.h> - #include <unordered_map> namespace storage { @@ -57,16 +55,12 @@ public: ~Distributor(); void onOpen() override; - void onClose() override; - bool onDown(const std::shared_ptr<api::StorageMessage>&) override; - void sendUp(const std::shared_ptr<api::StorageMessage>&) override; - void sendDown(const std::shared_ptr<api::StorageMessage>&) override; - virtual ChainedMessageSender& getMessageSender() override { + ChainedMessageSender& getMessageSender() override { return (_messageSender == 0 ? *this : *_messageSender); } @@ -99,8 +93,7 @@ public: bool handleReply(const std::shared_ptr<api::StorageReply>& reply) override; // StatusReporter implementation - vespalib::string getReportContentType( - const framework::HttpUrlPath&) const override; + vespalib::string getReportContentType(const framework::HttpUrlPath&) const override; bool reportStatus(std::ostream&, const framework::HttpUrlPath&) const override; bool handleStatusRequest(const DelegatedStatusRequest& request) const override; @@ -117,8 +110,7 @@ public: * Checks whether a bucket needs to be split, and sends a split * if so. */ - void checkBucketForSplit(const BucketDatabase::Entry& e, - uint8_t priority) override; + void checkBucketForSplit(const BucketDatabase::Entry& e, uint8_t priority) override; const lib::Distribution& getDistribution() const override; @@ -130,8 +122,9 @@ public: * @return Returns the states in which the distributors consider * storage nodes to be up. */ - const char* getStorageNodeUpStates() const override - { return _initializingIsUp ? "uri" : "ur"; } + const char* getStorageNodeUpStates() const override { + return _initializingIsUp ? "uri" : "ur"; + } /** * Called by bucket db updater after a merge has finished, and all the @@ -154,13 +147,10 @@ public: } int getDistributorIndex() const override; - const std::string& getClusterName() const override; - const PendingMessageTracker& getPendingMessageTracker() const override; - - virtual void sendCommand(const std::shared_ptr<api::StorageCommand>&) override; - virtual void sendReply(const std::shared_ptr<api::StorageReply>&) override; + void sendCommand(const std::shared_ptr<api::StorageCommand>&) override; + void sendReply(const std::shared_ptr<api::StorageReply>&) override; const BucketGcTimeCalculator::BucketIdHasher& getBucketIdHasher() const override { diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.h b/storage/src/vespa/storage/distributor/operations/external/getoperation.h index ecdfb1e677a..57a7c55fb81 100644 --- a/storage/src/vespa/storage/distributor/operations/external/getoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.h @@ -6,15 +6,11 @@ #include <vespa/storage/bucketdb/bucketcopy.h> #include <vespa/storageapi/messageapi/storagemessage.h> -namespace document { -class Document; -} +namespace document { class Document; } namespace storage { -namespace api { -class GetCommand; -} +namespace api { class GetCommand; } class PersistenceOperationMetricSet; @@ -30,13 +26,9 @@ public: PersistenceOperationMetricSet& metric); void onClose(DistributorMessageSender& sender) override; - void onStart(DistributorMessageSender& sender) override; - void onReceive(DistributorMessageSender& sender, const std::shared_ptr<api::StorageReply> & msg) override; - const char* getName() const override { return "get"; } - std::string getStatus() const override { return ""; } bool hasConsistentCopies() const; @@ -48,13 +40,9 @@ private: GroupId(const document::BucketId& id, uint32_t checksum, int node); bool operator<(const GroupId& other) const; - bool operator==(const GroupId& other) const; - const document::BucketId& getBucketId() const { return _id; } - int getNode() const { return _node; } - private: document::BucketId _id; uint32_t _checksum; @@ -65,7 +53,8 @@ private: public: BucketChecksumGroup(const BucketCopy& c) : copy(c), - sent(0), received(false), returnCode(api::ReturnCode::OK) {}; + sent(0), received(false), returnCode(api::ReturnCode::OK) + {} BucketCopy copy; api::StorageMessage::Id sent; @@ -105,8 +94,4 @@ private: }; } - - } - - diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.h b/storage/src/vespa/storage/distributor/operations/external/putoperation.h index 5043fed307f..6ba9e9c3dbf 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.h @@ -28,14 +28,9 @@ public: PersistenceOperationMetricSet& metric); void onStart(DistributorMessageSender& sender) override; - - const char* getName() const override { return "put"; } - - std::string getStatus() const override { return ""; } - - void onReceive(DistributorMessageSender& sender, - const std::shared_ptr<api::StorageReply> &) override; - + const char* getName() const override { return "put"; }; + std::string getStatus() const override { return ""; }; + void onReceive(DistributorMessageSender& sender, const std::shared_ptr<api::StorageReply> &) override; void onClose(DistributorMessageSender& sender) override; /** @@ -71,11 +66,9 @@ private: const uint16_t node, std::vector<PersistenceMessageTracker::ToSend>& putBatch); - bool shouldImplicitlyActivateReplica( - const OperationTargetList& targets) const; + bool shouldImplicitlyActivateReplica(const OperationTargetList& targets) const; std::shared_ptr<api::PutCommand> _msg; - DistributorComponent& _manager; }; diff --git a/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h b/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h index 25cc0825b7d..dce744b6648 100644 --- a/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h @@ -7,9 +7,7 @@ namespace storage { -namespace api { -class GetBucketListCommand; -} +namespace api { class GetBucketListCommand; } namespace distributor { @@ -23,22 +21,21 @@ public: const MaintenanceOperationGenerator& generator, uint16_t distributorIndex, const std::shared_ptr<api::GetBucketListCommand>& cmd); - virtual ~StatBucketListOperation() {} + ~StatBucketListOperation() {} - virtual const char* getName() const override { return "statBucketList"; } - virtual std::string getStatus() const override { return ""; } + const char* getName() const override { return "statBucketList"; } + std::string getStatus() const override { return ""; } - virtual void onStart(DistributorMessageSender& sender) override; - virtual void onReceive(DistributorMessageSender&, const std::shared_ptr<api::StorageReply>&) override { + void onStart(DistributorMessageSender& sender) override; + void onReceive(DistributorMessageSender&, const std::shared_ptr<api::StorageReply>&) override + { // Never called. assert(false); } - void onClose(DistributorMessageSender&) override { - } + void onClose(DistributorMessageSender&) override {} private: - void getBucketStatus(const BucketDatabase::Entry& entry, - std::ostream& os) const; + void getBucketStatus(const BucketDatabase::Entry& entry, std::ostream& os) const; const BucketDatabase& _bucketDb; const MaintenanceOperationGenerator& _generator; @@ -48,4 +45,3 @@ private: } // distributor } // storage - diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 15f8d06965a..7b1ff03b8af 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -52,24 +52,24 @@ struct IntermediateMessageSender : DistributorMessageSender { DistributorMessageSender & fwd); ~IntermediateMessageSender(); - virtual void sendCommand(const std::shared_ptr<api::StorageCommand>& cmd) override { + void sendCommand(const std::shared_ptr<api::StorageCommand>& cmd) override { msgMap.insert(cmd->getMsgId(), callback); forward.sendCommand(cmd); }; - virtual void sendReply(const std::shared_ptr<api::StorageReply>& reply) override { + void sendReply(const std::shared_ptr<api::StorageReply>& reply) override { _reply = reply; } - virtual int getDistributorIndex() const override { + int getDistributorIndex() const override { return forward.getDistributorIndex(); } - virtual const std::string& getClusterName() const override { + const std::string& getClusterName() const override { return forward.getClusterName(); } - virtual const PendingMessageTracker& getPendingMessageTracker() const override { + const PendingMessageTracker& getPendingMessageTracker() const override { return forward.getPendingMessageTracker(); } }; diff --git a/storage/src/vespa/storage/distributor/operations/external/updateoperation.h b/storage/src/vespa/storage/distributor/operations/external/updateoperation.h index 0a2a9e564a0..fef9c01efa9 100644 --- a/storage/src/vespa/storage/distributor/operations/external/updateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/updateoperation.h @@ -25,14 +25,9 @@ public: PersistenceOperationMetricSet& metric); void onStart(DistributorMessageSender& sender) override; - - const char* getName() const override { return "update"; } - - std::string getStatus() const override { return ""; } - - void onReceive(DistributorMessageSender& sender, - const std::shared_ptr<api::StorageReply> & msg) override; - + const char* getName() const override { return "update"; }; + std::string getStatus() const override { return ""; }; + void onReceive(DistributorMessageSender& sender, const std::shared_ptr<api::StorageReply> & msg) override; void onClose(DistributorMessageSender& sender) override; std::pair<document::BucketId, uint16_t> getNewestTimestampLocation() const { @@ -65,4 +60,3 @@ private: } } - diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h index 802bb0f4bac..fd133674bd1 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h @@ -2,8 +2,8 @@ #pragma once +#include "operationtargetresolver.h" #include <vespa/storage/bucketdb/bucketdatabase.h> -#include <vespa/storage/distributor/operationtargetresolver.h> #include <vespa/vdslib/distribution/idealnodecalculator.h> namespace storage { @@ -91,23 +91,20 @@ public: _idealNodeCalculator(idealNodeCalc), _minUsedBucketBits(minUsedBucketBits), _redundancy(redundancy) - { - } + {} BucketInstanceList getAllInstances(OperationType type, const document::BucketId& id); - BucketInstanceList getInstances(OperationType type, const document::BucketId& id) - { + BucketInstanceList getInstances(OperationType type, const document::BucketId& id) { BucketInstanceList result(getAllInstances(type, id)); result.limitToRedundancyCopies(_redundancy); return result; } - virtual OperationTargetList getTargets(OperationType type, - const document::BucketId& id) override - { return getInstances(type, id).createTargets(); } + OperationTargetList getTargets(OperationType type, const document::BucketId& id) override { + return getInstances(type, id).createTargets(); + } }; } // distributor } // storage - diff --git a/storage/src/vespa/storage/distributor/statecheckers.h b/storage/src/vespa/storage/distributor/statecheckers.h index 0113749f2fd..dbd9503f9f2 100644 --- a/storage/src/vespa/storage/distributor/statecheckers.h +++ b/storage/src/vespa/storage/distributor/statecheckers.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/storage/distributor/idealstatemanager.h> +#include "idealstatemanager.h" namespace storage { @@ -11,9 +11,7 @@ class SynchronizeAndMoveStateChecker : public StateChecker { public: std::string getStatusText() const override { return "Synchronization and moving"; } - Result check(Context& c) override; - const char* getName() const override { return "SynchronizeAndMove"; } }; @@ -21,9 +19,7 @@ class DeleteExtraCopiesStateChecker : public StateChecker { public: std::string getStatusText() const override { return "Delete extra copies"; } - Result check(Context& c) override; - const char* getName() const override { return "DeleteExtraCopies"; } private: @@ -49,9 +45,7 @@ class JoinBucketsStateChecker : public StateChecker { public: std::string getStatusText() const override { return "Join buckets"; } - Result check(Context& c) override; - const char* getName() const override { return "JoinBuckets"; } private: uint64_t getTotalUsedFileSize(const Context& c) const; @@ -69,13 +63,9 @@ class SplitBucketStateChecker : public StateChecker { public: std::string getStatusText() const override { return "Split buckets"; } - Result check(Context& c) override; - const char* getName() const override { return "SplitBucket"; } - private: - Result generateMinimumBucketSplitOperation(Context& c); Result generateMaxSizeExceededSplitOperation(Context& c); @@ -86,11 +76,8 @@ private: class SplitInconsistentStateChecker : public StateChecker { public: - std::string getStatusText() const override - { return "Fix inconsistently split buckets"; } - + std::string getStatusText() const override { return "Fix inconsistently split buckets"; } Result check(Context& c) override; - const char* getName() const override { return "SplitInconsistentBuckets"; } private: @@ -115,9 +102,7 @@ class BucketStateStateChecker : public StateChecker const StateChecker::Context& c) const; public: std::string getStatusText() const override { return "Set bucket copy state"; } - Result check(Context& c) override; - const char* getName() const override { return "SetBucketState"; } }; @@ -125,15 +110,11 @@ class GarbageCollectionStateChecker : public StateChecker { public: std::string getStatusText() const override { return "Garbage collection"; } - bool needsGarbageCollection(const Context& c) const; - Result check(Context& c) override; - const char* getName() const override { return "GarbageCollection"; } }; } } - diff --git a/storage/src/vespa/storage/distributor/statusreporterdelegate.h b/storage/src/vespa/storage/distributor/statusreporterdelegate.h index 0c836910765..93a1fe1cd48 100644 --- a/storage/src/vespa/storage/distributor/statusreporterdelegate.h +++ b/storage/src/vespa/storage/distributor/statusreporterdelegate.h @@ -1,9 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "delegatedstatusrequest.h" +#include "statusdelegator.h" #include <vespa/storageframework/storageframework.h> -#include <vespa/storage/distributor/delegatedstatusrequest.h> -#include <vespa/storage/distributor/statusdelegator.h> namespace storage { namespace distributor { @@ -20,9 +20,7 @@ public: const framework::StatusReporter& target); void registerStatusPage(); - - vespalib::string getReportContentType( - const framework::HttpUrlPath&) const override; + vespalib::string getReportContentType(const framework::HttpUrlPath&) const override; bool reportStatus(std::ostream&, const framework::HttpUrlPath&) const override; }; diff --git a/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h index 2cf0255bf9f..90ddb71d678 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h @@ -7,10 +7,10 @@ */ #pragma once +#include "storagecomponentregisterimpl.h" #include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <vespa/storage/common/distributorcomponent.h> #include <vespa/storage/common/nodestateupdater.h> -#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> namespace storage { namespace lib { @@ -36,18 +36,13 @@ public: DistributorComponentRegisterImpl(); ~DistributorComponentRegisterImpl(); - virtual void registerDistributorComponent(DistributorManagedComponent&) override; - + void registerDistributorComponent(DistributorManagedComponent&) override; void setTimeCalculator(UniqueTimeCalculator& calc); void setDistributorConfig(const DistributorConfig&); void setVisitorConfig(const VisitorConfig&); - private: - virtual void handleNewState() override; - - virtual void setNodeStateUpdater(NodeStateUpdater& updater) override; + void handleNewState() override; + void setNodeStateUpdater(NodeStateUpdater& updater) override; }; } // storage - - diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h index 7d322f027d5..d071ba8ba3e 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h @@ -7,10 +7,10 @@ */ #pragma once +#include "storagecomponentregisterimpl.h" #include <vespa/storage/bucketdb/minimumusedbitstracker.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/common/servicelayercomponent.h> -#include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> namespace storage { @@ -35,11 +35,8 @@ public: return _minUsedBitsTracker; } - virtual void registerServiceLayerComponent(ServiceLayerManagedComponent&) override; - + void registerServiceLayerComponent(ServiceLayerManagedComponent&) override; void setDiskCount(uint16_t count); }; } // storage - - diff --git a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp index 51a30d26def..4f2ed2bfc25 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp @@ -14,9 +14,9 @@ StorageComponentRegisterImpl::StorageComponentRegisterImpl() _index(0xffff), _loadTypes(new documentapi::LoadTypeSet), _nodeStateUpdater(0) -{ +{ } -} +StorageComponentRegisterImpl::~StorageComponentRegisterImpl() { } void StorageComponentRegisterImpl::registerStorageComponent(StorageComponent& smc) diff --git a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h index cf9abbb6864..38773be167f 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h @@ -40,6 +40,7 @@ public: typedef std::unique_ptr<StorageComponentRegisterImpl> UP; StorageComponentRegisterImpl(); + ~StorageComponentRegisterImpl(); const vespalib::string& getClusterName() const { return _clusterName; } const lib::NodeType& getNodeType() const @@ -47,13 +48,12 @@ public: uint16_t getIndex() const { return _index; } document::DocumentTypeRepo::SP getTypeRepo() { return _docTypeRepo; } documentapi::LoadTypeSet::SP getLoadTypes() { return _loadTypes; } - const document::BucketIdFactory& getBucketIdFactory() - { return _bucketIdFactory; } + const document::BucketIdFactory& getBucketIdFactory() { return _bucketIdFactory; } lib::Distribution::SP getDistribution() { return _distribution; } NodeStateUpdater& getNodeStateUpdater() { assert(_nodeStateUpdater != 0); return *_nodeStateUpdater; } - virtual void registerStorageComponent(StorageComponent&) override; + void registerStorageComponent(StorageComponent&) override; void setNodeInfo(vespalib::stringref clusterName, const lib::NodeType& nodeType, @@ -68,5 +68,3 @@ public: }; } // storage - - diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp index 16977f4fec5..883dc13cdb3 100644 --- a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp +++ b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.cpp @@ -1,14 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <vespa/storage/frameworkimpl/memory/memorystatusviewer.h> +#include "memorystatusviewer.h" #include <vespa/storage/storageutil/graph.h> #include <vespa/storage/storageutil/palette.h> #include <vespa/storage/storageutil/piechart.h> #include <vespa/metrics/metricmanager.h> #include <vespa/storageapi/messageapi/storagemessage.h> +#include <vespa/log/log.h> + LOG_SETUP(".memory.status.viewer"); using storage::framework::defaultimplementation::MemoryState; diff --git a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h index 120eea1a457..dd7ecd7ae59 100644 --- a/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h +++ b/storage/src/vespa/storage/frameworkimpl/memory/memorystatusviewer.h @@ -34,13 +34,14 @@ #pragma once -#include <deque> -#include <vespa/vespalib/util/document_runnable.h> #include <vespa/storage/common/storagecomponent.h> #include <vespa/storageframework/defaultimplementation/memory/memorystate.h> #include <vespa/storageframework/storageframework.h> -#include <vector> +#include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/sync.h> +#include <deque> +#include <vector> + namespace metrics { class MetricManager; @@ -110,7 +111,7 @@ private: _states.push_back(Entry::SP(new Entry(name, _component.getClock(), framework::SecondTime(maxAge)))); } - void run(framework::ThreadHandle&); + void run(framework::ThreadHandle&) override; void grabMemoryUsage(); void printSnapshot(std::ostream& out, Entry& entry, std::map<const framework::MemoryAllocationType*, @@ -123,10 +124,8 @@ public: StorageComponentRegister&); ~MemoryStatusViewer(); - virtual void reportHtmlHeaderAdditions(std::ostream&, - const framework::HttpUrlPath&) const; - virtual void reportHtmlStatus(std::ostream&, - const framework::HttpUrlPath&) const; + void reportHtmlHeaderAdditions(std::ostream&, const framework::HttpUrlPath&) const override; + void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; /** Useful for testing. */ framework::SecondTime getProcessedTime() const { return _processedTime; } diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp index 12e2c31068a..18e0f2280f8 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp @@ -236,7 +236,7 @@ StatusWebServer::WebServer::onGetRequest(const string & tmpurl, const string &se namespace { class IndexPageReporter : public framework::HtmlStatusReporter { std::ostringstream ost; - virtual void reportHtmlStatus(std::ostream& out, const framework::HttpUrlPath&) const override { + void reportHtmlStatus(std::ostream& out,const framework::HttpUrlPath&) const override{ out << ost.str(); } diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h index c89c286387b..d1a0a33da95 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.h @@ -10,13 +10,12 @@ #pragma once -#include <vespa/vespalib/stllike/string.h> -#include <vespa/fastlib/net/httpserver.h> -#include <list> #include <vespa/storage/config/config-stor-status.h> #include <vespa/storageframework/storageframework.h> #include <vespa/config/config.h> #include <vespa/config/helper/configfetcher.h> +#include <vespa/fastlib/net/httpserver.h> +#include <list> namespace storage { @@ -30,7 +29,7 @@ class StatusWebServer : private config::IFetcherCallback<vespa::config::content: public: WebServer(StatusWebServer&, uint16_t port); - virtual void onGetRequest(const string & url, + void onGetRequest(const string & url, const string & serverSpec, Fast_HTTPConnection& conn) override; const vespalib::string &getServerSpec() const { @@ -69,16 +68,12 @@ public: virtual ~StatusWebServer(); void handlePage(const framework::HttpUrlPath&, std::ostream& out); - static vespalib::string getServerSpec(const vespalib::string &requestSpec, const vespalib::string &serverSpec); - private: - virtual void configure(std::unique_ptr<vespa::config::content::core::StorStatusConfig> config) override; + void configure(std::unique_ptr<vespa::config::content::core::StorStatusConfig> config) override; void getPage(const char* url, Fast_HTTPConnection& conn); - virtual void run(framework::ThreadHandle&) override; - + void run(framework::ThreadHandle&) override; }; } - diff --git a/storage/src/vespa/storage/frameworkimpl/thread/appkiller.h b/storage/src/vespa/storage/frameworkimpl/thread/appkiller.h index b785c98e505..437927ccaf1 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/appkiller.h +++ b/storage/src/vespa/storage/frameworkimpl/thread/appkiller.h @@ -22,7 +22,7 @@ struct AppKiller { }; struct RealAppKiller : public AppKiller { - virtual void kill(); + void kill() override; }; } // storage diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp index 5d58a092053..2861ab904ff 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp +++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp @@ -2,8 +2,6 @@ #include "deadlockdetector.h" #include <vespa/storage/bucketdb/storbucketdb.h> -#include <vespa/storageframework/storageframework.h> -#include <vespa/vdslib/state/nodetype.h> #include <vespa/storage/bucketmover/htmltable.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -146,10 +144,10 @@ namespace { ThreadChecker(DeadLockDetector& d, const framework::MilliSecTime& time) : _detector(d), _currentTime(time) {} - virtual void visitThread(const vespalib::string& id, - const framework::ThreadProperties& tp, - const framework::ThreadTickData& tick, - DeadLockDetector::State& state) override + void visitThread(const vespalib::string& id, + const framework::ThreadProperties& tp, + const framework::ThreadTickData& tick, + DeadLockDetector::State& state) override { // In case we just got a new tick, ignore the thread if (tick._lastTickMs > _currentTime.getTime()) return; diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h index 146d0ccb751..c8cb3f35ddf 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h +++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.h @@ -12,9 +12,9 @@ #pragma once +#include "appkiller.h" #include <vespa/storage/common/distributorcomponent.h> #include <vespa/storage/common/servicelayercomponent.h> -#include <vespa/storage/frameworkimpl/thread/appkiller.h> #include <vespa/storageframework/storageframework.h> #include <vespa/vespalib/util/sync.h> #include <map> @@ -88,13 +88,9 @@ private: StorageComponent* _component; framework::Thread::UP _thread; - virtual void run(framework::ThreadHandle&) override; - - // Status implementation - virtual void reportHtmlStatus(std::ostream& out, - const framework::HttpUrlPath&) const override; + void run(framework::ThreadHandle&) override; + void reportHtmlStatus(std::ostream& out, const framework::HttpUrlPath&) const override; vespalib::string getBucketLockInfo() const; }; } - diff --git a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp index 002f0be2d3d..b26c6ec893f 100644 --- a/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp +++ b/storage/src/vespa/storage/persistence/bucketownershipnotifier.cpp @@ -1,15 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> - -#include <vespa/storageapi/message/bucket.h> -#include <vespa/vdslib/distribution/distribution.h> +#include "bucketownershipnotifier.h" #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/common/bucketoperationlogger.h> -#include <vespa/storage/persistence/bucketownershipnotifier.h> +#include <vespa/storageapi/message/bucket.h> +#include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/util/backtrace.h> +#include <vespa/log/log.h> LOG_SETUP(".persistence.bucketownershipnotifier"); namespace storage { diff --git a/storage/src/vespa/storage/persistence/diskthread.h b/storage/src/vespa/storage/persistence/diskthread.h index 3626bbb2c70..a1af83a65ef 100644 --- a/storage/src/vespa/storage/persistence/diskthread.h +++ b/storage/src/vespa/storage/persistence/diskthread.h @@ -60,7 +60,7 @@ public: bool operator==(const OperationCount& c) const { return (count == c.count && pending == c.pending); } - void print(std::ostream& out, bool, const std::string&) const + void print(std::ostream& out, bool, const std::string&) const override { out << "OperationCount(" << count << (pending ? ", pending" : "") << ")"; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp index 486051b2e95..34513687770 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp @@ -1,7 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/persistence/filestorage/filestorhandler.h> -#include <vespa/storage/persistence/filestorage/filestorhandlerimpl.h> +#include "filestorhandler.h" +#include "filestorhandlerimpl.h" namespace storage { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index e658599a3dc..b549aca35b6 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -13,9 +13,9 @@ #pragma once +#include "mergestatus.h" #include <vespa/document/bucket/bucketid.h> #include <ostream> -#include <vespa/storage/persistence/filestorage/mergestatus.h> #include <vespa/storage/storageutil/resumeguard.h> #include <vespa/storage/common/messagesender.h> diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index a6942102c42..39d9a60a950 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -8,6 +8,8 @@ #pragma once +#include "filestorhandler.h" +#include "filestormetrics.h" #include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/sync.h> #include <vespa/document/bucket/bucketid.h> @@ -20,8 +22,7 @@ #include <vespa/storage/common/storagelinkqueued.h> #include <vespa/config-stor-filestor.h> #include <vespa/storage/persistence/diskthread.h> -#include <vespa/storage/persistence/filestorage/filestorhandler.h> -#include <vespa/storage/persistence/filestorage/filestormetrics.h> + #include <vespa/storage/persistence/providershutdownwrapper.h> #include <vespa/storageframework/storageframework.h> #include <vespa/storage/common/nodestateupdater.h> @@ -104,7 +105,7 @@ public: ServiceLayerComponentRegister&); ~FileStorManager(); - virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; // Return true if we are currently merging the given bucket. bool isMerging(const document::BucketId& bucket) const; @@ -122,8 +123,7 @@ public: private: void configure(std::unique_ptr<vespa::config::content::StorFilestorConfig> config) override; - void replyWithBucketNotFound(api::StorageMessage&, - const document::BucketId&); + void replyWithBucketNotFound(api::StorageMessage&, const document::BucketId&); void replyDroppedOperation(api::StorageMessage& msg, const document::BucketId& bucket, @@ -135,17 +135,12 @@ private: api::StorageMessage& msg, const char* callerId); - bool validateApplyDiffCommandBucket(api::StorageMessage& msg, - const StorBucketDatabase::WrappedEntry&); - bool validateDiffReplyBucket(const StorBucketDatabase::WrappedEntry&, - const document::BucketId&); + bool validateApplyDiffCommandBucket(api::StorageMessage& msg, const StorBucketDatabase::WrappedEntry&); + bool validateDiffReplyBucket(const StorBucketDatabase::WrappedEntry&, const document::BucketId&); - StorBucketDatabase::WrappedEntry mapOperationToDisk( - api::StorageMessage&, const document::BucketId&); - StorBucketDatabase::WrappedEntry mapOperationToBucketAndDisk( - api::BucketCommand&, const document::DocumentId*); - bool handlePersistenceMessage(const std::shared_ptr<api::StorageMessage>&, - uint16_t disk); + StorBucketDatabase::WrappedEntry mapOperationToDisk(api::StorageMessage&, const document::BucketId&); + StorBucketDatabase::WrappedEntry mapOperationToBucketAndDisk(api::BucketCommand&, const document::DocumentId*); + bool handlePersistenceMessage(const std::shared_ptr<api::StorageMessage>&, uint16_t disk); // Document operations bool onPut(const std::shared_ptr<api::PutCommand>&) override; @@ -175,23 +170,15 @@ private: bool onInternal(const std::shared_ptr<api::InternalCommand>&) override; bool onInternalReply(const std::shared_ptr<api::InternalReply>&) override; - void handleAbortBucketOperations( - const std::shared_ptr<AbortBucketOperationsCommand>&); - + void handleAbortBucketOperations(const std::shared_ptr<AbortBucketOperationsCommand>&); void sendCommand(const std::shared_ptr<api::StorageCommand>&) override; void sendReply(const std::shared_ptr<api::StorageReply>&) override; - void sendUp(const std::shared_ptr<api::StorageMessage>&) override; - void onClose() override; void onFlush(bool downwards) override; - - virtual void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; - - virtual void storageDistributionChanged() override; - + void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; + void storageDistributionChanged() override; void updateState(); }; } // storage - diff --git a/storage/src/vespa/storage/persistence/filestorage/mergestatus.h b/storage/src/vespa/storage/persistence/filestorage/mergestatus.h index 9967fc24c7d..ff7e9e5dfb4 100644 --- a/storage/src/vespa/storage/persistence/filestorage/mergestatus.h +++ b/storage/src/vespa/storage/persistence/filestorage/mergestatus.h @@ -41,7 +41,7 @@ public: const std::vector<api::ApplyBucketDiffCommand::Entry>& part, uint16_t hasMask); void print(std::ostream& out, bool verbose, - const std::string& indent) const; + const std::string& indent) const override; bool isFirstNode() const { return (reply.get() != 0); } }; diff --git a/storage/src/vespa/storage/persistence/persistencethread.h b/storage/src/vespa/storage/persistence/persistencethread.h index f4c70353fab..d2c4dba3a5f 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.h +++ b/storage/src/vespa/storage/persistence/persistencethread.h @@ -2,15 +2,15 @@ #pragma once -#include <vespa/storage/common/statusmessages.h> -#include <vespa/storage/persistence/diskthread.h> -#include <vespa/storage/persistence/processallhandler.h> -#include <vespa/storage/persistence/mergehandler.h> -#include <vespa/storage/persistence/diskmoveoperationhandler.h> +#include "diskthread.h" +#include "processallhandler.h" +#include "mergehandler.h" +#include "diskmoveoperationhandler.h" +#include "persistenceutil.h" +#include "providershutdownwrapper.h" #include <vespa/storageframework/storageframework.h> #include <vespa/storage/common/storagecomponent.h> -#include <vespa/storage/persistence/persistenceutil.h> -#include <vespa/storage/persistence/providershutdownwrapper.h> +#include <vespa/storage/common/statusmessages.h> namespace storage { @@ -35,7 +35,7 @@ public: bool isMerging(const BucketId& bucket) const; - virtual framework::Thread& getThread() override { return *_thread; } + framework::Thread& getThread() override { return *_thread; } MessageTracker::UP handlePut(api::PutCommand& cmd); MessageTracker::UP handleRemove(api::RemoveCommand& cmd); @@ -99,10 +99,8 @@ private: void processMessages(FileStorHandler::LockedMessage & lock); // Thread main loop - virtual void run(framework::ThreadHandle&) override; - + void run(framework::ThreadHandle&) override; bool checkForError(const spi::Result& response, MessageTracker& tracker); - spi::Bucket getBucket(const DocumentId& id, const BucketId& bucket) const; void flushAllReplies(const document::BucketId& bucketId, @@ -114,4 +112,3 @@ private: }; } // storage - diff --git a/storage/src/vespa/storage/persistence/processallhandler.cpp b/storage/src/vespa/storage/persistence/processallhandler.cpp index 2095e19261c..c08b19a9e45 100644 --- a/storage/src/vespa/storage/persistence/processallhandler.cpp +++ b/storage/src/vespa/storage/persistence/processallhandler.cpp @@ -2,7 +2,6 @@ #include "processallhandler.h" #include "bucketprocessor.h" -#include <vespa/storage/common/bucketmessages.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> diff --git a/storage/src/vespa/storage/persistence/splitbitdetector.cpp b/storage/src/vespa/storage/persistence/splitbitdetector.cpp index 8d816760815..62039f36133 100644 --- a/storage/src/vespa/storage/persistence/splitbitdetector.cpp +++ b/storage/src/vespa/storage/persistence/splitbitdetector.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/storage/persistence/splitbitdetector.h> #include <vespa/storage/persistence/bucketprocessor.h> #include <vespa/document/bucket/bucketidfactory.h> diff --git a/storage/src/vespa/storage/storageutil/palette.h b/storage/src/vespa/storage/storageutil/palette.h index fe42eae0d31..338d8a3add7 100644 --- a/storage/src/vespa/storage/storageutil/palette.h +++ b/storage/src/vespa/storage/storageutil/palette.h @@ -11,6 +11,8 @@ #pragma once #include <vector> +#include <cstdint> +#include <iosfwd> namespace storage { diff --git a/storage/src/vespa/storage/tools/throttlingsim.h b/storage/src/vespa/storage/tools/throttlingsim.h index b583a7d2533..a7384d0f362 100644 --- a/storage/src/vespa/storage/tools/throttlingsim.h +++ b/storage/src/vespa/storage/tools/throttlingsim.h @@ -75,7 +75,7 @@ public: double max_diff; virtual void returnMessage(const Message& m) = 0; - virtual void run() override; + void run() override; virtual void print(double timenow); Client(Messaging& msgng, double windowSize, int to); @@ -91,17 +91,15 @@ class FixedClient : public Client { public: FixedClient(Messaging& msgng, int winsize, int to) : Client(msgng, winsize, to) {}; - - virtual void returnMessage(const Message& m) override; + void returnMessage(const Message& m) override; }; class LoadBalancingClient : public Client { public: LoadBalancingClient(Messaging& msgng, int winsize, int to); - - virtual void returnMessage(const Message& m) override; - virtual void run() override; - virtual void print(double timenow) override; + void returnMessage(const Message& m) override; + void run() override; + void print(double timenow) override; std::vector<double> weights; }; @@ -109,10 +107,9 @@ public: class BusyCounterBalancingClient : public Client { public: BusyCounterBalancingClient(Messaging& msgng, int winsize, int to); - - virtual void returnMessage(const Message& m) override; - virtual void run() override; - virtual void print(double timenow) override; + void returnMessage(const Message& m) override; + void run() override; + void print(double timenow) override; std::vector<int> busyCount; }; @@ -126,8 +123,7 @@ public: DynamicClient(Messaging& msgng, int maxWinSize, double to) : Client(msgng, 1, static_cast<int>(to)), maxwinsize(maxWinSize), threshold(maxWinSize / 2), lastFailTimestamp(0) {}; - - virtual void returnMessage(const Message& m) override; + void returnMessage(const Message& m) override; }; class LatencyControlClient : public Client { @@ -137,10 +133,8 @@ public: LatencyControlClient(Messaging& msgng, double to) : Client(msgng, 1, static_cast<int>(to)), count(0){}; - - virtual void returnMessage(const Message& m) override; - - virtual void print(double timenow) override; + void returnMessage(const Message& m) override; + void print(double timenow) override; }; @@ -154,5 +148,4 @@ public: ThrottlingApp() {}; int Main() override; - }; diff --git a/storage/src/vespa/storage/visiting/commandqueue.h b/storage/src/vespa/storage/visiting/commandqueue.h index 07677fdcd38..ba0141add45 100644 --- a/storage/src/vespa/storage/visiting/commandqueue.h +++ b/storage/src/vespa/storage/visiting/commandqueue.h @@ -22,229 +22,189 @@ namespace storage { - template<class Command> - class CommandQueue : public vespalib::Printable - { - public: - struct CommandEntry { - typedef typename Command::Priority PriorityType; - std::shared_ptr<Command> _command; - uint64_t _time; - uint64_t _sequenceId; - PriorityType _priority; - - CommandEntry(const std::shared_ptr<Command>& cmd, - uint64_t time, - uint64_t sequenceId, - PriorityType priority) - : _command(cmd), _time(time), _sequenceId(sequenceId), _priority(priority) - {} - - // Sort on both priority and sequence ID - bool operator<(const CommandEntry& entry) const { - if (_priority != entry._priority) { - return (_priority < entry._priority); - } - return (_sequenceId < entry._sequenceId); - } - }; - - private: - typedef boost::multi_index::multi_index_container< - CommandEntry, - boost::multi_index::indexed_by< - boost::multi_index::ordered_unique< - boost::multi_index::identity<CommandEntry> - >, - boost::multi_index::ordered_non_unique< - boost::multi_index::member<CommandEntry, uint64_t, &CommandEntry::_time> - > - > - > CommandList; - typedef typename boost::multi_index - ::nth_index<CommandList, 1>::type timelist; - - framework::Clock& _clock; - mutable CommandList _commands; +template<class Command> +class CommandQueue : public vespalib::Printable +{ +public: + struct CommandEntry { + typedef typename Command::Priority PriorityType; + std::shared_ptr<Command> _command; + uint64_t _time; uint64_t _sequenceId; - - public: - typedef typename CommandList::iterator iterator; - typedef typename CommandList::reverse_iterator reverse_iterator; - typedef typename CommandList::const_iterator const_iterator; - typedef typename CommandList::const_reverse_iterator const_reverse_iterator; - typedef typename timelist::const_iterator const_titerator; - - CommandQueue(framework::Clock& clock) - : _clock(clock), - _sequenceId(0) {} - - const framework::Clock& getTimer() const { return _clock; } - - iterator begin() { return _commands.begin(); } - iterator end() { return _commands.end(); } - - const_iterator begin() const { return _commands.begin(); } - const_iterator end() const { return _commands.end(); } - - const_titerator tbegin() const { - timelist& tl = boost::multi_index::get<1>(_commands); - return tl.begin(); - } - const_titerator tend() const { - timelist& tl = boost::multi_index::get<1>(_commands); - return tl.end(); + PriorityType _priority; + + CommandEntry(const std::shared_ptr<Command>& cmd, + uint64_t time, + uint64_t sequenceId, + PriorityType priority) + : _command(cmd), _time(time), _sequenceId(sequenceId), _priority(priority) + {} + + // Sort on both priority and sequence ID + bool operator<(const CommandEntry& entry) const { + if (_priority != entry._priority) { + return (_priority < entry._priority); + } + return (_sequenceId < entry._sequenceId); } + }; - bool empty() const; - - uint32_t size() const; - - std::pair<std::shared_ptr<Command>, time_t> releaseNextCommand(); - - std::shared_ptr<Command> peekNextCommand() const; - - void add(const std::shared_ptr<Command>& msg); +private: + typedef boost::multi_index::multi_index_container< + CommandEntry, + boost::multi_index::indexed_by< + boost::multi_index::ordered_unique< + boost::multi_index::identity<CommandEntry> + >, + boost::multi_index::ordered_non_unique< + boost::multi_index::member<CommandEntry, uint64_t, &CommandEntry::_time> + > + > + > CommandList; + typedef typename boost::multi_index + ::nth_index<CommandList, 1>::type timelist; - void erase(iterator it); + framework::Clock& _clock; + mutable CommandList _commands; + uint64_t _sequenceId; - std::list<CommandEntry> releaseTimedOut(); +public: + typedef typename CommandList::iterator iterator; + typedef typename CommandList::reverse_iterator reverse_iterator; + typedef typename CommandList::const_iterator const_iterator; + typedef typename CommandList::const_reverse_iterator const_reverse_iterator; + typedef typename timelist::const_iterator const_titerator; - std::pair<std::shared_ptr<Command>, time_t> - releaseLowestPriorityCommand(); + CommandQueue(framework::Clock& clock) + : _clock(clock), + _sequenceId(0) {} - std::shared_ptr<Command> peekLowestPriorityCommand() const; + const framework::Clock& getTimer() const { return _clock; } - void clear(); + iterator begin() { return _commands.begin(); } + iterator end() { return _commands.end(); } - void print(std::ostream& out, bool verbose, - const std::string& indent) const; - }; + const_iterator begin() const { return _commands.begin(); } + const_iterator end() const { return _commands.end(); } - template<class Command> - inline bool - CommandQueue<Command>::empty() const - { - return _commands.empty(); + const_titerator tbegin() const { + timelist& tl = boost::multi_index::get<1>(_commands); + return tl.begin(); } - - template<class Command> - inline uint32_t - CommandQueue<Command>::size() const - { - return _commands.size(); + const_titerator tend() const { + timelist& tl = boost::multi_index::get<1>(_commands); + return tl.end(); } - template<class Command> - inline std::pair<std::shared_ptr<Command>, time_t> - CommandQueue<Command>::releaseNextCommand() - { - std::pair<std::shared_ptr<Command>, time_t> retVal( - std::shared_ptr<Command>(), 0); - if (!_commands.empty()) { - iterator first = _commands.begin(); - retVal.first = first->_command; - retVal.second = first->_time; - _commands.erase(first); - } - return retVal; + bool empty() const { return _commands.empty(); } + uint32_t size() const { return _commands.size(); } + std::pair<std::shared_ptr<Command>, time_t> releaseNextCommand(); + std::shared_ptr<Command> peekNextCommand() const; + void add(const std::shared_ptr<Command>& msg); + void erase(iterator it) { _commands.erase(it); } + std::list<CommandEntry> releaseTimedOut(); + std::pair<std::shared_ptr<Command>, time_t> releaseLowestPriorityCommand(); + + std::shared_ptr<Command> peekLowestPriorityCommand() const; + void clear() { return _commands.clear(); } + void print(std::ostream& out, bool verbose, const std::string& indent) const override; +}; + + +template<class Command> +std::pair<std::shared_ptr<Command>, time_t> +CommandQueue<Command>::releaseNextCommand() +{ + std::pair<std::shared_ptr<Command>, time_t> retVal( + std::shared_ptr<Command>(), 0); + if (!_commands.empty()) { + iterator first = _commands.begin(); + retVal.first = first->_command; + retVal.second = first->_time; + _commands.erase(first); } - - template<class Command> - inline std::shared_ptr<Command> - CommandQueue<Command>::peekNextCommand() const - { - if (!_commands.empty()) { - const_iterator first = _commands.begin(); - return first->_command; - } else { - return std::shared_ptr<Command>(); - } - } - - template<class Command> - inline void - CommandQueue<Command>::add( - const std::shared_ptr<Command>& cmd) - { - framework::MicroSecTime time(_clock.getTimeInMicros() - + framework::MicroSecTime(cmd->getQueueTimeout() * 1000000)); - _commands.insert(CommandEntry(cmd, time.getTime(), ++_sequenceId, cmd->getPriority())); + return retVal; +} + +template<class Command> +std::shared_ptr<Command> +CommandQueue<Command>::peekNextCommand() const +{ + if (!_commands.empty()) { + const_iterator first = _commands.begin(); + return first->_command; + } else { + return std::shared_ptr<Command>(); } - - template<class Command> - inline void - CommandQueue<Command>::erase(iterator it) - { - _commands.erase(it); - } - - template<class Command> - inline std::list<typename CommandQueue<Command>::CommandEntry> - CommandQueue<Command>::releaseTimedOut() - { - std::list<CommandEntry> mylist; - framework::MicroSecTime time(_clock.getTimeInMicros()); - while (!empty() && tbegin()->_time <= time.getTime()) { - mylist.push_back(*tbegin()); - timelist& tl = boost::multi_index::get<1>(_commands); - tl.erase(tbegin()); - } - return mylist; +} + +template<class Command> +void +CommandQueue<Command>::add( + const std::shared_ptr<Command>& cmd) +{ + framework::MicroSecTime time(_clock.getTimeInMicros() + + framework::MicroSecTime(cmd->getQueueTimeout() * 1000000)); + _commands.insert(CommandEntry(cmd, time.getTime(), ++_sequenceId, cmd->getPriority())); +} + +template<class Command> +std::list<typename CommandQueue<Command>::CommandEntry> +CommandQueue<Command>::releaseTimedOut() +{ + std::list<CommandEntry> mylist; + framework::MicroSecTime time(_clock.getTimeInMicros()); + while (!empty() && tbegin()->_time <= time.getTime()) { + mylist.push_back(*tbegin()); + timelist& tl = boost::multi_index::get<1>(_commands); + tl.erase(tbegin()); } - - template <class Command> - inline std::pair<std::shared_ptr<Command>, time_t> - CommandQueue<Command>::releaseLowestPriorityCommand() - { - if (!_commands.empty()) { - iterator last = (++_commands.rbegin()).base(); - time_t time = last->_time; - std::shared_ptr<Command> cmd(last->_command); - _commands.erase(last); - return std::pair<std::shared_ptr<Command>, time_t>(cmd, time); - } else { - return std::pair<std::shared_ptr<Command>, time_t>( - std::shared_ptr<Command>(), 0); - } + return mylist; +} + +template <class Command> +std::pair<std::shared_ptr<Command>, time_t> +CommandQueue<Command>::releaseLowestPriorityCommand() +{ + if (!_commands.empty()) { + iterator last = (++_commands.rbegin()).base(); + time_t time = last->_time; + std::shared_ptr<Command> cmd(last->_command); + _commands.erase(last); + return std::pair<std::shared_ptr<Command>, time_t>(cmd, time); + } else { + return std::pair<std::shared_ptr<Command>, time_t>( + std::shared_ptr<Command>(), 0); } - - template <class Command> - inline std::shared_ptr<Command> - CommandQueue<Command>::peekLowestPriorityCommand() const - { - if (!_commands.empty()) { - const_reverse_iterator last = _commands.rbegin(); - return last->_command; - } else { - return std::shared_ptr<Command>(); - } +} + +template <class Command> +std::shared_ptr<Command> +CommandQueue<Command>::peekLowestPriorityCommand() const +{ + if (!_commands.empty()) { + const_reverse_iterator last = _commands.rbegin(); + return last->_command; + } else { + return std::shared_ptr<Command>(); } - - template<class Command> - inline void - CommandQueue<Command>::clear() - { - _commands.clear(); +} + +template<class Command> +void +CommandQueue<Command>::print(std::ostream& out, bool verbose, const std::string& indent) const +{ + (void) verbose; + out << "Insert order:\n"; + for (const_iterator it = begin(); it != end(); ++it) { + out << indent << *it->_command << ", priority " << it->_priority + << ", time " << it->_time << "\n"; } - - template<class Command> - inline void - CommandQueue<Command>::print(std::ostream& out, bool verbose, - const std::string& indent) const - { - (void) verbose; - out << "Insert order:\n"; - for (const_iterator it = begin(); it != end(); ++it) { - out << indent << *it->_command << ", priority " << it->_priority - << ", time " << it->_time << "\n"; - } - out << indent << "Time order:"; - for (const_titerator it = tbegin(); it != tend(); ++it) { - out << "\n" << indent << *it->_command << ", priority " << it->_priority - << ", time " << it->_time; - } + out << indent << "Time order:"; + for (const_titerator it = tbegin(); it != tend(); ++it) { + out << "\n" << indent << *it->_command << ", priority " << it->_priority + << ", time " << it->_time; } +} } // storage - diff --git a/storage/src/vespa/storage/visiting/countvisitor.h b/storage/src/vespa/storage/visiting/countvisitor.h index ecf51035a03..814cba9501b 100644 --- a/storage/src/vespa/storage/visiting/countvisitor.h +++ b/storage/src/vespa/storage/visiting/countvisitor.h @@ -9,7 +9,7 @@ */ #pragma once -#include <vespa/storage/visiting/visitor.h> +#include "visitor.h" namespace storage { diff --git a/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp b/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp index a36ace1d427..acafa30ef9d 100644 --- a/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp +++ b/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp @@ -3,12 +3,10 @@ #include "dumpvisitorsingle.h" #include <vespa/documentapi/messagebus/messages/multioperationmessage.h> #include <vespa/document/update/documentupdate.h> -#include <vespa/log/log.h> -#include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> #include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> -#include <memory> +#include <vespa/log/log.h> LOG_SETUP(".visitor.instance.dumpvisitorsingle"); namespace storage { diff --git a/storage/src/vespa/storage/visiting/dumpvisitorsingle.h b/storage/src/vespa/storage/visiting/dumpvisitorsingle.h index d7f7b63d851..b0fd4b2b9f0 100644 --- a/storage/src/vespa/storage/visiting/dumpvisitorsingle.h +++ b/storage/src/vespa/storage/visiting/dumpvisitorsingle.h @@ -9,7 +9,7 @@ */ #pragma once -#include <vespa/storage/visiting/visitor.h> +#include "visitor.h" namespace storage { @@ -19,8 +19,7 @@ public: const vdslib::Parameters& params); private: - void handleDocuments(const document::BucketId&, - std::vector<spi::DocEntry::UP>&, HitCounter&) override; + void handleDocuments(const document::BucketId&, std::vector<spi::DocEntry::UP>&, HitCounter&) override; }; struct DumpVisitorSingleFactory : public VisitorFactory { @@ -31,14 +30,10 @@ struct DumpVisitorSingleFactory : public VisitorFactory { }; Visitor* - makeVisitor(StorageComponent& c, - VisitorEnvironment&, const vdslib::Parameters& params) override - { + + makeVisitor(StorageComponent& c, VisitorEnvironment&, const vdslib::Parameters& params) override { return new DumpVisitorSingle(c, params); } }; } - - - diff --git a/storage/src/vespa/storage/visiting/messages.h b/storage/src/vespa/storage/visiting/messages.h index d646355895f..b79fae0575f 100644 --- a/storage/src/vespa/storage/visiting/messages.h +++ b/storage/src/vespa/storage/visiting/messages.h @@ -29,11 +29,9 @@ public: _config(config) {} std::unique_ptr<api::StorageReply> makeReply() override; - const vespa::config::content::core::StorVisitorConfig& getConfig() const { return _config; } - virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override - { + void print(std::ostream& out, bool verbose, const std::string& indent) const override { out << "PropagateVisitorConfig()"; if (verbose) { @@ -53,12 +51,8 @@ public: PropagateVisitorConfigReply(const PropagateVisitorConfig& cmd) : api::InternalReply(ID, cmd) - { - } - - - virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override - { + {} + void print(std::ostream& out, bool verbose, const std::string& indent) const override { out << "PropagateVisitorConfigReply()"; if (verbose) { @@ -76,4 +70,3 @@ PropagateVisitorConfig::makeReply() } } // storage - diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.h b/storage/src/vespa/storage/visiting/recoveryvisitor.h index dac6996028b..b9f54a5d0ea 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.h +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.h @@ -9,7 +9,7 @@ */ #pragma once -#include <vespa/storage/visiting/visitor.h> +#include "visitor.h" #include <vespa/storageapi/message/datagram.h> namespace documentapi { diff --git a/storage/src/vespa/storage/visiting/testvisitor.h b/storage/src/vespa/storage/visiting/testvisitor.h index c2740b11d56..db4e1d60939 100644 --- a/storage/src/vespa/storage/visiting/testvisitor.h +++ b/storage/src/vespa/storage/visiting/testvisitor.h @@ -8,7 +8,7 @@ */ #pragma once -#include <vespa/storage/visiting/visitor.h> +#include "visitor.h" namespace storage { @@ -30,10 +30,9 @@ private: } void completedVisiting(HitCounter& hitCounter) override; - void abortedVisiting() override; - // Send datagram with message back to client + // Send datagram with message back to client void report(const std::string& message); std::string _params; @@ -55,6 +54,3 @@ struct TestVisitorFactory : public VisitorFactory { }; } - - - diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index 3d5e9fc2c53..918d2d0e339 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -174,8 +174,7 @@ private: bool hasPendingIterators() const { return _pendingIterators > 0; } - void print(std::ostream& out, bool, const std::string& /*indent*/) const override - { + void print(std::ostream& out, bool, const std::string& ) const override { out << "BucketIterationState(" << _bucketId << ", pending GetIters: " << _pendingIterators diff --git a/storage/src/vespa/storage/visiting/visitormanager.h b/storage/src/vespa/storage/visiting/visitormanager.h index 9b750e3af4f..b966526ad7a 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.h +++ b/storage/src/vespa/storage/visiting/visitormanager.h @@ -19,19 +19,18 @@ #pragma once -#include <vespa/vespalib/util/document_runnable.h> +#include "commandqueue.h" +#include "visitor.h" +#include "visitormetrics.h" +#include "visitorthread.h" +#include "config-stor-visitor.h" +#include <vespa/storageframework/storageframework.h> +#include <vespa/storage/common/storagelink.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/internal.h> #include <vespa/storageapi/message/visitor.h> -#include <vespa/storage/common/storagelink.h> -#include <vespa/storage/visiting/commandqueue.h> -#include <vespa/storage/visiting/config-stor-visitor.h> -#include <vespa/storage/visiting/visitor.h> -#include <vespa/storage/visiting/visitormetrics.h> -#include <vespa/storage/visiting/visitorthread.h> -#include <vespa/storageframework/storageframework.h> -#include <vespa/storageframework/storageframework.h> #include <vespa/config/config.h> +#include <vespa/vespalib/util/document_runnable.h> namespace storage { namespace api { @@ -90,14 +89,11 @@ public: VisitorManager(const config::ConfigUri & configUri, StorageComponentRegister&, VisitorMessageSessionFactory&, const VisitorFactory::Map& external = VisitorFactory::Map()); - virtual ~VisitorManager(); - - virtual void onClose() override; - - virtual void print(std::ostream& out, bool verbose, const std::string& indent) const override; + ~VisitorManager(); + void onClose() override; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; uint32_t getActiveVisitorCount() const; - void setTimeBetweenTicks(uint32_t time); void setMaxConcurrentVisitors(uint32_t count) { // Used in unit testing @@ -126,7 +122,7 @@ public: private: void configure(std::unique_ptr<vespa::config::content::core::StorVisitorConfig>) override; - virtual void run(framework::ThreadHandle&) override; + void run(framework::ThreadHandle&) override; /** * Schedules a visitor for running. onCreateVisitor will typically call @@ -157,13 +153,12 @@ private: */ bool attemptScheduleQueuedVisitor(vespalib::MonitorGuard& visitorLock); - // VisitorMessageHandler implementation + // VisitorMessageHandler implementation void send(const std::shared_ptr<api::StorageCommand>& cmd, Visitor& visitor) override; void send(const std::shared_ptr<api::StorageReply>& reply) override; void closed(api::VisitorId id) override; - // Status::Reporter implementation - virtual void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; + void reportHtmlStatus(std::ostream&, const framework::HttpUrlPath&) const override; /** * The maximum amount of concurrent visitors for a priority is given @@ -179,4 +174,3 @@ private: }; } - diff --git a/storage/src/vespa/storage/visiting/visitorthread.h b/storage/src/vespa/storage/visiting/visitorthread.h index 2f0a125d6ce..5f9d8897d9f 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.h +++ b/storage/src/vespa/storage/visiting/visitorthread.h @@ -13,17 +13,17 @@ #pragma once -#include <deque> -#include <vespa/vespalib/util/document_runnable.h> -#include <vespa/metrics/metrictimer.h> +#include "visitor.h" +#include "visitormetrics.h" +#include "visitormessagesessionfactory.h" +#include <vespa/storageframework/storageframework.h> +#include <vespa/storage/persistence/messages.h> +#include <vespa/storage/common/storagecomponent.h> #include <vespa/storageapi/messageapi/messagehandler.h> +#include <vespa/metrics/metrictimer.h> +#include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/sync.h> -#include <vespa/storage/common/storagecomponent.h> -#include <vespa/storage/persistence/messages.h> -#include <vespa/storage/visiting/visitor.h> -#include <vespa/storage/visiting/visitormetrics.h> -#include <vespa/storage/visiting/visitormessagesessionfactory.h> -#include <vespa/storageframework/storageframework.h> +#include <deque> namespace storage { @@ -36,8 +36,7 @@ class VisitorThread : public framework::Runnable, typedef std::map<api::VisitorId, std::shared_ptr<Visitor> > VisitorMap; VisitorMap _visitors; - std::deque<std::pair<api::VisitorId, - framework::SecondTime> > _recentlyCompleted; + std::deque<std::pair<api::VisitorId, framework::SecondTime> > _recentlyCompleted; struct Event { enum Type { @@ -100,13 +99,9 @@ public: VisitorMessageHandler& sender); ~VisitorThread(); - void processMessage(api::VisitorId visitorId, - const std::shared_ptr<api::StorageMessage>& msg); - + void processMessage(api::VisitorId visitorId, const std::shared_ptr<api::StorageMessage>& msg); void shutdown(); - void setTimeBetweenTicks(uint32_t time) { _timeBetweenTicks = time; } - void handleMessageBusReply(std::unique_ptr<mbus::Reply> reply, Visitor& visitor); /** For unit tests needing to pause thread. */ @@ -117,7 +112,7 @@ public: } private: - virtual void run(framework::ThreadHandle&) override; + void run(framework::ThreadHandle&) override; /** * Attempt to fetch an event from the visitor thread's queue. If an event * was available, pop it from the queue and return it. If not, return @@ -127,12 +122,11 @@ private: Event popNextQueuedEventIfAvailable(); void tick(); void trimRecentlyCompletedList(framework::SecondTime currentTime); - void handleNonExistingVisitorCall(const Event& entry, - api::ReturnCode& code); + void handleNonExistingVisitorCall(const Event& entry, api::ReturnCode& code); std::shared_ptr<Visitor> createVisitor(const vespalib::stringref & libName, - const vdslib::Parameters& params, - vespalib::asciistream & error); + const vdslib::Parameters& params, + vespalib::asciistream & error); bool onCreateVisitor(const std::shared_ptr<api::CreateVisitorCommand>&) override; @@ -142,12 +136,9 @@ private: /** Deletes a visitor instance. */ void close(); - void getStatus(vespalib::asciistream & out, - const framework::HttpUrlPath& path) const; - + void getStatus(vespalib::asciistream & out, const framework::HttpUrlPath& path) const; void updateMetrics(const MetricLockGuard &) override; }; } // storage - |