diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-11-27 05:49:51 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2020-11-27 11:56:10 +0000 |
commit | 17d0bc7e91a0c055f57b002b6f501fea36235f99 (patch) | |
tree | 220ed4d472b0ceb4e928a955c007328425d99233 /storage | |
parent | 39d00826c3e7e30d59e6ca6d88eb3f36fb8358fd (diff) |
GC LoadType and PriorityMapper.
Diffstat (limited to 'storage')
15 files changed, 4 insertions, 131 deletions
diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index f816816a051..3f7e1036c08 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -134,15 +134,13 @@ void MetricsTest::createFakeLoad() { FileStorDiskMetrics& disk(*_filestorMetrics->disk); disk.queueSize.addValue(4 * n); - //disk.averageQueueWaitingTime[documentapi::LoadType::DEFAULT].addValue(10 * n); + disk.averageQueueWaitingTime.addValue(10 * n); disk.pendingMerges.addValue(4 * n); for (uint32_t j=0; j<disk.threads.size(); ++j) { FileStorThreadMetrics& thread(*disk.threads[j]); thread.operations.inc(120 * n); thread.failedOperations.inc(2 * n); - using documentapi::LoadType; - thread.put.count.inc(10 * n); thread.put.latency.addValue(5 * n); thread.get.count.inc(12 * n); @@ -239,7 +237,6 @@ TEST_F(MetricsTest, snapshot_presenting) { LOG(debug, "Adding to get metric"); - using documentapi::LoadType; thread0.get.count.inc(1); LOG(debug, "Waiting for 5 minute snapshot to be taken"); diff --git a/storage/src/tests/common/teststorageapp.cpp b/storage/src/tests/common/teststorageapp.cpp index b6acff804a8..81ef5a2e800 100644 --- a/storage/src/tests/common/teststorageapp.cpp +++ b/storage/src/tests/common/teststorageapp.cpp @@ -54,9 +54,6 @@ TestStorageApp::TestStorageApp(StorageComponentRegisterImpl::UP compReg, if (index == 0xffff) index = serverConfig->nodeIndex; redundancy = config::ConfigGetter<vespa::config::content::StorDistributionConfig>::getConfig(uri.getConfigId(), uri.getContext())->redundancy; nodeCount = config::ConfigGetter<vespa::config::content::FleetcontrollerConfig>::getConfig(uri.getConfigId(), uri.getContext())->totalStorageCount; - _compReg.setPriorityConfig( - *config::ConfigGetter<StorageComponent::PriorityConfig> - ::getConfig(uri.getConfigId(), uri.getContext())); } else { if (index == 0xffff) index = 0; } diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index b28f092904e..121195ba033 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -221,7 +221,6 @@ TEST_F(StateReporterTest, report_metrics) { LOG(debug, "Adding to get metric"); - using documentapi::LoadType; thread0.get.count.inc(1); LOG(debug, "Waiting for 5 minute snapshot to be taken"); diff --git a/storage/src/vespa/storage/common/storagecomponent.cpp b/storage/src/vespa/storage/common/storagecomponent.cpp index 8ad4e457b58..7e0543a0893 100644 --- a/storage/src/vespa/storage/common/storagecomponent.cpp +++ b/storage/src/vespa/storage/common/storagecomponent.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "storagecomponent.h" -#include <vespa/storage/storageserver/prioritymapper.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vdslib/distribution/distribution.h> @@ -40,13 +39,6 @@ StorageComponent::setDocumentTypeRepo(std::shared_ptr<const document::DocumentTy } void -StorageComponent::setPriorityConfig(const PriorityConfig& c) -{ - // Priority mapper is already thread safe. - _priorityMapper->setConfig(c); -} - -void StorageComponent::setBucketIdFactory(const document::BucketIdFactory& factory) { // Assumed to not be set dynamically. @@ -79,7 +71,6 @@ StorageComponent::StorageComponent(StorageComponentRegister& compReg, _nodeType(nullptr), _index(0), _repos(), - _priorityMapper(new PriorityMapper), _bucketIdFactory(), _distribution(), _nodeStateUpdater(nullptr), @@ -110,12 +101,6 @@ StorageComponent::getIdentity() const return name.str(); } -uint8_t -StorageComponent::getPriority(const documentapi::LoadType& lt) const -{ - return _priorityMapper->getPriority(lt); -} - std::shared_ptr<StorageComponent::Repos> StorageComponent::getTypeRepo() const { diff --git a/storage/src/vespa/storage/common/storagecomponent.h b/storage/src/vespa/storage/common/storagecomponent.h index 9b08bb7b242..8910c6da587 100644 --- a/storage/src/vespa/storage/common/storagecomponent.h +++ b/storage/src/vespa/storage/common/storagecomponent.h @@ -37,23 +37,16 @@ #include <vespa/vdslib/state/node.h> #include <mutex> -namespace vespa::config::content::core::internal { - class InternalStorPrioritymappingType; -} namespace document { class DocumentTypeRepo; class FieldSetRepo; } -namespace documentapi { - class LoadType; -} namespace storage { namespace lib { class Distribution; } struct NodeStateUpdater; -class PriorityMapper; struct StorageComponentRegister; class StorageComponent : public framework::Component { @@ -65,7 +58,6 @@ public: const std::shared_ptr<const document::FieldSetRepo> fieldSetRepo; }; using UP = std::unique_ptr<StorageComponent>; - using PriorityConfig = vespa::config::content::core::internal::InternalStorPrioritymappingType; using DistributionSP = std::shared_ptr<lib::Distribution>; /** @@ -81,7 +73,6 @@ public: */ void setNodeStateUpdater(NodeStateUpdater& updater); void setDocumentTypeRepo(std::shared_ptr<const document::DocumentTypeRepo>); - void setPriorityConfig(const PriorityConfig&); void setBucketIdFactory(const document::BucketIdFactory&); void setDistribution(DistributionSP); @@ -97,7 +88,6 @@ public: std::shared_ptr<Repos> getTypeRepo() const; const document::BucketIdFactory& getBucketIdFactory() const { return _bucketIdFactory; } - uint8_t getPriority(const documentapi::LoadType&) const; DistributionSP getDistribution() const; NodeStateUpdater& getStateUpdater() const; uint64_t getGeneration() const { return _generation.load(std::memory_order_relaxed); } @@ -107,7 +97,6 @@ private: uint16_t _index; std::shared_ptr<Repos> _repos; // TODO: move _distribution in to _repos so lock will only taken once and only copying one shared_ptr. - std::unique_ptr<PriorityMapper> _priorityMapper; document::BucketIdFactory _bucketIdFactory; DistributionSP _distribution; NodeStateUpdater* _nodeStateUpdater; diff --git a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp index 73b78ce8626..3ce02a73324 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.cpp @@ -16,7 +16,6 @@ StorageComponentRegisterImpl::StorageComponentRegisterImpl() _nodeType(nullptr), _index(0xffff), _docTypeRepo(), - _priorityConfig(), _bucketIdFactory(), _distribution(), _nodeStateUpdater(nullptr), @@ -37,7 +36,6 @@ StorageComponentRegisterImpl::registerStorageComponent(StorageComponent& smc) smc.setNodeStateUpdater(*_nodeStateUpdater); } smc.setDocumentTypeRepo(_docTypeRepo); - smc.setPriorityConfig(_priorityConfig); smc.setBucketIdFactory(_bucketIdFactory); smc.setDistribution(_distribution); } @@ -83,16 +81,6 @@ StorageComponentRegisterImpl::setDocumentTypeRepo(std::shared_ptr<const document } void -StorageComponentRegisterImpl::setPriorityConfig(const PriorityConfig& config) -{ - std::lock_guard lock(_componentLock); - _priorityConfig = config; - for (auto& component : _components) { - component->setPriorityConfig(config); - } -} - -void StorageComponentRegisterImpl::setBucketIdFactory(const document::BucketIdFactory& factory) { std::lock_guard lock(_componentLock); diff --git a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h index ce9a739689b..be3c89fe176 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h @@ -10,7 +10,6 @@ #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/storage/common/storagecomponent.h> #include <vespa/config-bucketspaces.h> -#include <vespa/storage/config/config-stor-prioritymapping.h> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> #include <vespa/vdslib/distribution/distribution.h> @@ -20,7 +19,6 @@ class StorageComponentRegisterImpl : public virtual StorageComponentRegister, public virtual framework::defaultimplementation::ComponentRegisterImpl { - using PriorityConfig = StorageComponent::PriorityConfig; using BucketspacesConfig = vespa::config::content::core::internal::InternalBucketspacesType; std::mutex _componentLock; @@ -29,7 +27,6 @@ class StorageComponentRegisterImpl const lib::NodeType* _nodeType; uint16_t _index; std::shared_ptr<const document::DocumentTypeRepo> _docTypeRepo; - PriorityConfig _priorityConfig; document::BucketIdFactory _bucketIdFactory; lib::Distribution::SP _distribution; NodeStateUpdater* _nodeStateUpdater; @@ -54,7 +51,6 @@ public: void setNodeInfo(vespalib::stringref clusterName, const lib::NodeType& nodeType, uint16_t index); virtual void setNodeStateUpdater(NodeStateUpdater& updater); virtual void setDocumentTypeRepo(std::shared_ptr<const document::DocumentTypeRepo>); - virtual void setPriorityConfig(const PriorityConfig&); virtual void setBucketIdFactory(const document::BucketIdFactory&); virtual void setDistribution(lib::Distribution::SP); virtual void setBucketSpacesConfig(const BucketspacesConfig&); diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index b1c8c1e43a4..70f47d8108b 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -185,7 +185,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentReply& fromReply, break; } - if (toMsg.get()) { + if (toMsg) { if (fromReply.hasErrors()) { toMsg->setResult(api::ReturnCode((api::ReturnCode::Result) fromReply.getError(0).getCode(), fromReply.getError(0).getMessage())); diff --git a/storage/src/vespa/storage/storageserver/priorityconverter.cpp b/storage/src/vespa/storage/storageserver/priorityconverter.cpp index 41bea1b44df..d4dbe943664 100644 --- a/storage/src/vespa/storage/storageserver/priorityconverter.cpp +++ b/storage/src/vespa/storage/storageserver/priorityconverter.cpp @@ -13,9 +13,7 @@ PriorityConverter::PriorityConverter(const config::ConfigUri & configUri) _configFetcher.start(); } -PriorityConverter::~PriorityConverter() -{ -} +PriorityConverter::~PriorityConverter() = default; uint8_t PriorityConverter::toStoragePriority(documentapi::Priority::Value documentApiPriority) const diff --git a/storage/src/vespa/storage/storageserver/prioritymapper.h b/storage/src/vespa/storage/storageserver/prioritymapper.h deleted file mode 100644 index f5aebcfb786..00000000000 --- a/storage/src/vespa/storage/storageserver/prioritymapper.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/documentapi/loadtypes/loadtype.h> -#include <vespa/storage/config/config-stor-prioritymapping.h> - -namespace storage { - -class PriorityMapper -{ - std::vector<uint8_t> _priorities; - -public: - typedef vespa::config::content::core::internal::InternalStorPrioritymappingType Config; - - PriorityMapper() : _priorities(16, 120) {} - - void setConfig(const Config c) { - _priorities[documentapi::Priority::PRI_HIGHEST] = c.highest; - _priorities[documentapi::Priority::PRI_VERY_HIGH] = c.veryHigh; - _priorities[documentapi::Priority::PRI_HIGH_1] = c.high1; - _priorities[documentapi::Priority::PRI_HIGH_2] = c.high2; - _priorities[documentapi::Priority::PRI_HIGH_3] = c.high3; - _priorities[documentapi::Priority::PRI_NORMAL_1] = c.normal1; - _priorities[documentapi::Priority::PRI_NORMAL_2] = c.normal2; - _priorities[documentapi::Priority::PRI_NORMAL_3] = c.normal3; - _priorities[documentapi::Priority::PRI_NORMAL_4] = c.normal4; - _priorities[documentapi::Priority::PRI_NORMAL_5] = c.normal5; - _priorities[documentapi::Priority::PRI_NORMAL_6] = c.normal6; - _priorities[documentapi::Priority::PRI_LOW_1] = c.low1; - _priorities[documentapi::Priority::PRI_LOW_2] = c.low2; - _priorities[documentapi::Priority::PRI_LOW_3] = c.low3; - _priorities[documentapi::Priority::PRI_VERY_LOW] = c.veryLow; - _priorities[documentapi::Priority::PRI_LOWEST] = c.lowest; - } - - uint8_t getPriority(const documentapi::LoadType& lt) const { - return _priorities[lt.getPriority()]; - } -}; - -} // storage diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index c6368ef0ee7..120965cc4fe 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -88,13 +88,11 @@ StorageNode::StorageNode( _serverConfig(), _clusterConfig(), _distributionConfig(), - _priorityConfig(), _doctypesConfig(), _bucketSpacesConfig(), _newServerConfig(), _newClusterConfig(), _newDistributionConfig(), - _newPriorityConfig(), _newDoctypesConfig(), _newBucketSpacesConfig(), _component(), @@ -111,7 +109,6 @@ StorageNode::subscribeToConfigs() _configFetcher->subscribe<StorDistributionConfig>(_configUri.getConfigId(), this); _configFetcher->subscribe<UpgradingConfig>(_configUri.getConfigId(), this); _configFetcher->subscribe<StorServerConfig>(_configUri.getConfigId(), this); - _configFetcher->subscribe<StorPrioritymappingConfig>(_configUri.getConfigId(), this); _configFetcher->subscribe<BucketspacesConfig>(_configUri.getConfigId(), this); _configFetcher->start(); @@ -120,7 +117,6 @@ StorageNode::subscribeToConfigs() _serverConfig = std::move(_newServerConfig); _clusterConfig = std::move(_newClusterConfig); _distributionConfig = std::move(_newDistributionConfig); - _priorityConfig = std::move(_newPriorityConfig); _bucketSpacesConfig = std::move(_newBucketSpacesConfig); } @@ -146,7 +142,6 @@ StorageNode::initialize() _context.getComponentRegister().setNodeInfo(_serverConfig->clusterName, getNodeType(), _serverConfig->nodeIndex); _context.getComponentRegister().setBucketIdFactory(document::BucketIdFactory()); _context.getComponentRegister().setDistribution(make_shared<lib::Distribution>(*_distributionConfig)); - _context.getComponentRegister().setPriorityConfig(*_priorityConfig); _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig); _metrics = std::make_shared<StorageMetricSet>(); @@ -333,10 +328,7 @@ StorageNode::handleLiveConfigUpdate(const InitialGuard & initGuard) } _newClusterConfig.reset(); } - if (_newPriorityConfig) { - _priorityConfig = std::move(_newPriorityConfig); - _context.getComponentRegister().setPriorityConfig(*_priorityConfig); - } + if (_newBucketSpacesConfig) { _bucketSpacesConfig = std::move(_newBucketSpacesConfig); _context.getComponentRegister().setBucketSpacesConfig(*_bucketSpacesConfig); @@ -488,19 +480,6 @@ void StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) { handleLiveConfigUpdate(concurrent_config_guard); } } - -void StorageNode::configure(std::unique_ptr<StorPrioritymappingConfig> config) { - log_config_received(*config); - { - std::lock_guard configLockGuard(_configLock); - _newPriorityConfig = std::move(config); - } - if (_priorityConfig) { - InitialGuard concurrent_config_guard(_initial_config_mutex); - handleLiveConfigUpdate(concurrent_config_guard); - } -} - void StorageNode::configure(std::unique_ptr<document::DocumenttypesConfig> config, bool hasChanged, int64_t generation) diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index 9450cdce5ff..259f48f3492 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -20,7 +20,6 @@ #include <vespa/document/config/config-documenttypes.h> #include <vespa/storage/common/doneinitializehandler.h> #include <vespa/config-bucketspaces.h> -#include <vespa/storage/config/config-stor-prioritymapping.h> #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/storageutil/resumeguard.h> #include <vespa/storageframework/defaultimplementation/component/componentregisterimpl.h> @@ -53,7 +52,6 @@ namespace lib { class NodeType; } class StorageNode : private config::IFetcherCallback<vespa::config::content::core::StorServerConfig>, private config::IFetcherCallback<vespa::config::content::StorDistributionConfig>, private config::IFetcherCallback<vespa::config::content::UpgradingConfig>, - private config::IFetcherCallback<vespa::config::content::core::StorPrioritymappingConfig>, private config::IFetcherCallback<vespa::config::content::core::BucketspacesConfig>, private framework::MetricUpdateHook, private DoneInitializeHandler, @@ -100,7 +98,6 @@ protected: using StorServerConfig = vespa::config::content::core::StorServerConfig; using UpgradingConfig = vespa::config::content::UpgradingConfig; using StorDistributionConfig = vespa::config::content::StorDistributionConfig; - using StorPrioritymappingConfig = vespa::config::content::core::StorPrioritymappingConfig; using BucketspacesConfig = vespa::config::content::core::BucketspacesConfig; private: bool _singleThreadedDebugMode; @@ -135,7 +132,6 @@ private: void configure(std::unique_ptr<StorServerConfig> config) override; void configure(std::unique_ptr<UpgradingConfig> config) override; void configure(std::unique_ptr<StorDistributionConfig> config) override; - void configure(std::unique_ptr<StorPrioritymappingConfig>) override; virtual void configure(std::unique_ptr<document::DocumenttypesConfig> config, bool hasChanged, int64_t generation); void configure(std::unique_ptr<BucketspacesConfig>) override; @@ -151,14 +147,12 @@ protected: std::unique_ptr<StorServerConfig> _serverConfig; std::unique_ptr<UpgradingConfig> _clusterConfig; std::unique_ptr<StorDistributionConfig> _distributionConfig; - std::unique_ptr<StorPrioritymappingConfig> _priorityConfig; std::unique_ptr<document::DocumenttypesConfig> _doctypesConfig; std::unique_ptr<BucketspacesConfig> _bucketSpacesConfig; // New configs gotten that has yet to have been handled std::unique_ptr<StorServerConfig> _newServerConfig; std::unique_ptr<UpgradingConfig> _newClusterConfig; std::unique_ptr<StorDistributionConfig> _newDistributionConfig; - std::unique_ptr<StorPrioritymappingConfig> _newPriorityConfig; std::unique_ptr<document::DocumenttypesConfig> _newDoctypesConfig; std::unique_ptr<BucketspacesConfig> _newBucketSpacesConfig; std::unique_ptr<StorageComponent> _component; diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index a8582fcf403..3618073da91 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -6,7 +6,6 @@ #include <vespa/storageapi/message/datagram.h> #include <vespa/storage/persistence/messages.h> #include <vespa/documentapi/messagebus/messages/visitor.h> -#include <vespa/documentapi/loadtypes/loadtype.h> #include <vespa/document/select/node.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/vespalib/stllike/hash_map.hpp> diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index 8eb02e6ccfc..6f2b33629e6 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -382,10 +382,6 @@ public: void setOwnNodeIndex(uint16_t nodeIndex) { _ownNodeIndex = nodeIndex; } void setBucketSpace(document::BucketSpace bucketSpace) { _bucketSpace = bucketSpace; } - const documentapi::LoadType& getLoadType() const { - return _initiatingCmd->getLoadType(); - } - /** Override this to know which buckets are currently being visited. */ virtual void startingVisitor(const std::vector<document::BucketId>&) {} diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index 52943fde419..1c89acf74b0 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -2,7 +2,6 @@ #include "visitorthread.h" #include "messages.h" -#include <vespa/documentapi/loadtypes/loadtype.h> #include <vespa/document/select/bodyfielddetector.h> #include <vespa/document/select/parser.h> #include <vespa/messagebus/rpcmessagebus.h> |