diff options
Diffstat (limited to 'searchcore/src')
10 files changed, 110 insertions, 24 deletions
diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp index 0fced6e0bff..b35027eac2a 100644 --- a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp @@ -3,9 +3,13 @@ LOG_SETUP("attribute_usage_filter_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> +#include <vespa/searchcore/proton/attribute/i_attribute_usage_listener.h> using proton::AttributeUsageFilter; using proton::AttributeUsageStats; +using proton::IAttributeUsageListener; +using search::AddressSpaceUsage; +using vespalib::AddressSpace; namespace { @@ -38,33 +42,47 @@ public: } }; +class MyListener : public IAttributeUsageListener { +public: + AttributeUsageStats stats; + MyListener() : stats() {} + void notify_attribute_usage(const AttributeUsageStats &stats_in) override { + stats = stats_in; + } +}; + struct Fixture { - AttributeUsageFilter _filter; + AttributeUsageFilter filter; + const MyListener* listener; using State = AttributeUsageFilter::State; using Config = AttributeUsageFilter::Config; Fixture() - : _filter() + : filter(), + listener() { + auto my_listener = std::make_unique<MyListener>(); + listener = my_listener.get(); + filter.set_listener(std::move(my_listener)); } void testWrite(const vespalib::string &exp) { if (exp.empty()) { - EXPECT_TRUE(_filter.acceptWriteOperation()); - State state = _filter.getAcceptState(); + EXPECT_TRUE(filter.acceptWriteOperation()); + State state = filter.getAcceptState(); EXPECT_TRUE(state.acceptWriteOperation()); EXPECT_EQUAL(exp, state.message()); } else { - EXPECT_FALSE(_filter.acceptWriteOperation()); - State state = _filter.getAcceptState(); + EXPECT_FALSE(filter.acceptWriteOperation()); + State state = filter.getAcceptState(); EXPECT_FALSE(state.acceptWriteOperation()); EXPECT_EQUAL(exp, state.message()); } } void setAttributeStats(const AttributeUsageStats &stats) { - _filter.setAttributeStats(stats); + filter.setAttributeStats(stats); } }; @@ -78,7 +96,7 @@ TEST_F("Check that default filter allows write", Fixture) TEST_F("Check that enum store limit can be reached", Fixture) { - f._filter.setConfig(Fixture::Config(0.8, 1.0)); + f.filter.setConfig(Fixture::Config(0.8, 1.0)); MyAttributeStats stats; stats.triggerEnumStoreLimit(); f.setAttributeStats(stats); @@ -95,7 +113,7 @@ TEST_F("Check that enum store limit can be reached", Fixture) TEST_F("Check that multivalue limit can be reached", Fixture) { - f._filter.setConfig(Fixture::Config(1.0, 0.8)); + f.filter.setConfig(Fixture::Config(1.0, 0.8)); MyAttributeStats stats; stats.triggerMultiValueLimit(); f.setAttributeStats(stats); @@ -113,7 +131,7 @@ TEST_F("Check that multivalue limit can be reached", Fixture) TEST_F("Check that both enumstore limit and multivalue limit can be reached", Fixture) { - f._filter.setConfig(Fixture::Config(0.8, 0.8)); + f.filter.setConfig(Fixture::Config(0.8, 0.8)); MyAttributeStats stats; stats.triggerEnumStoreLimit(); stats.triggerMultiValueLimit(); @@ -139,4 +157,13 @@ TEST_F("Check that both enumstore limit and multivalue limit can be reached", "attributeName: \"multiValueName\", subdb: \"ready\"}"); } +TEST_F("listener is updated when attribute stats change", Fixture) +{ + AttributeUsageStats stats; + AddressSpaceUsage usage(AddressSpace(12, 10, 15), AddressSpace(22, 20, 25)); + stats.merge(usage, "my_attr", "my_subdb"); + f.setAttributeStats(stats); + EXPECT_EQUAL(stats, f.listener->stats); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.cpp b/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.cpp index 3390447e26a..1acfa64285c 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "address_space_usage_stats.h" +#include <ostream> namespace proton { @@ -25,4 +26,13 @@ AddressSpaceUsageStats::merge(const vespalib::AddressSpace &usage, } } +std::ostream& +operator<<(std::ostream& out, const AddressSpaceUsageStats& rhs) +{ + out << "{usage=" << rhs.getUsage() << + ", attribute_name=" << rhs.getAttributeName() << + ", subdb_name=" << rhs.getSubDbName() << "}"; + return out; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.h b/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.h index 400c7cde03f..9ed68693ec1 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.h @@ -7,8 +7,8 @@ namespace proton { -/* - * class representing usage of a single address space (enum store or +/** + * Class representing usage of a single address space (enum store or * multi value) and the most largest attribute in that respect, relative * to the limit. */ @@ -28,6 +28,14 @@ public: const vespalib::AddressSpace &getUsage() const { return _usage; } const vespalib::string &getAttributeName() const { return _attributeName; } const vespalib::string &getSubDbName() const { return _subDbName; } + + bool operator==(const AddressSpaceUsageStats& rhs) const { + return (_usage == rhs._usage) && + (_attributeName == rhs._attributeName) && + (_subDbName == rhs._subDbName); + } }; +std::ostream& operator<<(std::ostream &out, const AddressSpaceUsageStats& rhs); + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp index 2852497cf3f..e19e7976227 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_usage_filter.h" +#include "i_attribute_usage_listener.h" #include <sstream> namespace proton { @@ -85,7 +86,8 @@ AttributeUsageFilter::AttributeUsageFilter() _attributeStats(), _config(), _state(), - _acceptWrite(true) + _acceptWrite(true), + _listener() { } @@ -97,6 +99,9 @@ AttributeUsageFilter::setAttributeStats(AttributeUsageStats attributeStats_in) Guard guard(_lock); _attributeStats = attributeStats_in; recalcState(guard); + if (_listener) { + _listener->notify_attribute_usage(_attributeStats); + } } AttributeUsageStats @@ -114,6 +119,13 @@ AttributeUsageFilter::setConfig(Config config_in) recalcState(guard); } +void +AttributeUsageFilter::set_listener(std::unique_ptr<IAttributeUsageListener> listener) +{ + Guard guard(_lock); + _listener = std::move(listener); +} + double AttributeUsageFilter::getEnumStoreUsedRatio() const { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h index ecc95d4ee02..cb9687e31a1 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h @@ -10,7 +10,9 @@ namespace proton { -/* +class IAttributeUsageListener; + +/** * Class to filter write operations based on sampled information about * attribute resource usage (e.g. enum store and multivalue mapping). * If resource limit is reached then further writes are denied in @@ -29,6 +31,7 @@ private: Config _config; State _state; std::atomic<bool> _acceptWrite; + std::unique_ptr<IAttributeUsageListener> _listener; void recalcState(const Guard &guard); // called with _lock held public: @@ -37,6 +40,7 @@ public: void setAttributeStats(AttributeUsageStats attributeStats_in); AttributeUsageStats getAttributeUsageStats() const; void setConfig(Config config); + void set_listener(std::unique_ptr<IAttributeUsageListener> listener); double getEnumStoreUsedRatio() const; double getMultiValueUsedRatio() const; bool acceptWriteOperation() const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp index 4d59c9081e0..f3da5486d3e 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_usage_stats.h" +#include <iostream> namespace proton { @@ -19,5 +20,12 @@ AttributeUsageStats::merge(const search::AddressSpaceUsage &usage, _multiValueUsage.merge(usage.multiValueUsage(), attributeName, subDbName); } +std::ostream& +operator<<(std::ostream& out, const AttributeUsageStats& rhs) +{ + out << "{enum_store=" << rhs.enumStoreUsage() << + ", multi_value=" << rhs.multiValueUsage() << "}"; + return out; +} } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h index f1805809e80..1eb6a9cc6be 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h @@ -7,7 +7,7 @@ namespace proton { -/* +/** * Class representing aggregated attribute usage, with info about * the most bloated attributes with regards to enum store and * multivalue mapping. @@ -23,10 +23,15 @@ public: const vespalib::string &attributeName, const vespalib::string &subDbName); - const AddressSpaceUsageStats & - enumStoreUsage() const { return _enumStoreUsage; } - const AddressSpaceUsageStats & - multiValueUsage() const { return _multiValueUsage; } + const AddressSpaceUsageStats& enumStoreUsage() const { return _enumStoreUsage; } + const AddressSpaceUsageStats& multiValueUsage() const { return _multiValueUsage; } + + bool operator==(const AttributeUsageStats& rhs) const { + return (_enumStoreUsage == rhs._enumStoreUsage) && + (_multiValueUsage == rhs._multiValueUsage); + } }; +std::ostream& operator<<(std::ostream& out, const AttributeUsageStats& rhs); + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index cd517fe7c60..60878c2b314 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -6,16 +6,17 @@ #include "document_subdb_collection_explorer.h" #include "documentdb.h" #include "documentdbconfigscout.h" +#include "feedhandler.h" #include "idocumentdbowner.h" #include "idocumentsubdb.h" #include "lid_space_compaction_handler.h" #include "maintenance_jobs_injector.h" #include "reconfig_params.h" -#include "feedhandler.h" -#include <vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/metrics/updatehook.h> #include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> +#include <vespa/searchcore/proton/attribute/i_attribute_usage_listener.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchcore/proton/common/statusreport.h> @@ -26,6 +27,7 @@ #include <vespa/searchcore/proton/initializer/task_runner.h> #include <vespa/searchcore/proton/metrics/executor_threading_service_stats.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> +#include <vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.h> #include <vespa/searchcore/proton/reference/document_db_reference_resolver.h> #include <vespa/searchcore/proton/reference/i_document_db_reference_registry.h> #include <vespa/searchlib/attribute/attributefactory.h> @@ -34,7 +36,6 @@ #include <vespa/searchlib/engine/searchreply.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/metrics/updatehook.h> #include <vespa/log/log.h> #include <vespa/searchcorespi/index/warmupconfig.h> @@ -1113,4 +1114,10 @@ DocumentDB::transient_memory_usage_provider() return _transient_memory_usage_provider; } +void +DocumentDB::set_attribute_usage_listener(std::unique_ptr<IAttributeUsageListener> listener) +{ + _writeFilter.set_listener(std::move(listener)); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 9c2facdddb0..b5d975884a3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -416,6 +416,8 @@ public: IDiskMemUsageListener *diskMemUsageListener() { return &_dmUsageForwarder; } std::shared_ptr<const ITransientMemoryUsageProvider> transient_memory_usage_provider(); ExecutorThreadingService & getWriteService() { return _writeService; } + + void set_attribute_usage_listener(std::unique_ptr<IAttributeUsageListener> listener); }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 6706581f52c..5aa5d88eda9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -18,14 +18,16 @@ #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/metrics/updatehook.h> +#include <vespa/searchcore/proton/attribute/i_attribute_usage_listener.h> #include <vespa/searchcore/proton/flushengine/flush_engine_explorer.h> #include <vespa/searchcore/proton/flushengine/flushengine.h> #include <vespa/searchcore/proton/flushengine/tls_stats_factory.h> #include <vespa/searchcore/proton/matchengine/matchengine.h> +#include <vespa/searchcore/proton/persistenceengine/persistenceengine.h> #include <vespa/searchcore/proton/reference/document_db_reference_registry.h> #include <vespa/searchcore/proton/summaryengine/docsum_by_slime.h> #include <vespa/searchcore/proton/summaryengine/summaryengine.h> -#include <vespa/searchcore/proton/persistenceengine/persistenceengine.h> #include <vespa/searchlib/common/packets.h> #include <vespa/searchlib/transactionlog/trans_log_server_explorer.h> #include <vespa/searchlib/transactionlog/translogserverapp.h> @@ -36,7 +38,6 @@ #include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/random.h> -#include <vespa/metrics/updatehook.h> #include <vespa/searchlib/aggregation/forcelink.hpp> #include <vespa/searchlib/expression/forcelink.hpp> @@ -630,6 +631,8 @@ Proton::addDocumentDB(const document::DocumentType &docType, } // TODO: Fix race with new cluster state setting. _persistenceEngine->putHandler(persistenceWGuard, bucketSpace, docTypeName, persistenceHandler); + ret->set_attribute_usage_listener( + _persistenceEngine->get_resource_usage_tracker().make_attribute_usage_listener(docTypeName.getName())); } auto searchHandler = std::make_shared<SearchHandlerProxy>(ret); _summaryEngine->putSearchHandler(docTypeName, searchHandler); |