diff options
author | Geir Storli <geirst@verizonmedia.com> | 2021-07-15 14:30:14 +0000 |
---|---|---|
committer | Geir Storli <geirst@verizonmedia.com> | 2021-07-15 14:30:14 +0000 |
commit | 63ff8a6794678fe9a50bfa6f5c3a3eebe52a9596 (patch) | |
tree | 6018dbf7b1d2e842fd44173f412ce4dce993335c /searchlib | |
parent | f505d2be5cda0c843063ab3624d6edf422070939 (diff) |
Generalize how address space usage is reported from attribute vectors.
This prepares for reporting address space usage for more attribute vector components,
not only enum store and multi-value mapping.
Diffstat (limited to 'searchlib')
13 files changed, 134 insertions, 68 deletions
diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 79e120d0683..0a4a562c5fb 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -6,22 +6,23 @@ #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/update/mapvalueupdate.h> #include <vespa/fastlib/io/bufferedfile.h> +#include <vespa/searchlib/attribute/address_space_components.h> #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/attributememorysavetarget.h> #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/attribute/attrvector.h> +#include <vespa/searchlib/attribute/multienumattribute.hpp> #include <vespa/searchlib/attribute/multinumericattribute.h> #include <vespa/searchlib/attribute/multistringattribute.h> +#include <vespa/searchlib/attribute/multivalueattribute.hpp> #include <vespa/searchlib/attribute/predicate_attribute.h> #include <vespa/searchlib/attribute/singlenumericpostattribute.h> #include <vespa/searchlib/attribute/singlestringattribute.h> -#include <vespa/searchlib/attribute/multivalueattribute.hpp> -#include <vespa/searchlib/attribute/multienumattribute.hpp> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/util/randomgenerator.h> #include <vespa/searchlib/test/weighted_type_test_utils.h> +#include <vespa/searchlib/util/randomgenerator.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/testkit/testapp.h> #include <cmath> @@ -2105,30 +2106,30 @@ AttributeTest::requireThatAddressSpaceUsageIsReported(const Config &config, bool AddressSpaceUsage after = attrPtr->getAddressSpaceUsage(); if (attrPtr->hasEnum()) { LOG(info, "requireThatAddressSpaceUsageIsReported(%s): Has enum", attrName.c_str()); - EXPECT_EQUAL(before.enumStoreUsage().used(), 1u); - EXPECT_EQUAL(before.enumStoreUsage().dead(), 1u); - EXPECT_GREATER(after.enumStoreUsage().used(), before.enumStoreUsage().used()); - EXPECT_GREATER_EQUAL(after.enumStoreUsage().limit(), before.enumStoreUsage().limit()); - EXPECT_GREATER(after.enumStoreUsage().limit(), 4200000000u); + EXPECT_EQUAL(before.enum_store_usage().used(), 1u); + EXPECT_EQUAL(before.enum_store_usage().dead(), 1u); + EXPECT_GREATER(after.enum_store_usage().used(), before.enum_store_usage().used()); + EXPECT_GREATER_EQUAL(after.enum_store_usage().limit(), before.enum_store_usage().limit()); + EXPECT_GREATER(after.enum_store_usage().limit(), 4200000000u); } else { LOG(info, "requireThatAddressSpaceUsageIsReported(%s): NOT enum", attrName.c_str()); - EXPECT_EQUAL(before.enumStoreUsage().used(), 0u); - EXPECT_EQUAL(before.enumStoreUsage().dead(), 0u); - EXPECT_EQUAL(after.enumStoreUsage(), before.enumStoreUsage()); - EXPECT_EQUAL(AddressSpaceUsage::defaultEnumStoreUsage(), after.enumStoreUsage()); + EXPECT_EQUAL(before.enum_store_usage().used(), 0u); + EXPECT_EQUAL(before.enum_store_usage().dead(), 0u); + EXPECT_EQUAL(after.enum_store_usage(), before.enum_store_usage()); + EXPECT_EQUAL(AddressSpaceComponents::default_enum_store_usage(), after.enum_store_usage()); } if (attrPtr->hasMultiValue()) { LOG(info, "requireThatAddressSpaceUsageIsReported(%s): Has multi-value", attrName.c_str()); - EXPECT_EQUAL(before.multiValueUsage().used(), 1u); - EXPECT_EQUAL(before.multiValueUsage().dead(), 1u); - EXPECT_GREATER_EQUAL(after.multiValueUsage().used(), before.multiValueUsage().used()); - EXPECT_GREATER(after.multiValueUsage().limit(), before.multiValueUsage().limit()); - EXPECT_GREATER((1ull << 32), after.multiValueUsage().limit()); + EXPECT_EQUAL(before.multi_value_usage().used(), 1u); + EXPECT_EQUAL(before.multi_value_usage().dead(), 1u); + EXPECT_GREATER_EQUAL(after.multi_value_usage().used(), before.multi_value_usage().used()); + EXPECT_GREATER(after.multi_value_usage().limit(), before.multi_value_usage().limit()); + EXPECT_GREATER((1ull << 32), after.multi_value_usage().limit()); } else { LOG(info, "requireThatAddressSpaceUsageIsReported(%s): NOT multi-value", attrName.c_str()); - EXPECT_EQUAL(before.multiValueUsage().used(), 0u); - EXPECT_EQUAL(after.multiValueUsage(), before.multiValueUsage()); - EXPECT_EQUAL(AddressSpaceUsage::defaultMultiValueUsage(), after.multiValueUsage()); + EXPECT_EQUAL(before.multi_value_usage().used(), 0u); + EXPECT_EQUAL(after.multi_value_usage(), before.multi_value_usage()); + EXPECT_EQUAL(AddressSpaceComponents::default_multi_value_usage(), after.multi_value_usage()); } } diff --git a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp index a029373821a..be3013c051e 100644 --- a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp +++ b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp @@ -154,7 +154,7 @@ public: return status; } const Config &getConfig() const { return _v->getConfig(); } - AddressSpace getMultiValueAddressSpaceUsage() const {return _v->getAddressSpaceUsage().multiValueUsage(); } + AddressSpace getMultiValueAddressSpaceUsage() const {return _v->getAddressSpaceUsage().multi_value_usage(); } AddressSpace getMultiValueAddressSpaceUsage(const vespalib::string &prefix) { AddressSpace usage(getMultiValueAddressSpaceUsage()); LOG(info, "address space usage %s: used=%zu, dead=%zu, limit=%zu, usage=%12.8f", diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index bf56b476cc9..e7267e4f6a4 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_attribute OBJECT SOURCES + address_space_components.cpp address_space_usage.cpp attribute.cpp attribute_blueprint_factory.cpp diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp b/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp new file mode 100644 index 00000000000..dd050fe0512 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp @@ -0,0 +1,21 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "address_space_components.h" +#include "i_enum_store.h" + +namespace search { + +using vespalib::AddressSpace; + +AddressSpace AddressSpaceComponents::default_enum_store_usage() { + return AddressSpace(0, 0, IEnumStore::InternalIndex::offsetSize()); +} + +AddressSpace AddressSpaceComponents::default_multi_value_usage() { + return AddressSpace(0, 0, (1ull << 32)); +} + +vespalib::string AddressSpaceComponents::enum_store = "enum-store"; +vespalib::string AddressSpaceComponents::multi_value = "multi-value"; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_components.h b/searchlib/src/vespa/searchlib/attribute/address_space_components.h new file mode 100644 index 00000000000..765ca169bca --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/address_space_components.h @@ -0,0 +1,21 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/address_space.h> + +namespace search { + +/** + * A set of components in an attribute vector that use address space. + */ +class AddressSpaceComponents { +public: + static vespalib::AddressSpace default_enum_store_usage(); + static vespalib::AddressSpace default_multi_value_usage(); + static vespalib::string enum_store; + static vespalib::string multi_value; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_usage.cpp b/searchlib/src/vespa/searchlib/attribute/address_space_usage.cpp index 692adaf3817..08b6666bbb0 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_usage.cpp +++ b/searchlib/src/vespa/searchlib/attribute/address_space_usage.cpp @@ -1,33 +1,52 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "address_space_components.h" #include "address_space_usage.h" -#include "i_enum_store.h" namespace search { using vespalib::AddressSpace; AddressSpaceUsage::AddressSpaceUsage() - : _enumStoreUsage(defaultEnumStoreUsage()), - _multiValueUsage(defaultMultiValueUsage()) { + : _map() +{ +} + +AddressSpaceUsage::AddressSpaceUsage(const AddressSpace& enum_store_usage, + const AddressSpace& multi_value_usage) + : _map() +{ + // TODO: Remove this constructor and instead add usage for each relevant component explicit. + add(AddressSpaceComponents::enum_store, enum_store_usage); + add(AddressSpaceComponents::multi_value, multi_value_usage); +} + +void +AddressSpaceUsage::add(const vespalib::string& component, const vespalib::AddressSpace& usage) +{ + _map[component] = usage; } -AddressSpaceUsage::AddressSpaceUsage(const AddressSpace &enumStoreUsage_, - const AddressSpace &multiValueUsage_) - : _enumStoreUsage(enumStoreUsage_), - _multiValueUsage(multiValueUsage_) { +AddressSpace +AddressSpaceUsage::get(const vespalib::string& component) const +{ + auto itr = _map.find(component); + if (itr != _map.end()) { + return itr->second; + } + return AddressSpace(); } AddressSpace -AddressSpaceUsage::defaultEnumStoreUsage() +AddressSpaceUsage::enum_store_usage() const { - return AddressSpace(0, 0, IEnumStore::InternalIndex::offsetSize()); + return get(AddressSpaceComponents::enum_store); } AddressSpace -AddressSpaceUsage::defaultMultiValueUsage() +AddressSpaceUsage::multi_value_usage() const { - return AddressSpace(0, 0, (1ull << 32)); + return get(AddressSpaceComponents::multi_value); } -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_usage.h b/searchlib/src/vespa/searchlib/attribute/address_space_usage.h index 3e6ad1d3f8e..1284340d4ff 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_usage.h +++ b/searchlib/src/vespa/searchlib/attribute/address_space_usage.h @@ -2,28 +2,30 @@ #pragma once +#include <vespa/vespalib/stllike/hash_fun.h> +#include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/address_space.h> +#include <unordered_map> namespace search { /** - * Represents the address space usage for enum store and multi value mapping. + * Represents the address space usage for a set of attribute vector components. */ class AddressSpaceUsage { private: - vespalib::AddressSpace _enumStoreUsage; - vespalib::AddressSpace _multiValueUsage; + using AddressSpaceMap = std::unordered_map<vespalib::string, vespalib::AddressSpace, vespalib::hash<vespalib::string>>; + AddressSpaceMap _map; public: AddressSpaceUsage(); - AddressSpaceUsage(const vespalib::AddressSpace &enumStoreUsage_, - const vespalib::AddressSpace &multiValueUsage_); - static vespalib::AddressSpace defaultEnumStoreUsage(); - static vespalib::AddressSpace defaultMultiValueUsage(); - const vespalib::AddressSpace &enumStoreUsage() const { return _enumStoreUsage; } - const vespalib::AddressSpace &multiValueUsage() const { return _multiValueUsage; } - + AddressSpaceUsage(const vespalib::AddressSpace& enum_store_usage, + const vespalib::AddressSpace& multi_value_usage); + void add(const vespalib::string& component, const vespalib::AddressSpace& usage); + vespalib::AddressSpace get(const vespalib::string& component) const; + vespalib::AddressSpace enum_store_usage() const; + vespalib::AddressSpace multi_value_usage() const; }; -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 08ddc01c15b..318fb563a4f 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -1,10 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "attributevector.h" +#include "address_space_components.h" #include "attribute_read_guard.h" #include "attributefilesavetarget.h" #include "attributeiterators.hpp" #include "attributesaver.h" +#include "attributevector.h" #include "attributevector.hpp" #include "floatbase.h" #include "interlock.h" @@ -14,14 +15,14 @@ #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/update/mapvalueupdate.h> #include <vespa/fastlib/io/bufferedfile.h> +#include <vespa/searchcommon/attribute/attribute_utils.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/query/query_term_decoder.h> #include <vespa/searchlib/queryeval/emptysearch.h> +#include <vespa/searchlib/util/logutil.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/searchlib/util/logutil.h> -#include <vespa/searchcommon/attribute/attribute_utils.h> #include <thread> #include <vespa/log/log.h> @@ -222,22 +223,20 @@ AttributeVector::getEnumStoreValuesMemoryUsage() const return vespalib::MemoryUsage(); } -vespalib::AddressSpace -AttributeVector::getEnumStoreAddressSpaceUsage() const -{ - return AddressSpaceUsage::defaultEnumStoreUsage(); -} - -vespalib::AddressSpace -AttributeVector::getMultiValueAddressSpaceUsage() const +void +AttributeVector::populate_address_space_usage(AddressSpaceUsage& usage) const { - return AddressSpaceUsage::defaultMultiValueUsage(); + // TODO: Stop inserting defaults here when code using AddressSpaceUsage no longer require these two components. + usage.add(AddressSpaceComponents::enum_store, AddressSpaceComponents::default_enum_store_usage()); + usage.add(AddressSpaceComponents::multi_value, AddressSpaceComponents::default_multi_value_usage()); } AddressSpaceUsage AttributeVector::getAddressSpaceUsage() const { - return AddressSpaceUsage(getEnumStoreAddressSpaceUsage(), getMultiValueAddressSpaceUsage()); + AddressSpaceUsage usage; + populate_address_space_usage(usage); + return usage; } bool diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index a4294df8cb7..3c27f00a022 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -381,8 +381,7 @@ protected: } virtual vespalib::MemoryUsage getEnumStoreValuesMemoryUsage() const; - virtual vespalib::AddressSpace getEnumStoreAddressSpaceUsage() const; - virtual vespalib::AddressSpace getMultiValueAddressSpaceUsage() const; + virtual void populate_address_space_usage(AddressSpaceUsage& usage) const; public: DECLARE_IDENTIFIABLE_ABSTRACT(AttributeVector); diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.h b/searchlib/src/vespa/searchlib/attribute/enumattribute.h index 443433757b3..cee091b1ac8 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.h @@ -64,7 +64,7 @@ protected: void insertNewUniqueValues(EnumStoreBatchUpdater& updater); virtual void considerAttributeChange(const Change & c, EnumStoreBatchUpdater & inserter) = 0; vespalib::MemoryUsage getEnumStoreValuesMemoryUsage() const override; - vespalib::AddressSpace getEnumStoreAddressSpaceUsage() const override; + void populate_address_space_usage(AddressSpaceUsage& usage) const override; public: EnumAttribute(const vespalib::string & baseFileName, const AttributeVector::Config & cfg); ~EnumAttribute(); diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp index 164bb411061..86870fc1772 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp @@ -2,6 +2,7 @@ #pragma once +#include "address_space_components.h" #include <vespa/vespalib/util/hdr_abort.h> #include <vespa/searchlib/attribute/enumattribute.h> #include <vespa/searchlib/attribute/enumstore.hpp> @@ -76,10 +77,11 @@ EnumAttribute<B>::getEnumStoreValuesMemoryUsage() const } template <typename B> -vespalib::AddressSpace -EnumAttribute<B>::getEnumStoreAddressSpaceUsage() const +void +EnumAttribute<B>::populate_address_space_usage(AddressSpaceUsage& usage) const { - return _enumStore.get_address_space_usage(); + B::populate_address_space_usage(usage); + usage.add(AddressSpaceComponents::enum_store, _enumStore.get_address_space_usage()); } } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h index d36777a25a9..8e6e4914012 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h @@ -48,7 +48,7 @@ protected: **/ bool onAddDoc(DocId doc) override { (void) doc; return false; } - vespalib::AddressSpace getMultiValueAddressSpaceUsage() const override; + void populate_address_space_usage(AddressSpaceUsage& usage) const override; public: MultiValueAttribute(const vespalib::string & baseFileName, const AttributeVector::Config & cfg); diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp index 2e73909ea1e..98cc0707ac0 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp @@ -2,6 +2,7 @@ #pragma once +#include "address_space_components.h" #include <vespa/searchlib/attribute/multivalueattribute.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/stllike/hash_map.hpp> @@ -200,13 +201,13 @@ MultiValueAttribute<B, M>::apply_attribute_changes_to_wset(DocumentValues& docVa } template <typename B, typename M> -vespalib::AddressSpace -MultiValueAttribute<B, M>::getMultiValueAddressSpaceUsage() const +void +MultiValueAttribute<B, M>::populate_address_space_usage(AddressSpaceUsage& usage) const { - return _mvMapping.getAddressSpaceUsage(); + B::populate_address_space_usage(usage); + usage.add(AddressSpaceComponents::multi_value, _mvMapping.getAddressSpaceUsage()); } - template <typename B, typename M> bool MultiValueAttribute<B, M>::addDoc(DocId & doc) |