From 23207ef234bf26140924dae31a1deefcbcf9d89d Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Thu, 2 May 2024 14:52:43 +0200 Subject: Expose imported attributes in state explorer. --- .../attributes_state_explorer_test.cpp | 110 ++++++++++++++++----- .../searchcore/proton/attribute/CMakeLists.txt | 1 + .../attribute/attribute_manager_explorer.cpp | 21 ++++ .../imported_attribute_vector_explorer.cpp | 28 ++++++ .../attribute/imported_attribute_vector_explorer.h | 26 +++++ 5 files changed, 164 insertions(+), 22 deletions(-) create mode 100644 searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.cpp create mode 100644 searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.h (limited to 'searchcore') diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp index fd202c24887..732a95e7c13 100644 --- a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp @@ -2,11 +2,19 @@ #include #include +#include +#include +#include #include #include +#include +#include +#include #include +#include #include #include +#include #include #include #include @@ -21,6 +29,12 @@ using namespace proton; using namespace proton::test; using search::AttributeVector; using search::DictionaryConfig; +using search::attribute::BasicType; +using search::attribute::Config; +using search::attribute::ImportedAttributeVector; +using search::attribute::ImportedAttributeVectorFactory; +using search::attribute::ReferenceAttribute; +using search::attribute::test::MockGidToLidMapperFactory; using vespalib::ForegroundThreadExecutor; using vespalib::ISequencedTaskExecutor; using vespalib::SequencedTaskExecutor; @@ -32,6 +46,10 @@ using vespalib::HwInfo; const vespalib::string TEST_DIR = "test_output"; +const vespalib::string ref_name("ref"); +const vespalib::string target_name("f3"); +const vespalib::string imported_name("my_f3"); + namespace { VESPA_THREAD_STACK_TAG(test_executor) } @@ -43,28 +61,13 @@ struct AttributesStateExplorerTest : public ::testing::Test std::unique_ptr _attribute_field_writer; ForegroundThreadExecutor _shared; HwInfo _hwInfo; + std::shared_ptr _parent_dms; + std::shared_ptr _dms; + std::shared_ptr _parent_mgr; AttributeManager::SP _mgr; AttributeManagerExplorer _explorer; - AttributesStateExplorerTest() - : _dirHandler(TEST_DIR), - _fileHeaderContext(), - _attribute_field_writer(SequencedTaskExecutor::create(test_executor, 1)), - _shared(), - _hwInfo(), - _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(), - _fileHeaderContext, - std::make_shared(), - *_attribute_field_writer, - _shared, - _hwInfo)), - _explorer(_mgr) - { - addAttribute("regular"); - addExtraAttribute("extra"); - add_fast_search_attribute("btree", DictionaryConfig::Type::BTREE); - add_fast_search_attribute("hybrid", DictionaryConfig::Type::BTREE_AND_HASH); - add_fast_search_attribute("hash", DictionaryConfig::Type::HASH); - } + AttributesStateExplorerTest() noexcept; + ~AttributesStateExplorerTest() override; void addAttribute(const vespalib::string &name) { _mgr->addAttribute({name, AttributeUtils::getInt32Config()}, 1); } @@ -84,16 +87,71 @@ struct AttributesStateExplorerTest : public ::testing::Test _explorer.get_child(name)->get_state(inserter, true); return result; } - + void add_reference_attribute() { + search::attribute::Config cfg(BasicType::REFERENCE); + _mgr->addAttribute({ ref_name, cfg }, 1); + auto& ref_attr = dynamic_cast(**_mgr->getAttribute(ref_name)); + ref_attr.setGidToLidMapperFactory(std::make_shared()); + } + std::shared_ptr get_reference_attribute() { + return std::dynamic_pointer_cast(_mgr->getAttribute(ref_name)->getSP()); + } + void add_imported_attributes() { + auto repo = std::make_unique(); + auto attr = ImportedAttributeVectorFactory::create(imported_name, + get_reference_attribute(), + _dms, + _parent_mgr->getAttribute(target_name)->getSP(), + _parent_dms, + false); + repo->add(imported_name, attr); + _mgr->setImportedAttributes(std::move(repo)); + } }; +AttributesStateExplorerTest::AttributesStateExplorerTest() noexcept + : _dirHandler(TEST_DIR), + _fileHeaderContext(), + _attribute_field_writer(SequencedTaskExecutor::create(test_executor, 1)), + _shared(), + _hwInfo(), + _parent_dms(std::make_shared(std::make_shared())), + _dms(), + _parent_mgr(std::make_shared + (TEST_DIR, "test.parent.subdb", TuneFileAttributes(), + _fileHeaderContext, + std::make_shared(), + *_attribute_field_writer, + _shared, + _hwInfo)), + _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(), + _fileHeaderContext, + std::make_shared(), + *_attribute_field_writer, + _shared, + _hwInfo)), + _explorer(_mgr) +{ + _parent_mgr->addAttribute({target_name, AttributeUtils::getInt32Config()}, 1); + addAttribute("regular"); + addExtraAttribute("extra"); + add_fast_search_attribute("btree", DictionaryConfig::Type::BTREE); + add_fast_search_attribute("hybrid", DictionaryConfig::Type::BTREE_AND_HASH); + add_fast_search_attribute("hash", DictionaryConfig::Type::HASH); + add_reference_attribute(); + add_imported_attributes(); +} + +AttributesStateExplorerTest::~AttributesStateExplorerTest() = default; + + using StringVector = std::vector; TEST_F(AttributesStateExplorerTest, require_that_attributes_are_exposed_as_children_names) { StringVector children = _explorer.get_children_names(); std::sort(children.begin(), children.end()); - EXPECT_EQ(StringVector({"btree", "hash", "hybrid", "regular"}), children); + EXPECT_EQ(StringVector({"btree", "hash", "hybrid", "my_f3", "ref", "regular"}), children); } TEST_F(AttributesStateExplorerTest, require_that_attributes_are_explorable) @@ -125,4 +183,12 @@ TEST_F(AttributesStateExplorerTest, require_that_dictionary_memory_usage_is_repo } } +TEST_F(AttributesStateExplorerTest, require_that_imported_attribute_shows_memory_usage) +{ + vespalib::string cache_memory_usage("cacheMemoryUsage"); + auto slime = explore_attribute(imported_name); + EXPECT_LT(0, slime[cache_memory_usage]["allocated"].asLong()); + EXPECT_LT(0, slime[cache_memory_usage]["used"].asLong()); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt index 3717c24650d..c1f10ebd80d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt @@ -34,6 +34,7 @@ vespa_add_library(searchcore_attribute STATIC document_field_retriever.cpp filter_attribute_manager.cpp flushableattribute.cpp + imported_attribute_vector_explorer.cpp imported_attributes_context.cpp imported_attributes_repo.cpp initialized_attributes_result.cpp diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp index 0b48afe4ab8..0797ddfb6cd 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.cpp @@ -3,9 +3,13 @@ #include "attribute_manager_explorer.h" #include "attribute_executor.h" #include "attribute_vector_explorer.h" +#include "imported_attribute_vector_explorer.h" +#include "imported_attributes_repo.h" #include +#include using search::AttributeVector; +using search::attribute::ImportedAttributeVector; using vespalib::slime::Inserter; namespace proton { @@ -32,6 +36,14 @@ AttributeManagerExplorer::get_children_names() const for (const auto &attr : attributes) { names.push_back(attr->getName()); } + auto imported = _mgr->getImportedAttributes(); + if (imported != nullptr) { + std::vector> i_list; + imported->getAll(i_list); + for (const auto& attr : i_list) { + names.push_back(attr->getName()); + } + } return names; } @@ -39,6 +51,15 @@ std::unique_ptr AttributeManagerExplorer::get_child(vespalib::stringref name) const { auto guard = _mgr->getAttribute(name); + if (!guard || !guard->getSP()) { + auto imported = _mgr->getImportedAttributes(); + if (imported != nullptr) { + auto& imported_attr = imported->get(name); + if (imported_attr) { + return std::make_unique(imported_attr); + } + } + } auto attr = guard ? guard->getSP() : std::shared_ptr(); if (attr && _mgr->getWritableAttribute(name) != nullptr) { auto executor = std::make_unique(_mgr, std::move(attr)); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.cpp new file mode 100644 index 00000000000..dd858b78934 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.cpp @@ -0,0 +1,28 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "imported_attribute_vector_explorer.h" +#include +#include +#include +#include + +using search::StateExplorerUtils; +using search::attribute::ImportedAttributeVector; +using namespace vespalib::slime; + +namespace proton { + +ImportedAttributeVectorExplorer::ImportedAttributeVectorExplorer(std::shared_ptr attr) + : _attr(std::move(attr)) +{ +} + +void +ImportedAttributeVectorExplorer::get_state(const vespalib::slime::Inserter &inserter, bool) const +{ + Cursor &object = inserter.insertObject(); + auto memory_usage = _attr->get_memory_usage(); + StateExplorerUtils::memory_usage_to_slime(memory_usage, object.setObject("cacheMemoryUsage")); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.h b/searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.h new file mode 100644 index 00000000000..ce8854b03d2 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/imported_attribute_vector_explorer.h @@ -0,0 +1,26 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include + +namespace search::attribute { class ImportedAttributeVector; } + +namespace proton { + +/** + * Class used to explore the state of an imported attribute vector. + */ +class ImportedAttributeVectorExplorer : public vespalib::StateExplorer +{ +private: + std::shared_ptr _attr; + +public: + ImportedAttributeVectorExplorer(std::shared_ptr attr); + + // Implements vespalib::StateExplorer + void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; +}; + +} -- cgit v1.2.3