diff options
author | Geir Storli <geirst@yahooinc.com> | 2023-09-04 15:13:31 +0000 |
---|---|---|
committer | Geir Storli <geirst@yahooinc.com> | 2023-09-04 15:27:33 +0000 |
commit | e6ca36975a9e4c2730b934e3d93bd8d91ddd7c4c (patch) | |
tree | 75d94c9051a0c7cc8710c87e7568196d0a9ce3cf /searchlib | |
parent | 3e4a2ecdd9ad0b3dab563fd8572644bb3cfae2e3 (diff) |
Add memory index write context details to index manager state explorer.
This shows which index fields that are assigned to each executor id.
Diffstat (limited to 'searchlib')
6 files changed, 79 insertions, 15 deletions
diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index e3b9cf9702d..3547bf6c9a8 100644 --- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -8,24 +8,25 @@ #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/i_field_length_inspector.h> -#include <vespa/searchlib/test/doc_builder.h> -#include <vespa/searchlib/test/schema_builder.h> -#include <vespa/searchlib/test/string_field_builder.h> #include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> +#include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/fake_searchable.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/searchiterator.h> -#include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/queryeval/simple_phrase_blueprint.h> -#include <vespa/searchlib/queryeval/blueprint.h> +#include <vespa/searchlib/queryeval/simpleresult.h> +#include <vespa/searchlib/test/doc_builder.h> +#include <vespa/searchlib/test/schema_builder.h> +#include <vespa/searchlib/test/string_field_builder.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("memory_index_test"); @@ -36,7 +37,6 @@ using document::FieldValue; using search::ScheduleTaskCallback; using search::index::FieldLengthInfo; using search::index::IFieldLengthInspector; -using vespalib::makeLambdaTask; using search::query::Node; using search::query::SimplePhrase; using search::query::SimpleStringTerm; @@ -45,6 +45,11 @@ using search::test::SchemaBuilder; using search::test::StringFieldBuilder; using vespalib::ISequencedTaskExecutor; using vespalib::SequencedTaskExecutor; +using vespalib::Slime; +using vespalib::makeLambdaTask; +using vespalib::slime::JsonFormat; +using vespalib::slime::SlimeInserter; + using namespace search::fef; using namespace search::index; using namespace search::memoryindex; @@ -542,4 +547,18 @@ TEST(MemoryIndexTest, field_length_info_can_be_retrieved_per_field) EXPECT_EQ(0, index.index.get_field_length_info("na").get_num_samples()); } +TEST(MemoryIndexTest, write_context_state_as_slime) +{ + Index index(MySetup().field(title).field(body)); + Slime act; + SlimeInserter inserter(act); + index.index.insert_write_context_state(inserter.insertObject()); + Slime exp; + JsonFormat::decode("{\"invert\": [{\"executor_id\": 0, \"fields\": [\"body\"]}," + "{\"executor_id\": 1, \"fields\": [\"title\"]}]," + "\"push\": [{\"executor_id\": 0, \"fields\": [\"body\"]}," + "{\"executor_id\": 1, \"fields\": [\"title\"]}]}", exp); + EXPECT_EQ(exp, act); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp index af7e19ee20d..4f9e88b323e 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp @@ -20,9 +20,10 @@ BundledFieldsContext::add_field(uint32_t field_id) } void -BundledFieldsContext::add_uri_field(uint32_t uri_field_id) +BundledFieldsContext::add_uri_field(uint32_t uri_field_id, uint32_t uri_all_field_id) { _uri_fields.emplace_back(uri_field_id); + _uri_all_field_ids.emplace_back(uri_all_field_id); } } diff --git a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h index fb1a68d7273..c058c14832d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h +++ b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h @@ -16,16 +16,18 @@ class BundledFieldsContext vespalib::ISequencedTaskExecutor::ExecutorId _id; std::vector<uint32_t> _fields; std::vector<uint32_t> _uri_fields; + std::vector<uint32_t> _uri_all_field_ids; protected: BundledFieldsContext(vespalib::ISequencedTaskExecutor::ExecutorId id); ~BundledFieldsContext(); public: void add_field(uint32_t field_id); - void add_uri_field(uint32_t uri_field_id); + void add_uri_field(uint32_t uri_field_id, uint32_t uri_all_field_id); void set_id(vespalib::ISequencedTaskExecutor::ExecutorId id) { _id = id; } vespalib::ISequencedTaskExecutor::ExecutorId get_id() const noexcept { return _id; } const std::vector<uint32_t>& get_fields() const noexcept { return _fields; } const std::vector<uint32_t>& get_uri_fields() const noexcept { return _uri_fields; } + const std::vector<uint32_t>& get_uri_all_field_ids() const noexcept { return _uri_all_field_ids; } }; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp index 8183cb005fe..93a12c24257 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp @@ -15,20 +15,20 @@ template <typename Context> void make_contexts(const index::Schema& schema, const SchemaIndexFields& schema_index_fields, ISequencedTaskExecutor& executor, std::vector<Context>& contexts) { using ExecutorId = ISequencedTaskExecutor::ExecutorId; - using IdMapping = std::vector<std::tuple<ExecutorId, bool, uint32_t>>; + using IdMapping = std::vector<std::tuple<ExecutorId, bool, uint32_t, uint32_t>>; IdMapping map; for (uint32_t field_id : schema_index_fields._textFields) { // TODO: Add bias when sharing sequenced task executor between document types auto& name = schema.getIndexField(field_id).getName(); auto id = executor.getExecutorIdFromName(name); - map.emplace_back(id, false, field_id); + map.emplace_back(id, false, field_id, 0); } uint32_t uri_field_id = 0; for (auto& uri_field : schema_index_fields._uriFields) { // TODO: Add bias when sharing sequenced task executor between document types auto& name = schema.getIndexField(uri_field._all).getName(); auto id = executor.getExecutorIdFromName(name); - map.emplace_back(id, true, uri_field_id); + map.emplace_back(id, true, uri_field_id, uri_field._all); ++uri_field_id; } std::sort(map.begin(), map.end()); @@ -39,7 +39,7 @@ void make_contexts(const index::Schema& schema, const SchemaIndexFields& schema_ prev_id = std::get<0>(entry); } if (std::get<1>(entry)) { - contexts.back().add_uri_field(std::get<2>(entry)); + contexts.back().add_uri_field(std::get<2>(entry), std::get<3>(entry)); } else { contexts.back().add_field(std::get<2>(entry)); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp index 643bcbb325e..86421711e32 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp @@ -7,13 +7,14 @@ #include "field_index_collection.h" #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/document.h> -#include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/searchlib/index/field_length_calculator.h> #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/queryeval/create_blueprint_visitor_helper.h> #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/vespalib/btree/btreenodeallocator.hpp> +#include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.memoryindex.memory_index"); @@ -47,6 +48,7 @@ using queryeval::FieldSpec; using queryeval::IRequestContext; using queryeval::Searchable; using vespalib::ISequencedTaskExecutor; +using vespalib::slime::Cursor; } @@ -251,4 +253,42 @@ MemoryIndex::get_field_length_info(const vespalib::string& field_name) const return FieldLengthInfo(); } +namespace { + +void +fields_to_slime(const std::vector<uint32_t>& field_ids, const Schema& schema, Cursor& array) +{ + for (uint32_t field_id : field_ids) { + assert(field_id < schema.getIndexFields().size()); + const auto& field = schema.getIndexField(field_id); + array.addString(field.getName()); + } +} + +void +write_context_to_slime(const BundledFieldsContext& ctx, const Schema& schema, Cursor& object) +{ + object.setLong("executor_id", ctx.get_id().getId()); + auto& fields = object.setArray("fields"); + fields_to_slime(ctx.get_fields(), schema, fields); + fields_to_slime(ctx.get_uri_all_field_ids(), schema, fields); +} + +} + +void +MemoryIndex::insert_write_context_state(Cursor& object) const +{ + auto& invert = object.setArray("invert"); + for (const auto& ctx : _inverter_context->get_invert_contexts()) { + auto& ctx_obj = invert.addObject(); + write_context_to_slime(ctx, _schema, ctx_obj); + } + auto& push = object.setArray("push"); + for (const auto& ctx : _inverter_context->get_push_contexts()) { + auto& ctx_obj = push.addObject(); + write_context_to_slime(ctx, _schema, ctx_obj); + } +} + } diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h index af76ed172ba..320c6fba277 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h @@ -17,7 +17,7 @@ namespace search::index { } namespace vespalib { class ISequencedTaskExecutor; } - +namespace vespalib::slime { struct Cursor; } namespace document { class Document; } namespace search::memoryindex { @@ -175,6 +175,8 @@ public: uint64_t getStaticMemoryFootprint() const { return _staticMemoryFootprint; } index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const; + + void insert_write_context_state(vespalib::slime::Cursor& object) const; }; } |