From e6ca36975a9e4c2730b934e3d93bd8d91ddd7c4c Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Mon, 4 Sep 2023 15:13:31 +0000 Subject: Add memory index write context details to index manager state explorer. This shows which index fields that are assigned to each executor id. --- .../memoryindex/memory_index/memory_index_test.cpp | 33 +++++++++++++---- .../memoryindex/bundled_fields_context.cpp | 3 +- .../searchlib/memoryindex/bundled_fields_context.h | 4 ++- .../memoryindex/document_inverter_context.cpp | 8 ++--- .../vespa/searchlib/memoryindex/memory_index.cpp | 42 +++++++++++++++++++++- .../src/vespa/searchlib/memoryindex/memory_index.h | 4 ++- 6 files changed, 79 insertions(+), 15 deletions(-) (limited to 'searchlib') diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index e3b9cf9702d..3547bf6c9a8 100644 --- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -8,24 +8,25 @@ #include #include #include -#include -#include -#include #include #include +#include #include #include #include #include #include -#include #include -#include +#include +#include +#include +#include +#include +#include #include #include #include #include -#include #include LOG_SETUP("memory_index_test"); @@ -36,7 +37,6 @@ using document::FieldValue; using search::ScheduleTaskCallback; using search::index::FieldLengthInfo; using search::index::IFieldLengthInspector; -using vespalib::makeLambdaTask; using search::query::Node; using search::query::SimplePhrase; using search::query::SimpleStringTerm; @@ -45,6 +45,11 @@ using search::test::SchemaBuilder; using search::test::StringFieldBuilder; using vespalib::ISequencedTaskExecutor; using vespalib::SequencedTaskExecutor; +using vespalib::Slime; +using vespalib::makeLambdaTask; +using vespalib::slime::JsonFormat; +using vespalib::slime::SlimeInserter; + using namespace search::fef; using namespace search::index; using namespace search::memoryindex; @@ -542,4 +547,18 @@ TEST(MemoryIndexTest, field_length_info_can_be_retrieved_per_field) EXPECT_EQ(0, index.index.get_field_length_info("na").get_num_samples()); } +TEST(MemoryIndexTest, write_context_state_as_slime) +{ + Index index(MySetup().field(title).field(body)); + Slime act; + SlimeInserter inserter(act); + index.index.insert_write_context_state(inserter.insertObject()); + Slime exp; + JsonFormat::decode("{\"invert\": [{\"executor_id\": 0, \"fields\": [\"body\"]}," + "{\"executor_id\": 1, \"fields\": [\"title\"]}]," + "\"push\": [{\"executor_id\": 0, \"fields\": [\"body\"]}," + "{\"executor_id\": 1, \"fields\": [\"title\"]}]}", exp); + EXPECT_EQ(exp, act); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp index af7e19ee20d..4f9e88b323e 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp @@ -20,9 +20,10 @@ BundledFieldsContext::add_field(uint32_t field_id) } void -BundledFieldsContext::add_uri_field(uint32_t uri_field_id) +BundledFieldsContext::add_uri_field(uint32_t uri_field_id, uint32_t uri_all_field_id) { _uri_fields.emplace_back(uri_field_id); + _uri_all_field_ids.emplace_back(uri_all_field_id); } } diff --git a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h index fb1a68d7273..c058c14832d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h +++ b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h @@ -16,16 +16,18 @@ class BundledFieldsContext vespalib::ISequencedTaskExecutor::ExecutorId _id; std::vector _fields; std::vector _uri_fields; + std::vector _uri_all_field_ids; protected: BundledFieldsContext(vespalib::ISequencedTaskExecutor::ExecutorId id); ~BundledFieldsContext(); public: void add_field(uint32_t field_id); - void add_uri_field(uint32_t uri_field_id); + void add_uri_field(uint32_t uri_field_id, uint32_t uri_all_field_id); void set_id(vespalib::ISequencedTaskExecutor::ExecutorId id) { _id = id; } vespalib::ISequencedTaskExecutor::ExecutorId get_id() const noexcept { return _id; } const std::vector& get_fields() const noexcept { return _fields; } const std::vector& get_uri_fields() const noexcept { return _uri_fields; } + const std::vector& get_uri_all_field_ids() const noexcept { return _uri_all_field_ids; } }; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp index 8183cb005fe..93a12c24257 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp @@ -15,20 +15,20 @@ template void make_contexts(const index::Schema& schema, const SchemaIndexFields& schema_index_fields, ISequencedTaskExecutor& executor, std::vector& contexts) { using ExecutorId = ISequencedTaskExecutor::ExecutorId; - using IdMapping = std::vector>; + using IdMapping = std::vector>; IdMapping map; for (uint32_t field_id : schema_index_fields._textFields) { // TODO: Add bias when sharing sequenced task executor between document types auto& name = schema.getIndexField(field_id).getName(); auto id = executor.getExecutorIdFromName(name); - map.emplace_back(id, false, field_id); + map.emplace_back(id, false, field_id, 0); } uint32_t uri_field_id = 0; for (auto& uri_field : schema_index_fields._uriFields) { // TODO: Add bias when sharing sequenced task executor between document types auto& name = schema.getIndexField(uri_field._all).getName(); auto id = executor.getExecutorIdFromName(name); - map.emplace_back(id, true, uri_field_id); + map.emplace_back(id, true, uri_field_id, uri_field._all); ++uri_field_id; } std::sort(map.begin(), map.end()); @@ -39,7 +39,7 @@ void make_contexts(const index::Schema& schema, const SchemaIndexFields& schema_ prev_id = std::get<0>(entry); } if (std::get<1>(entry)) { - contexts.back().add_uri_field(std::get<2>(entry)); + contexts.back().add_uri_field(std::get<2>(entry), std::get<3>(entry)); } else { contexts.back().add_field(std::get<2>(entry)); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp index 643bcbb325e..86421711e32 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp @@ -7,13 +7,14 @@ #include "field_index_collection.h" #include #include -#include #include #include #include #include #include #include +#include +#include #include LOG_SETUP(".searchlib.memoryindex.memory_index"); @@ -47,6 +48,7 @@ using queryeval::FieldSpec; using queryeval::IRequestContext; using queryeval::Searchable; using vespalib::ISequencedTaskExecutor; +using vespalib::slime::Cursor; } @@ -251,4 +253,42 @@ MemoryIndex::get_field_length_info(const vespalib::string& field_name) const return FieldLengthInfo(); } +namespace { + +void +fields_to_slime(const std::vector& field_ids, const Schema& schema, Cursor& array) +{ + for (uint32_t field_id : field_ids) { + assert(field_id < schema.getIndexFields().size()); + const auto& field = schema.getIndexField(field_id); + array.addString(field.getName()); + } +} + +void +write_context_to_slime(const BundledFieldsContext& ctx, const Schema& schema, Cursor& object) +{ + object.setLong("executor_id", ctx.get_id().getId()); + auto& fields = object.setArray("fields"); + fields_to_slime(ctx.get_fields(), schema, fields); + fields_to_slime(ctx.get_uri_all_field_ids(), schema, fields); +} + +} + +void +MemoryIndex::insert_write_context_state(Cursor& object) const +{ + auto& invert = object.setArray("invert"); + for (const auto& ctx : _inverter_context->get_invert_contexts()) { + auto& ctx_obj = invert.addObject(); + write_context_to_slime(ctx, _schema, ctx_obj); + } + auto& push = object.setArray("push"); + for (const auto& ctx : _inverter_context->get_push_contexts()) { + auto& ctx_obj = push.addObject(); + write_context_to_slime(ctx, _schema, ctx_obj); + } +} + } diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h index af76ed172ba..320c6fba277 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h @@ -17,7 +17,7 @@ namespace search::index { } namespace vespalib { class ISequencedTaskExecutor; } - +namespace vespalib::slime { struct Cursor; } namespace document { class Document; } namespace search::memoryindex { @@ -175,6 +175,8 @@ public: uint64_t getStaticMemoryFootprint() const { return _staticMemoryFootprint; } index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const; + + void insert_write_context_state(vespalib::slime::Cursor& object) const; }; } -- cgit v1.2.3