Add memory index write context details to index manager state explorer.

This shows which index fields that are assigned to each executor id.

6 files changed, 79 insertions, 15 deletions

diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp
index e3b9cf9702d..3547bf6c9a8 100644
--- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp
+++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp
@@ -8,24 +8,25 @@
 #include <vespa/searchlib/fef/matchdatalayout.h>
 #include <vespa/searchlib/fef/termfieldmatchdata.h>
 #include <vespa/searchlib/index/i_field_length_inspector.h>
-#include <vespa/searchlib/test/doc_builder.h>
-#include <vespa/searchlib/test/schema_builder.h>
-#include <vespa/searchlib/test/string_field_builder.h>
 #include <vespa/searchlib/memoryindex/memory_index.h>
 #include <vespa/searchlib/query/tree/simplequery.h>
+#include <vespa/searchlib/queryeval/blueprint.h>
 #include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h>
 #include <vespa/searchlib/queryeval/fake_requestcontext.h>
 #include <vespa/searchlib/queryeval/fake_searchable.h>
 #include <vespa/searchlib/queryeval/leaf_blueprints.h>
 #include <vespa/searchlib/queryeval/searchiterator.h>
-#include <vespa/searchlib/queryeval/simpleresult.h>
 #include <vespa/searchlib/queryeval/simple_phrase_blueprint.h>
-#include <vespa/searchlib/queryeval/blueprint.h>
+#include <vespa/searchlib/queryeval/simpleresult.h>
+#include <vespa/searchlib/test/doc_builder.h>
+#include <vespa/searchlib/test/schema_builder.h>
+#include <vespa/searchlib/test/string_field_builder.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/gtest/gtest.h>
 #include <vespa/vespalib/util/sequencedtaskexecutor.h>
 #include <vespa/vespalib/util/size_literals.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/threadstackexecutor.h>
-#include <vespa/vespalib/gtest/gtest.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("memory_index_test");
@@ -36,7 +37,6 @@ using document::FieldValue;
 using search::ScheduleTaskCallback;
 using search::index::FieldLengthInfo;
 using search::index::IFieldLengthInspector;
-using vespalib::makeLambdaTask;
 using search::query::Node;
 using search::query::SimplePhrase;
 using search::query::SimpleStringTerm;
@@ -45,6 +45,11 @@ using search::test::SchemaBuilder;
 using search::test::StringFieldBuilder;
 using vespalib::ISequencedTaskExecutor;
 using vespalib::SequencedTaskExecutor;
+using vespalib::Slime;
+using vespalib::makeLambdaTask;
+using vespalib::slime::JsonFormat;
+using vespalib::slime::SlimeInserter;
+
 using namespace search::fef;
 using namespace search::index;
 using namespace search::memoryindex;
@@ -542,4 +547,18 @@ TEST(MemoryIndexTest, field_length_info_can_be_retrieved_per_field)
     EXPECT_EQ(0, index.index.get_field_length_info("na").get_num_samples());
 }
 
+TEST(MemoryIndexTest, write_context_state_as_slime)
+{
+    Index index(MySetup().field(title).field(body));
+    Slime act;
+    SlimeInserter inserter(act);
+    index.index.insert_write_context_state(inserter.insertObject());
+    Slime exp;
+    JsonFormat::decode("{\"invert\": [{\"executor_id\": 0, \"fields\": [\"body\"]},"
+                                     "{\"executor_id\": 1, \"fields\": [\"title\"]}],"
+                        "\"push\": [{\"executor_id\": 0, \"fields\": [\"body\"]},"
+                                   "{\"executor_id\": 1, \"fields\": [\"title\"]}]}", exp);
+    EXPECT_EQ(exp, act);
+}
+
 GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp
index af7e19ee20d..4f9e88b323e 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp
@@ -20,9 +20,10 @@ BundledFieldsContext::add_field(uint32_t field_id)
 }
 
 void
-BundledFieldsContext::add_uri_field(uint32_t uri_field_id)
+BundledFieldsContext::add_uri_field(uint32_t uri_field_id, uint32_t uri_all_field_id)
 {
     _uri_fields.emplace_back(uri_field_id);
+    _uri_all_field_ids.emplace_back(uri_all_field_id);
 }
 
 }
diff --git a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h
index fb1a68d7273..c058c14832d 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h
@@ -16,16 +16,18 @@ class BundledFieldsContext
     vespalib::ISequencedTaskExecutor::ExecutorId _id;
     std::vector<uint32_t>                        _fields;
     std::vector<uint32_t>                        _uri_fields;
+    std::vector<uint32_t>                        _uri_all_field_ids;
 protected:
     BundledFieldsContext(vespalib::ISequencedTaskExecutor::ExecutorId id);
     ~BundledFieldsContext();
 public:
     void add_field(uint32_t field_id);
-    void add_uri_field(uint32_t uri_field_id);
+    void add_uri_field(uint32_t uri_field_id, uint32_t uri_all_field_id);
     void set_id(vespalib::ISequencedTaskExecutor::ExecutorId id) { _id = id; }
     vespalib::ISequencedTaskExecutor::ExecutorId get_id() const noexcept { return _id; }
     const std::vector<uint32_t>& get_fields() const noexcept { return _fields; }
     const std::vector<uint32_t>& get_uri_fields() const noexcept { return _uri_fields; }
+    const std::vector<uint32_t>& get_uri_all_field_ids() const noexcept { return _uri_all_field_ids; }
 };
 
 }
diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp
index 8183cb005fe..93a12c24257 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp
@@ -15,20 +15,20 @@ template <typename Context>
 void make_contexts(const index::Schema& schema, const SchemaIndexFields& schema_index_fields, ISequencedTaskExecutor& executor, std::vector<Context>& contexts)
 {
     using ExecutorId = ISequencedTaskExecutor::ExecutorId;
-    using IdMapping = std::vector<std::tuple<ExecutorId, bool, uint32_t>>;
+    using IdMapping = std::vector<std::tuple<ExecutorId, bool, uint32_t, uint32_t>>;
     IdMapping map;
     for (uint32_t field_id : schema_index_fields._textFields) {
         // TODO: Add bias when sharing sequenced task executor between document types
         auto& name = schema.getIndexField(field_id).getName();
         auto id = executor.getExecutorIdFromName(name);
-        map.emplace_back(id, false, field_id);
+        map.emplace_back(id, false, field_id, 0);
     }
     uint32_t uri_field_id = 0;
     for (auto& uri_field : schema_index_fields._uriFields) {
         // TODO: Add bias when sharing sequenced task executor between document types
         auto& name = schema.getIndexField(uri_field._all).getName();
         auto id = executor.getExecutorIdFromName(name);
-        map.emplace_back(id, true, uri_field_id);
+        map.emplace_back(id, true, uri_field_id, uri_field._all);
         ++uri_field_id;
     }
     std::sort(map.begin(), map.end());
@@ -39,7 +39,7 @@ void make_contexts(const index::Schema& schema, const SchemaIndexFields& schema_
             prev_id = std::get<0>(entry);
         }
         if (std::get<1>(entry)) {
-            contexts.back().add_uri_field(std::get<2>(entry));
+            contexts.back().add_uri_field(std::get<2>(entry), std::get<3>(entry));
         } else {
             contexts.back().add_field(std::get<2>(entry));
         }
diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp
index 643bcbb325e..86421711e32 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp
@@ -7,13 +7,14 @@
 #include "field_index_collection.h"
 #include <vespa/document/fieldvalue/arrayfieldvalue.h>
 #include <vespa/document/fieldvalue/document.h>
-#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 #include <vespa/searchlib/index/field_length_calculator.h>
 #include <vespa/searchlib/index/schemautil.h>
 #include <vespa/searchlib/queryeval/create_blueprint_visitor_helper.h>
 #include <vespa/searchlib/queryeval/emptysearch.h>
 #include <vespa/searchlib/queryeval/leaf_blueprints.h>
 #include <vespa/vespalib/btree/btreenodeallocator.hpp>
+#include <vespa/vespalib/data/slime/cursor.h>
+#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchlib.memoryindex.memory_index");
@@ -47,6 +48,7 @@ using queryeval::FieldSpec;
 using queryeval::IRequestContext;
 using queryeval::Searchable;
 using vespalib::ISequencedTaskExecutor;
+using vespalib::slime::Cursor;
 
 }
 
@@ -251,4 +253,42 @@ MemoryIndex::get_field_length_info(const vespalib::string& field_name) const
     return FieldLengthInfo();
 }
 
+namespace {
+
+void
+fields_to_slime(const std::vector<uint32_t>& field_ids, const Schema& schema, Cursor& array)
+{
+    for (uint32_t field_id : field_ids) {
+        assert(field_id < schema.getIndexFields().size());
+        const auto& field = schema.getIndexField(field_id);
+        array.addString(field.getName());
+    }
+}
+
+void
+write_context_to_slime(const BundledFieldsContext& ctx, const Schema& schema, Cursor& object)
+{
+    object.setLong("executor_id", ctx.get_id().getId());
+    auto& fields = object.setArray("fields");
+    fields_to_slime(ctx.get_fields(), schema, fields);
+    fields_to_slime(ctx.get_uri_all_field_ids(), schema, fields);
+}
+
+}
+
+void
+MemoryIndex::insert_write_context_state(Cursor& object) const
+{
+    auto& invert = object.setArray("invert");
+    for (const auto& ctx : _inverter_context->get_invert_contexts()) {
+        auto& ctx_obj = invert.addObject();
+        write_context_to_slime(ctx, _schema, ctx_obj);
+    }
+    auto& push = object.setArray("push");
+    for (const auto& ctx : _inverter_context->get_push_contexts()) {
+        auto& ctx_obj = push.addObject();
+        write_context_to_slime(ctx, _schema, ctx_obj);
+    }
+}
+
 }
diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h
index af76ed172ba..320c6fba277 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h
@@ -17,7 +17,7 @@ namespace search::index {
 }
 
 namespace vespalib { class ISequencedTaskExecutor; }
-
+namespace vespalib::slime { struct Cursor; }
 namespace document { class Document; }
 
 namespace search::memoryindex {
@@ -175,6 +175,8 @@ public:
     uint64_t getStaticMemoryFootprint() const { return _staticMemoryFootprint; }
 
     index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const;
+
+    void insert_write_context_state(vespalib::slime::Cursor& object) const;
 };
 
 }