Add explorer for the attribute writer.

This shows which attribute fields that are assigned to each executor id.

12 files changed, 146 insertions, 27 deletions

diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index 0b498a791b3..e558074f724 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -1033,20 +1033,23 @@ TEST_F("require that underlying components are explorable", StoreOnlyExplorerFix
 {
     assertExplorer({}, f._explorer);
     EXPECT_TRUE(f._explorer.get_child("attribute").get() == nullptr);
+    EXPECT_TRUE(f._explorer.get_child("attributewriter").get() == nullptr);
     EXPECT_TRUE(f._explorer.get_child("index").get() == nullptr);
 }
 
 TEST_F("require that underlying components are explorable", FastAccessExplorerFixture)
 {
-    assertExplorer({"attribute"}, f._explorer);
+    assertExplorer({"attribute", "attributewriter"}, f._explorer);
     EXPECT_TRUE(f._explorer.get_child("attribute").get() != nullptr);
+    EXPECT_TRUE(f._explorer.get_child("attributewriter").get() != nullptr);
     EXPECT_TRUE(f._explorer.get_child("index").get() == nullptr);
 }
 
 TEST_F("require that underlying components are explorable", SearchableExplorerFixture)
 {
-    assertExplorer({"attribute", "index"}, f._explorer);
+    assertExplorer({"attribute", "attributewriter", "index"}, f._explorer);
     EXPECT_TRUE(f._explorer.get_child("attribute").get() != nullptr);
+    EXPECT_TRUE(f._explorer.get_child("attributewriter").get() != nullptr);
     EXPECT_TRUE(f._explorer.get_child("index").get() != nullptr);
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt
index 70a91b418a9..7e5d3accbc3 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt
@@ -3,8 +3,8 @@ vespa_add_library(searchcore_attribute STATIC
     SOURCES
     address_space_usage_stats.cpp
     attribute_aspect_delayer.cpp
-    attribute_collection_spec_factory.cpp
     attribute_collection_spec.cpp
+    attribute_collection_spec_factory.cpp
     attribute_config_inspector.cpp
     attribute_directory.cpp
     attribute_executor.cpp
@@ -24,9 +24,10 @@ vespa_add_library(searchcore_attribute STATIC
     attribute_usage_stats.cpp
     attribute_vector_explorer.cpp
     attribute_writer.cpp
-    attributes_initializer_base.cpp
+    attribute_writer_explorer.cpp
     attributedisklayout.cpp
     attributemanager.cpp
+    attributes_initializer_base.cpp
     attributesconfigscout.cpp
     document_field_extractor.cpp
     document_field_populator.cpp
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.cpp
new file mode 100644
index 00000000000..0e26dcdfa21
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.cpp
@@ -0,0 +1,68 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "attribute_writer.h"
+#include "attribute_writer_explorer.h"
+#include <vespa/searchcommon/attribute/config.h>
+#include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/vespalib/data/slime/cursor.h>
+
+using search::attribute::BasicType;
+using search::attribute::CollectionType;
+using search::attribute::Config;
+using vespalib::slime::Cursor;
+using vespalib::slime::Inserter;
+
+namespace proton {
+
+AttributeWriterExplorer::AttributeWriterExplorer(std::shared_ptr<IAttributeWriter> writer)
+    : _writer(std::move(writer))
+{
+}
+
+AttributeWriterExplorer::~AttributeWriterExplorer() = default;
+
+namespace {
+
+vespalib::string
+type_to_string(const Config& cfg)
+{
+    if (cfg.basicType().type() == BasicType::TENSOR) {
+        return cfg.tensorType().to_spec();
+    }
+    if (cfg.collectionType().type() == CollectionType::SINGLE) {
+        return cfg.basicType().asString();
+    }
+    return vespalib::string(cfg.collectionType().asString()) +
+        "<" + vespalib::string(cfg.basicType().asString()) + ">";
+}
+
+void
+convert_to_slime(const AttributeWriter::WriteContext& context, Cursor& object)
+{
+    object.setLong("executor_id", context.getExecutorId().getId());
+    Cursor& fields = object.setArray("attribute_fields");
+    for (const auto& field : context.getFields()) {
+        Cursor& f = fields.addObject();
+        f.setString("name", field.getAttribute().getName());
+        f.setString("type", type_to_string(field.getAttribute().getConfig()));
+    }
+}
+
+}
+
+void
+AttributeWriterExplorer::get_state(const Inserter& inserter, bool full) const
+{
+    Cursor& object = inserter.insertObject();
+    if (full) {
+        auto* writer = dynamic_cast<AttributeWriter*>(_writer.get());
+        if (writer) {
+            Cursor& contexts = object.setArray("write_contexts");
+            for (const auto& context : writer->get_write_contexts()) {
+                convert_to_slime(context, contexts.addObject());
+            }
+        }
+    }
+}
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.h
new file mode 100644
index 00000000000..5569c2ae132
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.h
@@ -0,0 +1,25 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_attribute_writer.h"
+#include <vespa/vespalib/net/http/state_explorer.h>
+
+namespace proton {
+
+/**
+ * Class used to explore the state of an attribute writer and its write contexts.
+ */
+class AttributeWriterExplorer : public vespalib::StateExplorer {
+private:
+    std::shared_ptr<IAttributeWriter> _writer;
+
+public:
+    AttributeWriterExplorer(std::shared_ptr<IAttributeWriter> writer);
+    ~AttributeWriterExplorer();
+
+    void get_state(const vespalib::slime::Inserter& inserter, bool full) const override;
+};
+
+}
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp
index 8049e76d4b6..cc657b04459 100644
--- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.cpp
@@ -1,10 +1,10 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "document_subdb_explorer.h"
-
 #include <vespa/searchcore/proton/attribute/attribute_manager_explorer.h>
-#include <vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.h>
+#include <vespa/searchcore/proton/attribute/attribute_writer_explorer.h>
 #include <vespa/searchcore/proton/docsummary/document_store_explorer.h>
+#include <vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.h>
 #include <vespa/searchcorespi/index/index_manager_explorer.h>
 
 using searchcorespi::IndexManagerExplorer;
@@ -18,6 +18,7 @@ namespace {
 const vespalib::string DOCUMENT_META_STORE = "documentmetastore";
 const vespalib::string DOCUMENT_STORE = "documentstore";
 const vespalib::string ATTRIBUTE = "attribute";
+const vespalib::string ATTRIBUTE_WRITER = "attributewriter";
 const vespalib::string INDEX = "index";
 
 }
@@ -38,10 +39,13 @@ std::vector<vespalib::string>
 DocumentSubDBExplorer::get_children_names() const
 {
     std::vector<vespalib::string> children = {DOCUMENT_META_STORE, DOCUMENT_STORE};
-    if (_subDb.getAttributeManager().get() != nullptr) {
+    if (_subDb.getAttributeManager()) {
         children.push_back(ATTRIBUTE);
     }
-    if (_subDb.getIndexManager().get() != nullptr) {
+    if (_subDb.get_attribute_writer()) {
+        children.push_back(ATTRIBUTE_WRITER);
+    }
+    if (_subDb.getIndexManager()) {
         children.push_back(INDEX);
     }
     return children;
@@ -53,22 +57,27 @@ DocumentSubDBExplorer::get_child(vespalib::stringref name) const
     if (name == DOCUMENT_META_STORE) {
         // TODO(geirst): Avoid const cast by adding const interface to
         // IDocumentMetaStoreContext as seen from IDocumentSubDB.
-        return std::unique_ptr<StateExplorer>(new DocumentMetaStoreExplorer(
-                (const_cast<IDocumentSubDB &>(_subDb)).getDocumentMetaStoreContext().getReadGuard()));
+        return std::make_unique<DocumentMetaStoreExplorer>(
+                (const_cast<IDocumentSubDB &>(_subDb)).getDocumentMetaStoreContext().getReadGuard());
     } else if (name == DOCUMENT_STORE) {
-        return std::unique_ptr<StateExplorer>(new DocumentStoreExplorer(_subDb.getSummaryManager()));
+        return std::make_unique<DocumentStoreExplorer>(_subDb.getSummaryManager());
     } else if (name == ATTRIBUTE) {
-        proton::IAttributeManager::SP attrMgr = _subDb.getAttributeManager();
-        if (attrMgr.get() != nullptr) {
-            return std::unique_ptr<StateExplorer>(new AttributeManagerExplorer(attrMgr));
+        auto attrMgr = _subDb.getAttributeManager();
+        if (attrMgr) {
+            return std::make_unique<AttributeManagerExplorer>(attrMgr);
+        }
+    } else if (name == ATTRIBUTE_WRITER) {
+        auto writer = _subDb.get_attribute_writer();
+        if (writer) {
+            return std::make_unique<AttributeWriterExplorer>(std::move(writer));
         }
     } else if (name == INDEX) {
-        searchcorespi::IIndexManager::SP idxMgr = _subDb.getIndexManager();
-        if (idxMgr.get() != nullptr) {
-            return std::unique_ptr<StateExplorer>(new IndexManagerExplorer(std::move(idxMgr)));
+        auto idxMgr = _subDb.getIndexManager();
+        if (idxMgr) {
+            return std::make_unique<IndexManagerExplorer>(std::move(idxMgr));
         }
     }
-    return std::unique_ptr<StateExplorer>();
+    return {};
 }
 
-} // namespace proton
+}
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
index 47dd8c03706..140bc4d170c 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp
@@ -282,6 +282,12 @@ FastAccessDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const
     return tasks;
 }
 
+std::shared_ptr<IAttributeWriter>
+FastAccessDocSubDB::get_attribute_writer() const
+{
+    return _fastAccessFeedView.get()->getAttributeWriter();
+}
+
 proton::IAttributeManager::SP
 FastAccessDocSubDB::getAttributeManager() const
 {
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h
index 942d42b5ad3..3a6eeef7dac 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h
@@ -119,6 +119,7 @@ public:
     applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot,
                 SerialNum serialNum, const ReconfigParams &params, IDocumentDBReferenceResolver &resolver, const DocumentSubDBReconfig& prepared_reconfig) override;
 
+    std::shared_ptr<IAttributeWriter> get_attribute_writer() const override;
     std::shared_ptr<IAttributeManager> getAttributeManager() const override;
     IDocumentRetriever::UP getDocumentRetriever() override;
     void onReplayDone() override;
diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
index 506af3f6355..92fa0dcad43 100644
--- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
@@ -30,6 +30,7 @@ class DocumentSubDBReconfig;
 class DocumentSubDbInitializer;
 class DocumentSubDbInitializerResult;
 class FeedHandler;
+class IAttributeWriter;
 class IDocumentDBReference;
 class IDocumentRetriever;
 class IFeedView;
@@ -92,6 +93,7 @@ public:
     virtual std::shared_ptr<IFeedView> getFeedView() const = 0;
     virtual void clearViews() = 0;
     virtual const std::shared_ptr<ISummaryManager> &getSummaryManager() const = 0;
+    virtual std::shared_ptr<IAttributeWriter> get_attribute_writer() const = 0;
     virtual std::shared_ptr<IAttributeManager> getAttributeManager() const = 0;
     virtual const std::shared_ptr<searchcorespi::IIndexManager> &getIndexManager() const = 0;
     virtual const std::shared_ptr<ISummaryAdapter> &getSummaryAdapter() const = 0;
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
index 4f694bcfd38..b846a3eceff 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
@@ -348,6 +348,12 @@ SearchableDocSubDB::clearViews() {
     Parent::clearViews();
 }
 
+std::shared_ptr<IAttributeWriter>
+SearchableDocSubDB::get_attribute_writer() const
+{
+    return _rFeedView.get()->getAttributeWriter();
+}
+
 TransientResourceUsage
 SearchableDocSubDB::get_transient_resource_usage() const
 {
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
index 97172f23bcb..7cfe2b5f444 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h
@@ -109,6 +109,8 @@ public:
 
     void clearViews() override;
 
+    std::shared_ptr<IAttributeWriter> get_attribute_writer() const override;
+
     std::shared_ptr<IAttributeManager> getAttributeManager() const override {
         return _rSearchView.get()->getAttributeManager();
     }
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
index 1c956a4c45c..7a6bca2cd1f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
@@ -214,6 +214,7 @@ public:
     void clearViews() override;
 
     const ISummaryManager::SP &getSummaryManager() const override { return _iSummaryMgr; }
+    std::shared_ptr<IAttributeWriter> get_attribute_writer() const override { return {}; }
     IAttributeManager::SP getAttributeManager() const override;
     const std::shared_ptr<searchcorespi::IIndexManager> & getIndexManager() const override;
     const ISummaryAdapter::SP & getSummaryAdapter() const override { return _summaryAdapter; }
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
index d53016d7d8d..12603f1195a 100644
--- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
@@ -66,14 +66,9 @@ struct DummyDocumentSubDb : public IDocumentSubDB
     IFeedView::SP getFeedView() const override { return IFeedView::SP(); }
     void clearViews() override {}
     const ISummaryManager::SP &getSummaryManager() const override { return _summaryManager; }
-    proton::IAttributeManager::SP getAttributeManager() const override {
-        return proton::IAttributeManager::SP();
-    }
-
-    void validateDocStore(FeedHandler &, SerialNum ) const override {
-
-    }
-
+    std::shared_ptr<IAttributeWriter> get_attribute_writer() const override { return {}; }
+    proton::IAttributeManager::SP getAttributeManager() const override { return {}; }
+    void validateDocStore(FeedHandler &, SerialNum ) const override {}
     const IIndexManager::SP &getIndexManager() const override { return _indexManager; }
     const ISummaryAdapter::SP &getSummaryAdapter() const override { return _summaryAdapter; }
     const IIndexWriter::SP &getIndexWriter() const override { return _indexWriter; }