Report hash and B-tree memory usage separately for enum store dictionary.

3 files changed, 75 insertions, 18 deletions

diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt
index 8b53b4e093f..ca53d04c0cb 100644
--- a/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt
+++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/CMakeLists.txt
@@ -6,5 +6,6 @@ vespa_add_executable(searchcore_attributes_state_explorer_test_app TEST
     searchcore_attribute
     searchcore_pcommon
     searchcore_test
+    GTest::GTest
 )
 vespa_add_test(NAME searchcore_attributes_state_explorer_test_app COMMAND searchcore_attributes_state_explorer_test_app)
diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp
index 8745a4ffeaa..3fc80cfa5c5 100644
--- a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp
@@ -7,10 +7,10 @@
 #include <vespa/searchcore/proton/test/attribute_vectors.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
 #include <vespa/searchlib/test/directory_handler.h>
-#include <vespa/vespalib/test/insertion_operators.h>
-#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/gtest/gtest.h>
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
 #include <vespa/vespalib/util/foreground_thread_executor.h>
-#include <vespa/vespalib/util/foregroundtaskexecutor.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("attributes_state_explorer_test");
@@ -18,64 +18,107 @@ LOG_SETUP("attributes_state_explorer_test");
 using namespace proton;
 using namespace proton::test;
 using search::AttributeVector;
-using vespalib::ForegroundTaskExecutor;
+using search::DictionaryConfig;
 using vespalib::ForegroundThreadExecutor;
+using vespalib::ISequencedTaskExecutor;
+using vespalib::SequencedTaskExecutor;
+using vespalib::Slime;
 using search::TuneFileAttributes;
 using search::index::DummyFileHeaderContext;
 using search::test::DirectoryHandler;
 
 const vespalib::string TEST_DIR = "test_output";
 
-struct Fixture
+namespace {
+VESPA_THREAD_STACK_TAG(test_executor)
+}
+
+struct AttributesStateExplorerTest : public ::testing::Test
 {
     DirectoryHandler _dirHandler;
     DummyFileHeaderContext _fileHeaderContext;
-    ForegroundTaskExecutor _attributeFieldWriter;
+    std::unique_ptr<ISequencedTaskExecutor> _attribute_field_writer;
     ForegroundThreadExecutor _shared;
     HwInfo                 _hwInfo;
     AttributeManager::SP _mgr;
     AttributeManagerExplorer _explorer;
-    Fixture()
+    AttributesStateExplorerTest()
         : _dirHandler(TEST_DIR),
           _fileHeaderContext(),
-          _attributeFieldWriter(),
+          _attribute_field_writer(SequencedTaskExecutor::create(test_executor, 1)),
           _shared(),
           _hwInfo(),
           _mgr(new AttributeManager(TEST_DIR, "test.subdb", TuneFileAttributes(),
                                     _fileHeaderContext,
-                                    _attributeFieldWriter,
+                                    *_attribute_field_writer,
                                     _shared,
                                     _hwInfo)),
           _explorer(_mgr)
     {
         addAttribute("regular");
         addExtraAttribute("extra");
+        add_fast_search_attribute("btree", DictionaryConfig::Type::BTREE);
+        add_fast_search_attribute("hybrid", DictionaryConfig::Type::BTREE_AND_HASH);
+        add_fast_search_attribute("hash", DictionaryConfig::Type::HASH);
     }
     void addAttribute(const vespalib::string &name) {
         _mgr->addAttribute({name, AttributeUtils::getInt32Config()}, 1);
     }
+    void add_fast_search_attribute(const vespalib::string &name,
+                                   DictionaryConfig::Type dictionary_type) {
+        search::attribute::Config cfg = AttributeUtils::getInt32Config();
+        cfg.setFastSearch(true);
+        cfg.set_dictionary_config(search::DictionaryConfig(dictionary_type));
+        _mgr->addAttribute({name, cfg}, 1);
+    }
     void addExtraAttribute(const vespalib::string &name) {
         _mgr->addExtraAttribute(createInt32Attribute(name));
     }
+    Slime explore_attribute(const vespalib::string &name) {
+        Slime result;
+        vespalib::slime::SlimeInserter inserter(result);
+        _explorer.get_child(name)->get_state(inserter, true);
+        return result;
+    }
+
 };
 
 typedef std::vector<vespalib::string> StringVector;
 
-TEST_F("require that attributes are exposed as children names", Fixture)
+TEST_F(AttributesStateExplorerTest, require_that_attributes_are_exposed_as_children_names)
 {
-    StringVector children = f._explorer.get_children_names();
+    StringVector children = _explorer.get_children_names();
     std::sort(children.begin(), children.end());
-    EXPECT_EQUAL(StringVector({"extra", "regular"}), children);
+    EXPECT_EQ(StringVector({"btree", "extra", "hash", "hybrid", "regular"}), children);
 }
 
-TEST_F("require that attributes are explorable", Fixture)
+TEST_F(AttributesStateExplorerTest, require_that_attributes_are_explorable)
 {
-    EXPECT_TRUE(f._explorer.get_child("regular").get() != nullptr);
-    EXPECT_TRUE(f._explorer.get_child("extra").get() != nullptr);
-    EXPECT_TRUE(f._explorer.get_child("not").get() == nullptr);
+    EXPECT_TRUE(_explorer.get_child("regular").get() != nullptr);
+    EXPECT_TRUE(_explorer.get_child("extra").get() != nullptr);
+    EXPECT_TRUE(_explorer.get_child("not").get() == nullptr);
 }
 
-TEST_MAIN()
+TEST_F(AttributesStateExplorerTest, require_that_dictionary_memory_usage_is_reported)
 {
-    TEST_RUN_ALL();
+    {
+        auto slime = explore_attribute("btree");
+        auto& dictionary = slime.get()["enumStore"]["dictionary"];
+        EXPECT_LT(0, dictionary["btreeMemoryUsage"]["used"].asLong());
+        EXPECT_EQ(0, dictionary["hashMemoryUsage"]["used"].asLong());
+    }
+    {
+        auto slime = explore_attribute("hash");
+        auto& dictionary = slime.get()["enumStore"]["dictionary"];
+        EXPECT_EQ(0, dictionary["btreeMemoryUsage"]["used"].asLong());
+        EXPECT_LT(0, dictionary["hashMemoryUsage"]["used"].asLong());
+    }
+    {
+        auto slime = explore_attribute("hybrid");
+        auto& dictionary = slime.get()["enumStore"]["dictionary"];
+        EXPECT_LT(0, dictionary["btreeMemoryUsage"]["used"].asLong());
+        EXPECT_LT(0, dictionary["hashMemoryUsage"]["used"].asLong());
+    }
 }
+
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp
index 495fc5d63b3..2ffe3f2c901 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp
@@ -2,6 +2,7 @@
 
 #include "attribute_vector_explorer.h"
 #include <vespa/searchlib/attribute/i_enum_store.h>
+#include <vespa/searchlib/attribute/i_enum_store_dictionary.h>
 #include <vespa/searchlib/attribute/multi_value_mapping.h>
 #include <vespa/searchlib/attribute/attributevector.h>
 #include <vespa/searchlib/attribute/ipostinglistattributebase.h>
@@ -73,11 +74,23 @@ convertMemoryUsageToSlime(const MemoryUsage &usage, Cursor &object)
 }
 
 void
+convert_enum_store_dictionary_to_slime(const search::IEnumStoreDictionary &dictionary, Cursor &object)
+{
+    if (dictionary.get_has_btree_dictionary()) {
+        convertMemoryUsageToSlime(dictionary.get_btree_memory_usage(), object.setObject("btreeMemoryUsage"));
+    }
+    if (dictionary.get_has_hash_dictionary()) {
+        convertMemoryUsageToSlime(dictionary.get_hash_memory_usage(), object.setObject("hashMemoryUsage"));
+    }
+}
+
+void
 convertEnumStoreToSlime(const IEnumStore &enumStore, Cursor &object)
 {
     object.setLong("numUniques", enumStore.get_num_uniques());
     convertMemoryUsageToSlime(enumStore.get_values_memory_usage(), object.setObject("valuesMemoryUsage"));
     convertMemoryUsageToSlime(enumStore.get_dictionary_memory_usage(), object.setObject("dictionaryMemoryUsage"));
+    convert_enum_store_dictionary_to_slime(enumStore.get_dictionary(), object.setObject("dictionary"));
 }
 
 void