Add index manager stats that can be used by both state explorer and

metrics updater.

10 files changed, 279 insertions, 1 deletions

diff --git a/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt b/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt
index 65b006f8ea9..84891e37307 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt
+++ b/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt
@@ -3,11 +3,13 @@ vespa_add_library(searchcorespi_index OBJECT
     SOURCES
     activediskindexes.cpp
     diskindexcleaner.cpp
+    disk_index_stats.cpp
     eventlogger.cpp
     fusionrunner.cpp
     iindexmanager.cpp
     iindexcollection.cpp
     index_manager_explorer.cpp
+    index_manager_stats.cpp
     indexcollection.cpp
     indexdisklayout.cpp
     indexflushtarget.cpp
@@ -18,6 +20,8 @@ vespa_add_library(searchcorespi_index OBJECT
     indexmanagerconfig.cpp
     indexreadutilities.cpp
     indexsearchable.cpp
+    index_searchable_stats.cpp
+    memory_index_stats.cpp
     indexwriteutilities.cpp
     warmupindexcollection.cpp
     isearchableindexcollection.cpp
diff --git a/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.cpp b/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.cpp
new file mode 100644
index 00000000000..f9d5215d6f5
--- /dev/null
+++ b/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.cpp
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "disk_index_stats.h"
+#include "idiskindex.h"
+
+
+namespace searchcorespi {
+namespace index {
+
+DiskIndexStats::DiskIndexStats()
+    : IndexSearchableStats(),
+      _indexDir()
+{
+}
+
+DiskIndexStats::DiskIndexStats(const IDiskIndex &index)
+    : IndexSearchableStats(index),
+      _indexDir(index.getIndexDir())
+{
+}
+
+DiskIndexStats::~DiskIndexStats()
+{
+}
+
+} // namespace searchcorespi::index
+} // namespace searchcorespi
diff --git a/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.h b/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.h
new file mode 100644
index 00000000000..579a840b756
--- /dev/null
+++ b/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.h
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "index_searchable_stats.h"
+#include <vespa/vespalib/stllike/string.h>
+
+namespace searchcorespi {
+namespace index {
+
+struct IDiskIndex;
+
+/**
+ * Information about a disk index usable by state explorer.
+ */
+class DiskIndexStats : public IndexSearchableStats {
+    vespalib::string _indexDir;
+public:
+    DiskIndexStats();
+    DiskIndexStats(const IDiskIndex &index);
+    ~DiskIndexStats();
+
+    const vespalib::string &getIndexdir() const { return _indexDir; }
+};
+
+} // namespace searchcorespi::index
+} // namespace searchcorespi
diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.cpp b/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.cpp
index f3695f2bd52..448dde9bac2 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.cpp
@@ -4,11 +4,13 @@
 #include <vespa/log/log.h>
 LOG_SETUP(".searchcorespi.index.index_manager_explorer");
 #include "index_manager_explorer.h"
+#include "index_manager_stats.h"
 
 #include <vespa/vespalib/data/slime/cursor.h>
 
 using vespalib::slime::Cursor;
 using vespalib::slime::Inserter;
+using search::SearchableStats;
 
 namespace searchcorespi {
 
@@ -20,9 +22,27 @@ IndexManagerExplorer::IndexManagerExplorer(IIndexManager::SP mgr)
 void
 IndexManagerExplorer::get_state(const Inserter &inserter, bool full) const
 {
-    (void) full;
     Cursor &object = inserter.insertObject();
     object.setLong("lastSerialNum", _mgr->getCurrentSerialNum());
+    if (full) {
+        IndexManagerStats stats(*_mgr);
+        Cursor &diskIndexArrayCursor = object.setArray("diskIndexes");
+        for (const auto &diskIndex : stats.getDiskIndexes()) {
+            Cursor &diskIndexCursor = diskIndexArrayCursor.addObject();
+            const SearchableStats &sstats = diskIndex.getSearchableStats();
+            diskIndexCursor.setLong("serialNum", diskIndex.getSerialNum());
+            diskIndexCursor.setString("indexDir", diskIndex.getIndexdir());
+            diskIndexCursor.setLong("sizeOnDisk", sstats.sizeOnDisk());
+        }
+        Cursor &memoryIndexArrayCursor = object.setArray("memoryIndexes");
+        for (const auto &memoryIndex : stats.getMemoryIndexes()) {
+            Cursor &memoryIndexCursor = memoryIndexArrayCursor.addObject();
+            const SearchableStats &sstats = memoryIndex.getSearchableStats();
+            memoryIndexCursor.setLong("serialNum", memoryIndex.getSerialNum());
+            memoryIndexCursor.setLong("docsInMemory", sstats.docsInMemory());
+            memoryIndexCursor.setLong("memoryUsage", sstats.memoryUsage());
+        }
+    }
 }
 
 } // namespace searchcorespi
diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp b/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp
new file mode 100644
index 00000000000..035a5698c2f
--- /dev/null
+++ b/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "index_manager_stats.h"
+#include "iindexmanager.h"
+#include "indexsearchablevisitor.h"
+
+
+namespace searchcorespi {
+
+namespace {
+
+class Visitor : public IndexSearchableVisitor
+{
+public:
+    std::vector<index::DiskIndexStats> _diskIndexes;
+    std::vector<index::MemoryIndexStats> _memoryIndexes;
+
+    Visitor()
+        : _diskIndexes(),
+          _memoryIndexes()
+    {
+    }
+    virtual void visit(const index::IDiskIndex &index) override {
+        _diskIndexes.emplace_back(index);
+    }
+    virtual void visit(const index::IMemoryIndex &index) override {
+        _memoryIndexes.emplace_back(index);
+    }
+
+    void normalize() {
+        std::sort(_diskIndexes.begin(), _diskIndexes.end());
+        std::sort(_memoryIndexes.begin(), _memoryIndexes.end());
+    }
+};
+
+}
+
+IndexManagerStats::IndexManagerStats()
+    : _diskIndexes(),
+      _memoryIndexes()
+{
+}
+
+IndexManagerStats::IndexManagerStats(const IIndexManager &indexManager)
+    : _diskIndexes(),
+      _memoryIndexes()
+{
+    Visitor visitor;
+    IndexSearchable::SP searchable(indexManager.getSearchable());
+    searchable->accept(visitor);
+    visitor.normalize();
+    _diskIndexes = std::move(visitor._diskIndexes);
+    _memoryIndexes = std::move(visitor._memoryIndexes);
+}
+
+IndexManagerStats::~IndexManagerStats()
+{
+}
+
+} // namespace searchcorespi
diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.h b/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.h
new file mode 100644
index 00000000000..048124b688f
--- /dev/null
+++ b/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.h
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "disk_index_stats.h"
+#include "memory_index_stats.h"
+
+namespace searchcorespi {
+
+class IIndexManager;
+
+/**
+ * Information about an index manager usable by state explorer.
+ */
+class IndexManagerStats {
+    std::vector<index::DiskIndexStats> _diskIndexes;
+    std::vector<index::MemoryIndexStats> _memoryIndexes;
+public:
+    IndexManagerStats();
+    IndexManagerStats(const IIndexManager &indexManager);
+    ~IndexManagerStats();
+
+    const std::vector<index::DiskIndexStats> &getDiskIndexes() const {
+        return _diskIndexes;
+    }
+    const std::vector<index::MemoryIndexStats> &getMemoryIndexes() const {
+        return _memoryIndexes;
+    }
+};
+
+} // namespace searchcorespi
diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.cpp b/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.cpp
new file mode 100644
index 00000000000..c5ce5163aac
--- /dev/null
+++ b/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.cpp
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "index_searchable_stats.h"
+#include "indexsearchable.h"
+
+
+namespace searchcorespi {
+namespace index {
+
+IndexSearchableStats::IndexSearchableStats()
+    : _serialNum(0),
+      _searchableStats()
+{
+}
+
+IndexSearchableStats::IndexSearchableStats(const IndexSearchable &index)
+    : _serialNum(index.getSerialNum()),
+      _searchableStats(index.getSearchableStats())
+{
+}
+
+bool IndexSearchableStats::operator<(const IndexSearchableStats &rhs) const
+{
+    return _serialNum < rhs._serialNum;
+}
+
+} // namespace searchcorespi::index
+} // namespace searchcorespi
diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.h b/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.h
new file mode 100644
index 00000000000..dd904b3b286
--- /dev/null
+++ b/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.h
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/common/serialnum.h>
+#include <vespa/searchlib/util/searchable_stats.h>
+
+namespace searchcorespi {
+
+class IndexSearchable;
+
+namespace index {
+
+/**
+ * Information about a searchable index usable by state explorer.
+ */
+class IndexSearchableStats
+{
+    using SerialNum = search::SerialNum;
+    using SearchableStats = search::SearchableStats;
+    SerialNum _serialNum;
+    SearchableStats _searchableStats;
+public:
+    IndexSearchableStats();
+    IndexSearchableStats(const IndexSearchable &index);
+    bool operator<(const IndexSearchableStats &rhs) const;
+    SerialNum getSerialNum() const { return _serialNum; }
+    const SearchableStats &getSearchableStats() const { return _searchableStats; }
+};
+
+} // namespace searchcorespi::index
+} // namespace searchcorespi
diff --git a/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.cpp b/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.cpp
new file mode 100644
index 00000000000..7805702fd03
--- /dev/null
+++ b/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.cpp
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "memory_index_stats.h"
+#include "imemoryindex.h"
+
+
+namespace searchcorespi {
+namespace index {
+
+MemoryIndexStats::MemoryIndexStats()
+    : IndexSearchableStats()
+{
+}
+
+MemoryIndexStats::MemoryIndexStats(const IMemoryIndex &index)
+    : IndexSearchableStats(index)
+{
+}
+
+MemoryIndexStats::~MemoryIndexStats()
+{
+}
+
+} // namespace searchcorespi::index
+} // namespace searchcorespi
diff --git a/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.h b/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.h
new file mode 100644
index 00000000000..7508ecef3cc
--- /dev/null
+++ b/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.h
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "index_searchable_stats.h"
+
+namespace searchcorespi {
+namespace index {
+
+struct IMemoryIndex;
+
+/**
+ * Information about a memory index usable by state explorer.
+ */
+class MemoryIndexStats : public IndexSearchableStats {
+public:
+    MemoryIndexStats();
+    MemoryIndexStats(const IMemoryIndex &index);
+    ~MemoryIndexStats();
+};
+
+} // namespace searchcorespi::index
+} // namespace searchcorespi