diff options
author | Tor Egge <Tor.Egge@yahoo-inc.com> | 2016-09-21 09:45:44 +0000 |
---|---|---|
committer | Tor Egge <Tor.Egge@yahoo-inc.com> | 2016-09-21 10:33:06 +0000 |
commit | 97c19d53eb79e3ae13dab8ee89ea7b36ab4a7a7d (patch) | |
tree | 92722a487b4100bf62a92cb8045d0fb9adfbe224 /searchcorespi | |
parent | d52f2543aa0aa8c26b9719f8032174699e0c73e3 (diff) |
Add index manager stats that can be used by both state explorer and
metrics updater.
Diffstat (limited to 'searchcorespi')
10 files changed, 279 insertions, 1 deletions
diff --git a/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt b/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt index 65b006f8ea9..84891e37307 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt +++ b/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt @@ -3,11 +3,13 @@ vespa_add_library(searchcorespi_index OBJECT SOURCES activediskindexes.cpp diskindexcleaner.cpp + disk_index_stats.cpp eventlogger.cpp fusionrunner.cpp iindexmanager.cpp iindexcollection.cpp index_manager_explorer.cpp + index_manager_stats.cpp indexcollection.cpp indexdisklayout.cpp indexflushtarget.cpp @@ -18,6 +20,8 @@ vespa_add_library(searchcorespi_index OBJECT indexmanagerconfig.cpp indexreadutilities.cpp indexsearchable.cpp + index_searchable_stats.cpp + memory_index_stats.cpp indexwriteutilities.cpp warmupindexcollection.cpp isearchableindexcollection.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.cpp b/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.cpp new file mode 100644 index 00000000000..f9d5215d6f5 --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.cpp @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "disk_index_stats.h" +#include "idiskindex.h" + + +namespace searchcorespi { +namespace index { + +DiskIndexStats::DiskIndexStats() + : IndexSearchableStats(), + _indexDir() +{ +} + +DiskIndexStats::DiskIndexStats(const IDiskIndex &index) + : IndexSearchableStats(index), + _indexDir(index.getIndexDir()) +{ +} + +DiskIndexStats::~DiskIndexStats() +{ +} + +} // namespace searchcorespi::index +} // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.h b/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.h new file mode 100644 index 00000000000..579a840b756 --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.h @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "index_searchable_stats.h" +#include <vespa/vespalib/stllike/string.h> + +namespace searchcorespi { +namespace index { + +struct IDiskIndex; + +/** + * Information about a disk index usable by state explorer. + */ +class DiskIndexStats : public IndexSearchableStats { + vespalib::string _indexDir; +public: + DiskIndexStats(); + DiskIndexStats(const IDiskIndex &index); + ~DiskIndexStats(); + + const vespalib::string &getIndexdir() const { return _indexDir; } +}; + +} // namespace searchcorespi::index +} // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.cpp b/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.cpp index f3695f2bd52..448dde9bac2 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.cpp @@ -4,11 +4,13 @@ #include <vespa/log/log.h> LOG_SETUP(".searchcorespi.index.index_manager_explorer"); #include "index_manager_explorer.h" +#include "index_manager_stats.h" #include <vespa/vespalib/data/slime/cursor.h> using vespalib::slime::Cursor; using vespalib::slime::Inserter; +using search::SearchableStats; namespace searchcorespi { @@ -20,9 +22,27 @@ IndexManagerExplorer::IndexManagerExplorer(IIndexManager::SP mgr) void IndexManagerExplorer::get_state(const Inserter &inserter, bool full) const { - (void) full; Cursor &object = inserter.insertObject(); object.setLong("lastSerialNum", _mgr->getCurrentSerialNum()); + if (full) { + IndexManagerStats stats(*_mgr); + Cursor &diskIndexArrayCursor = object.setArray("diskIndexes"); + for (const auto &diskIndex : stats.getDiskIndexes()) { + Cursor &diskIndexCursor = diskIndexArrayCursor.addObject(); + const SearchableStats &sstats = diskIndex.getSearchableStats(); + diskIndexCursor.setLong("serialNum", diskIndex.getSerialNum()); + diskIndexCursor.setString("indexDir", diskIndex.getIndexdir()); + diskIndexCursor.setLong("sizeOnDisk", sstats.sizeOnDisk()); + } + Cursor &memoryIndexArrayCursor = object.setArray("memoryIndexes"); + for (const auto &memoryIndex : stats.getMemoryIndexes()) { + Cursor &memoryIndexCursor = memoryIndexArrayCursor.addObject(); + const SearchableStats &sstats = memoryIndex.getSearchableStats(); + memoryIndexCursor.setLong("serialNum", memoryIndex.getSerialNum()); + memoryIndexCursor.setLong("docsInMemory", sstats.docsInMemory()); + memoryIndexCursor.setLong("memoryUsage", sstats.memoryUsage()); + } + } } } // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp b/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp new file mode 100644 index 00000000000..035a5698c2f --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "index_manager_stats.h" +#include "iindexmanager.h" +#include "indexsearchablevisitor.h" + + +namespace searchcorespi { + +namespace { + +class Visitor : public IndexSearchableVisitor +{ +public: + std::vector<index::DiskIndexStats> _diskIndexes; + std::vector<index::MemoryIndexStats> _memoryIndexes; + + Visitor() + : _diskIndexes(), + _memoryIndexes() + { + } + virtual void visit(const index::IDiskIndex &index) override { + _diskIndexes.emplace_back(index); + } + virtual void visit(const index::IMemoryIndex &index) override { + _memoryIndexes.emplace_back(index); + } + + void normalize() { + std::sort(_diskIndexes.begin(), _diskIndexes.end()); + std::sort(_memoryIndexes.begin(), _memoryIndexes.end()); + } +}; + +} + +IndexManagerStats::IndexManagerStats() + : _diskIndexes(), + _memoryIndexes() +{ +} + +IndexManagerStats::IndexManagerStats(const IIndexManager &indexManager) + : _diskIndexes(), + _memoryIndexes() +{ + Visitor visitor; + IndexSearchable::SP searchable(indexManager.getSearchable()); + searchable->accept(visitor); + visitor.normalize(); + _diskIndexes = std::move(visitor._diskIndexes); + _memoryIndexes = std::move(visitor._memoryIndexes); +} + +IndexManagerStats::~IndexManagerStats() +{ +} + +} // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.h b/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.h new file mode 100644 index 00000000000..048124b688f --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.h @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "disk_index_stats.h" +#include "memory_index_stats.h" + +namespace searchcorespi { + +class IIndexManager; + +/** + * Information about an index manager usable by state explorer. + */ +class IndexManagerStats { + std::vector<index::DiskIndexStats> _diskIndexes; + std::vector<index::MemoryIndexStats> _memoryIndexes; +public: + IndexManagerStats(); + IndexManagerStats(const IIndexManager &indexManager); + ~IndexManagerStats(); + + const std::vector<index::DiskIndexStats> &getDiskIndexes() const { + return _diskIndexes; + } + const std::vector<index::MemoryIndexStats> &getMemoryIndexes() const { + return _memoryIndexes; + } +}; + +} // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.cpp b/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.cpp new file mode 100644 index 00000000000..c5ce5163aac --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.cpp @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "index_searchable_stats.h" +#include "indexsearchable.h" + + +namespace searchcorespi { +namespace index { + +IndexSearchableStats::IndexSearchableStats() + : _serialNum(0), + _searchableStats() +{ +} + +IndexSearchableStats::IndexSearchableStats(const IndexSearchable &index) + : _serialNum(index.getSerialNum()), + _searchableStats(index.getSearchableStats()) +{ +} + +bool IndexSearchableStats::operator<(const IndexSearchableStats &rhs) const +{ + return _serialNum < rhs._serialNum; +} + +} // namespace searchcorespi::index +} // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.h b/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.h new file mode 100644 index 00000000000..dd904b3b286 --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.h @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/common/serialnum.h> +#include <vespa/searchlib/util/searchable_stats.h> + +namespace searchcorespi { + +class IndexSearchable; + +namespace index { + +/** + * Information about a searchable index usable by state explorer. + */ +class IndexSearchableStats +{ + using SerialNum = search::SerialNum; + using SearchableStats = search::SearchableStats; + SerialNum _serialNum; + SearchableStats _searchableStats; +public: + IndexSearchableStats(); + IndexSearchableStats(const IndexSearchable &index); + bool operator<(const IndexSearchableStats &rhs) const; + SerialNum getSerialNum() const { return _serialNum; } + const SearchableStats &getSearchableStats() const { return _searchableStats; } +}; + +} // namespace searchcorespi::index +} // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.cpp b/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.cpp new file mode 100644 index 00000000000..7805702fd03 --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.cpp @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "memory_index_stats.h" +#include "imemoryindex.h" + + +namespace searchcorespi { +namespace index { + +MemoryIndexStats::MemoryIndexStats() + : IndexSearchableStats() +{ +} + +MemoryIndexStats::MemoryIndexStats(const IMemoryIndex &index) + : IndexSearchableStats(index) +{ +} + +MemoryIndexStats::~MemoryIndexStats() +{ +} + +} // namespace searchcorespi::index +} // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.h b/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.h new file mode 100644 index 00000000000..7508ecef3cc --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.h @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "index_searchable_stats.h" + +namespace searchcorespi { +namespace index { + +struct IMemoryIndex; + +/** + * Information about a memory index usable by state explorer. + */ +class MemoryIndexStats : public IndexSearchableStats { +public: + MemoryIndexStats(); + MemoryIndexStats(const IMemoryIndex &index); + ~MemoryIndexStats(); +}; + +} // namespace searchcorespi::index +} // namespace searchcorespi |