Merge pull request #1337 from yahoo/geirst/index-memory-usage-metrics

Geirst/index memory usage metrics

9 files changed, 41 insertions, 23 deletions

diff --git a/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp b/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp
index 29df68b2d17..36cdd7ff89f 100644
--- a/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp
+++ b/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp
@@ -61,8 +61,8 @@ TEST("require that the metric proton.diskusage is the sum of the documentDB disk
 
     DocumentDBMetricsCollection metrics1("type1", 1);
     DocumentDBMetricsCollection metrics2("type2", 1);
-    metrics1.getMetrics().index.diskUsage.addValue(100);
-    metrics2.getMetrics().index.diskUsage.addValue(1000);
+    metrics1.getLegacyMetrics().index.diskUsage.addValue(100);
+    metrics2.getLegacyMetrics().index.diskUsage.addValue(1000);
 
     metrics_engine.addDocumentDBMetrics(metrics1);
     metrics_engine.addDocumentDBMetrics(metrics2);
diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h
index a4ef714d09d..6521c98bcdb 100644
--- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h
+++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h
@@ -51,7 +51,7 @@ public:
     }
     virtual search::SearchableStats getSearchableStats() const override {
         return search::SearchableStats()
-            .memoryUsage(getMemoryUsage().allocatedBytes())
+            .memoryUsage(getMemoryUsage())
             .docsInMemory(_index.getNumDocs())
             .sizeOnDisk(0);
     }
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics.cpp
index 7afc7b0ab0a..e6a03c8df52 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics.cpp
+++ b/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics.cpp
@@ -8,7 +8,7 @@ namespace proton {
 using Entry = AttributeMetrics::Entry;
 
 AttributeMetrics::Entry::Entry(const vespalib::string &attrName)
-    : metrics::MetricSet("attribute", {{"fieldname", attrName}}, "Metrics for a given attribute vector", nullptr),
+    : metrics::MetricSet("attribute", {{"field", attrName}}, "Metrics for a given attribute vector", nullptr),
       memoryUsage(this)
 {
 }
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_metrics_collection.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_metrics_collection.h
index 20e72a23edf..3a2b0ff8633 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_metrics_collection.h
+++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_metrics_collection.h
@@ -21,7 +21,7 @@ public:
           _taggedMetrics(docTypeName)
     {
     }
-    LegacyDocumentDBMetrics &getMetrics() { return _metrics; }
+    LegacyDocumentDBMetrics &getLegacyMetrics() { return _metrics; }
     DocumentDBTaggedMetrics &getTaggedMetrics() { return _taggedMetrics; }
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp
index 68580d90974..213c869a8e4 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp
+++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp
@@ -76,10 +76,17 @@ DocumentDBTaggedMetrics::AttributeMetrics::ResourceUsageMetrics::ResourceUsageMe
 {
 }
 
+DocumentDBTaggedMetrics::IndexMetrics::IndexMetrics(MetricSet *parent)
+    : MetricSet("index", "", "Index metrics (memory and disk) for this document db", parent),
+      memoryUsage(this)
+{
+}
+
 DocumentDBTaggedMetrics::DocumentDBTaggedMetrics(const vespalib::string &docTypeName)
     : MetricSet("documentdb", {{"documenttype", docTypeName}}, "Document DB metrics", nullptr),
       job(this),
       attribute(this),
+      index(this),
       ready("ready", this),
       notReady("notready", this),
       removed("removed", this)
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h
index 39f4733f66c..323d0766e6f 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h
+++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h
@@ -76,8 +76,16 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet
         AttributeMetrics(metrics::MetricSet *parent);
     };
 
+    struct IndexMetrics : metrics::MetricSet
+    {
+        MemoryUsageMetrics memoryUsage;
+
+        IndexMetrics(metrics::MetricSet *parent);
+    };
+
     JobMetrics job;
     AttributeMetrics attribute;
+    IndexMetrics index;
     SubDBMetrics ready;
     SubDBMetrics notReady;
     SubDBMetrics removed;
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp
index 59d33675d34..23daf44d98a 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp
+++ b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp
@@ -112,7 +112,7 @@ void
 MetricsEngine::addDocumentDBMetrics(DocumentDBMetricsCollection &child)
 {
     metrics::MetricLockGuard guard(_manager->getMetricLock());
-    addLegacyDocumentDBMetrics(_legacyRoot, child.getMetrics());
+    addLegacyDocumentDBMetrics(_legacyRoot, child.getLegacyMetrics());
 
     _root.registerMetric(child.getTaggedMetrics());
 }
@@ -121,7 +121,7 @@ void
 MetricsEngine::removeDocumentDBMetrics(DocumentDBMetricsCollection &child)
 {
     metrics::MetricLockGuard guard(_manager->getMetricLock());
-    removeLegacyDocumentDBMetrics(_legacyRoot, child.getMetrics());
+    removeLegacyDocumentDBMetrics(_legacyRoot, child.getLegacyMetrics());
 
     _root.unregisterMetric(child.getTaggedMetrics());
 }
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index b21d6c1eba8..86a2f3460ee 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -132,7 +132,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir,
       _feedHandler(_writeService,
                    tlsSpec,
                    docTypeName,
-                   getMetricsCollection().getMetrics().feed,
+                   getMetricsCollection().getLegacyMetrics().feed,
                    _state,
                    *this,
                    _writeFilter,
@@ -559,7 +559,7 @@ DocumentDB::close()
 
     // The attributes in the ready sub db is also the total set of attributes.
     DocumentDBTaggedMetrics &metrics = getMetricsCollection().getTaggedMetrics();
-    LegacyDocumentDBMetrics &legacyMetrics = getMetricsCollection().getMetrics();
+    LegacyDocumentDBMetrics &legacyMetrics = getMetricsCollection().getLegacyMetrics();
     AttributeMetricsCollection ready(metrics.ready.attributes, legacyMetrics.ready.attributes);
     AttributeMetricsCollection notReady(metrics.notReady.attributes, legacyMetrics.notReady.attributes);
     _metricsWireService.cleanAttributes(ready, &legacyMetrics.attributes);
@@ -1245,12 +1245,16 @@ DocumentDB::getIndexManagerFactory(const vespalib::stringref &name) const
 namespace {
 
 void
-updateIndexMetrics(LegacyDocumentDBMetrics::IndexMetrics &metrics,
+updateIndexMetrics(DocumentDBMetricsCollection &metrics,
                    const search::SearchableStats &stats)
 {
-    metrics.memoryUsage.set(stats.memoryUsage());
-    metrics.docsInMemory.set(stats.docsInMemory());
-    metrics.diskUsage.set(stats.sizeOnDisk());
+    DocumentDBTaggedMetrics::IndexMetrics &indexMetrics = metrics.getTaggedMetrics().index;
+    indexMetrics.memoryUsage.update(stats.memoryUsage());
+
+    LegacyDocumentDBMetrics::IndexMetrics &legacyIndexMetrics = metrics.getLegacyMetrics().index;
+    legacyIndexMetrics.memoryUsage.set(stats.memoryUsage().allocatedBytes());
+    legacyIndexMetrics.docsInMemory.set(stats.docsInMemory());
+    legacyIndexMetrics.diskUsage.set(stats.sizeOnDisk());
 }
 
 struct TempAttributeMetric
@@ -1362,9 +1366,9 @@ updateAttributeMetrics(DocumentDBMetricsCollection &metrics,
     TempAttributeMetrics notReadyMetrics;
     fillTempAttributeMetrics(totalMetrics, readyMetrics, notReadyMetrics, subDbs);
 
-    updateLegacyAttributeMetrics(metrics.getMetrics().attributes, totalMetrics);
-    updateLegacyAttributeMetrics(metrics.getMetrics().ready.attributes, readyMetrics);
-    updateLegacyAttributeMetrics(metrics.getMetrics().notReady.attributes, notReadyMetrics);
+    updateLegacyAttributeMetrics(metrics.getLegacyMetrics().attributes, totalMetrics);
+    updateLegacyAttributeMetrics(metrics.getLegacyMetrics().ready.attributes, readyMetrics);
+    updateLegacyAttributeMetrics(metrics.getLegacyMetrics().notReady.attributes, notReadyMetrics);
 
     updateAttributeMetrics(metrics.getTaggedMetrics().ready.attributes, readyMetrics);
     updateAttributeMetrics(metrics.getTaggedMetrics().notReady.attributes, notReadyMetrics);
@@ -1455,7 +1459,8 @@ updateLidSpaceMetrics(MetricSetType &metrics,
 void
 DocumentDB::updateMetrics(DocumentDBMetricsCollection &metrics)
 {
-    updateLegacyMetrics(metrics.getMetrics());
+    updateLegacyMetrics(metrics.getLegacyMetrics());
+    updateIndexMetrics(metrics, _subDBs.getReadySubDB()->getSearchableStats());
     updateAttributeMetrics(metrics, _subDBs);
     updateMetrics(metrics.getTaggedMetrics());
 }
@@ -1463,8 +1468,6 @@ DocumentDB::updateMetrics(DocumentDBMetricsCollection &metrics)
 void
 DocumentDB::updateLegacyMetrics(LegacyDocumentDBMetrics &metrics)
 {
-    updateIndexMetrics(metrics.index,
-                       _subDBs.getReadySubDB()->getSearchableStats());
     updateMatchingMetrics(metrics.matching, *_subDBs.getReadySubDB());
     metrics.executor.update(_writeService.getMasterExecutor().getStats());
     metrics.indexExecutor.update(_writeService.getIndexExecutor().getStats());
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
index dfb85ac47f1..06d6162baee 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
@@ -65,7 +65,7 @@ DocumentSubDBCollection::DocumentSubDBCollection(
                                        summaryExecutor,
                                        _bucketDB,
                                        *_bucketDBHandler,
-                                       metrics.getMetrics(),
+                                       metrics.getLegacyMetrics(),
                                        configLock,
                                        hwInfo);
     _subDBs.push_back
@@ -85,8 +85,8 @@ DocumentSubDBCollection::DocumentSubDBCollection(
                 SearchableDocSubDB::Context(FastAccessDocSubDB::Context
                         (context,
                          AttributeMetricsCollection(metrics.getTaggedMetrics().ready.attributes,
-                                                    metrics.getMetrics().ready.attributes),
-                        &metrics.getMetrics().attributes,
+                                                    metrics.getLegacyMetrics().ready.attributes),
+                        &metrics.getLegacyMetrics().attributes,
                         metricsWireService),
                         queryLimiter,
                         clock,
@@ -114,7 +114,7 @@ DocumentSubDBCollection::DocumentSubDBCollection(
                         true),
                 FastAccessDocSubDB::Context(context,
                         AttributeMetricsCollection(metrics.getTaggedMetrics().notReady.attributes,
-                                                   metrics.getMetrics().notReady.attributes),
+                                                   metrics.getLegacyMetrics().notReady.attributes),
                         NULL,
                         metricsWireService)));
 }