Move control of internal limits closer to where the limitation is.

9 files changed, 42 insertions, 76 deletions

diff --git a/searchcorespi/src/vespa/searchcorespi/flush/flushstats.h b/searchcorespi/src/vespa/searchcorespi/flush/flushstats.h
index 43636f34075..d3b09b00d1e 100644
--- a/searchcorespi/src/vespa/searchcorespi/flush/flushstats.h
+++ b/searchcorespi/src/vespa/searchcorespi/flush/flushstats.h
@@ -12,7 +12,7 @@ class FlushStats
 {
 private:
     vespalib::string _path; // path to data flushed
-    size_t      _pathElementsToLog;
+    size_t           _pathElementsToLog;
 
 public:
     FlushStats();
diff --git a/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt b/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt
index 06e492e59e4..cd9f033f203 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt
+++ b/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt
@@ -20,7 +20,6 @@ vespa_add_library(searchcorespi_index OBJECT
     indexmanagerconfig.cpp
     indexreadutilities.cpp
     index_searchable_stats.cpp
-    memory_index_stats.cpp
     indexwriteutilities.cpp
     warmupindexcollection.cpp
     isearchableindexcollection.cpp
diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp b/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp
index 3ff19497949..e1fd5a98e10 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp
@@ -3,6 +3,7 @@
 #include "index_manager_stats.h"
 #include "iindexmanager.h"
 #include "indexsearchablevisitor.h"
+#include "imemoryindex.h"
 
 namespace searchcorespi {
 
@@ -14,15 +15,12 @@ public:
     std::vector<index::DiskIndexStats> _diskIndexes;
     std::vector<index::MemoryIndexStats> _memoryIndexes;
 
-    Visitor()
-        : _diskIndexes(),
-          _memoryIndexes()
-    {
-    }
-    virtual void visit(const index::IDiskIndex &index) override {
+    Visitor();
+    ~Visitor();
+    void visit(const index::IDiskIndex &index) override {
         _diskIndexes.emplace_back(index);
     }
-    virtual void visit(const index::IMemoryIndex &index) override {
+    void visit(const index::IMemoryIndex &index) override {
         _memoryIndexes.emplace_back(index);
     }
 
@@ -32,6 +30,9 @@ public:
     }
 };
 
+Visitor::Visitor() = default;
+Visitor::~Visitor() = default;
+
 }
 
 IndexManagerStats::IndexManagerStats()
@@ -52,8 +53,6 @@ IndexManagerStats::IndexManagerStats(const IIndexManager &indexManager)
     _memoryIndexes = std::move(visitor._memoryIndexes);
 }
 
-IndexManagerStats::~IndexManagerStats()
-{
-}
+IndexManagerStats::~IndexManagerStats() = default;
 
-} // namespace searchcorespi
+}
diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.h b/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.h
index 587a560f663..f37a7422e3d 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.h
+++ b/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.h
@@ -5,11 +5,9 @@
 #include <vespa/searchlib/common/serialnum.h>
 #include <vespa/searchlib/util/searchable_stats.h>
 
-namespace searchcorespi {
+namespace searchcorespi { class IndexSearchable; }
 
-class IndexSearchable;
-
-namespace index {
+namespace searchcorespi::index {
 
 /**
  * Information about a searchable index usable by state explorer.
@@ -18,7 +16,7 @@ class IndexSearchableStats
 {
     using SerialNum = search::SerialNum;
     using SearchableStats = search::SearchableStats;
-    SerialNum _serialNum;
+    SerialNum       _serialNum;
     SearchableStats _searchableStats;
 public:
     IndexSearchableStats();
@@ -28,5 +26,4 @@ public:
     const SearchableStats &getSearchableStats() const { return _searchableStats; }
 };
 
-} // namespace searchcorespi::index
-} // namespace searchcorespi
+}
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexflushtarget.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexflushtarget.cpp
index 1b07f2b2c4f..8ae6e18a28a 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/indexflushtarget.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/indexflushtarget.cpp
@@ -7,10 +7,10 @@ LOG_SETUP(".searchcorespi.index.indexflushtarget");
 
 namespace searchcorespi::index {
 
-IndexFlushTarget::IndexFlushTarget(IndexMaintainer &indexMaintainer)
+IndexFlushTarget::IndexFlushTarget(IndexMaintainer &indexMaintainer, IndexMaintainer::FlushStats flushStats)
     : IFlushTarget("memoryindex.flush", Type::FLUSH, Component::INDEX),
       _indexMaintainer(indexMaintainer),
-      _flushStats(indexMaintainer.getFlushStats()),
+      _flushStats(flushStats),
       _numFrozenMemoryIndexes(indexMaintainer.getNumFrozenMemoryIndexes()),
       _maxFrozenMemoryIndexes(indexMaintainer.getMaxFrozenMemoryIndexes()),
       _lastStats()
@@ -18,6 +18,10 @@ IndexFlushTarget::IndexFlushTarget(IndexMaintainer &indexMaintainer)
     _lastStats.setPathElementsToLog(7);
 }
 
+IndexFlushTarget::IndexFlushTarget(IndexMaintainer &indexMaintainer)
+    : IndexFlushTarget(indexMaintainer, indexMaintainer.getFlushStats())
+{}
+
 IndexFlushTarget::~IndexFlushTarget() = default;
 
 IFlushTarget::MemoryGain
@@ -32,18 +36,20 @@ IndexFlushTarget::getApproxDiskGain() const
     return DiskGain(0, 0);
 }
 
-
 bool
 IndexFlushTarget::needUrgentFlush() const
 {
-    bool urgent = _numFrozenMemoryIndexes > _maxFrozenMemoryIndexes;
+    // Due to limitation of 16G address space of single datastore
+    // TODO: Even better if urgency was decided by memory index itself.
+    constexpr int64_t G = 1024*1024*1024l;
+    bool urgent = (_numFrozenMemoryIndexes > _maxFrozenMemoryIndexes) ||
+                  (getApproxMemoryGain().gain() > 16*G);
     SerialNum flushedSerial = _indexMaintainer.getFlushedSerialNum();
-    LOG(debug, "Num frozen: %u Urgent: %d, flushedSerial=%" PRIu64,
-        _numFrozenMemoryIndexes, static_cast<int>(urgent), flushedSerial);
+    LOG(debug, "Num frozen: %u Memory gain: %ld Urgent: %d, flushedSerial=%" PRIu64,
+        _numFrozenMemoryIndexes, getApproxMemoryGain().gain(), static_cast<int>(urgent), flushedSerial);
     return urgent;
 }
 
-
 IFlushTarget::Time
 IndexFlushTarget::getLastFlushTime() const
 {
@@ -63,12 +69,10 @@ IndexFlushTarget::initFlush(SerialNum serialNum, std::shared_ptr<search::IFlushT
     return _indexMaintainer.initFlush(serialNum, &_lastStats);
 }
 
-
 uint64_t
 IndexFlushTarget::getApproxBytesToWriteToDisk() const
 {
-    MemoryGain gain(_flushStats.memory_before_bytes,
-                    _flushStats.memory_after_bytes);
+    MemoryGain gain(_flushStats.memory_before_bytes, _flushStats.memory_after_bytes);
     if (gain.getAfter() < gain.getBefore()) {
         return gain.getBefore() - gain.getAfter();
     } else {
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexflushtarget.h b/searchcorespi/src/vespa/searchcorespi/index/indexflushtarget.h
index 5573a646d1b..0d4947bc69a 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/indexflushtarget.h
+++ b/searchcorespi/src/vespa/searchcorespi/index/indexflushtarget.h
@@ -11,15 +11,16 @@ namespace searchcorespi::index {
  **/
 class IndexFlushTarget : public IFlushTarget {
 private:
-    IndexMaintainer &_indexMaintainer;
-    IndexMaintainer::FlushStats _flushStats;
-    uint32_t _numFrozenMemoryIndexes;
-    uint32_t _maxFrozenMemoryIndexes;
-    FlushStats _lastStats;
+    IndexMaintainer                   &_indexMaintainer;
+    const IndexMaintainer::FlushStats  _flushStats;
+    uint32_t                           _numFrozenMemoryIndexes;
+    uint32_t                           _maxFrozenMemoryIndexes;
+    FlushStats                         _lastStats;
 
 public:
-    IndexFlushTarget(IndexMaintainer &indexMaintainer);
-    ~IndexFlushTarget();
+    explicit IndexFlushTarget(IndexMaintainer &indexMaintainer);
+    IndexFlushTarget(IndexMaintainer &indexMaintainer, IndexMaintainer::FlushStats flushStats);
+    ~IndexFlushTarget() override;
 
     // Implements IFlushTarget
     MemoryGain getApproxMemoryGain() const override;
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h
index f002d6253c3..58ea36d685b 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h
+++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h
@@ -285,8 +285,8 @@ public:
     void removeOldDiskIndexes();
 
     struct FlushStats {
-        FlushStats() :
-            memory_before_bytes(0),
+        explicit FlushStats(uint64_t memory_before=0) :
+            memory_before_bytes(memory_before),
             memory_after_bytes(0),
             disk_write_bytes(0),
             cpu_time_required(0)
diff --git a/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.cpp b/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.cpp
deleted file mode 100644
index 6034c09a9be..00000000000
--- a/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.cpp
+++ /dev/null
@@ -1,22 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "memory_index_stats.h"
-#include "imemoryindex.h"
-
-namespace searchcorespi::index {
-
-MemoryIndexStats::MemoryIndexStats()
-    : IndexSearchableStats()
-{
-}
-
-MemoryIndexStats::MemoryIndexStats(const IMemoryIndex &index)
-    : IndexSearchableStats(index)
-{
-}
-
-MemoryIndexStats::~MemoryIndexStats()
-{
-}
-
-} // namespace searchcorespi::index
diff --git a/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.h b/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.h
index a32b0ec32a0..62863bcb726 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.h
+++ b/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.h
@@ -3,20 +3,8 @@
 
 #include "index_searchable_stats.h"
 
-namespace searchcorespi {
-namespace index {
+namespace searchcorespi::index {
 
-struct IMemoryIndex;
+using MemoryIndexStats = IndexSearchableStats;
 
-/**
- * Information about a memory index usable by state explorer.
- */
-class MemoryIndexStats : public IndexSearchableStats {
-public:
-    MemoryIndexStats();
-    MemoryIndexStats(const IMemoryIndex &index);
-    ~MemoryIndexStats();
-};
-
-} // namespace searchcorespi::index
-} // namespace searchcorespi
+}