Propagate target-active-docs in monitor reply.

20 files changed, 61 insertions, 51 deletions

diff --git a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java
index 6b510aadd3f..09ddccc4204 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java
@@ -184,7 +184,7 @@ public class Coverage {
         int degradation = com.yahoo.container.logging.Coverage.toDegradation(isDegradedByMatchPhase(),
                 isDegradedByTimeout(),
                 isDegradedByAdapativeTimeout());
-        return new com.yahoo.container.logging.Coverage(getDocs(), getActive(), getSoonActive(), degradation);
+        return new com.yahoo.container.logging.Coverage(getDocs(), getActive(), getTargetActive(), degradation);
     }
 
 }
diff --git a/container-core/src/main/java/com/yahoo/container/logging/Coverage.java b/container-core/src/main/java/com/yahoo/container/logging/Coverage.java
index bf0cb2d524b..d952cbd7490 100644
--- a/container-core/src/main/java/com/yahoo/container/logging/Coverage.java
+++ b/container-core/src/main/java/com/yahoo/container/logging/Coverage.java
@@ -41,7 +41,7 @@ public class Coverage {
         return v;
     }
 
-    public long getSoonActive() { return soonActive; }
+    public long getTargetActive() { return soonActive; }
 
     public boolean isDegraded() { return (degradedReason != 0) || isDegradedByNonIdealState(); }
     public boolean isDegradedByMatchPhase() { return (degradedReason & DEGRADED_BY_MATCH_PHASE) != 0; }
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java
index 99a2bc34a14..7b993e51ffb 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java
@@ -268,7 +268,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM
     private void collectCoverage(Coverage source) {
         answeredDocs += source.getDocs();
         answeredActiveDocs += source.getActive();
-        answeredSoonActiveDocs += source.getSoonActive();
+        answeredSoonActiveDocs += source.getTargetActive();
         answeredNodesParticipated += source.getNodes();
         answeredNodes++;
         degradedByMatchPhase |= source.isDegradedByMatchPhase();
@@ -280,7 +280,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM
 
         Coverage coverage = new Coverage(answeredDocs, answeredActiveDocs, answeredNodesParticipated, 1);
         coverage.setNodesTried(askedNodes);
-        coverage.setSoonActive(answeredSoonActiveDocs);
+        coverage.setTargetActive(answeredSoonActiveDocs);
         int degradedReason = 0;
         if (timedOut) {
             degradedReason |= (adaptiveTimeoutCalculated ? DEGRADED_BY_ADAPTIVE_TIMEOUT : DEGRADED_BY_TIMEOUT);
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java
index 09a101e3bff..73a005d809d 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java
@@ -263,7 +263,7 @@ public class ProtobufSerialization {
 
     private static Coverage convertToCoverage(SearchProtocol.SearchReply protobuf) {
         var coverage = new Coverage(protobuf.getCoverageDocs(), protobuf.getActiveDocs(), 1);
-        coverage.setNodesTried(1).setSoonActive(protobuf.getSoonActiveDocs());
+        coverage.setNodesTried(1).setTargetActive(protobuf.getSoonActiveDocs());
 
         int degradedReason = 0;
         if (protobuf.getDegradedByMatchPhase())
@@ -280,7 +280,7 @@ public class ProtobufSerialization {
 
         var coverage = result.getCoverage(false);
         if (coverage != null) {
-            builder.setCoverageDocs(coverage.getDocs()).setActiveDocs(coverage.getActive()).setSoonActiveDocs(coverage.getSoonActive())
+            builder.setCoverageDocs(coverage.getDocs()).setActiveDocs(coverage.getActive()).setSoonActiveDocs(coverage.getTargetActive())
                     .setDegradedBySoftTimeout(coverage.isDegradedByTimeout()).setDegradedByMatchPhase(coverage.isDegradedByMatchPhase());
         }
 
diff --git a/container-search/src/main/java/com/yahoo/search/result/Coverage.java b/container-search/src/main/java/com/yahoo/search/result/Coverage.java
index aa561f58d7a..bab5af8974e 100644
--- a/container-search/src/main/java/com/yahoo/search/result/Coverage.java
+++ b/container-search/src/main/java/com/yahoo/search/result/Coverage.java
@@ -30,7 +30,7 @@ public class Coverage extends com.yahoo.container.handler.Coverage {
      * @return self for chaining
      */
     @Beta
-    public Coverage setSoonActive(long soonActive) { this.soonActive = soonActive; return this; }
+    public Coverage setTargetActive(long soonActive) { this.soonActive = soonActive; return this; }
 
     /**
      * Will set the reasons for degraded coverage as reported by vespa backend.
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
index 1293975deb5..a225254079e 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
@@ -438,7 +438,7 @@ public class InterleavedSearchInvokerTest {
 
     private static Coverage createCoverage(int docs, int activeDocs, int soonActiveDocs, int nodes, int nodesTried, int degradedReason) {
         Coverage coverage = new Coverage(docs, activeDocs, nodes);
-        coverage.setSoonActive(soonActiveDocs);
+        coverage.setTargetActive(soonActiveDocs);
         coverage.setNodesTried(nodesTried);
         coverage.setDegradedReason(degradedReason);
         return coverage;
diff --git a/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java
index c0b0a90b682..787880f6a4e 100644
--- a/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java
@@ -74,7 +74,7 @@ public class CoverageTestCase {
         com.yahoo.container.logging.Coverage lc = c.toLoggingCoverage();
         assertEquals(lc.getDocs(), c.getDocs());
         assertEquals(lc.getActive(), c.getActive());
-        assertEquals(lc.getSoonActive(), c.getSoonActive());
+        assertEquals(lc.getSoonActive(), c.getTargetActive());
         assertEquals(lc.getResultPercentage(), c.getResultPercentage());
         assertEquals(lc.isDegraded(), c.isDegraded());
         assertEquals(lc.isDegradedByNonIdealState(), c.isDegradedByNonIdealState());
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h
index 2ad13576601..fbdde710277 100644
--- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h
@@ -34,6 +34,7 @@ public:
     Guard takeGuard() {
         return Guard(&_bucketDB, _mutex);
     }
+    size_t getActiveDocs() const { return _bucketDB.getNumActiveDocs(); }
 private:
     BucketDB   _bucketDB;
     std::mutex _mutex;
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
index 6dc1b2b354e..26ab2d10671 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
@@ -277,7 +277,7 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl
         SearchReply::Coverage & coverage = reply->coverage;
         coverage.setActive(numActiveLids);
         //TODO this should be calculated with ClusterState calculator.
-        coverage.setSoonActive(numActiveLids);
+        coverage.setTargetActive(numActiveLids);
         coverage.setCovered(covered);
         if (wasLimited) {
             coverage.degradeMatchPhase();
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index 51c02c818b2..81edf9bd659 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -619,13 +619,13 @@ DocumentDB::getNumDocs() const
     }
 }
 
-size_t
+std::pair<size_t, size_t>
 DocumentDB::getNumActiveDocs() const
 {
     if (_state.get_load_done()) {
-        return _subDBs.getReadySubDB()->getNumActiveDocs();
+        return { _subDBs.getReadySubDB()->getNumActiveDocs(), _subDBs.getBucketDB().getActiveDocs() };
     } else {
-        return 0u;
+        return {0u, 0u};
     }
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
index 96944ba25e7..d05fef00f4b 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
@@ -297,11 +297,12 @@ public:
     size_t getNumDocs() const;
 
     /**
-     * Returns the number of documents that are active for search in this database.
+     * Returns the number of documents that are active for search in this database,
+     * and the number of documents that will be active once ideal state is reached.
      *
-     * @return The active-document count.
+     * @return The active and target-active document count.
      */
-    size_t getNumActiveDocs() const;
+    std::pair<size_t, size_t> getNumActiveDocs() const;
 
     /**
      * Returns the base directory that this document database uses when
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
index f5654fb661d..8c1e670454c 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
@@ -118,6 +118,7 @@ public:
     const_iterator end() const { return _subDBs.end(); }
 
     bucketdb::BucketDBOwner &getBucketDB() { return *_bucketDB; }
+    const bucketdb::BucketDBOwner &getBucketDB() const { return *_bucketDB; }
 
     bucketdb::IBucketDBHandler &getBucketDBHandler() {
         return *_bucketDBHandler;
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index c4b30d50dbd..51e131d5548 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -115,11 +115,10 @@ setFS4Compression(const ProtonConfig & proton)
 DiskMemUsageSampler::Config
 diskMemUsageSamplerConfig(const ProtonConfig &proton, const HwInfo &hwInfo)
 {
-    return DiskMemUsageSampler::Config(
-            proton.writefilter.memorylimit,
-            proton.writefilter.disklimit,
-            vespalib::from_s(proton.writefilter.sampleinterval),
-            hwInfo);
+    return { proton.writefilter.memorylimit,
+             proton.writefilter.disklimit,
+             vespalib::from_s(proton.writefilter.sampleinterval),
+             hwInfo };
 }
 
 uint32_t
@@ -135,7 +134,7 @@ computeRpcTransportThreads(const ProtonConfig & cfg, const HwInfo::Cpu &cpuInfo)
 struct MetricsUpdateHook : metrics::UpdateHook
 {
     Proton &self;
-    MetricsUpdateHook(Proton &s)
+    explicit MetricsUpdateHook(Proton &s)
         : metrics::UpdateHook("proton-hook"),
           self(s)
     {}
@@ -427,14 +426,14 @@ Proton::addDocumentDB(const DocTypeName &docTypeName,
                 "Did not find document type '%s' in the document manager. "
                 "Skipping creating document database for this type",
                 docTypeName.toString().c_str());
-            return std::shared_ptr<DocumentDBConfigOwner>();
+            return {};
         }
     } catch (const document::DocumentTypeNotFoundException & e) {
         LOG(warning,
             "Did not find document type '%s' in the document manager. "
             "Skipping creating document database for this type",
             docTypeName.toString().c_str());
-        return std::shared_ptr<DocumentDBConfigOwner>();
+        return {};
     }
 }
 
@@ -537,14 +536,17 @@ size_t Proton::getNumDocs() const
     return numDocs;
 }
 
-size_t Proton::getNumActiveDocs() const
+std::pair<size_t, size_t>
+Proton::getNumActiveDocs() const
 {
-    size_t numDocs(0);
+    size_t activeDocs(0), targetActiveDocs(0);
     std::shared_lock<std::shared_mutex> guard(_mutex);
     for (const auto &kv : _documentDBMap) {
-        numDocs += kv.second->getNumActiveDocs();
+        const auto & docs = kv.second->getNumActiveDocs();
+        activeDocs += docs.first;
+        targetActiveDocs += docs.second;
     }
-    return numDocs;
+    return {activeDocs, targetActiveDocs};
 }
 
 search::engine::SearchServer &
@@ -725,8 +727,16 @@ Proton::ping(std::unique_ptr<MonitorRequest>, MonitorClient &)
     BootstrapConfig::SP configSnapshot = getActiveConfigSnapshot();
     const ProtonConfig &protonConfig = configSnapshot->getProtonConfig();
     ret.distribution_key = protonConfig.distributionkey;
-    ret.timestamp = (_matchEngine->isOnline()) ? 42 : 0;
-    ret.activeDocs = (_matchEngine->isOnline()) ? getNumActiveDocs() : 0;
+    if (_matchEngine->isOnline()) {
+        ret.timestamp = 42;
+        auto [active, targetActive] = getNumActiveDocs();
+        ret.activeDocs = active;
+        ret.targetActiveDocs = targetActive;
+    } else {
+        ret.timestamp = 0;
+        ret.activeDocs = 0;
+        ret.targetActiveDocs = 0; // TODO vekterli hmm... or target anyway ...
+    }
     ret.is_blocking_writes = !_diskMemUsageSampler->writeFilter().acceptWriteOperation();
     return reply;
 }
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h
index 6e154159ecc..08a1b17705f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.h
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.h
@@ -198,7 +198,8 @@ public:
     int64_t getConfigGeneration();
 
     size_t getNumDocs() const;
-    size_t getNumActiveDocs() const;
+    // Active (searchable), and targetActive that will be searchable when idealstate is reached
+    std::pair<size_t, size_t> getNumActiveDocs() const;
 
     search::engine::SearchServer &get_search_server();
     search::engine::DocsumServer &get_docsum_server();
diff --git a/searchlib/src/protobuf/search_protocol.proto b/searchlib/src/protobuf/search_protocol.proto
index 3cf2f01a7a8..1be68abad04 100644
--- a/searchlib/src/protobuf/search_protocol.proto
+++ b/searchlib/src/protobuf/search_protocol.proto
@@ -46,7 +46,7 @@ message SearchReply {
     int64 total_hit_count = 1;
     int64 coverage_docs = 2;
     int64 active_docs = 3;
-    int64 soon_active_docs = 4;
+    int64 target_active_docs = 4;
     bool degraded_by_match_phase = 5;
     bool degraded_by_soft_timeout = 6;
     repeated Hit hits = 7;
@@ -104,4 +104,5 @@ message MonitorReply {
     int64 active_docs = 2;
     int32 distribution_key = 3;
     bool is_blocking_writes = 4;
+    int64 target_active_docs = 5;
 }
diff --git a/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp b/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp
index 3091d167399..693ef74086a 100644
--- a/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp
+++ b/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp
@@ -247,9 +247,9 @@ TEST_F(SearchReplyTest, require_that_active_docs_is_converted) {
 }
 
 TEST_F(SearchReplyTest, require_that_soon_active_docs_is_converted) {
-    reply.coverage.setSoonActive(250000);
+    reply.coverage.setTargetActive(250000);
     convert();
-    EXPECT_EQ(proto.soon_active_docs(), 250000);
+    EXPECT_EQ(proto.target_active_docs(), 250000);
 }
 
 TEST_F(SearchReplyTest, require_that_degraded_by_match_phase_is_converted) {
diff --git a/searchlib/src/vespa/searchlib/engine/monitorreply.cpp b/searchlib/src/vespa/searchlib/engine/monitorreply.cpp
index 44a00638661..139c377a626 100644
--- a/searchlib/src/vespa/searchlib/engine/monitorreply.cpp
+++ b/searchlib/src/vespa/searchlib/engine/monitorreply.cpp
@@ -6,6 +6,7 @@ namespace search::engine {
 
 MonitorReply::MonitorReply()
     : activeDocs(0),
+      targetActiveDocs(0),
       distribution_key(-1),
       timestamp(),
       is_blocking_writes(false)
diff --git a/searchlib/src/vespa/searchlib/engine/monitorreply.h b/searchlib/src/vespa/searchlib/engine/monitorreply.h
index 52948e9f963..7e0af9be63a 100644
--- a/searchlib/src/vespa/searchlib/engine/monitorreply.h
+++ b/searchlib/src/vespa/searchlib/engine/monitorreply.h
@@ -9,6 +9,7 @@ namespace search::engine {
 struct MonitorReply
 {
     uint64_t                  activeDocs;
+    uint64_t                  targetActiveDocs;
     int32_t                   distribution_key;
     uint32_t                  timestamp;
     bool                      is_blocking_writes;
diff --git a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp
index 8d06dcc10e6..842ced502c6 100644
--- a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp
+++ b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp
@@ -101,7 +101,7 @@ ProtoConverter::search_reply_to_proto(const SearchReply &reply, ProtoSearchReply
     proto.set_total_hit_count(reply.totalHitCount);
     proto.set_coverage_docs(reply.coverage.getCovered());
     proto.set_active_docs(reply.coverage.getActive());
-    proto.set_soon_active_docs(reply.coverage.getSoonActive());
+    proto.set_target_active_docs(reply.coverage.getTargetActive());
     proto.set_degraded_by_match_phase(reply.coverage.wasDegradedByMatchPhase());
     proto.set_degraded_by_soft_timeout(reply.coverage.wasDegradedByTimeout());
     bool has_sort_data = ! reply.sortIndex.empty();
@@ -228,6 +228,7 @@ ProtoConverter::monitor_reply_to_proto(const MonitorReply &reply, ProtoMonitorRe
 {
     proto.set_online(reply.timestamp != 0);
     proto.set_active_docs(reply.activeDocs);
+    proto.set_target_active_docs(reply.targetActiveDocs);
     proto.set_distribution_key(reply.distribution_key);
     proto.set_is_blocking_writes(reply.is_blocking_writes);
 }
diff --git a/searchlib/src/vespa/searchlib/engine/searchreply.h b/searchlib/src/vespa/searchlib/engine/searchreply.h
index 25418a698c2..fc64566acf1 100644
--- a/searchlib/src/vespa/searchlib/engine/searchreply.h
+++ b/searchlib/src/vespa/searchlib/engine/searchreply.h
@@ -28,38 +28,30 @@ public:
     class Coverage {
     public:
         Coverage() noexcept : Coverage(0) { }
-        Coverage(uint64_t active) noexcept : Coverage(active, active) { }
+        explicit Coverage(uint64_t active) noexcept : Coverage(active, active) { }
         Coverage(uint64_t active, uint64_t covered) noexcept
-            : _covered(covered), _active(active), _soonActive(active),
-              _degradeReason(0), _nodesQueried(1), _nodesReplied(1)
+            : _covered(covered), _active(active), _targetActive(active),
+              _degradeReason(0)
         { }
         uint64_t getCovered() const { return _covered; }
         uint64_t getActive() const { return _active; }
-        uint64_t getSoonActive() const { return _soonActive; }
-        uint32_t getDegradeReason() const { return _degradeReason; }
-        uint16_t getNodesQueried() const { return _nodesQueried; }
-        uint16_t getNodesReplied() const { return _nodesReplied; }
+        uint64_t getTargetActive() const { return _targetActive; }
+
         bool wasDegradedByMatchPhase() const { return ((_degradeReason & MATCH_PHASE) != 0); }
         bool wasDegradedByTimeout() const { return ((_degradeReason & TIMEOUT) != 0); }
 
         Coverage & setCovered(uint64_t v) { _covered = v; return *this; }
         Coverage & setActive(uint64_t v) { _active = v; return *this; }
-        Coverage & setSoonActive(uint64_t v) { _soonActive = v; return *this; }
-        Coverage & setDegradeReason(uint32_t v) { _degradeReason = v; return *this; }
-        Coverage & setNodesQueried(uint16_t v) { _nodesQueried = v; return *this; }
-        Coverage & setNodesReplied(uint16_t v) { _nodesReplied = v; return *this; }
+        Coverage & setTargetActive(uint64_t v) { _targetActive = v; return *this; }
 
         Coverage & degradeMatchPhase() { _degradeReason |= MATCH_PHASE; return *this; }
         Coverage & degradeTimeout() { _degradeReason |= TIMEOUT; return *this; }
-        Coverage & degradeAdaptiveTimeout() { _degradeReason |= ADAPTIVE_TIMEOUT; return *this; }
-        enum DegradeReason {MATCH_PHASE=0x01, TIMEOUT=0x02, ADAPTIVE_TIMEOUT=0x04};
+        enum DegradeReason {MATCH_PHASE=0x01, TIMEOUT=0x02};
     private:
         uint64_t _covered;
         uint64_t _active;
-        uint64_t _soonActive;
+        uint64_t _targetActive;
         uint32_t _degradeReason;
-        uint16_t _nodesQueried;
-        uint16_t _nodesReplied;
     };
 
 private: