diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2022-09-09 13:03:08 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2022-09-09 13:03:08 +0000 |
commit | e0dcdda7286506373b2f60e657afb931ca5edf8b (patch) | |
tree | f11d28e54b1b8b8eded744dd411ea8808e4ceecf | |
parent | cbba6fbd513bdbc9d1696f3b3c17f28abd2b4367 (diff) |
Propagate target-active-docs in monitor reply.
20 files changed, 61 insertions, 51 deletions
diff --git a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java index 6b510aadd3f..09ddccc4204 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java +++ b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java @@ -184,7 +184,7 @@ public class Coverage { int degradation = com.yahoo.container.logging.Coverage.toDegradation(isDegradedByMatchPhase(), isDegradedByTimeout(), isDegradedByAdapativeTimeout()); - return new com.yahoo.container.logging.Coverage(getDocs(), getActive(), getSoonActive(), degradation); + return new com.yahoo.container.logging.Coverage(getDocs(), getActive(), getTargetActive(), degradation); } } diff --git a/container-core/src/main/java/com/yahoo/container/logging/Coverage.java b/container-core/src/main/java/com/yahoo/container/logging/Coverage.java index bf0cb2d524b..d952cbd7490 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/Coverage.java +++ b/container-core/src/main/java/com/yahoo/container/logging/Coverage.java @@ -41,7 +41,7 @@ public class Coverage { return v; } - public long getSoonActive() { return soonActive; } + public long getTargetActive() { return soonActive; } public boolean isDegraded() { return (degradedReason != 0) || isDegradedByNonIdealState(); } public boolean isDegradedByMatchPhase() { return (degradedReason & DEGRADED_BY_MATCH_PHASE) != 0; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index 99a2bc34a14..7b993e51ffb 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -268,7 +268,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM private void collectCoverage(Coverage source) { answeredDocs += source.getDocs(); answeredActiveDocs += source.getActive(); - answeredSoonActiveDocs += source.getSoonActive(); + answeredSoonActiveDocs += source.getTargetActive(); answeredNodesParticipated += source.getNodes(); answeredNodes++; degradedByMatchPhase |= source.isDegradedByMatchPhase(); @@ -280,7 +280,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM Coverage coverage = new Coverage(answeredDocs, answeredActiveDocs, answeredNodesParticipated, 1); coverage.setNodesTried(askedNodes); - coverage.setSoonActive(answeredSoonActiveDocs); + coverage.setTargetActive(answeredSoonActiveDocs); int degradedReason = 0; if (timedOut) { degradedReason |= (adaptiveTimeoutCalculated ? DEGRADED_BY_ADAPTIVE_TIMEOUT : DEGRADED_BY_TIMEOUT); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index 09a101e3bff..73a005d809d 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -263,7 +263,7 @@ public class ProtobufSerialization { private static Coverage convertToCoverage(SearchProtocol.SearchReply protobuf) { var coverage = new Coverage(protobuf.getCoverageDocs(), protobuf.getActiveDocs(), 1); - coverage.setNodesTried(1).setSoonActive(protobuf.getSoonActiveDocs()); + coverage.setNodesTried(1).setTargetActive(protobuf.getSoonActiveDocs()); int degradedReason = 0; if (protobuf.getDegradedByMatchPhase()) @@ -280,7 +280,7 @@ public class ProtobufSerialization { var coverage = result.getCoverage(false); if (coverage != null) { - builder.setCoverageDocs(coverage.getDocs()).setActiveDocs(coverage.getActive()).setSoonActiveDocs(coverage.getSoonActive()) + builder.setCoverageDocs(coverage.getDocs()).setActiveDocs(coverage.getActive()).setSoonActiveDocs(coverage.getTargetActive()) .setDegradedBySoftTimeout(coverage.isDegradedByTimeout()).setDegradedByMatchPhase(coverage.isDegradedByMatchPhase()); } diff --git a/container-search/src/main/java/com/yahoo/search/result/Coverage.java b/container-search/src/main/java/com/yahoo/search/result/Coverage.java index aa561f58d7a..bab5af8974e 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Coverage.java +++ b/container-search/src/main/java/com/yahoo/search/result/Coverage.java @@ -30,7 +30,7 @@ public class Coverage extends com.yahoo.container.handler.Coverage { * @return self for chaining */ @Beta - public Coverage setSoonActive(long soonActive) { this.soonActive = soonActive; return this; } + public Coverage setTargetActive(long soonActive) { this.soonActive = soonActive; return this; } /** * Will set the reasons for degraded coverage as reported by vespa backend. diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java index 1293975deb5..a225254079e 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java @@ -438,7 +438,7 @@ public class InterleavedSearchInvokerTest { private static Coverage createCoverage(int docs, int activeDocs, int soonActiveDocs, int nodes, int nodesTried, int degradedReason) { Coverage coverage = new Coverage(docs, activeDocs, nodes); - coverage.setSoonActive(soonActiveDocs); + coverage.setTargetActive(soonActiveDocs); coverage.setNodesTried(nodesTried); coverage.setDegradedReason(degradedReason); return coverage; diff --git a/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java index c0b0a90b682..787880f6a4e 100644 --- a/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java @@ -74,7 +74,7 @@ public class CoverageTestCase { com.yahoo.container.logging.Coverage lc = c.toLoggingCoverage(); assertEquals(lc.getDocs(), c.getDocs()); assertEquals(lc.getActive(), c.getActive()); - assertEquals(lc.getSoonActive(), c.getSoonActive()); + assertEquals(lc.getSoonActive(), c.getTargetActive()); assertEquals(lc.getResultPercentage(), c.getResultPercentage()); assertEquals(lc.isDegraded(), c.isDegraded()); assertEquals(lc.isDegradedByNonIdealState(), c.isDegradedByNonIdealState()); diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h index 2ad13576601..fbdde710277 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_owner.h @@ -34,6 +34,7 @@ public: Guard takeGuard() { return Guard(&_bucketDB, _mutex); } + size_t getActiveDocs() const { return _bucketDB.getNumActiveDocs(); } private: BucketDB _bucketDB; std::mutex _mutex; diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 6dc1b2b354e..26ab2d10671 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -277,7 +277,7 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl SearchReply::Coverage & coverage = reply->coverage; coverage.setActive(numActiveLids); //TODO this should be calculated with ClusterState calculator. - coverage.setSoonActive(numActiveLids); + coverage.setTargetActive(numActiveLids); coverage.setCovered(covered); if (wasLimited) { coverage.degradeMatchPhase(); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 51c02c818b2..81edf9bd659 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -619,13 +619,13 @@ DocumentDB::getNumDocs() const } } -size_t +std::pair<size_t, size_t> DocumentDB::getNumActiveDocs() const { if (_state.get_load_done()) { - return _subDBs.getReadySubDB()->getNumActiveDocs(); + return { _subDBs.getReadySubDB()->getNumActiveDocs(), _subDBs.getBucketDB().getActiveDocs() }; } else { - return 0u; + return {0u, 0u}; } } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 96944ba25e7..d05fef00f4b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -297,11 +297,12 @@ public: size_t getNumDocs() const; /** - * Returns the number of documents that are active for search in this database. + * Returns the number of documents that are active for search in this database, + * and the number of documents that will be active once ideal state is reached. * - * @return The active-document count. + * @return The active and target-active document count. */ - size_t getNumActiveDocs() const; + std::pair<size_t, size_t> getNumActiveDocs() const; /** * Returns the base directory that this document database uses when diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index f5654fb661d..8c1e670454c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -118,6 +118,7 @@ public: const_iterator end() const { return _subDBs.end(); } bucketdb::BucketDBOwner &getBucketDB() { return *_bucketDB; } + const bucketdb::BucketDBOwner &getBucketDB() const { return *_bucketDB; } bucketdb::IBucketDBHandler &getBucketDBHandler() { return *_bucketDBHandler; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index c4b30d50dbd..51e131d5548 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -115,11 +115,10 @@ setFS4Compression(const ProtonConfig & proton) DiskMemUsageSampler::Config diskMemUsageSamplerConfig(const ProtonConfig &proton, const HwInfo &hwInfo) { - return DiskMemUsageSampler::Config( - proton.writefilter.memorylimit, - proton.writefilter.disklimit, - vespalib::from_s(proton.writefilter.sampleinterval), - hwInfo); + return { proton.writefilter.memorylimit, + proton.writefilter.disklimit, + vespalib::from_s(proton.writefilter.sampleinterval), + hwInfo }; } uint32_t @@ -135,7 +134,7 @@ computeRpcTransportThreads(const ProtonConfig & cfg, const HwInfo::Cpu &cpuInfo) struct MetricsUpdateHook : metrics::UpdateHook { Proton &self; - MetricsUpdateHook(Proton &s) + explicit MetricsUpdateHook(Proton &s) : metrics::UpdateHook("proton-hook"), self(s) {} @@ -427,14 +426,14 @@ Proton::addDocumentDB(const DocTypeName &docTypeName, "Did not find document type '%s' in the document manager. " "Skipping creating document database for this type", docTypeName.toString().c_str()); - return std::shared_ptr<DocumentDBConfigOwner>(); + return {}; } } catch (const document::DocumentTypeNotFoundException & e) { LOG(warning, "Did not find document type '%s' in the document manager. " "Skipping creating document database for this type", docTypeName.toString().c_str()); - return std::shared_ptr<DocumentDBConfigOwner>(); + return {}; } } @@ -537,14 +536,17 @@ size_t Proton::getNumDocs() const return numDocs; } -size_t Proton::getNumActiveDocs() const +std::pair<size_t, size_t> +Proton::getNumActiveDocs() const { - size_t numDocs(0); + size_t activeDocs(0), targetActiveDocs(0); std::shared_lock<std::shared_mutex> guard(_mutex); for (const auto &kv : _documentDBMap) { - numDocs += kv.second->getNumActiveDocs(); + const auto & docs = kv.second->getNumActiveDocs(); + activeDocs += docs.first; + targetActiveDocs += docs.second; } - return numDocs; + return {activeDocs, targetActiveDocs}; } search::engine::SearchServer & @@ -725,8 +727,16 @@ Proton::ping(std::unique_ptr<MonitorRequest>, MonitorClient &) BootstrapConfig::SP configSnapshot = getActiveConfigSnapshot(); const ProtonConfig &protonConfig = configSnapshot->getProtonConfig(); ret.distribution_key = protonConfig.distributionkey; - ret.timestamp = (_matchEngine->isOnline()) ? 42 : 0; - ret.activeDocs = (_matchEngine->isOnline()) ? getNumActiveDocs() : 0; + if (_matchEngine->isOnline()) { + ret.timestamp = 42; + auto [active, targetActive] = getNumActiveDocs(); + ret.activeDocs = active; + ret.targetActiveDocs = targetActive; + } else { + ret.timestamp = 0; + ret.activeDocs = 0; + ret.targetActiveDocs = 0; // TODO vekterli hmm... or target anyway ... + } ret.is_blocking_writes = !_diskMemUsageSampler->writeFilter().acceptWriteOperation(); return reply; } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 6e154159ecc..08a1b17705f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -198,7 +198,8 @@ public: int64_t getConfigGeneration(); size_t getNumDocs() const; - size_t getNumActiveDocs() const; + // Active (searchable), and targetActive that will be searchable when idealstate is reached + std::pair<size_t, size_t> getNumActiveDocs() const; search::engine::SearchServer &get_search_server(); search::engine::DocsumServer &get_docsum_server(); diff --git a/searchlib/src/protobuf/search_protocol.proto b/searchlib/src/protobuf/search_protocol.proto index 3cf2f01a7a8..1be68abad04 100644 --- a/searchlib/src/protobuf/search_protocol.proto +++ b/searchlib/src/protobuf/search_protocol.proto @@ -46,7 +46,7 @@ message SearchReply { int64 total_hit_count = 1; int64 coverage_docs = 2; int64 active_docs = 3; - int64 soon_active_docs = 4; + int64 target_active_docs = 4; bool degraded_by_match_phase = 5; bool degraded_by_soft_timeout = 6; repeated Hit hits = 7; @@ -104,4 +104,5 @@ message MonitorReply { int64 active_docs = 2; int32 distribution_key = 3; bool is_blocking_writes = 4; + int64 target_active_docs = 5; } diff --git a/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp b/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp index 3091d167399..693ef74086a 100644 --- a/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp +++ b/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp @@ -247,9 +247,9 @@ TEST_F(SearchReplyTest, require_that_active_docs_is_converted) { } TEST_F(SearchReplyTest, require_that_soon_active_docs_is_converted) { - reply.coverage.setSoonActive(250000); + reply.coverage.setTargetActive(250000); convert(); - EXPECT_EQ(proto.soon_active_docs(), 250000); + EXPECT_EQ(proto.target_active_docs(), 250000); } TEST_F(SearchReplyTest, require_that_degraded_by_match_phase_is_converted) { diff --git a/searchlib/src/vespa/searchlib/engine/monitorreply.cpp b/searchlib/src/vespa/searchlib/engine/monitorreply.cpp index 44a00638661..139c377a626 100644 --- a/searchlib/src/vespa/searchlib/engine/monitorreply.cpp +++ b/searchlib/src/vespa/searchlib/engine/monitorreply.cpp @@ -6,6 +6,7 @@ namespace search::engine { MonitorReply::MonitorReply() : activeDocs(0), + targetActiveDocs(0), distribution_key(-1), timestamp(), is_blocking_writes(false) diff --git a/searchlib/src/vespa/searchlib/engine/monitorreply.h b/searchlib/src/vespa/searchlib/engine/monitorreply.h index 52948e9f963..7e0af9be63a 100644 --- a/searchlib/src/vespa/searchlib/engine/monitorreply.h +++ b/searchlib/src/vespa/searchlib/engine/monitorreply.h @@ -9,6 +9,7 @@ namespace search::engine { struct MonitorReply { uint64_t activeDocs; + uint64_t targetActiveDocs; int32_t distribution_key; uint32_t timestamp; bool is_blocking_writes; diff --git a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp index 8d06dcc10e6..842ced502c6 100644 --- a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp +++ b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp @@ -101,7 +101,7 @@ ProtoConverter::search_reply_to_proto(const SearchReply &reply, ProtoSearchReply proto.set_total_hit_count(reply.totalHitCount); proto.set_coverage_docs(reply.coverage.getCovered()); proto.set_active_docs(reply.coverage.getActive()); - proto.set_soon_active_docs(reply.coverage.getSoonActive()); + proto.set_target_active_docs(reply.coverage.getTargetActive()); proto.set_degraded_by_match_phase(reply.coverage.wasDegradedByMatchPhase()); proto.set_degraded_by_soft_timeout(reply.coverage.wasDegradedByTimeout()); bool has_sort_data = ! reply.sortIndex.empty(); @@ -228,6 +228,7 @@ ProtoConverter::monitor_reply_to_proto(const MonitorReply &reply, ProtoMonitorRe { proto.set_online(reply.timestamp != 0); proto.set_active_docs(reply.activeDocs); + proto.set_target_active_docs(reply.targetActiveDocs); proto.set_distribution_key(reply.distribution_key); proto.set_is_blocking_writes(reply.is_blocking_writes); } diff --git a/searchlib/src/vespa/searchlib/engine/searchreply.h b/searchlib/src/vespa/searchlib/engine/searchreply.h index 25418a698c2..fc64566acf1 100644 --- a/searchlib/src/vespa/searchlib/engine/searchreply.h +++ b/searchlib/src/vespa/searchlib/engine/searchreply.h @@ -28,38 +28,30 @@ public: class Coverage { public: Coverage() noexcept : Coverage(0) { } - Coverage(uint64_t active) noexcept : Coverage(active, active) { } + explicit Coverage(uint64_t active) noexcept : Coverage(active, active) { } Coverage(uint64_t active, uint64_t covered) noexcept - : _covered(covered), _active(active), _soonActive(active), - _degradeReason(0), _nodesQueried(1), _nodesReplied(1) + : _covered(covered), _active(active), _targetActive(active), + _degradeReason(0) { } uint64_t getCovered() const { return _covered; } uint64_t getActive() const { return _active; } - uint64_t getSoonActive() const { return _soonActive; } - uint32_t getDegradeReason() const { return _degradeReason; } - uint16_t getNodesQueried() const { return _nodesQueried; } - uint16_t getNodesReplied() const { return _nodesReplied; } + uint64_t getTargetActive() const { return _targetActive; } + bool wasDegradedByMatchPhase() const { return ((_degradeReason & MATCH_PHASE) != 0); } bool wasDegradedByTimeout() const { return ((_degradeReason & TIMEOUT) != 0); } Coverage & setCovered(uint64_t v) { _covered = v; return *this; } Coverage & setActive(uint64_t v) { _active = v; return *this; } - Coverage & setSoonActive(uint64_t v) { _soonActive = v; return *this; } - Coverage & setDegradeReason(uint32_t v) { _degradeReason = v; return *this; } - Coverage & setNodesQueried(uint16_t v) { _nodesQueried = v; return *this; } - Coverage & setNodesReplied(uint16_t v) { _nodesReplied = v; return *this; } + Coverage & setTargetActive(uint64_t v) { _targetActive = v; return *this; } Coverage & degradeMatchPhase() { _degradeReason |= MATCH_PHASE; return *this; } Coverage & degradeTimeout() { _degradeReason |= TIMEOUT; return *this; } - Coverage & degradeAdaptiveTimeout() { _degradeReason |= ADAPTIVE_TIMEOUT; return *this; } - enum DegradeReason {MATCH_PHASE=0x01, TIMEOUT=0x02, ADAPTIVE_TIMEOUT=0x04}; + enum DegradeReason {MATCH_PHASE=0x01, TIMEOUT=0x02}; private: uint64_t _covered; uint64_t _active; - uint64_t _soonActive; + uint64_t _targetActive; uint32_t _degradeReason; - uint16_t _nodesQueried; - uint16_t _nodesReplied; }; private: |