From c6552160827cd02d3aa0c0439f15be1823dc754a Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 9 Sep 2022 16:37:46 +0200 Subject: soonActive => targetActive Wire in targetActive in MonitorReply/Pong. --- .../src/main/java/com/yahoo/prelude/Pong.java | 35 ++++++++++++---------- .../search/dispatch/InterleavedSearchInvoker.java | 10 +++---- .../search/dispatch/rpc/ProtobufSerialization.java | 4 +-- .../com/yahoo/search/dispatch/rpc/RpcPing.java | 2 +- .../yahoo/search/dispatch/searchcluster/Group.java | 8 ++++- .../yahoo/search/dispatch/searchcluster/Node.java | 11 ++++--- .../dispatch/searchcluster/SearchCluster.java | 1 + .../java/com/yahoo/search/result/Coverage.java | 7 ++--- 8 files changed, 44 insertions(+), 34 deletions(-) (limited to 'container-search/src/main') diff --git a/container-search/src/main/java/com/yahoo/prelude/Pong.java b/container-search/src/main/java/com/yahoo/prelude/Pong.java index ecd6e302ccc..69bd89ecd57 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Pong.java +++ b/container-search/src/main/java/com/yahoo/prelude/Pong.java @@ -4,9 +4,7 @@ package com.yahoo.prelude; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.statistics.ElapsedTime; -import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; /** * An answer from Ping. @@ -16,42 +14,47 @@ import java.util.stream.Collectors; public class Pong { private final ElapsedTime elapsed = new ElapsedTime(); - private final Optional activeDocuments; + private final Long activeDocuments; + private final Long targetActiveDocuments; private final boolean isBlockingWrites; - private final Optional error; + private final ErrorMessage error; public Pong() { - this(Optional.empty(), false, Optional.empty()); + this(null, null,false, null); } public Pong(ErrorMessage error) { - this(Optional.empty(), false, Optional.of(error)); + this(null, null,false, error); } - public Pong(long activeDocuments) { - this(Optional.of(activeDocuments), false, Optional.empty()); + public Pong(long activeDocuments, long targetActiveDocuments) { + this(activeDocuments, targetActiveDocuments, false, null); } - public Pong(long activeDocuments, boolean isBlockingWrites) { - this(Optional.of(activeDocuments), isBlockingWrites, Optional.empty()); + public Pong(long activeDocuments, long targetActiveDocuments, boolean isBlockingWrites) { + this(activeDocuments, targetActiveDocuments, isBlockingWrites, null); } - private Pong(Optional activeDocuments, boolean isBlockingWrites, Optional error) { + private Pong(Long activeDocuments, Long targetActiveDocuments, boolean isBlockingWrites, ErrorMessage error) { this.activeDocuments = activeDocuments; + this.targetActiveDocuments = targetActiveDocuments; this.isBlockingWrites = isBlockingWrites; this.error = error; } - public Optional error() { return error; } + public Optional error() { return Optional.ofNullable(error); } /** Returns the number of active documents in the backend responding in this Pong, if available */ - public Optional activeDocuments() { return activeDocuments; } + public Optional activeDocuments() { return Optional.ofNullable(activeDocuments); } + + /** Returns the number of target active documents in the backend responding in this Pong, if available */ + public Optional targetActiveDocuments() { return Optional.ofNullable(targetActiveDocuments); } /** Returns true if the pinged node is currently blocking write operations due to being full */ public boolean isBlockingWrites() { return isBlockingWrites; } /** Returns whether there is an error or not */ - public boolean badResponse() { return error.isPresent(); } + public boolean badResponse() { return error != null; } public ElapsedTime getElapsedTime() { return elapsed; } @@ -59,10 +62,10 @@ public class Pong { @Override public String toString() { StringBuilder m = new StringBuilder("Ping result"); - activeDocuments.ifPresent(docCount -> m.append(" active docs: ").append(docCount)); + activeDocuments().ifPresent(docCount -> m.append(" active docs: ").append(docCount)); if (isBlockingWrites) m.append(" blocking writes: true"); - error.ifPresent(e -> m.append(" error: ").append(error)); + error().ifPresent(e -> m.append(" error: ").append(error)); return m.toString(); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index 7b993e51ffb..8c92e8b0270 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -54,7 +54,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM private long answeredDocs = 0; private long answeredActiveDocs = 0; - private long answeredSoonActiveDocs = 0; + private long answeredTargetActiveDocs = 0; private int askedNodes = 0; private int answeredNodes = 0; private int answeredNodesParticipated = 0; @@ -268,7 +268,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM private void collectCoverage(Coverage source) { answeredDocs += source.getDocs(); answeredActiveDocs += source.getActive(); - answeredSoonActiveDocs += source.getTargetActive(); + answeredTargetActiveDocs += source.getTargetActive(); answeredNodesParticipated += source.getNodes(); answeredNodes++; degradedByMatchPhase |= source.isDegradedByMatchPhase(); @@ -280,7 +280,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM Coverage coverage = new Coverage(answeredDocs, answeredActiveDocs, answeredNodesParticipated, 1); coverage.setNodesTried(askedNodes); - coverage.setTargetActive(answeredSoonActiveDocs); + coverage.setTargetActive(answeredTargetActiveDocs); int degradedReason = 0; if (timedOut) { degradedReason |= (adaptiveTimeoutCalculated ? DEGRADED_BY_ADAPTIVE_TIMEOUT : DEGRADED_BY_TIMEOUT); @@ -300,14 +300,14 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM if (adaptiveTimeoutCalculated && answeredNodesParticipated > 0) { answeredActiveDocs += (notAnswered * answeredActiveDocs / answeredNodesParticipated); - answeredSoonActiveDocs += (notAnswered * answeredSoonActiveDocs / answeredNodesParticipated); + answeredTargetActiveDocs += (notAnswered * answeredTargetActiveDocs / answeredNodesParticipated); } else { if (askedNodes > answeredNodesParticipated) { int searchableCopies = (int) searchCluster.dispatchConfig().searchableCopies(); int missingNodes = notAnswered - (searchableCopies - 1); if (answeredNodesParticipated > 0) { answeredActiveDocs += (missingNodes * answeredActiveDocs / answeredNodesParticipated); - answeredSoonActiveDocs += (missingNodes * answeredSoonActiveDocs / answeredNodesParticipated); + answeredTargetActiveDocs += (missingNodes * answeredTargetActiveDocs / answeredNodesParticipated); timedOut = true; } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index 73a005d809d..de4f4f45eed 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -263,7 +263,7 @@ public class ProtobufSerialization { private static Coverage convertToCoverage(SearchProtocol.SearchReply protobuf) { var coverage = new Coverage(protobuf.getCoverageDocs(), protobuf.getActiveDocs(), 1); - coverage.setNodesTried(1).setTargetActive(protobuf.getSoonActiveDocs()); + coverage.setNodesTried(1).setTargetActive(protobuf.getTargetActiveDocs()); int degradedReason = 0; if (protobuf.getDegradedByMatchPhase()) @@ -280,7 +280,7 @@ public class ProtobufSerialization { var coverage = result.getCoverage(false); if (coverage != null) { - builder.setCoverageDocs(coverage.getDocs()).setActiveDocs(coverage.getActive()).setSoonActiveDocs(coverage.getTargetActive()) + builder.setCoverageDocs(coverage.getDocs()).setActiveDocs(coverage.getActive()).setTargetActiveDocs(coverage.getTargetActive()) .setDegradedBySoftTimeout(coverage.isDegradedByTimeout()).setDegradedByMatchPhase(coverage.isDegradedByMatchPhase()); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java index 8e99f4948ce..44f0af2aca1 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java @@ -81,7 +81,7 @@ public class RpcPing implements Pinger, Client.ResponseReceiver { } else if (!reply.getOnline()) { return new Pong(ErrorMessage.createBackendCommunicationError("Node id " + node.key() + " reports being offline")); } else { - return new Pong(reply.getActiveDocs(), reply.getIsBlockingWrites()); + return new Pong(reply.getActiveDocs(), reply.getTargetActiveDocs(), reply.getIsBlockingWrites()); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java index d30abd1d047..cf161638104 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java @@ -26,6 +26,7 @@ public class Group { private final AtomicBoolean hasSufficientCoverage = new AtomicBoolean(true); private final AtomicBoolean hasFullCoverage = new AtomicBoolean(true); private final AtomicLong activeDocuments = new AtomicLong(0); + private final AtomicLong targetActiveDocuments = new AtomicLong(0); private final AtomicBoolean isBlockingWrites = new AtomicBoolean(false); private final AtomicBoolean isBalanced = new AtomicBoolean(true); @@ -68,6 +69,8 @@ public class Group { public void aggregateNodeValues() { long activeDocs = nodes.stream().filter(node -> node.isWorking() == Boolean.TRUE).mapToLong(Node::getActiveDocuments).sum(); activeDocuments.set(activeDocs); + long targetActiveDocs = nodes.stream().filter(node -> node.isWorking() == Boolean.TRUE).mapToLong(Node::getTargetActiveDocuments).sum(); + targetActiveDocuments.set(targetActiveDocs); isBlockingWrites.set(nodes.stream().anyMatch(Node::isBlockingWrites)); int numWorkingNodes = workingNodes(); if (numWorkingNodes > 0) { @@ -90,6 +93,9 @@ public class Group { /** Returns the active documents on this group. If unknown, 0 is returned. */ long activeDocuments() { return activeDocuments.get(); } + /** Returns the target active documents on this group. If unknown, 0 is returned. */ + long targetActiveDocuments() { return targetActiveDocuments.get(); } + /** Returns whether any node in this group is currently blocking write operations */ public boolean isBlockingWrites() { return isBlockingWrites.get(); } @@ -99,7 +105,7 @@ public class Group { /** Returns whether this group has too few documents per node to expect it to be balanced */ public boolean isSparse() { if (nodes.isEmpty()) return false; - return activeDocuments.get() / nodes.size() < minDocsPerNodeToRequireLowSkew; + return activeDocuments() / nodes.size() < minDocsPerNodeToRequireLowSkew; } public boolean fullCoverageStatusChanged(boolean hasFullCoverageNow) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index 0dce3d84273..d9c0198a472 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -21,6 +21,7 @@ public class Node { private final AtomicBoolean statusIsKnown = new AtomicBoolean(false); private final AtomicBoolean working = new AtomicBoolean(true); private final AtomicLong activeDocuments = new AtomicLong(0); + private final AtomicLong targetActiveDocuments = new AtomicLong(0); private final AtomicLong pingSequence = new AtomicLong(0); private final AtomicLong lastPong = new AtomicLong(0); private final AtomicBoolean isBlockingWrites = new AtomicBoolean(false); @@ -62,6 +63,7 @@ public class Node { this.working.lazySet(working); if ( ! working ) { activeDocuments.set(0); + targetActiveDocuments.set(0); } } @@ -71,10 +73,12 @@ public class Node { } /** Updates the active documents on this node */ - public void setActiveDocuments(long activeDocuments) { this.activeDocuments.set(activeDocuments); } + public void setActiveDocuments(long documents) { this.activeDocuments.set(documents); } + public void setTargetActiveDocuments(long documents) { this.targetActiveDocuments.set(documents); } /** Returns the active documents on this node. If unknown, 0 is returned. */ long getActiveDocuments() { return activeDocuments.get(); } + long getTargetActiveDocuments() { return targetActiveDocuments.get(); } public void setBlockingWrites(boolean isBlockingWrites) { this.isBlockingWrites.set(isBlockingWrites); } @@ -86,8 +90,7 @@ public class Node { @Override public boolean equals(Object o) { if (o == this) return true; - if ( ! (o instanceof Node)) return false; - Node other = (Node)o; + if ( ! (o instanceof Node other)) return false; if ( ! Objects.equals(this.hostname, other.hostname)) return false; if ( ! Objects.equals(this.key, other.key)) return false; if ( ! Objects.equals(this.pathIndex, other.pathIndex)) return false; @@ -100,7 +103,7 @@ public class Node { public String toString() { return "search node key = " + key + " hostname = "+ hostname + " path = " + pathIndex + " in group " + group + " statusIsKnown = " + statusIsKnown.get() + " working = " + working.get() + - " activeDocs = " + activeDocuments.get(); + " activeDocs = " + getActiveDocuments() + " targetActiveDocs = " + getTargetActiveDocuments(); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index f8c4627473d..82d3d98d9ef 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -339,6 +339,7 @@ public class SearchCluster implements NodeManager { } else { if (pong.activeDocuments().isPresent()) { node.setActiveDocuments(pong.activeDocuments().get()); + node.setTargetActiveDocuments(pong.targetActiveDocuments().get()); node.setBlockingWrites(pong.isBlockingWrites()); } clusterMonitor.responded(node); diff --git a/container-search/src/main/java/com/yahoo/search/result/Coverage.java b/container-search/src/main/java/com/yahoo/search/result/Coverage.java index bab5af8974e..390c6ec4435 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Coverage.java +++ b/container-search/src/main/java/com/yahoo/search/result/Coverage.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.result; -import com.yahoo.api.annotations.Beta; - /** * The coverage report for a result set. * @@ -26,11 +24,10 @@ public class Coverage extends com.yahoo.container.handler.Coverage { /** * Will set number of documents present in ideal state * - * @param soonActive number of documents active in ideal state + * @param targetActive number of documents active in ideal state * @return self for chaining */ - @Beta - public Coverage setTargetActive(long soonActive) { this.soonActive = soonActive; return this; } + public Coverage setTargetActive(long targetActive) { this.targetActive = targetActive; return this; } /** * Will set the reasons for degraded coverage as reported by vespa backend. -- cgit v1.2.3