diff options
author | Jon Bratseth <bratseth@verizonmedia.com> | 2020-01-23 22:28:33 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@verizonmedia.com> | 2020-01-23 22:28:33 +0100 |
commit | 70456b0068094e1d4e1b6ef04710409af32e6108 (patch) | |
tree | 7325f90700c111e6411a08ca499c72d9b2be29fe | |
parent | 51d4b6fcc2dd00909365ebe9e9c797d341a12d33 (diff) |
Record the specific change agent in the node history
11 files changed, 38 insertions, 16 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 3778b7aa625..0123a09f141 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -250,8 +250,8 @@ public final class Node { } /** Returns a copy of this with a history record saying it was detected to be down at this instant */ - public Node downAt(Instant instant) { - return with(history.with(new History.Event(History.Event.Type.down, Agent.system, instant))); + public Node downAt(Instant instant, Agent agent) { + return with(history.with(new History.Event(History.Event.Type.down, agent, instant))); } /** Returns a copy of this with any history record saying it has been detected down removed */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index d5aaf5dbad7..7d875434e1c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -684,7 +684,7 @@ public class NodeRepository extends AbstractComponent { * Writes these nodes after they have changed some internal state but NOT changed their state field. * This does NOT lock the node repository implicitly, but callers are expected to already hold the lock. * - * @param lock Already acquired lock + * @param lock already acquired lock * @return the written nodes for convenience */ public List<Node> write(List<Node> nodes, @SuppressWarnings("unused") Mutex lock) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java index ae2f68a3143..f428e276df8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java @@ -33,7 +33,7 @@ public class DirtyExpirer extends Expirer { @Override protected void expire(List<Node> expired) { for (Node expiredNode : expired) - nodeRepository.fail(expiredNode.hostname(), Agent.system, "Node is stuck in dirty"); + nodeRepository.fail(expiredNode.hostname(), Agent.DirtyExpirer, "Node is stuck in dirty"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index 438732ad4a8..264df716fa8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -107,7 +107,7 @@ public class FailedExpirer extends Maintainer { .collect(Collectors.toList()); if (unparkedChildren.isEmpty()) { - nodeRepository.park(candidate.hostname(), false, Agent.system, + nodeRepository.park(candidate.hostname(), false, Agent.FailedExpirer, "Parked by FailedExpirer due to hardware issue"); } else { log.info(String.format("Expired failed node %s with hardware issue was not parked because of " + @@ -118,7 +118,7 @@ public class FailedExpirer extends Maintainer { nodesToRecycle.add(candidate); } } - nodeRepository.setDirty(nodesToRecycle, Agent.system, "Expired by FailedExpirer"); + nodeRepository.setDirty(nodesToRecycle, Agent.FailedExpirer, "Expired by FailedExpirer"); } /** Returns whether the current node fail count should be used as an indicator of hardware issue */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java index a6b88d50acb..21746c96411 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java @@ -41,9 +41,9 @@ public class InactiveExpirer extends Expirer { expired.forEach(node -> { if (node.status().wantToRetire() && node.history().event(History.Event.Type.wantToRetire).get().agent() == Agent.operator) { - nodeRepository.park(node.hostname(), false, Agent.system, "Expired by InactiveExpirer"); + nodeRepository.park(node.hostname(), false, Agent.InactiveExpirer, "Expired by InactiveExpirer"); } else { - nodeRepository.setDirty(node, Agent.system, "Expired by InactiveExpirer"); + nodeRepository.setDirty(node, Agent.InactiveExpirer, "Expired by InactiveExpirer"); } }); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 424d6409670..b80446b06da 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -323,7 +323,7 @@ public class NodeFailer extends Maintainer { try (Mutex lock = nodeRepository().lock(node.allocation().get().owner())) { node = nodeRepository().getNode(node.hostname(), Node.State.active).get(); // re-get inside lock - return nodeRepository().write(node.downAt(clock.instant()), lock); + return nodeRepository().write(node.downAt(clock.instant(), Agent.NodeFailer), lock); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java index 3109e55df5c..e1407f2a41d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java @@ -27,7 +27,7 @@ public class ProvisionedExpirer extends Expirer { @Override protected void expire(List<Node> expired) { for (Node expiredNode : expired) - nodeRepository.parkRecursively(expiredNode.hostname(), Agent.system, "Node is stuck in provisioned"); + nodeRepository.parkRecursively(expiredNode.hostname(), Agent.ProvisionedExpirer, "Node is stuck in provisioned"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java index d7dd93522e4..675e3400722 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java @@ -115,7 +115,7 @@ public class Rebalancer extends Maintainer { if (nodeToMove.get().status().wantToRetire() == wantToRetire) return false; - nodeRepository().write(nodeToMove.get().withWantToRetire(wantToRetire, Agent.system, clock.instant()), lock); + nodeRepository().write(nodeToMove.get().withWantToRetire(wantToRetire, Agent.Rebalancer, clock.instant()), lock); return true; } } @@ -154,7 +154,7 @@ public class Rebalancer extends Maintainer { // Immediately clean up if we reserved the node but could not activate or reserved a node on the wrong host expectedNewNode.flatMap(node -> nodeRepository().getNode(node.hostname(), Node.State.reserved)) - .ifPresent(node -> nodeRepository().setDirty(node, Agent.system, "Expired by Rebalancer")); + .ifPresent(node -> nodeRepository().setDirty(node, Agent.Rebalancer, "Expired by Rebalancer")); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java index b0aa389fe7d..03d466dbf09 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java @@ -28,6 +28,6 @@ public class ReservationExpirer extends Expirer { } @Override - protected void expire(List<Node> expired) { nodeRepository.setDirty(expired, Agent.system, "Expired by ReservationExpirer"); } + protected void expire(List<Node> expired) { nodeRepository.setDirty(expired, Agent.ReservationExpirer, "Expired by ReservationExpirer"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java index f46e2f501bc..7522e411e42 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java @@ -7,5 +7,15 @@ package com.yahoo.vespa.hosted.provision.node; * @author bratseth */ public enum Agent { - system, application, operator, NodeFailer + operator, // A hosted Vespa operator. Some logic recognizes these events. + application, // An application package change depoyment + system, // An unspecified system agent + // Specific system agents: + NodeFailer, + Rebalancer, + DirtyExpirer, + FailedExpirer, + InactiveExpirer, + ProvisionedExpirer, + ReservationExpirer } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 728ec344c3f..b6a276e5e57 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -399,19 +399,31 @@ public class NodeSerializer { private Agent eventAgentFromSlime(Inspector eventAgentField) { switch (eventAgentField.asString()) { + case "operator" : return Agent.operator; case "application" : return Agent.application; case "system" : return Agent.system; - case "operator" : return Agent.operator; case "NodeFailer" : return Agent.NodeFailer; + case "Rebalancer" : return Agent.Rebalancer; + case "DirtyExpirer" : return Agent.DirtyExpirer; + case "FailedExpirer" : return Agent.FailedExpirer; + case "InactiveExpirer" : return Agent.InactiveExpirer; + case "ProvisionedExpirer" : return Agent.ProvisionedExpirer; + case "ReservationExpirer" : return Agent.ReservationExpirer; } throw new IllegalArgumentException("Unknown node event agent '" + eventAgentField.asString() + "'"); } private String toString(Agent agent) { switch (agent) { + case operator : return "operator"; case application : return "application"; case system : return "system"; - case operator : return "operator"; case NodeFailer : return "NodeFailer"; + case Rebalancer : return "Rebalancer"; + case DirtyExpirer : return "DirtyExpirer"; + case FailedExpirer : return "FailedExpirer"; + case InactiveExpirer : return "InactiveExpirer"; + case ProvisionedExpirer : return "ProvisionedExpirer"; + case ReservationExpirer : return "ReservationExpirer"; } throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined"); } |