diff options
author | Jon Bratseth <bratseth@gmail.com> | 2020-11-12 22:37:04 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2020-11-12 22:37:04 +0100 |
commit | ee332d88dbf3acdc778b5c5a5d38685c00a27297 (patch) | |
tree | 05f4a61708ac69b301a23e32a257d743f535e9f7 | |
parent | ccbb05c22ac610c2f6cb59024edcc3c90586b1a7 (diff) |
Use node repository clock
21 files changed, 76 insertions, 131 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java index eb5973f11a9..88bf3426ee0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java @@ -7,7 +7,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; -import java.time.Clock; import java.time.Duration; import java.util.List; @@ -24,17 +23,14 @@ import java.util.List; */ public class DirtyExpirer extends Expirer { - private final NodeRepository nodeRepository; - - DirtyExpirer(NodeRepository nodeRepository, Clock clock, Duration dirtyTimeout, Metric metric) { - super(Node.State.dirty, History.Event.Type.deallocated, nodeRepository, clock, dirtyTimeout, metric); - this.nodeRepository = nodeRepository; + DirtyExpirer(NodeRepository nodeRepository, Duration dirtyTimeout, Metric metric) { + super(Node.State.dirty, History.Event.Type.deallocated, nodeRepository, dirtyTimeout, metric); } @Override protected void expire(List<Node> expired) { for (Node expiredNode : expired) - nodeRepository.fail(expiredNode.hostname(), Agent.DirtyExpirer, "Node is stuck in dirty"); + nodeRepository().fail(expiredNode.hostname(), Agent.DirtyExpirer, "Node is stuck in dirty"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java index 8503d9adab6..ba59ab02780 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java @@ -6,7 +6,6 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.History; -import java.time.Clock; import java.time.Duration; import java.util.List; import java.util.logging.Logger; @@ -29,16 +28,14 @@ public abstract class Expirer extends NodeRepositoryMaintainer { private final History.Event.Type eventType; private final Metric metric; - private final Clock clock; private final Duration expiryTime; Expirer(Node.State fromState, History.Event.Type eventType, NodeRepository nodeRepository, - Clock clock, Duration expiryTime, Metric metric) { + Duration expiryTime, Metric metric) { super(nodeRepository, min(Duration.ofMinutes(10), expiryTime), metric); this.fromState = fromState; this.eventType = eventType; this.metric = metric; - this.clock = clock; this.expiryTime = expiryTime; } @@ -58,7 +55,7 @@ public abstract class Expirer extends NodeRepositoryMaintainer { } protected boolean isExpired(Node node) { - return node.history().hasEventBefore(eventType, clock.instant().minus(expiryTime)); + return node.history().hasEventBefore(eventType, clock().instant().minus(expiryTime)); } /** Implement this callback to take action to expire these nodes */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index ec04464fde0..b55f49722cb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; -import java.time.Clock; import java.time.Duration; import java.util.ArrayList; import java.util.List; @@ -49,16 +48,12 @@ public class FailedExpirer extends NodeRepositoryMaintainer { private static final int maxAllowedFailures = 50; private final NodeRepository nodeRepository; - private final Zone zone; - private final Clock clock; private final Duration defaultExpiry; // Grace period to allow recovery of data private final Duration containerExpiry; // Stateless nodes, no data to recover - FailedExpirer(NodeRepository nodeRepository, Zone zone, Clock clock, Duration interval, Metric metric) { + FailedExpirer(NodeRepository nodeRepository, Zone zone, Duration interval, Metric metric) { super(nodeRepository, interval, metric); this.nodeRepository = nodeRepository; - this.zone = zone; - this.clock = clock; if (zone.system().isCd()) { defaultExpiry = containerExpiry = Duration.ofMinutes(30); } else { @@ -81,9 +76,9 @@ public class FailedExpirer extends NodeRepositoryMaintainer { recycleIf(remainingNodes, node -> node.allocation().isEmpty()); recycleIf(remainingNodes, node -> node.allocation().get().membership().cluster().type() == ClusterSpec.Type.container && - node.history().hasEventBefore(History.Event.Type.failed, clock.instant().minus(containerExpiry))); + node.history().hasEventBefore(History.Event.Type.failed, clock().instant().minus(containerExpiry))); recycleIf(remainingNodes, node -> - node.history().hasEventBefore(History.Event.Type.failed, clock.instant().minus(defaultExpiry))); + node.history().hasEventBefore(History.Event.Type.failed, clock().instant().minus(defaultExpiry))); return true; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java index 389fc0ee907..392146f6ead 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.Status; -import java.time.Clock; import java.time.Duration; import java.util.List; @@ -37,8 +36,8 @@ public class InactiveExpirer extends Expirer { private final NodeRepository nodeRepository; - InactiveExpirer(NodeRepository nodeRepository, Clock clock, Duration inactiveTimeout, Metric metric) { - super(Node.State.inactive, History.Event.Type.deactivated, nodeRepository, clock, inactiveTimeout, metric); + InactiveExpirer(NodeRepository nodeRepository, Duration inactiveTimeout, Metric metric) { + super(Node.State.inactive, History.Event.Type.deactivated, nodeRepository, inactiveTimeout, metric); this.nodeRepository = nodeRepository; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index b2fc875a17f..e5284a7b56d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -22,7 +22,6 @@ import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; -import java.time.Clock; import java.time.Duration; import java.util.HashMap; import java.util.List; @@ -44,21 +43,18 @@ public class MetricsReporter extends NodeRepositoryMaintainer { private final ServiceMonitor serviceMonitor; private final Map<Map<String, String>, Metric.Context> contextMap = new HashMap<>(); private final Supplier<Integer> pendingRedeploymentsSupplier; - private final Clock clock; MetricsReporter(NodeRepository nodeRepository, Metric metric, Orchestrator orchestrator, ServiceMonitor serviceMonitor, Supplier<Integer> pendingRedeploymentsSupplier, - Duration interval, - Clock clock) { + Duration interval) { super(nodeRepository, interval, metric); this.metric = metric; this.orchestrator = orchestrator; this.serviceMonitor = serviceMonitor; this.pendingRedeploymentsSupplier = pendingRedeploymentsSupplier; - this.clock = clock; } @Override @@ -157,7 +153,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { metric.set("suspended", suspended, context); metric.set("allowedToBeDown", suspended, context); // remove summer 2020. long suspendedSeconds = info.suspendedSince() - .map(suspendedSince -> Duration.between(suspendedSince, clock.instant()).getSeconds()) + .map(suspendedSince -> Duration.between(suspendedSince, clock().instant()).getSeconds()) .orElse(0L); metric.set("suspendedSeconds", suspendedSeconds, context); }); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index fa9ed9bed26..9c7e6496ccd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Deployment; -import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TransientException; import com.yahoo.jdisc.Metric; @@ -18,10 +17,8 @@ import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException; import com.yahoo.vespa.orchestrator.HostNameNotFoundException; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; -import com.yahoo.vespa.service.monitor.ServiceMonitor; import com.yahoo.yolean.Exceptions; -import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.HashMap; @@ -58,22 +55,20 @@ public class NodeFailer extends NodeRepositoryMaintainer { private final Deployer deployer; private final Duration downTimeLimit; - private final Clock clock; private final Orchestrator orchestrator; private final Instant constructionTime; private final ThrottlePolicy throttlePolicy; private final Metric metric; public NodeFailer(Deployer deployer, NodeRepository nodeRepository, - Duration downTimeLimit, Duration interval, Clock clock, Orchestrator orchestrator, + Duration downTimeLimit, Duration interval, Orchestrator orchestrator, ThrottlePolicy throttlePolicy, Metric metric) { // check ping status every interval, but at least twice as often as the down time limit super(nodeRepository, min(downTimeLimit.dividedBy(2), interval), metric); this.deployer = deployer; this.downTimeLimit = downTimeLimit; - this.clock = clock; this.orchestrator = orchestrator; - this.constructionTime = clock.instant(); + this.constructionTime = nodeRepository.clock().instant(); this.throttlePolicy = throttlePolicy; this.metric = metric; } @@ -124,12 +119,12 @@ public class NodeFailer extends NodeRepositoryMaintainer { private Map<Node, String> getReadyNodesByFailureReason() { Instant oldestAcceptableRequestTime = // Allow requests some time to be registered in case all config servers have been down - constructionTime.isAfter(clock.instant().minus(nodeRequestInterval.multipliedBy(2))) ? + constructionTime.isAfter(clock().instant().minus(nodeRequestInterval.multipliedBy(2))) ? Instant.EPOCH : // Nodes are taken as dead if they have not made a config request since this instant. // Add 10 minutes to the down time limit to allow nodes to make a request that infrequently. - clock.instant().minus(downTimeLimit).minus(nodeRequestInterval); + clock().instant().minus(downTimeLimit).minus(nodeRequestInterval); Map<Node, String> nodesByFailureReason = new HashMap<>(); for (Node node : nodeRepository().getNodes(Node.State.ready)) { @@ -152,7 +147,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { private Map<Node, String> getActiveNodesByFailureReason() { List<Node> activeNodes = nodeRepository().getNodes(Node.State.active); - Instant graceTimeEnd = clock.instant().minus(downTimeLimit); + Instant graceTimeEnd = clock().instant().minus(downTimeLimit); Map<Node, String> nodesByFailureReason = new HashMap<>(); for (Node node : activeNodes) { if (node.history().hasEventBefore(History.Event.Type.down, graceTimeEnd) && ! applicationSuspended(node)) { @@ -303,7 +298,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { /** Returns true if node failing should be throttled */ private boolean throttle(Node node) { if (throttlePolicy == ThrottlePolicy.disabled) return false; - Instant startOfThrottleWindow = clock.instant().minus(throttlePolicy.throttleWindow); + Instant startOfThrottleWindow = clock().instant().minus(throttlePolicy.throttleWindow); List<Node> nodes = nodeRepository().getNodes(); NodeList recentlyFailedNodes = nodes.stream() .filter(n -> n.state() == Node.State.failed) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java index c76239527bd..07889599c4c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter; -import java.time.Clock; import java.time.Duration; import java.util.Comparator; import java.util.EnumSet; @@ -29,14 +28,12 @@ import java.util.stream.Collectors; public class NodeRebooter extends NodeRepositoryMaintainer { private final IntFlag rebootIntervalInDays; - private final Clock clock; private final Random random; - NodeRebooter(NodeRepository nodeRepository, Clock clock, FlagSource flagSource, Metric metric) { + NodeRebooter(NodeRepository nodeRepository, FlagSource flagSource, Metric metric) { super(nodeRepository, Duration.ofMinutes(25), metric); this.rebootIntervalInDays = Flags.REBOOT_INTERVAL_IN_DAYS.bindTo(flagSource); - this.clock = clock; - this.random = new Random(clock.millis()); // seed with clock for test determinism + this.random = new Random(nodeRepository.clock().millis()); // seed with clock for test determinism } @Override @@ -62,7 +59,7 @@ public class NodeRebooter extends NodeRepositoryMaintainer { .filter(event -> rebootEvents.contains(event.type())) .map(History.Event::at) .max(Comparator.naturalOrder()) - .map(lastReboot -> Duration.between(lastReboot, clock.instant()).minus(rebootInterval)); + .map(lastReboot -> Duration.between(lastReboot, clock().instant()).minus(rebootInterval)); if (overdue.isEmpty()) // should never happen as all !docker-container should have provisioned timestamp return random.nextDouble() < interval().getSeconds() / (double) rebootInterval.getSeconds(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index c14cd18425d..a2774333645 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -57,31 +57,22 @@ public class NodeRepositoryMaintenance extends AbstractComponent { HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, Zone zone, Orchestrator orchestrator, Metric metric, ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource, - MetricsFetcher nodeMetrics, MetricsDb metricsDb) { - this(nodeRepository, deployer, infraDeployer, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(), - orchestrator, metric, provisionServiceProvider, flagSource, nodeMetrics, metricsDb); - } - - public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, InfraDeployer infraDeployer, - HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor, - Zone zone, Clock clock, Orchestrator orchestrator, Metric metric, - ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource, MetricsFetcher metricsFetcher, MetricsDb metricsDb) { DefaultTimes defaults = new DefaultTimes(zone, deployer); - nodeFailer = new NodeFailer(deployer, nodeRepository, defaults.failGrace, defaults.nodeFailerInterval, clock, orchestrator, defaults.throttlePolicy, metric); + nodeFailer = new NodeFailer(deployer, nodeRepository, defaults.failGrace, defaults.nodeFailerInterval, orchestrator, defaults.throttlePolicy, metric); nodeFailureStatusUpdater = new NodeFailureStatusUpdater(hostLivenessTracker, serviceMonitor, nodeRepository, defaults.nodeFailureStatusUpdateInterval, metric); periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, metric, nodeRepository, defaults.redeployMaintainerInterval, defaults.periodicRedeployInterval, flagSource); operatorChangeApplicationMaintainer = new OperatorChangeApplicationMaintainer(deployer, metric, nodeRepository, defaults.operatorChangeRedeployInterval); - reservationExpirer = new ReservationExpirer(nodeRepository, clock, defaults.reservationExpiry, metric); - retiredExpirer = new RetiredExpirer(nodeRepository, orchestrator, deployer, metric, clock, defaults.retiredInterval, defaults.retiredExpiry); - inactiveExpirer = new InactiveExpirer(nodeRepository, clock, defaults.inactiveExpiry, metric); - failedExpirer = new FailedExpirer(nodeRepository, zone, clock, defaults.failedExpirerInterval, metric); - dirtyExpirer = new DirtyExpirer(nodeRepository, clock, defaults.dirtyExpiry, metric); - provisionedExpirer = new ProvisionedExpirer(nodeRepository, clock, defaults.provisionedExpiry, metric); - nodeRebooter = new NodeRebooter(nodeRepository, clock, flagSource, metric); - metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval, clock); + reservationExpirer = new ReservationExpirer(nodeRepository, defaults.reservationExpiry, metric); + retiredExpirer = new RetiredExpirer(nodeRepository, orchestrator, deployer, metric, defaults.retiredInterval, defaults.retiredExpiry); + inactiveExpirer = new InactiveExpirer(nodeRepository, defaults.inactiveExpiry, metric); + failedExpirer = new FailedExpirer(nodeRepository, zone, defaults.failedExpirerInterval, metric); + dirtyExpirer = new DirtyExpirer(nodeRepository, defaults.dirtyExpiry, metric); + provisionedExpirer = new ProvisionedExpirer(nodeRepository, defaults.provisionedExpiry, metric); + nodeRebooter = new NodeRebooter(nodeRepository, flagSource, metric); + metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval); infrastructureProvisioner = new InfrastructureProvisioner(nodeRepository, infraDeployer, defaults.infrastructureProvisionInterval, metric); loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService(nodeRepository).map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService, metric)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java index d38bff091b0..00cccab6f74 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java @@ -7,7 +7,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; -import java.time.Clock; import java.time.Duration; import java.util.List; @@ -18,17 +17,14 @@ import java.util.List; */ public class ProvisionedExpirer extends Expirer { - private final NodeRepository nodeRepository; - - ProvisionedExpirer(NodeRepository nodeRepository, Clock clock, Duration dirtyTimeout, Metric metric) { - super(Node.State.provisioned, History.Event.Type.provisioned, nodeRepository, clock, dirtyTimeout, metric); - this.nodeRepository = nodeRepository; + ProvisionedExpirer(NodeRepository nodeRepository, Duration dirtyTimeout, Metric metric) { + super(Node.State.provisioned, History.Event.Type.provisioned, nodeRepository, dirtyTimeout, metric); } @Override protected void expire(List<Node> expired) { for (Node expiredNode : expired) - nodeRepository.parkRecursively(expiredNode.hostname(), Agent.ProvisionedExpirer, "Node is stuck in provisioned"); + nodeRepository().parkRecursively(expiredNode.hostname(), Agent.ProvisionedExpirer, "Node is stuck in provisioned"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java index 27f77dd08a3..c6427123d09 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirer.java @@ -7,7 +7,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; -import java.time.Clock; import java.time.Duration; import java.util.List; @@ -21,14 +20,11 @@ import java.util.List; */ public class ReservationExpirer extends Expirer { - private final NodeRepository nodeRepository; - - public ReservationExpirer(NodeRepository nodeRepository, Clock clock, Duration reservationPeriod, Metric metric) { - super(Node.State.reserved, History.Event.Type.reserved, nodeRepository, clock, reservationPeriod, metric); - this.nodeRepository = nodeRepository; + public ReservationExpirer(NodeRepository nodeRepository, Duration reservationPeriod, Metric metric) { + super(Node.State.reserved, History.Event.Type.reserved, nodeRepository, reservationPeriod, metric); } @Override - protected void expire(List<Node> expired) { nodeRepository.setDirty(expired, Agent.ReservationExpirer, "Expired by ReservationExpirer"); } + protected void expire(List<Node> expired) { nodeRepository().setDirty(expired, Agent.ReservationExpirer, "Expired by ReservationExpirer"); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java index 62c90ffe433..abda7ed6120 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.Orchestrator; -import java.time.Clock; import java.time.Duration; import java.util.List; import java.util.Map; @@ -30,13 +29,11 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { private final Metric metric; private final Orchestrator orchestrator; private final Duration retiredExpiry; - private final Clock clock; public RetiredExpirer(NodeRepository nodeRepository, Orchestrator orchestrator, Deployer deployer, Metric metric, - Clock clock, Duration maintenanceInterval, Duration retiredExpiry) { super(nodeRepository, maintenanceInterval, metric); @@ -44,7 +41,6 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { this.metric = metric; this.orchestrator = orchestrator; this.retiredExpiry = retiredExpiry; - this.clock = clock; } @Override @@ -98,7 +94,7 @@ public class RetiredExpirer extends NodeRepositoryMaintainer { return false; } - if (node.history().hasEventBefore(History.Event.Type.retired, clock.instant().minus(retiredExpiry))) { + if (node.history().hasEventBefore(History.Event.Type.retired, clock().instant().minus(retiredExpiry))) { log.info("Node " + node + " has been retired longer than " + retiredExpiry); return true; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java index ceedb41ab31..940404fa605 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java @@ -256,7 +256,7 @@ public class FailedExpirerTest { this.clock = tester.clock(); this.nodeRepository = tester.nodeRepository(); this.provisioner = tester.provisioner(); - this.expirer = new FailedExpirer(nodeRepository, zone, clock, Duration.ofMinutes(30), new TestMetric()); + this.expirer = new FailedExpirer(nodeRepository, zone, Duration.ofMinutes(30), new TestMetric()); } public ManualClock clock() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index 056fe041377..20c1bd78a52 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -64,7 +64,7 @@ public class InactiveAndFailedExpirerTest { // Inactive times out tester.advanceTime(Duration.ofMinutes(14)); - new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new TestMetric()).run(); + new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run(); assertEquals(0, tester.nodeRepository().getNodes(Node.State.inactive).size()); List<Node> dirty = tester.nodeRepository().getNodes(Node.State.dirty); assertEquals(2, dirty.size()); @@ -79,7 +79,7 @@ public class InactiveAndFailedExpirerTest { // Dirty times out for the other one tester.advanceTime(Duration.ofMinutes(14)); - new DirtyExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new TestMetric()).run(); + new DirtyExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run(); assertEquals(0, tester.nodeRepository().getNodes(NodeType.tenant, Node.State.dirty).size()); List<Node> failed = tester.nodeRepository().getNodes(NodeType.tenant, Node.State.failed); assertEquals(1, failed.size()); @@ -107,7 +107,7 @@ public class InactiveAndFailedExpirerTest { // Inactive times out and node is moved to dirty tester.advanceTime(Duration.ofMinutes(14)); - new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new TestMetric()).run(); + new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run(); List<Node> dirty = tester.nodeRepository().getNodes(Node.State.dirty); assertEquals(2, dirty.size()); @@ -152,13 +152,13 @@ public class InactiveAndFailedExpirerTest { ); Orchestrator orchestrator = mock(Orchestrator.class); doThrow(new RuntimeException()).when(orchestrator).acquirePermissionToRemove(any()); - new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, new TestMetric(), tester.clock(), Duration.ofDays(30), - Duration.ofMinutes(10)).run(); + new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, new TestMetric(), + Duration.ofDays(30), Duration.ofMinutes(10)).run(); assertEquals(1, tester.nodeRepository().getNodes(Node.State.inactive).size()); // Inactive times out and one node is moved to parked tester.advanceTime(Duration.ofMinutes(11)); // Trigger InactiveExpirer - new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new TestMetric()).run(); + new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run(); assertEquals(1, tester.nodeRepository().getNodes(Node.State.parked).size()); } @@ -180,7 +180,7 @@ public class InactiveAndFailedExpirerTest { assertEquals(1, inactiveNodes.size()); // See that nodes are moved to dirty immediately. - new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new TestMetric()).run(); + new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run(); assertEquals(0, tester.nodeRepository().getNodes(Node.State.inactive).size()); List<Node> dirty = tester.nodeRepository().getNodes(Node.State.dirty); assertEquals(1, dirty.size()); @@ -205,7 +205,7 @@ public class InactiveAndFailedExpirerTest { // Nodes marked for deprovisioning are moved to parked tester.patchNodes(inactiveNodes, (node) -> node.withWantToRetire(true, true, Agent.system, tester.clock().instant())); tester.advanceTime(Duration.ofMinutes(11)); - new InactiveExpirer(tester.nodeRepository(), tester.clock(), Duration.ofMinutes(10), new TestMetric()).run(); + new InactiveExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()).run(); assertEquals(2, tester.nodeRepository().getNodes(Node.State.parked).size()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 653791c971a..194ff070425 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -125,21 +125,19 @@ public class MetricsReporterTest { expectedMetrics.put("cache.curator.evictionCount", 0L); expectedMetrics.put("cache.curator.size", 12L); - ManualClock clock = new ManualClock(Instant.ofEpochSecond(124)); + tester.clock().setInstant(Instant.ofEpochSecond(124)); Orchestrator orchestrator = mock(Orchestrator.class); when(orchestrator.getHostInfo(eq(reference), any())).thenReturn( HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.ofEpochSecond(1))); TestMetric metric = new TestMetric(); - MetricsReporter metricsReporter = new MetricsReporter( - nodeRepository, - metric, - orchestrator, - serviceMonitor, - () -> 42, - Duration.ofMinutes(1), - clock); + MetricsReporter metricsReporter = new MetricsReporter(nodeRepository, + metric, + orchestrator, + serviceMonitor, + () -> 42, + Duration.ofMinutes(1)); metricsReporter.maintain(); // Verify sum of values across dimensions, and remove these metrics to avoid checking against @@ -218,15 +216,12 @@ public class MetricsReporterTest { when(orchestrator.getHostInfo(eq(reference), any())).thenReturn(HostInfo.createNoRemarks()); TestMetric metric = new TestMetric(); - ManualClock clock = new ManualClock(); - MetricsReporter metricsReporter = new MetricsReporter( - nodeRepository, - metric, - orchestrator, - serviceMonitor, - () -> 42, - Duration.ofMinutes(1), - clock); + MetricsReporter metricsReporter = new MetricsReporter(nodeRepository, + metric, + orchestrator, + serviceMonitor, + () -> 42, + Duration.ofMinutes(1)); metricsReporter.maintain(); assertEquals(0, metric.values.get("hostedVespa.readyHosts")); // Only tenants counts diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index d759ceba4f6..3bb281f5b96 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -200,7 +200,7 @@ public class NodeFailTester { public NodeFailer createFailer() { return new NodeFailer(deployer, nodeRepository, downtimeLimitOneHour, - Duration.ofMinutes(5), clock, orchestrator, NodeFailer.ThrottlePolicy.hosted, metric); + Duration.ofMinutes(5), orchestrator, NodeFailer.ThrottlePolicy.hosted, metric); } public NodeFailureStatusUpdater createUpdater() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java index a77eaa172bb..0ee60f4c27f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooterTest.java @@ -33,7 +33,7 @@ public class NodeRebooterTest { makeReadyHosts(15, tester); NodeRepository nodeRepository = tester.nodeRepository(); - NodeRebooter rebooter = new NodeRebooter(nodeRepository, tester.clock(), flagSource, new TestMetric()); + NodeRebooter rebooter = new NodeRebooter(nodeRepository, flagSource, new TestMetric()); assertReadyHosts(15, nodeRepository, 0L); @@ -79,7 +79,7 @@ public class NodeRebooterTest { makeReadyHosts(2, tester); NodeRepository nodeRepository = tester.nodeRepository(); - NodeRebooter rebooter = new NodeRebooter(nodeRepository, tester.clock(), flagSource, new TestMetric()); + NodeRebooter rebooter = new NodeRebooter(nodeRepository, flagSource, new TestMetric()); assertReadyHosts(2, nodeRepository, 0L); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java index 1f401983219..4d29343214e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java @@ -48,7 +48,7 @@ public class ReservationExpirerTest { // Reservation times out clock.advance(Duration.ofMinutes(14)); // Reserved but not used time out - new ReservationExpirer(nodeRepository, clock, Duration.ofMinutes(10), metric).run(); + new ReservationExpirer(nodeRepository, Duration.ofMinutes(10), metric).run(); // Assert nothing is reserved assertEquals(0, nodeRepository.getNodes(NodeType.tenant, Node.State.reserved).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index bdae9d28fe2..952f52c44e0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -158,14 +158,12 @@ public class RetiredExpirerTest { } private RetiredExpirer createRetiredExpirer(Deployer deployer) { - return new RetiredExpirer( - nodeRepository, - orchestrator, - deployer, - new TestMetric(), - clock, - Duration.ofDays(30), /* Maintenance interval, use large value so it never runs by itself */ - RETIRED_EXPIRATION); + return new RetiredExpirer(nodeRepository, + orchestrator, + deployer, + new TestMetric(), + Duration.ofDays(30), /* Maintenance interval, use large value so it never runs by itself */ + RETIRED_EXPIRATION); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java index aba7ac2a530..9e2b453bf78 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java @@ -240,7 +240,6 @@ public class MultigroupProvisioningTest { tester.orchestrator(), deployer, new TestMetric(), - tester.clock(), Duration.ofDays(30), Duration.ofHours(12)).run(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java index d9265153596..7b964b4c7f3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java @@ -90,16 +90,16 @@ public class NodeTypeProvisioningTest { @Test public void retire_proxy() { - MockDeployer deployer = new MockDeployer( - tester.provisioner(), - tester.clock(), - Collections.singletonMap( - application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity))); + MockDeployer deployer = new MockDeployer(tester.provisioner(), + tester.clock(), + Collections.singletonMap(application, + new MockDeployer.ApplicationContext(application, + clusterSpec, + capacity))); RetiredExpirer retiredExpirer = new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, new TestMetric(), - tester.clock(), Duration.ofDays(30), Duration.ofMinutes(10)); @@ -168,7 +168,6 @@ public class NodeTypeProvisioningTest { tester.orchestrator(), deployer, new TestMetric(), - tester.clock(), Duration.ofDays(30), Duration.ofMinutes(10)); final int numNodesToRetire = 5; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 9c1ced856cc..020849e66d8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -852,7 +852,7 @@ public class ProvisioningTest { // Over 10 minutes pass since first reservation. First set of reserved nodes are not expired tester.clock().advance(Duration.ofMinutes(8).plus(Duration.ofSeconds(1))); - ReservationExpirer expirer = new ReservationExpirer(tester.nodeRepository(), tester.clock(), + ReservationExpirer expirer = new ReservationExpirer(tester.nodeRepository(), Duration.ofMinutes(10), new TestMetric()); expirer.run(); assertEquals("Nodes remain reserved", 4, |