diff options
author | Jon Bratseth <bratseth@gmail.com> | 2020-04-30 10:27:32 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2020-04-30 10:27:32 +0200 |
commit | 2756ec125d173170b987aa43aeea716ebd61fa6b (patch) | |
tree | 488819a75da795a6dce554c0f2624946eb994624 /node-repository | |
parent | eb9f0add11504d49151b005ee03cd80e1369ec8b (diff) | |
parent | a5e6a9170120c5efa9dd620d35eba5377581fbb1 (diff) |
Merge branch 'master' into bratseth/allow-non-allocatable-limits
Diffstat (limited to 'node-repository')
27 files changed, 96 insertions, 431 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index fe80ed489f4..a459cc2826f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -5,6 +5,7 @@ import com.google.inject.Inject; import com.yahoo.collections.ListMap; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; +import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; @@ -23,7 +24,6 @@ import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; -import com.yahoo.vespa.hosted.provision.maintenance.JobControl; import com.yahoo.vespa.hosted.provision.maintenance.NodeFailer; import com.yahoo.vespa.hosted.provision.maintenance.PeriodicApplicationMaintainer; import com.yahoo.vespa.hosted.provision.node.Agent; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index b58fd3f67f1..06300e45e9e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -5,7 +5,6 @@ import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployer; import com.yahoo.jdisc.Metric; -import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; @@ -15,12 +14,13 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; /** * @author bratseth * @author mpolden */ -public abstract class ApplicationMaintainer extends Maintainer { +public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer { private final Deployer deployer; private final Metric metric; @@ -91,8 +91,8 @@ public abstract class ApplicationMaintainer extends Maintainer { } @Override - public void deconstruct() { - super.deconstruct(); + public void close() { + super.close(); this.deploymentExecutor.shutdownNow(); try { // Give deployments in progress some time to complete diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 34f6014095f..fa8e8375e23 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -1,27 +1,22 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import com.yahoo.collections.Pair; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.NodeResources; import com.yahoo.jdisc.Metric; -import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import java.time.Duration; -import java.time.Instant; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -32,7 +27,7 @@ import java.util.stream.Collectors; * * @author bratseth */ -public class AutoscalingMaintainer extends Maintainer { +public class AutoscalingMaintainer extends NodeRepositoryMaintainer { private final Autoscaler autoscaler; private final Deployer deployer; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java index b4c0d469272..abd1b477d4f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java @@ -21,7 +21,7 @@ import java.util.*; * * @author mgimle */ -public class CapacityReportMaintainer extends Maintainer { +public class CapacityReportMaintainer extends NodeRepositoryMaintainer { private final Metric metric; private final NodeRepository nodeRepository; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 07de9c5d443..85909daa240 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -38,7 +38,7 @@ import java.util.stream.IntStream; /** * @author freva */ -public class DynamicProvisioningMaintainer extends Maintainer { +public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { private static final Logger log = Logger.getLogger(DynamicProvisioningMaintainer.class.getName()); private static final ApplicationId preprovisionAppId = ApplicationId.from("hosted-vespa", "tenant-host", "preprovision"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java index ca576be8a69..dc5155312e7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java @@ -17,7 +17,7 @@ import java.util.logging.Logger; * * @author bratseth */ -public abstract class Expirer extends Maintainer { +public abstract class Expirer extends NodeRepositoryMaintainer { protected static final Logger log = Logger.getLogger(Expirer.class.getName()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index af37aa5becf..7e02eedc572 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -45,7 +45,7 @@ import java.util.stream.Collectors; * @author bratseth * @author mpolden */ -public class FailedExpirer extends Maintainer { +public class FailedExpirer extends NodeRepositoryMaintainer { private static final Logger log = Logger.getLogger(FailedExpirer.class.getName()); private static final int maxAllowedFailures = 5; // Stop recycling nodes after this number of failures diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java index 932c435221c..b933e549357 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java @@ -14,7 +14,7 @@ import java.util.logging.Logger; * * @author freva */ -public class InfrastructureProvisioner extends Maintainer { +public class InfrastructureProvisioner extends NodeRepositoryMaintainer { private static final Logger logger = Logger.getLogger(InfrastructureProvisioner.class.getName()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/JobControl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/JobControl.java deleted file mode 100644 index 2d641ef57ab..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/JobControl.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance; - -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; - -import java.util.Collections; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentSkipListMap; - -/** - * Provides status and control over running maintenance jobs. - * - * This is multi-thread safe. - * - * @author bratseth - */ -public class JobControl { - - /** This is not stored in ZooKeeper as all nodes start all jobs */ - private final Map<String, Maintainer> startedJobs = new ConcurrentSkipListMap<>(); - - /** Used to store deactivation in ZooKeeper to make changes take effect on all nodes */ - private final CuratorDatabaseClient db; - - public JobControl(CuratorDatabaseClient db) { - this.db = db; - } - - /** Notifies this that a job was started */ - public void started(String jobSimpleClassName, Maintainer maintainer) { - startedJobs.put(jobSimpleClassName, maintainer); - } - - /** - * Returns a snapshot of the set of jobs started on this system (whether deactivated or not). - * Each job is represented by its simple (omitting package) class name. - */ - public Set<String> jobs() { return Collections.unmodifiableSet(startedJobs.keySet()); } - - /** Returns a snapshot containing the currently inactive jobs in this */ - public Set<String> inactiveJobs() { return db.readInactiveJobs(); } - - /** Returns true if this job is not currently deactivated */ - public boolean isActive(String jobSimpleClassName) { - return ! inactiveJobs().contains(jobSimpleClassName); - } - - /** Set a job active or inactive */ - public void setActive(String jobSimpleClassName, boolean active) { - try (Lock lock = db.lockInactiveJobs()) { - Set<String> inactiveJobs = db.readInactiveJobs(); - if (active) - inactiveJobs.remove(jobSimpleClassName); - else - inactiveJobs.add(jobSimpleClassName); - db.writeInactiveJobs(inactiveJobs); - } - } - - /** Run given job (inactive or not) immediately */ - public void run(String jobSimpleClassName) { - var job = startedJobs.get(jobSimpleClassName); - if (job == null) throw new IllegalArgumentException("No such job '" + jobSimpleClassName + "'"); - job.runWithLock(); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index 55152f52744..d793874119a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -30,7 +30,7 @@ import java.util.stream.Collectors; * * @author mpolden */ -public class LoadBalancerExpirer extends Maintainer { +public class LoadBalancerExpirer extends NodeRepositoryMaintainer { private static final Duration reservedExpiry = Duration.ofHours(1); private static final Duration inactiveExpiry = Duration.ofHours(1); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java deleted file mode 100644 index 17c73282b4b..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance; - -import com.google.common.util.concurrent.UncheckedTimeoutException; -import com.yahoo.component.AbstractComponent; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.NodeRepository; - -import java.time.Duration; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * A maintainer is some job which runs at a fixed rate to perform some maintenance task on the node repo. - * - * @author bratseth - */ -public abstract class Maintainer extends AbstractComponent implements Runnable { - - protected final Logger log = Logger.getLogger(this.getClass().getName()); - - private final NodeRepository nodeRepository; - private final Duration interval; - private final JobControl jobControl; - - private final ScheduledExecutorService service; - - public Maintainer(NodeRepository nodeRepository, Duration interval) { - this.nodeRepository = nodeRepository; - this.interval = interval; - this.jobControl = nodeRepository.jobControl(); - - HostName hostname = HostName.from(com.yahoo.net.HostName.getLocalhost()); - long delay = staggeredDelay(nodeRepository.database().cluster(), hostname, nodeRepository.clock().instant(), interval); - service = new ScheduledThreadPoolExecutor(1, r -> new Thread(r, name() + "-worker")); - service.scheduleAtFixedRate(this, delay, interval.toMillis(), TimeUnit.MILLISECONDS); - jobControl.started(name(), this); - } - - /** Returns the node repository */ - protected NodeRepository nodeRepository() { return nodeRepository; } - - protected static Duration min(Duration a, Duration b) { - return a.toMillis() < b.toMillis() ? a : b; - } - - /** Returns the interval at which this job is set to run */ - protected Duration interval() { return interval; } - - @Override - public void run() { - try { - if (jobControl.isActive(name())) { - runWithLock(); - } - } catch (UncheckedTimeoutException ignored) { - // Another config server or operator is running this job - } catch (Throwable e) { - log.log(Level.WARNING, this + " failed. Will retry in " + interval.toMinutes() + " minutes", e); - } - } - - @Override - public void deconstruct() { - var timeout = Duration.ofSeconds(30); - service.shutdown(); - try { - if (!service.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS)) { - log.log(Level.WARNING, "Maintainer " + name() + " failed to shutdown " + - "within " + timeout); - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - /** Returns the simple name of this job */ - @Override - public final String toString() { return name(); } - - /** Run this while holding the job lock */ - public void runWithLock() { - try (var lock = nodeRepository.database().lockMaintenanceJob(name())) { - maintain(); - } - } - - /** Called once each time this maintenance job should run */ - protected abstract void maintain(); - - private String name() { return this.getClass().getSimpleName(); } - - /** A utility to group active tenant nodes by application */ - protected Map<ApplicationId, List<Node>> activeNodesByApplication() { - return nodeRepository().list().nodeType(NodeType.tenant).state(Node.State.active).asList() - .stream() - .filter(node -> ! node.allocation().get().owner().instance().isTester()) - .collect(Collectors.groupingBy(node -> node.allocation().get().owner())); - } - - static long staggeredDelay(List<HostName> cluster, HostName host, Instant now, Duration interval) { - if ( ! cluster.contains(host)) - return interval.toMillis(); - - long offset = cluster.indexOf(host) * interval.toMillis() / cluster.size(); - return Math.floorMod(offset - now.toEpochMilli(), interval.toMillis()); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 098d706bf05..c631de5f17b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -34,7 +34,7 @@ import static com.yahoo.vespa.hosted.provision.Node.State.active; /** * @author oyving */ -public class MetricsReporter extends Maintainer { +public class MetricsReporter extends NodeRepositoryMaintainer { private final Metric metric; private final Orchestrator orchestrator; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index cffbfec8f4b..7ce674cbbbf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -31,7 +31,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -45,7 +44,7 @@ import static java.util.stream.Collectors.counting; * @author bratseth * @author mpolden */ -public class NodeFailer extends Maintainer { +public class NodeFailer extends NodeRepositoryMaintainer { private static final Logger log = Logger.getLogger(NodeFailer.class.getName()); private static final Duration nodeRequestInterval = Duration.ofMinutes(10); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java index adc1911a169..eb2b46dd53e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -16,7 +16,7 @@ import java.util.logging.Level; * * @author bratseth */ -public class NodeMetricsDbMaintainer extends Maintainer { +public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { private static final int maxWarningsPerInvocation = 2; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java index d1e15c78a47..c78ed72ff42 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java @@ -26,7 +26,7 @@ import java.util.stream.Collectors; * * @author bratseth */ -public class NodeRebooter extends Maintainer { +public class NodeRebooter extends NodeRepositoryMaintainer { private final IntFlag rebootIntervalInDays; private final Clock clock; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java new file mode 100644 index 00000000000..8368569cda0 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java @@ -0,0 +1,44 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.concurrent.maintenance.Maintainer; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * A maintainer is some job which runs at a fixed rate to perform some maintenance task on the node repo. + * + * @author bratseth + */ +public abstract class NodeRepositoryMaintainer extends Maintainer { + + private final NodeRepository nodeRepository; + + public NodeRepositoryMaintainer(NodeRepository nodeRepository, Duration interval) { + super(null, interval, nodeRepository.clock().instant(), nodeRepository.jobControl(), nodeRepository.database().cluster()); + this.nodeRepository = nodeRepository; + } + + protected static Duration min(Duration a, Duration b) { + return a.toMillis() < b.toMillis() ? a : b; + } + + /** Returns the node repository */ + protected NodeRepository nodeRepository() { return nodeRepository; } + + /** A utility to group active tenant nodes by application */ + protected Map<ApplicationId, List<Node>> activeNodesByApplication() { + return nodeRepository().list().nodeType(NodeType.tenant).state(Node.State.active).asList() + .stream() + .filter(node -> ! node.allocation().get().owner().instance().isTester()) + .collect(Collectors.groupingBy(node -> node.allocation().get().owner())); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index a1a6ee2e49d..d388fb5a967 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -101,26 +101,26 @@ public class NodeRepositoryMaintenance extends AbstractComponent { @Override public void deconstruct() { - nodeFailer.deconstruct(); - periodicApplicationMaintainer.deconstruct(); - operatorChangeApplicationMaintainer.deconstruct(); - reservationExpirer.deconstruct(); - inactiveExpirer.deconstruct(); - retiredExpirer.deconstruct(); - failedExpirer.deconstruct(); - dirtyExpirer.deconstruct(); - nodeRebooter.deconstruct(); - capacityReportMaintainer.deconstruct(); - provisionedExpirer.deconstruct(); - metricsReporter.deconstruct(); - infrastructureProvisioner.deconstruct(); - loadBalancerExpirer.ifPresent(Maintainer::deconstruct); - dynamicProvisioningMaintainer.ifPresent(Maintainer::deconstruct); - osUpgradeActivator.deconstruct(); - rebalancer.deconstruct(); - nodeMetricsDbMaintainer.deconstruct(); - autoscalingMaintainer.deconstruct(); - scalingSuggestionsMaintainer.deconstruct(); + nodeFailer.close(); + periodicApplicationMaintainer.close(); + operatorChangeApplicationMaintainer.close(); + reservationExpirer.close(); + inactiveExpirer.close(); + retiredExpirer.close(); + failedExpirer.close(); + dirtyExpirer.close(); + nodeRebooter.close(); + capacityReportMaintainer.close(); + provisionedExpirer.close(); + metricsReporter.close(); + infrastructureProvisioner.close(); + loadBalancerExpirer.ifPresent(NodeRepositoryMaintainer::close); + dynamicProvisioningMaintainer.ifPresent(NodeRepositoryMaintainer::close); + osUpgradeActivator.close(); + rebalancer.close(); + nodeMetricsDbMaintainer.close(); + autoscalingMaintainer.close(); + scalingSuggestionsMaintainer.close(); } private static Optional<NodeFailer.ThrottlePolicy> throttlePolicyFromEnv() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java index c22bda8a5cf..b98b03671c4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OsUpgradeActivator.java @@ -15,7 +15,7 @@ import java.time.Duration; * * @author mpolden */ -public class OsUpgradeActivator extends Maintainer { +public class OsUpgradeActivator extends NodeRepositoryMaintainer { public OsUpgradeActivator(NodeRepository nodeRepository, Duration interval) { super(nodeRepository, interval); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java index a496f496975..7ffb541be2a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java @@ -22,7 +22,7 @@ import java.util.Optional; /** * @author bratseth */ -public class Rebalancer extends Maintainer { +public class Rebalancer extends NodeRepositoryMaintainer { private final Deployer deployer; private final Optional<HostProvisioner> hostProvisioner; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java index 3c01c8bc23c..73869fb098d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java @@ -24,7 +24,7 @@ import java.util.stream.Collectors; * * @author hakon */ -public class RetiredExpirer extends Maintainer { +public class RetiredExpirer extends NodeRepositoryMaintainer { private final Deployer deployer; private final Metric metric; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index d626f4f486a..b68e8eacbaa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; @@ -26,7 +25,7 @@ import java.util.stream.Collectors; * * @author bratseth */ -public class ScalingSuggestionsMaintainer extends Maintainer { +public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { private final Autoscaler autoscaler; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 347c66d5426..4defbb55485 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.provision.persistence; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.component.Version; +import com.yahoo.concurrent.maintenance.JobControl; +import com.yahoo.concurrent.maintenance.StringSetSerializer; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.DockerImage; @@ -57,7 +59,7 @@ import static java.util.stream.Collectors.toMap; * * @author bratseth */ -public class CuratorDatabaseClient { +public class CuratorDatabaseClient implements JobControl.Db { private static final Logger log = Logger.getLogger(CuratorDatabaseClient.class.getName()); @@ -90,8 +92,8 @@ public class CuratorDatabaseClient { initZK(); } - public List<HostName> cluster() { - return db.cluster(); + public List<String> cluster() { + return db.cluster().stream().map(HostName::value).collect(Collectors.toUnmodifiableList()); } private void initZK() { @@ -435,10 +437,12 @@ public class CuratorDatabaseClient { // Maintenance jobs ----------------------------------------------------------- + @Override public Lock lockMaintenanceJob(String jobName) { return db.lock(lockPath.append("maintenanceJobLocks").append(jobName), defaultLockTimeout); } + @Override public Set<String> readInactiveJobs() { try { return read(inactiveJobsPath, stringSetSerializer::fromJson).orElseGet(HashSet::new); @@ -450,6 +454,7 @@ public class CuratorDatabaseClient { } } + @Override public void writeInactiveJobs(Set<String> inactiveJobs) { NestedTransaction transaction = new NestedTransaction(); CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); @@ -457,7 +462,8 @@ public class CuratorDatabaseClient { stringSetSerializer.toJson(inactiveJobs))); transaction.commit(); } - + + @Override public Lock lockInactiveJobs() { return db.lock(lockPath.append("inactiveJobsLock"), defaultLockTimeout); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java deleted file mode 100644 index 1149b15a2b0..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.persistence; - -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Set; - -/** - * Serialization of a set of strings to/from Json bytes using Slime. - * - * The set is serialized as an array of string. - * - * @author bratseth - */ -public class StringSetSerializer { - - public byte[] toJson(Set<String> stringSet) { - try { - Slime slime = new Slime(); - Cursor array = slime.setArray(); - for (String element : stringSet) - array.addString(element); - return SlimeUtils.toJsonBytes(slime); - } - catch (IOException e) { - throw new RuntimeException("Serialization of a string set failed", e); - } - - } - - public Set<String> fromJson(byte[] data) { - Inspector inspector = SlimeUtils.jsonToSlime(data).get(); - Set<String> stringSet = new HashSet<>(); - inspector.traverse((ArrayTraverser) (index, name) -> stringSet.add(name.asString())); - return stringSet; - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java index 3b6de41f142..eb425095978 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java @@ -1,11 +1,11 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; +import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.provision.maintenance.JobControl; import java.io.IOException; import java.io.OutputStream; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/JobControlTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/JobControlTest.java deleted file mode 100644 index 396fcd67034..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/JobControlTest.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance; - -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; -import org.junit.Test; - -import java.time.Duration; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author bratseth - */ -public class JobControlTest { - - @Test - public void testJobControl() { - NodeRepositoryTester tester = new NodeRepositoryTester(); - JobControl jobControl = new JobControl(tester.nodeRepository().database()); - - MockMaintainer maintainer1 = new MockMaintainer(tester.nodeRepository()); - MockMaintainer maintainer2 = new MockMaintainer(tester.nodeRepository()); - assertTrue(jobControl.jobs().isEmpty()); - - String job1 = "Job1"; - String job2 = "Job2"; - - jobControl.started(job1, maintainer1); - jobControl.started(job2, maintainer2); - assertEquals(2, jobControl.jobs().size()); - assertTrue(jobControl.jobs().contains(job1)); - assertTrue(jobControl.jobs().contains(job2)); - - assertTrue(jobControl.isActive(job1)); - assertTrue(jobControl.isActive(job2)); - - jobControl.setActive(job1, false); - assertFalse(jobControl.isActive(job1)); - assertTrue(jobControl.isActive(job2)); - - jobControl.setActive(job2, false); - assertFalse(jobControl.isActive(job1)); - assertFalse(jobControl.isActive(job2)); - - jobControl.setActive(job1, true); - assertTrue(jobControl.isActive(job1)); - assertFalse(jobControl.isActive(job2)); - - jobControl.setActive(job2, true); - assertTrue(jobControl.isActive(job1)); - assertTrue(jobControl.isActive(job2)); - - // Run jobs on-demand - jobControl.run(job1); - jobControl.run(job1); - assertEquals(2, maintainer1.maintenanceInvocations); - jobControl.run(job2); - assertEquals(1, maintainer2.maintenanceInvocations); - - // Running jobs on-demand ignores inactive flag - jobControl.setActive(job1, false); - jobControl.run(job1); - assertEquals(3, maintainer1.maintenanceInvocations); - } - - @Test - public void testJobControlMayDeactivateJobs() { - NodeRepositoryTester tester = new NodeRepositoryTester(); - JobControl jobControl = tester.nodeRepository().jobControl(); - MockMaintainer mockMaintainer = new MockMaintainer(tester.nodeRepository()); - - assertTrue(jobControl.jobs().contains("MockMaintainer")); - - assertEquals(0, mockMaintainer.maintenanceInvocations); - - mockMaintainer.run(); - assertEquals(1, mockMaintainer.maintenanceInvocations); - - jobControl.setActive("MockMaintainer", false); - mockMaintainer.run(); - assertEquals(1, mockMaintainer.maintenanceInvocations); - - jobControl.setActive("MockMaintainer", true); - mockMaintainer.run(); - assertEquals(2, mockMaintainer.maintenanceInvocations); - } - - private static class MockMaintainer extends Maintainer { - - int maintenanceInvocations = 0; - - private MockMaintainer(NodeRepository nodeRepository) { - super(nodeRepository, Duration.ofHours(1)); - } - - @Override - protected void maintain() { - maintenanceInvocations++; - } - - } - -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintainerTest.java deleted file mode 100644 index 7ce64093491..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintainerTest.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance; - -import com.yahoo.config.provision.HostName; -import org.junit.Test; - -import java.time.Duration; -import java.time.Instant; -import java.util.Arrays; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author freva - */ -public class MaintainerTest { - - @Test - public void staggering() { - List<HostName> cluster = Arrays.asList(HostName.from("cfg1"), HostName.from("cfg2"), HostName.from("cfg3")); - Duration interval = Duration.ofMillis(300); - Instant now = Instant.ofEpochMilli(1000); - assertEquals(200, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval)); - assertEquals( 0, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval)); - assertEquals(100, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval)); - - now = Instant.ofEpochMilli(1001); - assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval)); - assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval)); - assertEquals( 99, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval)); - - now = Instant.ofEpochMilli(1101); - assertEquals( 99, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval)); - assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval)); - assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval)); - - assertEquals(300, Maintainer.staggeredDelay(cluster, HostName.from("cfg0"), now, interval)); - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java index d339700d28f..8a2a69bb437 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java @@ -76,7 +76,7 @@ public class PeriodicApplicationMaintainerTest { @After public void after() { - this.fixture.maintainer.deconstruct(); + this.fixture.maintainer.close(); } @Test(timeout = 60_000) |