diff options
author | Harald Musum <musum@verizonmedia.com> | 2021-03-10 15:48:19 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-03-10 15:48:19 +0100 |
commit | 34868d7ae17aac147c9652983d31080797c9bca3 (patch) | |
tree | 85419a176d2df3e9db19542220864a809812d0eb /node-repository | |
parent | ce58e0548f27f3084b63c6d0b00e539099ce59d4 (diff) | |
parent | 949df298d5a012aa5900e8c23f92e6e78a25b120 (diff) |
Merge pull request #16818 from vespa-engine/jonmv/cluster-controller-migration-cleanup
Always use dedicated cluster controllers when hosted (except in some …
Diffstat (limited to 'node-repository')
4 files changed, 0 insertions, 233 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DedicatedClusterControllerClusterMigrator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DedicatedClusterControllerClusterMigrator.java deleted file mode 100644 index 4588fc13875..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DedicatedClusterControllerClusterMigrator.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Deployer; -import com.yahoo.config.provision.Environment; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.orchestrator.Orchestrator; - -import java.time.Duration; -import java.time.ZonedDateTime; -import java.util.List; -import java.util.Set; -import java.util.logging.Level; - -import static java.time.DayOfWeek.SATURDAY; -import static java.time.DayOfWeek.SUNDAY; -import static java.util.stream.Collectors.toUnmodifiableSet; - -/** - * @author jonmv - */ -public class DedicatedClusterControllerClusterMigrator extends ApplicationMaintainer { - - private final BooleanFlag flag; - private final Orchestrator orchestrator; - - protected DedicatedClusterControllerClusterMigrator(Deployer deployer, Metric metric, NodeRepository nodeRepository, - Duration interval, FlagSource flags, Orchestrator orchestrator) { - super(deployer, metric, nodeRepository, interval); - this.flag = Flags.DEDICATED_CLUSTER_CONTROLLER_CLUSTER.bindTo(flags); - this.orchestrator = orchestrator; - } - - @Override - protected Set<ApplicationId> applicationsNeedingMaintenance() { - if (deployer().bootstrapping()) return Set.of(); - - ZonedDateTime date = ZonedDateTime.ofInstant(clock().instant(), java.time.ZoneId.of("Europe/Oslo")); - if ( ! nodeRepository().zone().system().isCd() - && nodeRepository().zone().environment() != Environment.staging - && (List.of(SATURDAY, SUNDAY).contains(date.getDayOfWeek()) || date.getHour() < 8 || 13 < date.getHour())) - return Set.of(); - - return nodeRepository().applications().ids().stream() - .sorted() - .filter(id -> ! id.instance().isTester()) - .filter(this::isEligible) - .filter(this::hasNotSwitched) - .filter(this::isQuiescent) - .limit(1) - .collect(toUnmodifiableSet()); - } - - @Override - protected void deploy(ApplicationId id) { - migrate(id); - super.deploy(id); - } - - private boolean isEligible(ApplicationId id) { - return deployer().lastDeployTime(id).map(at -> at.isBefore(clock().instant().minus(Duration.ofMinutes(10)))).orElse(false) - && flag.with(FetchVector.Dimension.APPLICATION_ID, id.serializedForm()).value(); - } - - private boolean hasNotSwitched(ApplicationId id) { - return ! deployer().getDedicatedClusterControllerCluster(id); - } - - private boolean isQuiescent(ApplicationId id) { - return orchestrator.isQuiescent(id); // Check all content nodes are UP, have wanted state UP, and can be moved to MAINTENANCE. - } - - private void migrate(ApplicationId id) { - log.log(Level.INFO, "Migrating " + id + " to dedicated cluster controller cluster"); - deployer().setDedicatedClusterControllerCluster(id); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 08106a94205..7f41f89f664 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -68,7 +68,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent { maintainers.add(new AutoscalingMaintainer(nodeRepository, metricsDb, deployer, metric, defaults.autoscalingInterval)); maintainers.add(new ScalingSuggestionsMaintainer(nodeRepository, metricsDb, defaults.scalingSuggestionsInterval, metric)); maintainers.add(new SwitchRebalancer(nodeRepository, defaults.switchRebalancerInterval, metric, deployer)); - maintainers.add(new DedicatedClusterControllerClusterMigrator(deployer, metric, nodeRepository, defaults.dedicatedClusterControllerMigratorInterval, flagSource, orchestrator)); provisionServiceProvider.getLoadBalancerService(nodeRepository) .map(lbService -> new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService, metric)) @@ -120,7 +119,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration autoscalingInterval; private final Duration scalingSuggestionsInterval; private final Duration switchRebalancerInterval; - private final Duration dedicatedClusterControllerMigratorInterval; private final NodeFailer.ThrottlePolicy throttlePolicy; @@ -149,8 +147,6 @@ public class NodeRepositoryMaintenance extends AbstractComponent { switchRebalancerInterval = Duration.ofHours(1); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; retiredExpiry = Duration.ofDays(4); // give up migrating data after 4 days - dedicatedClusterControllerMigratorInterval = zone.environment() == Environment.staging || zone.system().isCd() ? Duration.ofMinutes(3) - : Duration.ofHours(2); inactiveConfigServerExpiry = Duration.ofMinutes(5); inactiveControllerExpiry = Duration.ofMinutes(5); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DedicatedClusterControllerClusterMigratorTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DedicatedClusterControllerClusterMigratorTest.java deleted file mode 100644 index 22db730d69f..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DedicatedClusterControllerClusterMigratorTest.java +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.ClusterResources; -import com.yahoo.config.provision.Deployer; -import com.yahoo.config.provision.Deployment; -import com.yahoo.config.provision.HostFilter; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.jdisc.test.MockMetric; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.hosted.provision.applications.Application; -import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; -import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; -import org.junit.Test; - -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Phaser; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * @author jonmv - */ -public class DedicatedClusterControllerClusterMigratorTest { - - @Test - public void testMigration() throws InterruptedException, TimeoutException { - ApplicationId id1 = ApplicationId.from("t", "a", "i1"), id2 = ApplicationId.from("t", "a", "i2"); - ProvisioningTester tester = new ProvisioningTester.Builder().build(); - tester.clock().setInstant(Instant.EPOCH); // EPOCH was a week-day. - tester.makeReadyNodes(4, new NodeResources(1.5, 8, 50, 0.3)); - tester.makeReadyHosts(1, NodeResources.unspecified()); - tester.deploy(id1, Capacity.from(new ClusterResources(2, 1, NodeResources.unspecified()))); - tester.deploy(id2, Capacity.from(new ClusterResources(2, 1, NodeResources.unspecified()))); - MockDeployer deployer = new MockDeployer(); - InMemoryFlagSource flags = new InMemoryFlagSource(); - AtomicBoolean isQuiescent = new AtomicBoolean(); - OrchestratorMock orchestrator = new OrchestratorMock() { - @Override public boolean isQuiescent(ApplicationId id) { return isQuiescent.get(); } - }; - - DedicatedClusterControllerClusterMigrator migrator = new DedicatedClusterControllerClusterMigrator(deployer, - new MockMetric(), - tester.nodeRepository(), - Duration.ofDays(365), - flags, - orchestrator); - assertFalse(deployer.getDedicatedClusterControllerCluster(id1)); - - // Set all conditions true except time window. - flags.withBooleanFlag(Flags.DEDICATED_CLUSTER_CONTROLLER_CLUSTER.id(), true); - isQuiescent.set(true); - deployer.deployedAt.set(tester.clock().instant().minus(Duration.ofMinutes(15))); - migrator.maintain(); - assertFalse(deployer.getDedicatedClusterControllerCluster(id1)); - - // Enter time window, but no longer quiescent. - tester.clock().advance(Duration.ofHours(8)); - isQuiescent.set(false); - migrator.maintain(); - assertFalse(deployer.getDedicatedClusterControllerCluster(id1)); - - // Quiescent, but no longer flagged. - isQuiescent.set(true); - flags.withBooleanFlag(Flags.DEDICATED_CLUSTER_CONTROLLER_CLUSTER.id(), false); - migrator.maintain(); - assertFalse(deployer.getDedicatedClusterControllerCluster(id1)); - - // Flagged, but recently deployed. - flags.withBooleanFlag(Flags.DEDICATED_CLUSTER_CONTROLLER_CLUSTER.id(), true); - deployer.deployedAt.set(tester.clock().instant().minus(Duration.ofMinutes(5))); - migrator.maintain(); - assertFalse(deployer.getDedicatedClusterControllerCluster(id1)); - - // Finally, all stars align. - deployer.deployedAt.set(tester.clock().instant().minus(Duration.ofMinutes(15))); - migrator.maintain(); - assertTrue(deployer.getDedicatedClusterControllerCluster(id1)); // Lex sorting, t.a.i1 before t.a.i2. - assertFalse(deployer.getDedicatedClusterControllerCluster(id2)); - assertEquals(1, deployer.phaser.awaitAdvanceInterruptibly(deployer.phaser.arrive(), 10, TimeUnit.SECONDS)); - - migrator.maintain(); - assertTrue(deployer.getDedicatedClusterControllerCluster(id2)); - assertEquals(2, deployer.phaser.awaitAdvanceInterruptibly(deployer.phaser.arrive(), 10, TimeUnit.SECONDS)); - - assertEquals(Set.of(), migrator.applicationsNeedingMaintenance()); - } - - - private static class MockDeployer implements Deployer { - - final Phaser phaser = new Phaser(2); // Test thread and deployer. - final Set<ApplicationId> dedicatedCCC = new ConcurrentSkipListSet<>(); - final AtomicReference<Instant> deployedAt = new AtomicReference<>(); - - @Override - public Optional<Deployment> deployFromLocalActive(ApplicationId application, boolean bootstrap) { - return Optional.of(new Deployment() { - @Override public void prepare() { fail("Shouldn't run"); } - @Override public long activate() { return phaser.arriveAndAwaitAdvance(); } - @Override public void restart(HostFilter filter) { fail("Shouldn't run"); } - }); - } - - @Override - public Optional<Deployment> deployFromLocalActive(ApplicationId application, Duration timeout, boolean bootstrap) { - return deployFromLocalActive(application, bootstrap); - } - - @Override - public Optional<Instant> lastDeployTime(ApplicationId application) { - return Optional.ofNullable(deployedAt.get()); - } - - @Override - public void setDedicatedClusterControllerCluster(ApplicationId id) { - dedicatedCCC.add(id); - } - - @Override - public boolean getDedicatedClusterControllerCluster(ApplicationId id) { - return dedicatedCCC.contains(id); - } - - } - -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json index bd4029ec0c0..b31c597e2b0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json @@ -4,9 +4,6 @@ "name": "AutoscalingMaintainer" }, { - "name": "DedicatedClusterControllerClusterMigrator" - }, - { "name": "DirtyExpirer" }, { |