diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-10-25 13:38:27 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-10-25 13:38:27 +0200 |
commit | 1cf9d739771c850fe5ed3366612e39372efdc0df (patch) | |
tree | aa1e7d4077cae5ce545fb8a24845547352bdfe11 | |
parent | caf2513a89cd29fb473f404e4389f0265ee8b594 (diff) | |
parent | ea2df47c46abd2b641e4d2efc41d063a64894179 (diff) |
Merge pull request #7447 from vespa-engine/mpolden/pr-instance-cleanup
PR instance handling cleanup
18 files changed, 6 insertions, 162 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index e46df3c5a4a..c7e7165cb91 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -139,14 +139,6 @@ public class ApplicationList { .anyMatch(d -> d.version().isBefore(version)))); } - /** - * Returns the subset of applications which are not pull requests: - * Pull requests changes the application instance name to (default-pr)?[pull-request-number] - */ - public ApplicationList notPullRequest() { - return listOf(list.stream().filter(a -> ! a.id().instance().value().matches("^(default-pr)?\\d+$"))); - } - /** Returns the subset of applications which have a project ID */ public ApplicationList withProjectId() { return listOf(list.stream().filter(a -> a.deploymentJobs().projectId().isPresent())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index cf9b2370671..8bc61d6bc3b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -275,7 +275,6 @@ public class DeploymentTrigger { /** Returns the set of all jobs which have changes to propagate from the upstream steps. */ private List<Job> computeReadyJobs() { return ApplicationList.from(applications().asList()) - .notPullRequest() .withProjectId() .deploying() .idList().stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index b2b69a81dbc..219517cfd30 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; -import java.io.UncheckedIOException; import java.time.Duration; import java.util.NoSuchElementException; import java.util.Optional; @@ -44,7 +43,6 @@ public class ApplicationOwnershipConfirmer extends Maintainer { /** File an ownership issue with the owners of all applications we know about. */ private void confirmApplicationOwnerships() { ApplicationList.from(controller().applications().asList()) - .notPullRequest() .withProjectId() .hasProductionDeployment() .asList() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java index aa0ad8b63e0..4a49d5df5ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -89,7 +88,7 @@ public class ClusterInfoMaintainer extends Maintainer { @Override protected void maintain() { - for (Application application : ApplicationList.from(controller().applications().asList()).notPullRequest().asList()) { + for (Application application : controller().applications().asList()) { for (Deployment deployment : application.deployments().values()) { DeploymentId deploymentId = new DeploymentId(application.id(), deployment.zone()); try { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java index a046ed87a05..135a05e2725 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java @@ -3,11 +3,10 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -44,7 +43,7 @@ public class ClusterUtilizationMaintainer extends Maintainer { @Override protected void maintain() { - for (Application application : ApplicationList.from(controller().applications().asList()).notPullRequest().asList()) { + for (Application application : controller().applications().asList()) { for (Deployment deployment : application.deployments().values()) { Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization = getUpdatedClusterUtilizations(application.id(), deployment.zone()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index a00c2ccb4ed..33555c08a43 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -48,7 +48,6 @@ public class ControllerMaintenance extends AbstractComponent { private final OsVersionStatusUpdater osVersionStatusUpdater; private final JobRunner jobRunner; private final ContactInformationMaintainer contactInformationMaintainer; - private final DatabaseMaintainer databaseMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator, @@ -75,7 +74,6 @@ public class ControllerMaintenance extends AbstractComponent { osUpgraders = osUpgraders(controller, jobControl); osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl); contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, organization, apiAuthorityConfig); - databaseMaintainer = new DatabaseMaintainer(controller, maintenanceInterval, jobControl); } public Upgrader upgrader() { return upgrader; } @@ -102,7 +100,6 @@ public class ControllerMaintenance extends AbstractComponent { osVersionStatusUpdater.deconstruct(); jobRunner.deconstruct(); contactInformationMaintainer.deconstruct(); - databaseMaintainer.deconstruct(); } /** Create one OS upgrader per cloud found in the zone registry of controller */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DatabaseMaintainer.java deleted file mode 100644 index ac368b00b82..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DatabaseMaintainer.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; - -import java.time.Duration; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Runs maintenance operations for the controller database, such as removing stale entries. - * - * @author mpolden - */ -public class DatabaseMaintainer extends Maintainer { - - public DatabaseMaintainer(Controller controller, Duration interval, JobControl jobControl) { - super(controller, interval, jobControl); - } - - @Override - protected void maintain() { - removePullRequestInstances(); - } - - /** Pull request instances are no longer created. This removes all existing entries */ - // TODO: Remove after 2018-11-01 - // TODO: Remove ApplicationList#notPullRequest and its usages - private void removePullRequestInstances() { - List<Application> pullRequestInstances = controller().applications().asList().stream() - .filter(a -> a.id().instance().value().matches("^(default-pr)?\\d+$")) - .collect(Collectors.toList()); - - pullRequestInstances.forEach(application -> { - controller().applications().lockIfPresent(application.id(), (lockedApplication) -> controller().curator().removeApplication(lockedApplication.get().id())); - }); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 11e0ada6c36..1745e013a40 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; -import java.io.UncheckedIOException; import java.time.Duration; import java.util.Collection; import java.util.List; @@ -57,7 +56,6 @@ public class DeploymentIssueReporter extends Maintainer { private List<Application> applications() { return ApplicationList.from(controller().applications().asList()) .withProjectId() - .notPullRequest() .asList(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 0b56916a0de..72936f8a679 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -6,7 +6,6 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.RotationStatus; @@ -48,7 +47,7 @@ public class DeploymentMetricsMaintainer extends Maintainer { protected void maintain() { AtomicInteger failures = new AtomicInteger(0); AtomicReference<Exception> lastException = new AtomicReference<>(null); - List<Application> applicationList = ApplicationList.from(applications.asList()).notPullRequest().asList(); + List<Application> applicationList = applications.asList(); // Run parallel stream inside a custom ForkJoinPool so that we can control the number of threads used ForkJoinPool pool = new ForkJoinPool(applicationsToUpdateInParallel); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 258a72c8d01..305241a7d0e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -117,7 +117,6 @@ public class MetricsReporter extends Maintainer { private void reportDeploymentMetrics() { ApplicationList applications = ApplicationList.from(controller().applications().asList()) - .notPullRequest() .hasProductionDeployment(); metric.set(deploymentFailMetric, deploymentFailRatio(applications) * 100, metric.createContext(Collections.emptyMap())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java index 63361e5dcc4..f87a0d625c0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; import java.time.Duration; @@ -20,8 +19,7 @@ public class OutstandingChangeDeployer extends Maintainer { @Override protected void maintain() { - ApplicationList applications = ApplicationList.from(controller().applications().asList()).notPullRequest(); - for (Application application : applications.asList()) { + for (Application application : controller().applications().asList()) { if ( ! application.change().isPresent() && application.outstandingChange().isPresent()) { controller().applications().deploymentTrigger().triggerChange(application.id(), application.outstandingChange()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 676fe808bfe..06712ad7862 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -92,7 +92,6 @@ public class Upgrader extends Maintainer { private ApplicationList applications() { return ApplicationList.from(controller().applications().asList()); } private void upgrade(ApplicationList applications, Version version) { - applications = applications.notPullRequest(); // Pull requests are deployed as separate applications to test then deleted; No need to upgrade applications = applications.hasProductionDeployment(); applications = applications.onLowerVersionThan(version); applications = applications.allowMajorVersion(version.getMajor()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 8b70a49576b..e367de35d46 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -188,7 +188,6 @@ public class VersionStatus { } ApplicationList applicationList = ApplicationList.from(applications) - .notPullRequest() .hasProductionDeployment(); for (Application application : applicationList.asList()) { // Note that each version deployed on this application in production exists diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index 06896412652..ffbf24be12a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -51,8 +51,7 @@ public class VespaVersion implements Comparable<VespaVersion> { .notFailing(); ApplicationList failingOnThis = ApplicationList.from(statistics.failing(), controller.applications()); ApplicationList all = ApplicationList.from(controller.applications().asList()) - .hasDeployment() - .notPullRequest(); + .hasDeployment(); // 'broken' if any Canary fails if ( ! failingOnThis.with(UpgradePolicy.canary).isEmpty()) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 046d4af3899..70a47934262 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -954,25 +954,6 @@ public class DeploymentTriggerTest { } @Test - public void ignoresPullRequestInstances() throws Exception { - tester.controllerTester().zoneRegistry().setSystemName(SystemName.cd); - - // Current system version, matches version in test data - Version version = Version.fromString("6.42.1"); - tester.upgradeSystem(version); - assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); - - // Load test data data - byte[] json = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json")); - Slime slime = SlimeUtils.jsonToSlime(json); - tester.controllerTester().createApplication(slime); - - // Failure redeployer does not restart deployment - tester.readyJobTrigger().maintain(); - assertTrue("No jobs scheduled", tester.buildService().jobs().isEmpty()); - } - - @Test public void applicationWithoutProjectIdIsNotTriggered() throws Exception { // Current system version, matches version in test data Version version = Version.fromString("6.42.1"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DatabaseMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DatabaseMaintainerTest.java deleted file mode 100644 index bdb55d0b546..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DatabaseMaintainerTest.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import org.junit.Test; - -import java.time.Duration; -import java.time.Instant; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author mpolden - */ -public class DatabaseMaintainerTest { - - @Test - public void remove_pull_request_instances() { - DeploymentTester tester = new DeploymentTester(); - DatabaseMaintainer maintainer = new DatabaseMaintainer(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator())); - - // Deploy regular application - Application app1 = tester.createApplication("app1", "tenant1", 1, 1); - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .environment(Environment.prod) - .region("us-east-3") - .build(); - tester.deployCompletely(app1, applicationPackage); - - // Create pr instance (no longer permitted through ApplicationController so we have to cheat) - Application app2 = new Application(ApplicationId.from("tenant1", "app1", "default-pr1"), Instant.now()); - tester.controller().curator().writeApplication(app2); - assertTrue(tester.applications().get(app2.id()).isPresent()); - - maintainer.maintain(); - assertTrue("Regular application is not deleted", tester.applications().get(app1.id()).isPresent()); - assertFalse("PR instance is deleted", tester.applications().get(app2.id()).isPresent()); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json deleted file mode 100644 index 425b9d4512d..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "id": "tenant1:app1:default-pr1", - "deploymentSpecField": "<deployment version='1.0'>\n <test />\n <staging />\n <prod global-service-id=\"default\">\n <region active=\"true\">us-central-1</region>\n </prod>\n</deployment>\n", - "validationOverrides": "<deployment version='1.0'/>", - "deployments": [], - "deploymentJobs": { - "projectId": 42, - "jobStatus": [ - { - "jobType": "system-test", - "lastTriggered": { - "version": "6.42.1", - "revision": { - "applicationPackageHash": "dead" - }, - "upgrade": false, - "at": 1499075576005 - } - } - ] - }, - "outstandingChangeField": false -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 707b71b772f..6a71e524ae4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -13,9 +13,6 @@ "name": "ContactInformationMaintainer" }, { - "name": "DatabaseMaintainer" - }, - { "name": "DefaultOsUpgrader" }, { |