diff options
Diffstat (limited to 'controller-server/src/test/java/com/yahoo')
52 files changed, 1668 insertions, 660 deletions
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerProxyMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerProxyMock.java new file mode 100644 index 00000000000..cc915d4d9a1 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerProxyMock.java @@ -0,0 +1,43 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; +import com.yahoo.vespa.hosted.controller.proxy.ProxyException; +import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; +import com.yahoo.vespa.hosted.controller.restapi.StringResponse; + +import java.io.InputStream; +import java.util.Optional; +import java.util.Scanner; + +/** + * @author mpolden + */ +public class ConfigServerProxyMock extends AbstractComponent implements ConfigServerRestExecutor { + + private volatile ProxyRequest lastReceived = null; + private volatile String requestBody = null; + + @Override + public HttpResponse handle(ProxyRequest proxyRequest) throws ProxyException { + lastReceived = proxyRequest; + // Copy request body as the input stream is drained once the request completes + requestBody = asString(proxyRequest.getData()); + return new StringResponse("ok"); + } + + public Optional<ProxyRequest> lastReceived() { + return Optional.ofNullable(lastReceived); + } + + public Optional<String> lastRequestBody() { + return Optional.ofNullable(requestBody); + } + + private static String asString(InputStream inputStream) { + Scanner scanner = new Scanner(inputStream).useDelimiter("\\A"); + return scanner.hasNext() ? scanner.next() : ""; + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index aea66f3cd67..d0c1fd95427 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -30,7 +30,6 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.athenz.NToken; @@ -80,6 +79,12 @@ public class ControllerTest { .region("corp-us-east-1") .build(); + private static final ApplicationPackage applicationPackage2 = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("corp-us-east-1") + .region("us-west-1") + .build(); + @Test public void testDeployment() { // Setup system @@ -94,7 +99,7 @@ public class ControllerTest { // staging job - succeeding Version version1 = Version.fromString("6.1"); // Set in config server mock Application app1 = tester.createApplication("app1", "tenant1", 1, 11L); - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); assertFalse("Revision is currently not known", ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision().isPresent()); tester.deployAndNotify(app1, applicationPackage, true, systemTest); @@ -104,12 +109,12 @@ public class ControllerTest { Optional<ApplicationRevision> revision = ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision(); assertTrue("Revision has been set during deployment", revision.isPresent()); assertStatus(JobStatus.initial(stagingTest) - .withTriggering(-1, version1, revision, false, "", tester.clock().instant()) - .withCompletion(-1, Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); + .withTriggering(version1, revision, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) + .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); // Causes first deployment job to be triggered assertStatus(JobStatus.initial(productionCorpUsEast1) - .withTriggering(-1, version1, revision, false, "", tester.clock().instant()), app1.id(), tester.controller()); + .withTriggering(version1, revision, false, "", tester.clock().instant()), app1.id(), tester.controller()); tester.clock().advance(Duration.ofSeconds(1)); // production job (failing) @@ -117,10 +122,10 @@ public class ControllerTest { assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); JobStatus expectedJobStatus = JobStatus.initial(productionCorpUsEast1) - .withTriggering(-1, version1, revision, false, "", tester.clock().instant()) // Triggered first without revision info - .withCompletion(-1, Optional.of(JobError.unknown), tester.clock().instant(), tester.controller()) - .withTriggering(-1, version1, revision, false, "", tester.clock().instant()); // Re-triggering (due to failure) has revision info - + .withTriggering(version1, revision, false, "", tester.clock().instant()) // Triggered first without revision info + .withCompletion(42, Optional.of(JobError.unknown), tester.clock().instant(), tester.controller()) + .withTriggering(version1, revision, false, "", tester.clock().instant()); // Re-triggering (due to failure) has revision info + assertStatus(expectedJobStatus, app1.id(), tester.controller()); // Simulate restart @@ -133,26 +138,29 @@ public class ControllerTest { InstanceName.from("default")))); assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); - tester.clock().advance(Duration.ofSeconds(1)); + + tester.clock().advance(Duration.ofHours(1)); + + tester.notifyJobCompletion(productionCorpUsEast1, app1, false); // Need to complete the job, or new jobs won't start. // system and staging test job - succeeding - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); tester.deployAndNotify(app1, applicationPackage, true, false, systemTest); assertStatus(JobStatus.initial(systemTest) - .withTriggering(-1, version1, revision, false, "", tester.clock().instant()) - .withCompletion(-1, Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); + .withTriggering(version1, revision, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) + .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); tester.deployAndNotify(app1, applicationPackage, true, stagingTest); // production job succeeding now tester.deployAndNotify(app1, applicationPackage, true, productionCorpUsEast1); expectedJobStatus = expectedJobStatus - .withTriggering(-1, version1, revision, false, "", tester.clock().instant()) - .withCompletion(-1, Optional.empty(), tester.clock().instant(), tester.controller()); + .withTriggering(version1, revision, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) + .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()); assertStatus(expectedJobStatus, app1.id(), tester.controller()); // causes triggering of next production job assertStatus(JobStatus.initial(productionUsEast3) - .withTriggering(-1, version1, revision, false, "", tester.clock().instant()), + .withTriggering(version1, revision, false, "", tester.clock().instant()), app1.id(), tester.controller()); tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3); @@ -163,7 +171,7 @@ public class ControllerTest { .environment(Environment.prod) .region("us-east-3") .build(); - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); try { tester.deploy(systemTest, app1, applicationPackage); fail("Expected exception due to unallowed production deployment removal"); @@ -176,7 +184,7 @@ public class ControllerTest { JobStatus jobStatus = applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1); assertNotNull("Deployment job was not removed", jobStatus); assertEquals(42, jobStatus.lastCompleted().get().id()); - assertEquals("stagingTest completed successfully in build 42", jobStatus.lastCompleted().get().reason()); + assertEquals("staging-test completed", jobStatus.lastCompleted().get().reason()); // prod zone removal is allowed with override applicationPackage = new ApplicationPackageBuilder() @@ -205,13 +213,13 @@ public class ControllerTest { Application app1 = tester.createApplication("application1", "tenant1", 1, 1L); // First deployment: An application change - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); tester.deployAndNotify(app1, applicationPackage, true, systemTest); tester.deployAndNotify(app1, applicationPackage, true, stagingTest); tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1); app1 = applications.require(app1.id()); - assertEquals("First deployment gets system version", systemVersion, app1.deployedVersion().get()); + assertEquals("First deployment gets system version", systemVersion, app1.oldestDeployedVersion().get()); assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get()); // Unexpected deployment @@ -228,19 +236,19 @@ public class ControllerTest { .region("us-west-1") .region("us-east-3") .build(); - applications.notifyJobCompletion(mockReport(app1, component, true)); + tester.notifyJobCompletion(component, app1, true); tester.deployAndNotify(app1, applicationPackage, true, systemTest); tester.deployAndNotify(app1, applicationPackage, true, stagingTest); tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1); app1 = applications.require(app1.id()); - assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get()); + assertEquals("Application change preserves version", systemVersion, app1.oldestDeployedVersion().get()); assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get()); // A deployment to the new region gets the same version tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3); app1 = applications.require(app1.id()); - assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get()); + assertEquals("Application change preserves version", systemVersion, app1.oldestDeployedVersion().get()); assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get()); assertFalse("Change deployed", app1.deploying().isPresent()); @@ -253,7 +261,7 @@ public class ControllerTest { tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3); app1 = applications.require(app1.id()); - assertEquals("Version upgrade changes version", newSystemVersion, app1.deployedVersion().get()); + assertEquals("Version upgrade changes version", newSystemVersion, app1.oldestDeployedVersion().get()); assertEquals(newSystemVersion, tester.configServer().lastPrepareVersion().get()); } @@ -322,13 +330,13 @@ public class ControllerTest { tester.notifyJobCompletion(component, app, true); tester.deployAndNotify(app, applicationPackage, false, systemTest); assertEquals("Failure age is right at initial failure", - initialFailure, firstFailing(app, tester).get().at()); + initialFailure.plus(Duration.ofMillis(2)), firstFailing(app, tester).get().at()); // Failure again -- failingSince should remain the same tester.clock().advance(Duration.ofMillis(1000)); tester.deployAndNotify(app, applicationPackage, false, systemTest); assertEquals("Failure age is right at second consecutive failure", - initialFailure, firstFailing(app, tester).get().at()); + initialFailure.plus(Duration.ofMillis(2)), firstFailing(app, tester).get().at()); // Success resets failingSince tester.clock().advance(Duration.ofMillis(1000)); @@ -346,13 +354,13 @@ public class ControllerTest { tester.notifyJobCompletion(component, app, true); tester.deployAndNotify(app, applicationPackage, false, systemTest); assertEquals("Failure age is right at initial failure", - initialFailure, firstFailing(app, tester).get().at()); + initialFailure.plus(Duration.ofMillis(2)), firstFailing(app, tester).get().at()); // Failure again -- failingSince should remain the same tester.clock().advance(Duration.ofMillis(1000)); tester.deployAndNotify(app, applicationPackage, false, systemTest); assertEquals("Failure age is right at second consecutive failure", - initialFailure, firstFailing(app, tester).get().at()); + initialFailure.plus(Duration.ofMillis(2)), firstFailing(app, tester).get().at()); } private Optional<JobStatus.JobRun> firstFailing(Application application, DeploymentTester tester) { @@ -425,7 +433,7 @@ public class ControllerTest { // app1: staging-test job fails with out of capacity and is added to the front of the queue tester.deploy(stagingTest, app1, applicationPackage); tester.notifyJobCompletion(stagingTest, app1, Optional.of(JobError.outOfCapacity)); - assertEquals(stagingTest.id(), buildSystem.jobs().get(0).jobName()); + assertEquals(stagingTest.jobName(), buildSystem.jobs().get(0).jobName()); assertEquals(project1, buildSystem.jobs().get(0).projectId()); // app2 and app3: Completes deployment @@ -437,6 +445,7 @@ public class ControllerTest { // app1: 15 minutes pass, staging-test job is still failing due out of capacity, but is no longer re-queued by // out of capacity retry mechanism tester.clock().advance(Duration.ofMinutes(15)); + tester.notifyJobCompletion(stagingTest, app1, Optional.of(JobError.outOfCapacity)); // Clear the previous staging test tester.notifyJobCompletion(component, app1, true); tester.deployAndNotify(app1, applicationPackage, true, false, systemTest); tester.deploy(stagingTest, app1, applicationPackage); @@ -444,12 +453,13 @@ public class ControllerTest { tester.notifyJobCompletion(stagingTest, app1, Optional.of(JobError.outOfCapacity)); assertTrue("No jobs queued", buildSystem.jobs().isEmpty()); - // app2 and app3: New change triggers staging-test jobs + // app2 and app3: New change triggers system-test jobs + // Provide a changed application package, too, or the deployment is a no-op. tester.notifyJobCompletion(component, app2, true); - tester.deployAndNotify(app2, applicationPackage, true, systemTest); + tester.deployAndNotify(app2, applicationPackage2, true, systemTest); tester.notifyJobCompletion(component, app3, true); - tester.deployAndNotify(app3, applicationPackage, true, systemTest); + tester.deployAndNotify(app3, applicationPackage2, true, systemTest); assertEquals(2, buildSystem.jobs().size()); @@ -457,19 +467,19 @@ public class ControllerTest { // back of the queue tester.clock().advance(Duration.ofHours(3)); tester.clock().advance(Duration.ofMinutes(50)); - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); List<BuildJob> nextJobs = buildSystem.takeJobsToRun(); assertEquals(2, nextJobs.size()); - assertEquals(stagingTest.id(), nextJobs.get(0).jobName()); + assertEquals(stagingTest.jobName(), nextJobs.get(0).jobName()); assertEquals(project2, nextJobs.get(0).projectId()); - assertEquals(stagingTest.id(), nextJobs.get(1).jobName()); + assertEquals(stagingTest.jobName(), nextJobs.get(1).jobName()); assertEquals(project3, nextJobs.get(1).projectId()); // And finally the requeued job for app1 nextJobs = buildSystem.takeJobsToRun(); assertEquals(1, nextJobs.size()); - assertEquals(stagingTest.id(), nextJobs.get(0).jobName()); + assertEquals(stagingTest.jobName(), nextJobs.get(0).jobName()); assertEquals(project1, nextJobs.get(0).projectId()); } @@ -480,20 +490,6 @@ public class ControllerTest { assertEquals(expectedStatus, existingStatus); } - private JobReport mockReport(Application application, JobType jobType, Optional<JobError> jobError) { - return new JobReport( - application.id(), - jobType, - application.deploymentJobs().projectId().get(), - 42, - jobError - ); - } - - private JobReport mockReport(Application application, JobType jobType, boolean success) { - return mockReport(application, jobType, JobError.from(success)); - } - @Test public void testGlobalRotations() throws IOException { // Setup tester and app def @@ -527,8 +523,7 @@ public class ControllerTest { TenantId tenant = tester.createTenant("tenant1", "domain1", 11L); Application app = tester.createApplication(tenant, "app1", "default", 1); - try (Lock lock = tester.controller().applications().lock(app.id())) { - LockedApplication application = tester.controller().applications().require(app.id(), lock); + tester.controller().applications().lockedOrThrow(app.id(), application -> { application = application.withDeploying(Optional.of(new Change.VersionChange(Version.fromString("6.3")))); applications.store(application); try { @@ -537,7 +532,7 @@ public class ControllerTest { } catch (IllegalArgumentException e) { assertEquals("Rejecting deployment of application 'tenant1.app1' to zone prod.us-east-3 as version change to 6.3 is not tested", e.getMessage()); } - } + }); } @Test @@ -557,11 +552,7 @@ public class ControllerTest { // Load test data data ApplicationSerializer serializer = new ApplicationSerializer(); byte[] json = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json")); - Slime slime = SlimeUtils.jsonToSlime(json); - Application application = serializer.fromSlime(slime); - try (Lock lock = tester.controller().applications().lock(application.id())) { - tester.controller().applications().store(new LockedApplication(application, lock)); - } + Application application = tester.controllerTester().createApplication(SlimeUtils.jsonToSlime(json)); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .upgradePolicy("canary") diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 4e3c15ea1a4..8f9c22f8b81 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -181,12 +181,9 @@ public final class ControllerTester { public Application createApplication(TenantId tenant, String applicationName, String instanceName, long projectId) { ApplicationId applicationId = applicationId(tenant.id(), applicationName, instanceName); controller().applications().createApplication(applicationId, Optional.of(TestIdentities.userNToken)); - try (Lock lock = controller().applications().lock(applicationId)) { - LockedApplication lockedApplication = controller().applications().require(applicationId, lock) - .withProjectId(projectId); - controller().applications().store(lockedApplication); - return lockedApplication; - } + controller().applications().lockedOrThrow(applicationId, lockedApplication -> + controller().applications().store(lockedApplication.withProjectId(projectId))); + return controller().applications().require(applicationId); } public void deploy(Application application, Zone zone) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java index bf21467bc8d..18332942c24 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; @@ -10,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.net.URI; import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -19,15 +22,39 @@ import java.util.Optional; /** * @author mpolden */ -public class ZoneRegistryMock implements ZoneRegistry { +public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry { private final Map<Zone, Duration> deploymentTimeToLive = new HashMap<>(); + private final Map<Environment, RegionName> defaultRegionForEnvironment = new HashMap<>(); + private List<Zone> zones = new ArrayList<>(); + private SystemName system = SystemName.main; + + @Inject + public ZoneRegistryMock() { + this.zones.add(new Zone(SystemName.main, Environment.from("prod"), RegionName.from("corp-us-east-1"))); + this.zones.add(new Zone(SystemName.main, Environment.from("prod"), RegionName.from("us-east-3"))); + this.zones.add(new Zone(SystemName.main, Environment.from("prod"), RegionName.from("us-west-1"))); + } - public void setDeploymentTimeToLive(Zone zone, Duration duration) { + public ZoneRegistryMock setDeploymentTimeToLive(Zone zone, Duration duration) { deploymentTimeToLive.put(zone, duration); + return this; } - private SystemName system = SystemName.main; + public ZoneRegistryMock setDefaultRegionForEnvironment(Environment environment, RegionName region) { + defaultRegionForEnvironment.put(environment, region); + return this; + } + + public ZoneRegistryMock setZones(List<Zone> zones) { + this.zones = zones; + return this; + } + + public ZoneRegistryMock setSystem(SystemName system) { + this.system = system; + return this; + } @Override public SystemName system() { @@ -36,12 +63,13 @@ public class ZoneRegistryMock implements ZoneRegistry { @Override public List<Zone> zones() { - return Collections.singletonList(new Zone(SystemName.main, Environment.from("prod"), RegionName.from("corp-us-east-1"))); + return Collections.unmodifiableList(zones); } @Override public Optional<Zone> getZone(Environment environment, RegionName region) { - return zones().stream().filter(z -> z.environment().equals(environment) && z.region().equals(region)).findFirst(); + return zones().stream().filter(z -> z.environment().equals(environment) && + z.region().equals(region)).findFirst(); } @Override @@ -64,6 +92,11 @@ public class ZoneRegistryMock implements ZoneRegistry { } @Override + public Optional<RegionName> getDefaultRegion(Environment environment) { + return Optional.ofNullable(defaultRegionForEnvironment.get(environment)); + } + + @Override public URI getMonitoringSystemUri(Environment environment, RegionName name, ApplicationId application) { return URI.create("http://monitoring-system.test/?environment=" + environment.value() + "®ion=" + name.value() + "&application=" + application.toShortString()); @@ -74,7 +107,4 @@ public class ZoneRegistryMock implements ZoneRegistry { return URI.create("http://dashboard.test"); } - public void setSystem(SystemName system) { - this.system = system; - } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 79fd717a24f..2b0e953c12c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -16,17 +16,19 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; -import com.yahoo.vespa.hosted.controller.maintenance.FailureRedeployer; +import com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import java.time.Duration; import java.util.List; +import java.util.NoSuchElementException; import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; +import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.unknown; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -44,7 +46,7 @@ public class DeploymentTester { private final ControllerTester tester; private final Upgrader upgrader; - private final FailureRedeployer failureRedeployer; + private final ReadyJobsTrigger readyJobTrigger; public DeploymentTester() { this(new ControllerTester()); @@ -55,18 +57,20 @@ public class DeploymentTester { tester.curator().writeUpgradesPerMinute(100); this.upgrader = new Upgrader(tester.controller(), maintenanceInterval, new JobControl(tester.curator()), tester.curator()); - this.failureRedeployer = new FailureRedeployer(tester.controller(), maintenanceInterval, - new JobControl(tester.curator())); + this.readyJobTrigger = new ReadyJobsTrigger(tester.controller(), maintenanceInterval, + new JobControl(tester.curator())); } public Upgrader upgrader() { return upgrader; } - public FailureRedeployer failureRedeployer() { return failureRedeployer; } + public ReadyJobsTrigger readyJobTrigger() { return readyJobTrigger; } public Controller controller() { return tester.controller(); } public ApplicationController applications() { return tester.controller().applications(); } + // TODO: This thing simulates the wrong thing: the build system won't hold the jobs that are running, + // and so these should be consumed immediately upon triggering, and be "somewhere else" while running. public BuildSystem buildSystem() { return tester.controller().applications().deploymentTrigger().buildSystem(); } public DeploymentTrigger deploymentTrigger() { return tester.controller().applications().deploymentTrigger(); } @@ -115,8 +119,13 @@ public class DeploymentTester { /** Simulate the full lifecycle of an application deployment as declared in given application package */ public Application createAndDeploy(String applicationName, int projectId, ApplicationPackage applicationPackage) { - tester.createTenant("tenant1", "domain1", 1L); - Application application = tester.createApplication(new TenantId("tenant1"), applicationName, "default", projectId); + TenantId tenantId = tester.createTenant("tenant1", "domain1", 1L); + return createAndDeploy(tenantId, applicationName, projectId, applicationPackage); + } + + /** Simulate the full lifecycle of an application deployment as declared in given application package */ + public Application createAndDeploy(TenantId tenantId, String applicationName, int projectId, ApplicationPackage applicationPackage) { + Application application = tester.createApplication(tenantId, applicationName, "default", projectId); deployCompletely(application, applicationPackage); return applications().require(application.id()); } @@ -126,6 +135,11 @@ public class DeploymentTester { return createAndDeploy(applicationName, projectId, applicationPackage(upgradePolicy)); } + /** Simulate the full lifecycle of an application deployment to prod.us-west-1 with the given upgrade policy */ + public Application createAndDeploy(TenantId tenantId, String applicationName, int projectId, String upgradePolicy) { + return createAndDeploy(tenantId, applicationName, projectId, applicationPackage(upgradePolicy)); + } + /** Complete an ongoing deployment */ public void deployCompletely(String applicationName) { deployCompletely(applications().require(ApplicationId.from("tenant1", applicationName, "default")), @@ -139,6 +153,20 @@ public class DeploymentTester { completeDeployment(application, applicationPackage, Optional.empty(), true); } + public static DeploymentJobs.JobReport jobReport(Application application, JobType jobType, boolean success) { + return jobReport(application, jobType, Optional.ofNullable(success ? null : unknown)); + } + + public static DeploymentJobs.JobReport jobReport(Application application, JobType jobType, Optional<DeploymentJobs.JobError> jobError) { + return new DeploymentJobs.JobReport( + application.id(), + jobType, + application.deploymentJobs().projectId().get(), + 42, + jobError + ); + } + /** Deploy application using the given application package, but expecting to stop after test phases */ public void deployTestOnly(Application application, ApplicationPackage applicationPackage) { notifyJobCompletion(JobType.component, application, true); @@ -154,7 +182,7 @@ public class DeploymentTester { jobs = jobs.stream().filter(job -> ! job.isProduction()).collect(Collectors.toList()); for (JobType job : jobs) { boolean failJob = failOnJob.map(j -> j.equals(job)).orElse(false); - deployAndNotify(application, applicationPackage, !failJob, false, job); + deployAndNotify(application, applicationPackage, ! failJob, false, job); if (failJob) { break; } @@ -171,10 +199,11 @@ public class DeploymentTester { } public void notifyJobCompletion(JobType jobType, Application application, boolean success) { - notifyJobCompletion(jobType, application, DeploymentJobs.JobError.from(success)); + notifyJobCompletion(jobType, application, Optional.ofNullable(success ? null : unknown)); } public void notifyJobCompletion(JobType jobType, Application application, Optional<DeploymentJobs.JobError> jobError) { + clock().advance(Duration.ofMillis(1)); applications().notifyJobCompletion(jobReport(application, jobType, jobError)); } @@ -211,7 +240,7 @@ public class DeploymentTester { deployAndNotify(application, applicationPackage, success, true, jobs); } - public void deployAndNotify(Application application, ApplicationPackage applicationPackage, boolean success, + public void deployAndNotify(Application application, ApplicationPackage applicationPackage, boolean success, boolean expectOnlyTheseJobs, JobType... jobs) { consumeJobs(application, expectOnlyTheseJobs, jobs); for (JobType job : jobs) { @@ -225,21 +254,20 @@ public class DeploymentTester { /** Assert that the sceduled jobs of this application are exactly those given, and take them */ private void consumeJobs(Application application, boolean expectOnlyTheseJobs, JobType... jobs) { for (JobType job : jobs) { - Optional<BuildService.BuildJob> buildJob = findJob(application, job); - assertTrue(String.format("Job %s is scheduled for %s", job, application), buildJob.isPresent()); - assertEquals((long) application.deploymentJobs().projectId().get(), buildJob.get().projectId()); - assertEquals(job.id(), buildJob.get().jobName()); + BuildService.BuildJob buildJob = findJob(application, job); + assertEquals((long) application.deploymentJobs().projectId().get(), buildJob.projectId()); + assertEquals(job.jobName(), buildJob.jobName()); } if (expectOnlyTheseJobs) assertEquals(jobs.length, countJobsOf(application)); buildSystem().removeJobs(application.id()); } - private Optional<BuildService.BuildJob> findJob(Application application, JobType jobType) { + private BuildService.BuildJob findJob(Application application, JobType jobType) { for (BuildService.BuildJob job : buildSystem().jobs()) - if (job.projectId() == application.deploymentJobs().projectId().get() && job.jobName().equals(jobType.id())) - return Optional.of(job); - return Optional.empty(); + if (job.projectId() == application.deploymentJobs().projectId().get() && job.jobName().equals(jobType.jobName())) + return job; + throw new NoSuchElementException(jobType + " is not scheduled for " + application); } private int countJobsOf(Application application) { @@ -247,17 +275,8 @@ public class DeploymentTester { .filter(job -> job.projectId() == application.deploymentJobs().projectId().get()) .count(); } - private DeploymentJobs.JobReport jobReport(Application application, JobType jobType, Optional<DeploymentJobs.JobError> jobError) { - return new DeploymentJobs.JobReport( - application.id(), - jobType, - application.deploymentJobs().projectId().get(), - 42, - jobError - ); - } - private static ApplicationPackage applicationPackage(String upgradePolicy) { + public static ApplicationPackage applicationPackage(String upgradePolicy) { return new ApplicationPackageBuilder() .upgradePolicy(upgradePolicy) .environment(Environment.prod) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 022fa705def..10f8e80f318 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; -import com.yahoo.vespa.hosted.controller.maintenance.BlockedChangeDeployer; +import com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import org.junit.Test; @@ -59,10 +59,11 @@ public class DeploymentTriggerTest { // system-test fails and is retried tester.deployAndNotify(app, applicationPackage, false, JobType.systemTest); assertEquals("Retried immediately", 1, tester.buildSystem().jobs().size()); - tester.buildSystem().takeJobsToRun(); - assertEquals("Job removed", 0, tester.buildSystem().jobs().size()); - tester.clock().advance(Duration.ofHours(4).plus(Duration.ofSeconds(1))); - tester.failureRedeployer().maintain(); // Causes retry of systemTests + tester.clock().advance(Duration.ofHours(1)); + tester.deployAndNotify(app, applicationPackage, false, JobType.systemTest); + tester.clock().advance(Duration.ofHours(1)); + assertEquals("Nothing scheduled", 0, tester.buildSystem().jobs().size()); + tester.readyJobTrigger().maintain(); // Causes retry of systemTests assertEquals("Scheduled retry", 1, tester.buildSystem().jobs().size()); tester.deployAndNotify(app, applicationPackage, true, JobType.systemTest); @@ -70,9 +71,9 @@ public class DeploymentTriggerTest { // staging-test times out and is retried tester.buildSystem().takeJobsToRun(); tester.clock().advance(Duration.ofHours(12).plus(Duration.ofSeconds(1))); - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); assertEquals("Retried dead job", 1, tester.buildSystem().jobs().size()); - assertEquals(JobType.stagingTest.id(), tester.buildSystem().jobs().get(0).jobName()); + assertEquals(JobType.stagingTest.jobName(), tester.buildSystem().jobs().get(0).jobName()); } @Test @@ -127,16 +128,16 @@ public class DeploymentTriggerTest { // 30 seconds pass, us-west-1 is triggered tester.clock().advance(Duration.ofSeconds(30)); - tester.deploymentTrigger().triggerDelayed(); + tester.deploymentTrigger().triggerReadyJobs(); // Consume us-west-1 job without reporting completion assertEquals(1, buildSystem.jobs().size()); - assertEquals(JobType.productionUsWest1.id(), buildSystem.jobs().get(0).jobName()); + assertEquals(JobType.productionUsWest1.jobName(), buildSystem.jobs().get(0).jobName()); buildSystem.takeJobsToRun(); // 3 minutes pass, delayed trigger does nothing as us-west-1 is still in progress tester.clock().advance(Duration.ofMinutes(3)); - tester.deploymentTrigger().triggerDelayed(); + tester.deploymentTrigger().triggerReadyJobs(); assertTrue("No more jobs triggered at this time", buildSystem.jobs().isEmpty()); // us-west-1 completes @@ -144,18 +145,18 @@ public class DeploymentTriggerTest { tester.notifyJobCompletion(JobType.productionUsWest1, application, true); // Delayed trigger does nothing as not enough time has passed after us-west-1 completion - tester.deploymentTrigger().triggerDelayed(); + tester.deploymentTrigger().triggerReadyJobs(); assertTrue("No more jobs triggered at this time", buildSystem.jobs().isEmpty()); // 3 minutes pass, us-central-1 is triggered tester.clock().advance(Duration.ofMinutes(3)); - tester.deploymentTrigger().triggerDelayed(); + tester.deploymentTrigger().triggerReadyJobs(); tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsCentral1); assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); // Delayed trigger job runs again, with nothing to trigger tester.clock().advance(Duration.ofMinutes(10)); - tester.deploymentTrigger().triggerDelayed(); + tester.deploymentTrigger().triggerReadyJobs(); assertTrue("All jobs consumed", buildSystem.jobs().isEmpty()); } @@ -184,8 +185,8 @@ public class DeploymentTriggerTest { // Deploys in two regions in parallel assertEquals(2, tester.buildSystem().jobs().size()); - assertEquals(JobType.productionUsEast3.id(), tester.buildSystem().jobs().get(0).jobName()); - assertEquals(JobType.productionUsWest1.id(), tester.buildSystem().jobs().get(1).jobName()); + assertEquals(JobType.productionUsEast3.jobName(), tester.buildSystem().jobs().get(0).jobName()); + assertEquals(JobType.productionUsWest1.jobName(), tester.buildSystem().jobs().get(1).jobName()); tester.buildSystem().takeJobsToRun(); tester.deploy(JobType.productionUsWest1, application, applicationPackage, false); @@ -269,9 +270,9 @@ public class DeploymentTriggerTest { public void testBlockRevisionChange() { ManualClock clock = new ManualClock(Instant.parse("2017-09-26T17:30:00.00Z")); // Tuesday, 17:30 DeploymentTester tester = new DeploymentTester(new ControllerTester(clock)); - BlockedChangeDeployer blockedChangeDeployer = new BlockedChangeDeployer(tester.controller(), - Duration.ofHours(1), - new JobControl(tester.controllerTester().curator())); + ReadyJobsTrigger readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), + Duration.ofHours(1), + new JobControl(tester.controllerTester().curator())); Version version = Version.fromString("5.0"); tester.updateVersionStatus(version); @@ -290,7 +291,7 @@ public class DeploymentTriggerTest { tester.clock().advance(Duration.ofHours(1)); // --------------- Enter block window: 18:30 - blockedChangeDeployer.run(); + readyJobsTrigger.run(); assertEquals(0, tester.buildSystem().jobs().size()); String searchDefinition = @@ -304,7 +305,7 @@ public class DeploymentTriggerTest { tester.deployTestOnly(app, changedApplication); - blockedChangeDeployer.run(); + readyJobsTrigger.run(); assertEquals(0, tester.buildSystem().jobs().size()); tester.clock().advance(Duration.ofHours(2)); // ---------------- Exit block window: 20:30 @@ -317,14 +318,14 @@ public class DeploymentTriggerTest { @Test public void testUpgradingButNoJobStarted() { DeploymentTester tester = new DeploymentTester(); - BlockedChangeDeployer blockedChangeDeployer = new BlockedChangeDeployer(tester.controller(), - Duration.ofHours(1), - new JobControl(tester.controllerTester().curator())); + ReadyJobsTrigger readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), + Duration.ofHours(1), + new JobControl(tester.controllerTester().curator())); LockedApplication app = (LockedApplication)tester.createAndDeploy("default0", 3, "default"); // Store that we are upgrading but don't start the system-tests job tester.controller().applications().store(app.withDeploying(Optional.of(new Change.VersionChange(Version.fromString("6.2"))))); assertEquals(0, tester.buildSystem().jobs().size()); - blockedChangeDeployer.run(); + readyJobsTrigger.run(); assertEquals(1, tester.buildSystem().jobs().size()); assertEquals("system-test", tester.buildSystem().jobs().get(0).jobName()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java index 0293ea08d65..1b1a4feaa4e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/MockBuildService.java @@ -13,6 +13,7 @@ import java.time.Duration; import java.util.EnumMap; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.function.Supplier; import static com.yahoo.vespa.hosted.controller.deployment.MockBuildService.JobStatus.QUEUED; @@ -161,11 +162,11 @@ public class MockBuildService implements BuildService { jobType, projectId, 42, - JobError.from(success) + Optional.ofNullable(success ? null : JobError.unknown) )); } - private BuildJob buildJob() { return new BuildJob(projectId, jobType.id()); } + private BuildJob buildJob() { return new BuildJob(projectId, jobType.jobName()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java new file mode 100644 index 00000000000..f5be882fcb8 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java @@ -0,0 +1,121 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.Tenant; +import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; +import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; +import com.yahoo.vespa.hosted.controller.api.integration.organization.User; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; +import java.util.Optional; +import java.util.function.Supplier; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author jvenstad + */ +public class ApplicationOwnershipConfirmerTest { + + private MockOwnershipIssues issues; + private ApplicationOwnershipConfirmer confirmer; + private DeploymentTester tester; + + @Before + public void setup() { + tester = new DeploymentTester(); + issues = new MockOwnershipIssues(); + confirmer = new ApplicationOwnershipConfirmer(tester.controller(), Duration.ofDays(1), new JobControl(new MockCuratorDb()), issues); + } + + @Test + public void testConfirmation() { + TenantId property = tester.controllerTester().createTenant("property", "domain", 1L); + tester.createAndDeploy(property, "application", 1, "default"); + Supplier<Application> propertyApp = () -> tester.controller().applications().require(ApplicationId.from("property", "application", "default")); + + TenantId user = new TenantId("by-user"); + tester.controller().tenants().addTenant(Tenant.createUserTenant(user), Optional.empty()); + tester.createAndDeploy(user, "application", 2, "default"); + Supplier<Application> userApp = () -> tester.controller().applications().require(ApplicationId.from("by-user", "application", "default")); + + assertFalse("No issue is initially stored for a new application.", propertyApp.get().ownershipIssueId().isPresent()); + assertFalse("No issue is initially stored for a new application.", userApp.get().ownershipIssueId().isPresent()); + assertFalse("No escalation has been attempted for a new application", issues.escalatedForProperty || issues.escalatedForUser); + + // Set response from the issue mock, which will be obtained by the maintainer on issue filing. + Optional<IssueId> issueId = Optional.of(IssueId.from("1")); + issues.response = issueId; + confirmer.maintain(); + confirmer.maintain(); + + assertEquals("Confirmation issue has been filed for property owned application.", issueId, propertyApp.get().ownershipIssueId()); + assertEquals("Confirmation issue has been filed for user owned application.", issueId, userApp.get().ownershipIssueId()); + assertTrue("Both applications have had their responses ensured.", issues.escalatedForProperty && issues.escalatedForUser); + + // No new issue is created, so return empty now. + issues.response = Optional.empty(); + confirmer.maintain(); + confirmer.maintain(); + + assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, propertyApp.get().ownershipIssueId()); + assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, userApp.get().ownershipIssueId()); + + // The user deletes its production deployment — see that the issue is forgotten. + assertEquals("Confirmation issue for user is sitll open.", issueId, userApp.get().ownershipIssueId()); + tester.controller().applications().deactivate(userApp.get(), userApp.get().productionDeployments().keySet().stream().findAny().get()); + assertTrue("No production deployments are listed for user.", userApp.get().productionDeployments().isEmpty()); + confirmer.maintain(); + confirmer.maintain(); + + assertEquals("Confirmation issue has been forgotten for application without production deployments.", Optional.empty(), userApp.get().ownershipIssueId()); + + // Time has passed, and a new confirmation issue is in order for the property which is still in production. + Optional<IssueId> issueId2 = Optional.of(IssueId.from("2")); + issues.response = issueId2; + confirmer.maintain(); + confirmer.maintain(); + + assertEquals("A new confirmation issue id is stored when something is returned to the maintainer.", issueId2, propertyApp.get().ownershipIssueId()); + assertEquals("Confirmation issue for application without production deployments has not been filed.", Optional.empty(), userApp.get().ownershipIssueId()); + + } + + private class MockOwnershipIssues implements OwnershipIssues { + + private Optional<IssueId> response; + private boolean escalatedForProperty = false; + private boolean escalatedForUser = false; + + @Override + public Optional<IssueId> confirmOwnership(Optional<IssueId> issueId, ApplicationId applicationId, PropertyId propertyId) { + return response; + } + + @Override + public Optional<IssueId> confirmOwnership(Optional<IssueId> issueId, ApplicationId applicationId, User owner) { + return response; + } + + @Override + public void ensureResponse(IssueId issueId, Optional<PropertyId> propertyId) { + if (propertyId.isPresent()) escalatedForProperty = true; + else escalatedForUser = true; + } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java index b5ea8e0a36f..e57edcf6da0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java @@ -54,7 +54,7 @@ public class DeploymentIssueReporterTest { public void setup() { tester = new DeploymentTester(); issues = new MockDeploymentIssues(); - reporter = new DeploymentIssueReporter(tester.controller(), issues, Duration.ofMinutes(5), new JobControl(new MockCuratorDb())); + reporter = new DeploymentIssueReporter(tester.controller(), issues, Duration.ofDays(1), new JobControl(new MockCuratorDb())); } @Test @@ -135,9 +135,7 @@ public class DeploymentIssueReporterTest { // app3 now has a new failure past max failure age; see that a new issue is filed. tester.notifyJobCompletion(component, app3, true); - tester.deployAndNotify(app3, applicationPackage, true, systemTest); - tester.deployAndNotify(app3, applicationPackage, true, stagingTest); - tester.deployAndNotify(app3, applicationPackage, false, productionCorpUsEast1); + tester.deployAndNotify(app3, applicationPackage, false, systemTest); tester.clock().advance(maxInactivity.plus(maxFailureAge)); reporter.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java index d3907e27456..148d11e8b38 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java @@ -5,14 +5,16 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import org.junit.Assert; import org.junit.Test; import java.time.Duration; +import static org.junit.Assert.assertEquals; + /** * @author smorgrav */ @@ -23,20 +25,24 @@ public class DeploymentMetricsMaintainerTest { ControllerTester tester = new ControllerTester(); ApplicationId app = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id(); - // Pre condition: no metric info on the deployment + // Pre condition: no metric info on neither application nor deployment + assertEquals(0, tester.controller().applications().require(app).metrics().queryServiceQuality(), 0); Deployment deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get(); - Assert.assertEquals(0, deployment.metrics().documentCount(), Double.MIN_VALUE); + assertEquals(0, deployment.metrics().documentCount(), 0); DeploymentMetricsMaintainer maintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofMinutes(10), new JobControl(new MockCuratorDb())); maintainer.maintain(); // Post condition: - deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get(); - Assert.assertEquals(1, deployment.metrics().queriesPerSecond(), Double.MIN_VALUE); - Assert.assertEquals(2, deployment.metrics().writesPerSecond(), Double.MIN_VALUE); - Assert.assertEquals(3, deployment.metrics().documentCount(), Double.MIN_VALUE); - Assert.assertEquals(4, deployment.metrics().queryLatencyMillis(), Double.MIN_VALUE); - Assert.assertEquals(5, deployment.metrics().writeLatencyMillis(), Double.MIN_VALUE); + Application application = tester.controller().applications().require(app); + assertEquals(0.5, application.metrics().queryServiceQuality(), Double.MIN_VALUE); + assertEquals(0.7, application.metrics().writeServiceQuality(), Double.MIN_VALUE); + deployment = application.deployments().values().stream().findAny().get(); + assertEquals(1, deployment.metrics().queriesPerSecond(), Double.MIN_VALUE); + assertEquals(2, deployment.metrics().writesPerSecond(), Double.MIN_VALUE); + assertEquals(3, deployment.metrics().documentCount(), Double.MIN_VALUE); + assertEquals(4, deployment.metrics().queryLatencyMillis(), Double.MIN_VALUE); + assertEquals(5, deployment.metrics().writeLatencyMillis(), Double.MIN_VALUE); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java index 2782dd6ec3b..fd00123c697 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -62,15 +62,16 @@ public class FailureRedeployerTest { // Another version is released, which cancels any pending upgrades to lower versions version = Version.fromString("5.2"); tester.updateVersionStatus(version); + tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3); // Finish previous production job. tester.upgrader().maintain(); assertEquals("Application starts upgrading to new version", 1, tester.buildSystem().jobs().size()); assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version()); // Failure redeployer does not retry failing job for prod.us-east-3 as there's an ongoing deployment tester.clock().advance(Duration.ofMinutes(1)); - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); assertFalse("Job is not retried", tester.buildSystem().jobs().stream() - .anyMatch(j -> j.jobName().equals(DeploymentJobs.JobType.productionUsEast3.id()))); + .anyMatch(j -> j.jobName().equals(DeploymentJobs.JobType.productionUsEast3.jobName()))); // Test environments pass tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); @@ -86,7 +87,7 @@ public class FailureRedeployerTest { // Failure redeployer retries job tester.clock().advance(Duration.ofMinutes(5)); - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); assertEquals("Job is retried", 1, tester.buildSystem().jobs().size()); // Production job finally succeeds @@ -109,14 +110,14 @@ public class FailureRedeployerTest { tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); // staging-test starts, but does not complete - assertEquals(DeploymentJobs.JobType.stagingTest.id(), tester.buildSystem().takeJobsToRun().get(0).jobName()); - tester.failureRedeployer().maintain(); + assertEquals(DeploymentJobs.JobType.stagingTest.jobName(), tester.buildSystem().takeJobsToRun().get(0).jobName()); + tester.readyJobTrigger().maintain(); assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); // Just over 12 hours pass, job is retried tester.clock().advance(Duration.ofHours(12).plus(Duration.ofSeconds(1))); - tester.failureRedeployer().maintain(); - assertEquals(DeploymentJobs.JobType.stagingTest.id(), tester.buildSystem().takeJobsToRun().get(0).jobName()); + tester.readyJobTrigger().maintain(); + assertEquals(DeploymentJobs.JobType.stagingTest.jobName(), tester.buildSystem().takeJobsToRun().get(0).jobName()); // Deployment completes tester.deploy(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true); @@ -168,7 +169,7 @@ public class FailureRedeployerTest { // Failure re-deployer does not retry failing system-test job as it failed for an older change tester.clock().advance(Duration.ofMinutes(5)); - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); } @@ -212,11 +213,11 @@ public class FailureRedeployerTest { // Production job starts, but does not complete assertEquals(1, tester.buildSystem().jobs().size()); - assertEquals("Production job triggered", DeploymentJobs.JobType.productionCdUsCentral1.id(), tester.buildSystem().jobs().get(0).jobName()); + assertEquals("Production job triggered", DeploymentJobs.JobType.productionCdUsCentral1.jobName(), tester.buildSystem().jobs().get(0).jobName()); tester.buildSystem().takeJobsToRun(); // Failure re-deployer runs - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); // Deployment completes @@ -241,7 +242,7 @@ public class FailureRedeployerTest { Application application = tester.controllerTester().createApplication(slime); // Failure redeployer does not restart deployment - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); assertTrue("No jobs scheduled", tester.buildSystem().jobs().isEmpty()); } @@ -261,7 +262,7 @@ public class FailureRedeployerTest { tester.controllerTester().createApplication(slime); // Failure redeployer does not restart deployment - tester.failureRedeployer().maintain(); + tester.readyJobTrigger().maintain(); assertTrue("No jobs scheduled", tester.buildSystem().jobs().isEmpty()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 621e189ba37..a7458f9f8ed 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -71,7 +71,7 @@ public class MetricsReporterTest { metricsReporter.maintain(); assertEquals(0.0, metricsMock.getMetric(MetricsReporter.deploymentFailMetric)); - // Deploy 3 apps successfully + // Deploy all apps successfully Application app1 = tester.createApplication("app1", "tenant1", 1, 11L); Application app2 = tester.createApplication("app2", "tenant1", 2, 22L); Application app3 = tester.createApplication("app3", "tenant1", 3, 33L); @@ -79,6 +79,7 @@ public class MetricsReporterTest { tester.deployCompletely(app1, applicationPackage); tester.deployCompletely(app2, applicationPackage); tester.deployCompletely(app3, applicationPackage); + tester.deployCompletely(app4, applicationPackage); metricsReporter.maintain(); assertEquals(0.0, metricsMock.getMetric(MetricsReporter.deploymentFailMetric)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 4886eba40b6..13636122cfd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -49,7 +49,7 @@ public class OutstandingChangeDeployerTest { List<BuildService.BuildJob> jobs = tester.buildSystem().jobs(); assertEquals(1, jobs.size()); assertEquals(11, jobs.get(0).projectId()); - assertEquals(DeploymentJobs.JobType.systemTest.id(), jobs.get(0).jobName()); + assertEquals(DeploymentJobs.JobType.systemTest.jobName(), jobs.get(0).jobName()); assertFalse(tester.application("app1").hasOutstandingChange()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 0414cda3f55..8839f6a5a18 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -51,7 +51,7 @@ public class UpgraderTest { tester.upgrader().maintain(); assertEquals("All already on the right version: Nothing to do", 0, tester.buildSystem().jobs().size()); - // --- A new version is released - everything goes smoothly + // --- 5.1 is released - everything goes smoothly version = Version.fromString("5.1"); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); @@ -86,7 +86,7 @@ public class UpgraderTest { tester.upgrader().maintain(); assertEquals("Nothing to do", 0, tester.buildSystem().jobs().size()); - // --- A new version is released - which fails a Canary + // --- 5.2 is released - which fails a Canary version = Version.fromString("5.2"); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); @@ -95,12 +95,23 @@ public class UpgraderTest { assertEquals("New system version: Should upgrade Canaries", 2, tester.buildSystem().jobs().size()); tester.completeUpgradeWithError(canary0, version, "canary", DeploymentJobs.JobType.stagingTest); assertEquals("Other Canary was cancelled", 2, tester.buildSystem().jobs().size()); + // TODO: Cancelled would mean it was triggerd, removed from the build system, but never reported in. + // Thus, the expected number of jobs should be 1, above: the retrying canary0. + // Further, canary1 should be retried after the timeout period of 12 hours, but verifying this is + // not possible when jobs are consumed form the build system on notification, rather than on deploy. tester.updateVersionStatus(version); assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence()); tester.upgrader().maintain(); assertEquals("Version broken, but Canaries should keep trying", 2, tester.buildSystem().jobs().size()); + // Exhaust canary retries. + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, canary1, false); + tester.clock().advance(Duration.ofHours(1)); + tester.deployAndNotify(canary0, DeploymentTester.applicationPackage("canary"), false, DeploymentJobs.JobType.stagingTest); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, canary1, false); + //tester.deployAndNotify(canary1, DeploymentTester.applicationPackage("canary"), false, DeploymentJobs.JobType.stagingTest); + // --- A new version is released - which repairs the Canary app and fails a default version = Version.fromString("5.3"); tester.updateVersionStatus(version); @@ -128,11 +139,15 @@ public class UpgraderTest { tester.completeUpgrade(default2, version, "default"); tester.updateVersionStatus(version); - assertEquals("Not enough evidence to mark this neither broken nor high", + assertEquals("Not enough evidence to mark this as neither broken nor high", VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); - tester.upgrader().maintain(); + assertEquals("Upgrade with error should retry", 1, tester.buildSystem().jobs().size()); + // Finish previous run, with exhausted retry. + tester.clock().advance(Duration.ofHours(1)); + tester.notifyJobCompletion(DeploymentJobs.JobType.stagingTest, default0, false); + // --- Failing application is repaired by changing the application, causing confidence to move above 'high' threshold // Deploy application change tester.deployCompletely("default0"); @@ -148,51 +163,59 @@ public class UpgraderTest { assertEquals("Applications are on 5.3 - nothing to do", 0, tester.buildSystem().jobs().size()); // --- Starting upgrading to a new version which breaks, causing upgrades to commence on the previous version - version = Version.fromString("5.4"); + Version version54 = Version.fromString("5.4"); Application default3 = tester.createAndDeploy("default3", 5, "default"); // need 4 to break a version Application default4 = tester.createAndDeploy("default4", 5, "default"); - tester.updateVersionStatus(version); + tester.updateVersionStatus(version54); tester.upgrader().maintain(); // cause canary upgrades to 5.4 - tester.completeUpgrade(canary0, version, "canary"); - tester.completeUpgrade(canary1, version, "canary"); - tester.updateVersionStatus(version); + tester.completeUpgrade(canary0, version54, "canary"); + tester.completeUpgrade(canary1, version54, "canary"); + tester.updateVersionStatus(version54); assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); tester.upgrader().maintain(); assertEquals("Upgrade of defaults are scheduled", 5, tester.buildSystem().jobs().size()); - assertEquals(version, ((Change.VersionChange)tester.application(default0.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default1.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default2.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default3.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default4.id()).deploying().get()).version()); - tester.completeUpgrade(default0, version, "default"); + assertEquals(version54, ((Change.VersionChange)tester.application(default0.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default1.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default2.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default3.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default4.id()).deploying().get()).version()); + tester.completeUpgrade(default0, version54, "default"); // State: Default applications started upgrading to 5.4 (and one completed) - version = Version.fromString("5.5"); - tester.updateVersionStatus(version); + Version version55 = Version.fromString("5.5"); + tester.updateVersionStatus(version55); tester.upgrader().maintain(); // cause canary upgrades to 5.5 - tester.completeUpgrade(canary0, version, "canary"); - tester.completeUpgrade(canary1, version, "canary"); - tester.updateVersionStatus(version); + tester.completeUpgrade(canary0, version55, "canary"); + tester.completeUpgrade(canary1, version55, "canary"); + tester.updateVersionStatus(version55); assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); tester.upgrader().maintain(); assertEquals("Upgrade of defaults are scheduled", 5, tester.buildSystem().jobs().size()); - assertEquals(version, ((Change.VersionChange)tester.application(default0.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default1.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default2.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default3.id()).deploying().get()).version()); - assertEquals(version, ((Change.VersionChange)tester.application(default4.id()).deploying().get()).version()); + assertEquals(version55, ((Change.VersionChange)tester.application(default0.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default1.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default2.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default3.id()).deploying().get()).version()); + assertEquals(version54, ((Change.VersionChange)tester.application(default4.id()).deploying().get()).version()); + tester.completeUpgrade(default1, version54, "default"); + tester.completeUpgrade(default2, version54, "default"); + tester.completeUpgradeWithError(default3, version54, "default", DeploymentJobs.JobType.stagingTest); + tester.completeUpgradeWithError(default4, version54, "default", DeploymentJobs.JobType.productionUsWest1); // State: Default applications started upgrading to 5.5 - tester.completeUpgradeWithError(default0, version, "default", DeploymentJobs.JobType.stagingTest); - tester.completeUpgradeWithError(default1, version, "default", DeploymentJobs.JobType.stagingTest); - tester.completeUpgradeWithError(default2, version, "default", DeploymentJobs.JobType.stagingTest); - tester.completeUpgradeWithError(default3, version, "default", DeploymentJobs.JobType.productionUsWest1); - tester.completeUpgrade(default4, version, "default"); - tester.updateVersionStatus(version); + tester.upgrader().maintain(); + tester.completeUpgradeWithError(default0, version55, "default", DeploymentJobs.JobType.stagingTest); + tester.completeUpgradeWithError(default1, version55, "default", DeploymentJobs.JobType.stagingTest); + tester.completeUpgradeWithError(default2, version55, "default", DeploymentJobs.JobType.stagingTest); + tester.completeUpgradeWithError(default3, version55, "default", DeploymentJobs.JobType.productionUsWest1); + tester.updateVersionStatus(version55); assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence()); + + // Finish running job, without retry. + tester.clock().advance(Duration.ofHours(1)); + tester.notifyJobCompletion(DeploymentJobs.JobType.productionUsWest1, default3, false); + tester.upgrader().maintain(); - assertEquals("Upgrade of defaults are scheduled on 5.4 instead, since 5.5 broken", - 3, tester.buildSystem().jobs().size()); - assertEquals("5.4", ((Change.VersionChange)tester.application(default1.id()).deploying().get()).version().toString()); - assertEquals("5.4", ((Change.VersionChange)tester.application(default2.id()).deploying().get()).version().toString()); + assertEquals("Upgrade of defaults are scheduled on 5.4 instead, since 5.5 broken: " + + "This is default3 since it failed upgrade on both 5.4 and 5.5", + 1, tester.buildSystem().jobs().size()); assertEquals("5.4", ((Change.VersionChange)tester.application(default3.id()).deploying().get()).version().toString()); } @@ -451,9 +474,9 @@ public class UpgraderTest { public void testBlockVersionChangeHalfwayThough() { ManualClock clock = new ManualClock(Instant.parse("2017-09-26T17:00:00.00Z")); // Tuesday, 17:00 DeploymentTester tester = new DeploymentTester(new ControllerTester(clock)); - BlockedChangeDeployer blockedChangeDeployer = new BlockedChangeDeployer(tester.controller(), - Duration.ofHours(1), - new JobControl(tester.controllerTester().curator())); + ReadyJobsTrigger readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), + Duration.ofHours(1), + new JobControl(tester.controllerTester().curator())); Version version = Version.fromString("5.0"); tester.updateVersionStatus(version); @@ -483,12 +506,12 @@ public class UpgraderTest { // One hour passes, time is 19:00, still no upgrade tester.clock().advance(Duration.ofHours(1)); - blockedChangeDeployer.maintain(); + readyJobsTrigger.maintain(); assertTrue("No jobs scheduled", tester.buildSystem().jobs().isEmpty()); // Another hour pass, time is 20:00 and application upgrades tester.clock().advance(Duration.ofHours(1)); - blockedChangeDeployer.maintain(); + readyJobsTrigger.maintain(); tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsCentral1); tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3); assertTrue("All jobs consumed", tester.buildSystem().jobs().isEmpty()); @@ -505,9 +528,9 @@ public class UpgraderTest { public void testBlockVersionChangeHalfwayThoughThenNewVersion() { ManualClock clock = new ManualClock(Instant.parse("2017-09-29T16:00:00.00Z")); // Friday, 16:00 DeploymentTester tester = new DeploymentTester(new ControllerTester(clock)); - BlockedChangeDeployer blockedChangeDeployer = new BlockedChangeDeployer(tester.controller(), - Duration.ofHours(1), - new JobControl(tester.controllerTester().curator())); + ReadyJobsTrigger readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), + Duration.ofHours(1), + new JobControl(tester.controllerTester().curator())); Version version = Version.fromString("5.0"); tester.updateVersionStatus(version); @@ -542,14 +565,14 @@ public class UpgraderTest { version = Version.fromString("5.2"); tester.updateVersionStatus(version); tester.upgrader().maintain(); - blockedChangeDeployer.maintain(); + readyJobsTrigger.maintain(); assertTrue("Nothing is scheduled", tester.buildSystem().jobs().isEmpty()); // Monday morning: We are not blocked tester.clock().advance(Duration.ofDays(1)); // Sunday, 17:00 tester.clock().advance(Duration.ofHours(17)); // Monday, 10:00 tester.upgrader().maintain(); - blockedChangeDeployer.maintain(); + readyJobsTrigger.maintain(); // We proceed with the new version in the expected order, not starting with the previously blocked version: // Test jobs are run with the new version, but not production as we are in the block window tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest); @@ -584,7 +607,7 @@ public class UpgraderTest { Application default3 = tester.createAndDeploy("default3", 6, "default"); Application default4 = tester.createAndDeploy("default4", 7, "default"); - assertEquals(version, default0.deployedVersion().get()); + assertEquals(version, default0.oldestDeployedVersion().get()); // New version is released version = Version.fromString("5.1"); @@ -610,8 +633,16 @@ public class UpgraderTest { tester.completeUpgradeWithError(default3, version, "default", DeploymentJobs.JobType.systemTest); tester.updateVersionStatus(version); assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence()); + tester.upgrader().maintain(); + // Exhaust retries and finish runs + tester.clock().advance(Duration.ofHours(1)); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, default0, false); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, default1, false); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, default2, false); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, default3, false); + // 5th app never reports back and has a dead job, but no ongoing change Application deadLocked = tester.applications().require(default4.id()); assertTrue("Jobs in progress", deadLocked.deploymentJobs().isRunning(tester.controller().applications().deploymentTrigger().jobTimeoutLimit())); @@ -633,20 +664,10 @@ public class UpgraderTest { tester.completeUpgrade(default2, version, "default"); tester.completeUpgrade(default3, version, "default"); - assertEquals(version, tester.application(default0.id()).deployedVersion().get()); - assertEquals(version, tester.application(default1.id()).deployedVersion().get()); - assertEquals(version, tester.application(default2.id()).deployedVersion().get()); - assertEquals(version, tester.application(default3.id()).deployedVersion().get()); - - // Over 12 hours pass and upgrade is rescheduled for 5th app - assertEquals(0, tester.buildSystem().jobs().size()); - tester.clock().advance(Duration.ofHours(12).plus(Duration.ofSeconds(1))); - tester.upgrader().maintain(); - assertEquals(1, tester.buildSystem().jobs().size()); - assertEquals("Upgrade is rescheduled", DeploymentJobs.JobType.systemTest.id(), - tester.buildSystem().jobs().get(0).jobName()); - tester.deployCompletely(default4, applicationPackage); - assertEquals(version, tester.application(default4.id()).deployedVersion().get()); + assertEquals(version, tester.application(default0.id()).oldestDeployedVersion().get()); + assertEquals(version, tester.application(default1.id()).oldestDeployedVersion().get()); + assertEquals(version, tester.application(default2.id()).oldestDeployedVersion().get()); + assertEquals(version, tester.application(default3.id()).oldestDeployedVersion().get()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json index 32d34edd576..425b9d4512d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/pr-instance-with-dead-locked-job.json @@ -4,7 +4,7 @@ "validationOverrides": "<deployment version='1.0'/>", "deployments": [], "deploymentJobs": { - "projectId": 0, + "projectId": 42, "jobStatus": [ { "jobType": "system-test", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index b38a38c3120..2c1471b29b6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -13,6 +13,8 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; @@ -69,10 +71,10 @@ public class ApplicationSerializerTest { List<JobStatus> statusList = new ArrayList<>(); statusList.add(JobStatus.initial(DeploymentJobs.JobType.systemTest) - .withTriggering(37, Version.fromString("5.6.7"), Optional.empty(), true, "Test", Instant.ofEpochMilli(7)) + .withTriggering(Version.fromString("5.6.7"), Optional.empty(), true, "Test", Instant.ofEpochMilli(7)) .withCompletion(30, Optional.empty(), Instant.ofEpochMilli(8), tester.controller())); statusList.add(JobStatus.initial(DeploymentJobs.JobType.stagingTest) - .withTriggering(12, Version.fromString("5.6.6"), Optional.empty(), true, "Test 2", Instant.ofEpochMilli(5)) + .withTriggering(Version.fromString("5.6.6"), Optional.empty(), true, "Test 2", Instant.ofEpochMilli(5)) .withCompletion(11, Optional.of(JobError.unknown), Instant.ofEpochMilli(6), tester.controller())); DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, Optional.empty()); @@ -82,7 +84,9 @@ public class ApplicationSerializerTest { validationOverrides, deployments, deploymentJobs, Optional.of(new Change.VersionChange(Version.fromString("6.7"))), - true); + true, + Optional.of(IssueId.from("1234")), + new MetricsService.ApplicationMetrics(0.5, 0.9)); Application serialized = applicationSerializer.fromSlime(applicationSerializer.toSlime(original)); @@ -105,10 +109,11 @@ public class ApplicationSerializerTest { serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest)); assertEquals( original.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest), serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest)); - assertEquals(original.deploymentJobs().failingSince(), serialized.deploymentJobs().failingSince()); assertEquals(original.hasOutstandingChange(), serialized.hasOutstandingChange()); + assertEquals(original.ownershipIssueId(), serialized.ownershipIssueId()); + assertEquals(original.deploying(), serialized.deploying()); // Test cluster utilization @@ -129,6 +134,9 @@ public class ApplicationSerializerTest { assertEquals(50, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorDisk(), Double.MIN_VALUE); // Test metrics + assertEquals(original.metrics().queryServiceQuality(), serialized.metrics().queryServiceQuality(), Double.MIN_VALUE); + assertEquals(original.metrics().writeServiceQuality(), serialized.metrics().writeServiceQuality(), Double.MIN_VALUE); + assertEquals(2, serialized.deployments().get(zone2).metrics().queriesPerSecond(), Double.MIN_VALUE); assertEquals(3, serialized.deployments().get(zone2).metrics().writesPerSecond(), Double.MIN_VALUE); assertEquals(4, serialized.deployments().get(zone2).metrics().documentCount(), Double.MIN_VALUE); @@ -199,14 +207,11 @@ public class ApplicationSerializerTest { Application application = applicationSerializer.fromSlime(applicationSlime(false)); assertFalse(application.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest).lastCompleted().get().upgrade()); } - - // TODO: Remove after October 2017 + @Test - public void testLegacySerializationWithZeroProjectId() { - Application original = applicationSerializer.fromSlime(applicationSlime(0, false)); - assertFalse(original.deploymentJobs().projectId().isPresent()); - Application serialized = applicationSerializer.fromSlime(applicationSerializer.toSlime(original)); - assertFalse(serialized.deploymentJobs().projectId().isPresent()); + public void testCompleteApplicationDeserialization() { + Application application = applicationSerializer.fromSlime(SlimeUtils.jsonToSlime(longApplicationJson.getBytes(StandardCharsets.UTF_8))); + // ok if no error } private Slime applicationSlime(boolean error) { @@ -245,4 +250,6 @@ public class ApplicationSerializerTest { " }\n" + "}\n"; } + + private final String longApplicationJson = "{\"id\":\"tripod:service-aggregation-vespa:default\",\"deploymentSpecField\":\"<deployment version='1.0'>\\n <test />\\n <!--<staging />-->\\n <prod global-service-id=\\\"tripod\\\">\\n <region active=\\\"true\\\">us-east-3</region>\\n <region active=\\\"true\\\">us-west-1</region>\\n </prod>\\n</deployment>\\n\",\"validationOverrides\":\"<validation-overrides>\\n <allow until=\\\"2016-04-28\\\" comment=\\\"Renaming content cluster\\\">content-cluster-removal</allow>\\n <allow until=\\\"2016-08-22\\\" comment=\\\"Migrating us-east-3 to C-2E\\\">cluster-size-reduction</allow>\\n <allow until=\\\"2017-06-30\\\" comment=\\\"Test Vespa upgrade tests\\\">force-automatic-tenant-upgrade-test</allow>\\n</validation-overrides>\\n\",\"deployments\":[{\"zone\":{\"environment\":\"prod\",\"region\":\"us-west-1\"},\"version\":\"6.173.62\",\"deployTime\":1510837817704,\"applicationPackageRevision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"clusterInfo\":{\"tripod\":{\"flavor\":\"d-3-16-100\",\"cost\":9,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"container\",\"hostnames\":[\"oxy-oxygen-2001-4998-c-2942--10d1.gq1.yahoo.com\",\"oxy-oxygen-2001-4998-c-2942--10e2.gq1.yahoo.com\"]},\"tripodaggregation\":{\"flavor\":\"d-12-64-400\",\"cost\":38,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"content\",\"hostnames\":[\"oxy-oxygen-2001-4998-c-2941--106a.gq1.yahoo.com\",\"zt74700-v6-23.ostk.bm2.prod.gq1.yahoo.com\",\"zt74714-v6-28.ostk.bm2.prod.gq1.yahoo.com\",\"zt74730-v6-13.ostk.bm2.prod.gq1.yahoo.com\",\"zt74717-v6-7.ostk.bm2.prod.gq1.yahoo.com\",\"2080260-v6-12.ostk.bm2.prod.gq1.yahoo.com\",\"zt74719-v6-23.ostk.bm2.prod.gq1.yahoo.com\",\"zt74722-v6-26.ostk.bm2.prod.gq1.yahoo.com\",\"zt74704-v6-9.ostk.bm2.prod.gq1.yahoo.com\",\"oxy-oxygen-2001-4998-c-2942--107d.gq1.yahoo.com\"]},\"tripodaggregationstream\":{\"flavor\":\"d-12-64-400\",\"cost\":38,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"content\",\"hostnames\":[\"zt74727-v6-21.ostk.bm2.prod.gq1.yahoo.com\",\"zt74773-v6-8.ostk.bm2.prod.gq1.yahoo.com\",\"zt74699-v6-25.ostk.bm2.prod.gq1.yahoo.com\",\"zt74766-v6-27.ostk.bm2.prod.gq1.yahoo.com\"]}},\"clusterUtils\":{\"tripod\":{\"cpu\":0.1720353499228221,\"mem\":0.4986146831512451,\"disk\":0.0617671330041831,\"diskbusy\":0},\"tripodaggregation\":{\"cpu\":0.07505730001866318,\"mem\":0.7936344432830811,\"disk\":0.2260549694485994,\"diskbusy\":0},\"tripodaggregationstream\":{\"cpu\":0.01712671480989384,\"mem\":0.0225852754983035,\"disk\":0.006084436856721915,\"diskbusy\":0}},\"metrics\":{\"queriesPerSecond\":1.25,\"writesPerSecond\":43.83199977874756,\"documentCount\":525880277.9999999,\"queryLatencyMillis\":5.607503938674927,\"writeLatencyMillis\":20.57866265104621}},{\"zone\":{\"environment\":\"test\",\"region\":\"us-east-1\"},\"version\":\"6.173.62\",\"deployTime\":1511256872316,\"applicationPackageRevision\":{\"applicationPackageHash\":\"ec548fa61cbfab7a270a51d46b1263ec1be5d9a8\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"234f3e4e77049d0b9538c9e1b356d17eb1dedb6a\"}},\"clusterInfo\":{},\"clusterUtils\":{},\"metrics\":{\"queriesPerSecond\":0,\"writesPerSecond\":0,\"documentCount\":0,\"queryLatencyMillis\":0,\"writeLatencyMillis\":0}},{\"zone\":{\"environment\":\"dev\",\"region\":\"us-east-1\"},\"version\":\"6.173.62\",\"deployTime\":1510597489464,\"applicationPackageRevision\":{\"applicationPackageHash\":\"59b883f263c2a3c23dfab249730097d7e0e1ed32\"},\"clusterInfo\":{\"tripod\":{\"flavor\":\"d-2-8-50\",\"cost\":5,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"container\",\"hostnames\":[\"zt40807-v6-29.ostk.bm2.prod.bf1.yahoo.com\"]},\"tripodaggregation\":{\"flavor\":\"d-2-8-50\",\"cost\":5,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"content\",\"hostnames\":[\"zt40807-v6-24.ostk.bm2.prod.bf1.yahoo.com\"]},\"tripodaggregationstream\":{\"flavor\":\"d-2-8-50\",\"cost\":5,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"content\",\"hostnames\":[\"zt40694-v6-21.ostk.bm2.prod.bf1.yahoo.com\"]}},\"clusterUtils\":{\"tripod\":{\"cpu\":0.191833330678661,\"mem\":0.4625738318415235,\"disk\":0.05582004563850269,\"diskbusy\":0},\"tripodaggregation\":{\"cpu\":0.2227037978608054,\"mem\":0.2051752598416401,\"disk\":0.05471533698695047,\"diskbusy\":0},\"tripodaggregationstream\":{\"cpu\":0.1869410834020498,\"mem\":0.1691722576000564,\"disk\":0.04977374774258153,\"diskbusy\":0}},\"metrics\":{\"queriesPerSecond\":0,\"writesPerSecond\":0,\"documentCount\":30916,\"queryLatencyMillis\":0,\"writeLatencyMillis\":0}},{\"zone\":{\"environment\":\"prod\",\"region\":\"us-east-3\"},\"version\":\"6.173.62\",\"deployTime\":1510817190016,\"applicationPackageRevision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"clusterInfo\":{\"tripod\":{\"flavor\":\"d-3-16-100\",\"cost\":9,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"container\",\"hostnames\":[\"zt40738-v6-13.ostk.bm2.prod.bf1.yahoo.com\",\"zt40783-v6-31.ostk.bm2.prod.bf1.yahoo.com\"]},\"tripodaggregation\":{\"flavor\":\"d-12-64-400\",\"cost\":38,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"content\",\"hostnames\":[\"zt40819-v6-7.ostk.bm2.prod.bf1.yahoo.com\",\"zt40661-v6-3.ostk.bm2.prod.bf1.yahoo.com\",\"zt40805-v6-30.ostk.bm2.prod.bf1.yahoo.com\",\"zt40702-v6-32.ostk.bm2.prod.bf1.yahoo.com\",\"zt40706-v6-3.ostk.bm2.prod.bf1.yahoo.com\",\"zt40691-v6-27.ostk.bm2.prod.bf1.yahoo.com\",\"zt40676-v6-15.ostk.bm2.prod.bf1.yahoo.com\",\"zt40788-v6-23.ostk.bm2.prod.bf1.yahoo.com\",\"zt40782-v6-30.ostk.bm2.prod.bf1.yahoo.com\",\"zt40802-v6-32.ostk.bm2.prod.bf1.yahoo.com\"]},\"tripodaggregationstream\":{\"flavor\":\"d-12-64-400\",\"cost\":38,\"flavorCpu\":0,\"flavorMem\":0,\"flavorDisk\":0,\"clusterType\":\"content\",\"hostnames\":[\"zt40779-v6-27.ostk.bm2.prod.bf1.yahoo.com\",\"zt40791-v6-15.ostk.bm2.prod.bf1.yahoo.com\",\"zt40733-v6-31.ostk.bm2.prod.bf1.yahoo.com\",\"zt40724-v6-30.ostk.bm2.prod.bf1.yahoo.com\"]}},\"clusterUtils\":{\"tripod\":{\"cpu\":0.2295038983007097,\"mem\":0.4627357390237263,\"disk\":0.05559941525894966,\"diskbusy\":0},\"tripodaggregation\":{\"cpu\":0.05340429087579549,\"mem\":0.8107630891552372,\"disk\":0.226444914138854,\"diskbusy\":0},\"tripodaggregationstream\":{\"cpu\":0.02148227413975218,\"mem\":0.02162174219104161,\"disk\":0.006057760545243265,\"diskbusy\":0}},\"metrics\":{\"queriesPerSecond\":1.734000012278557,\"writesPerSecond\":44.59999895095825,\"documentCount\":525868193.9999999,\"queryLatencyMillis\":5.65284947195106,\"writeLatencyMillis\":17.34593812832452}}],\"deploymentJobs\":{\"projectId\":102889,\"jobStatus\":[{\"jobType\":\"staging-test\",\"lastTriggered\":{\"id\":-1,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"system-test completed\",\"at\":1510830134259},\"lastCompleted\":{\"id\":1184,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"system-test completed\",\"at\":1510830684960},\"lastSuccess\":{\"id\":1184,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"system-test completed\",\"at\":1510830684960}},{\"jobType\":\"component\",\"lastCompleted\":{\"id\":849,\"version\":\"6.174.156\",\"upgrade\":false,\"reason\":\"Application commit\",\"at\":1511217733555},\"lastSuccess\":{\"id\":849,\"version\":\"6.174.156\",\"upgrade\":false,\"reason\":\"Application commit\",\"at\":1511217733555}},{\"jobType\":\"production-us-east-3\",\"lastTriggered\":{\"id\":-1,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"staging-test completed\",\"at\":1510830685127},\"lastCompleted\":{\"id\":923,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"staging-test completed\",\"at\":1510837650046},\"lastSuccess\":{\"id\":923,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"staging-test completed\",\"at\":1510837650046}},{\"jobType\":\"production-us-west-1\",\"lastTriggered\":{\"id\":-1,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"production-us-east-3 completed\",\"at\":1510837650139},\"lastCompleted\":{\"id\":646,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"production-us-east-3 completed\",\"at\":1510843559162},\"lastSuccess\":{\"id\":646,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"production-us-east-3 completed\",\"at\":1510843559162}},{\"jobType\":\"system-test\",\"jobError\":\"unknown\",\"lastTriggered\":{\"id\":-1,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"ec548fa61cbfab7a270a51d46b1263ec1be5d9a8\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"234f3e4e77049d0b9538c9e1b356d17eb1dedb6a\"}},\"upgrade\":false,\"reason\":\"Available change in component\",\"at\":1511256608649},\"lastCompleted\":{\"id\":1686,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"ec548fa61cbfab7a270a51d46b1263ec1be5d9a8\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"234f3e4e77049d0b9538c9e1b356d17eb1dedb6a\"}},\"upgrade\":false,\"reason\":\"Available change in component\",\"at\":1511256603353},\"firstFailing\":{\"id\":1659,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"ec548fa61cbfab7a270a51d46b1263ec1be5d9a8\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"234f3e4e77049d0b9538c9e1b356d17eb1dedb6a\"}},\"upgrade\":false,\"reason\":\"component completed\",\"at\":1511219070725},\"lastSuccess\":{\"id\":1658,\"version\":\"6.173.62\",\"revision\":{\"applicationPackageHash\":\"9db423e1021d7b452d37ec6372bc757d9c1bda87\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"49cd7bbb1ed9f4b922083cb042590b0885ffe22b\"}},\"upgrade\":true,\"reason\":\"Upgrading to 6.173.62\",\"at\":1511175754163}}]},\"deployingField\":{\"applicationPackageHash\":\"ec548fa61cbfab7a270a51d46b1263ec1be5d9a8\",\"sourceRevision\":{\"repositoryField\":\"git@git.ouroath.com:Tripod/service-aggregation-vespa.git\",\"branchField\":\"origin/master\",\"commitField\":\"234f3e4e77049d0b9538c9e1b356d17eb1dedb6a\"}},\"outstandingChangeField\":false,\"queryQuality\":100,\"writeQuality\":99.99894341115082}"; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index f5f43265cb8..189b3a97a80 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -27,6 +27,8 @@ public class VersionStatusSerializerTest { Version.fromString("5.0"), Arrays.asList(ApplicationId.from("tenant1", "failing1", "default")), Arrays.asList(ApplicationId.from("tenant2", "success1", "default"), + ApplicationId.from("tenant2", "success2", "default")), + Arrays.asList(ApplicationId.from("tenant1", "failing1", "default"), ApplicationId.from("tenant2", "success2", "default")) ); vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java new file mode 100644 index 00000000000..04a987d98d1 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java @@ -0,0 +1,83 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.proxy; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.IOException; +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author Haakon Dybdahl + */ +public class ProxyRequestTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + @Test + public void testEmpty() throws Exception { + exception.expectMessage("Request not set."); + testRequest(null, "/zone/v2/"); + } + + @Test + public void testBadUri() throws Exception { + exception.expectMessage("Request not starting with /zone/v2/"); + testRequest(URI.create("http://foo"), "/zone/v2/"); + } + + @Test + public void testConfigRequestEmpty() throws Exception { + ProxyRequest proxyRequest = testRequest(URI.create("http://foo/zone/v2/foo/bar"), "/zone/v2/"); + assertEquals("foo", proxyRequest.getEnvironment()); + assertEquals("bar", proxyRequest.getRegion()); + assertFalse(proxyRequest.isDiscoveryRequest()); + assertTrue(proxyRequest.getConfigServerRequest().isEmpty()); + + } + + @Test + public void testDiscoveryRequest() throws Exception { + ProxyRequest proxyRequest = testRequest(URI.create("http://foo/zone/v2/foo"), "/zone/v2/"); + assertEquals("foo", proxyRequest.getEnvironment()); + assertTrue(proxyRequest.isDiscoveryRequest()); + + } + + @Test + public void testProxyRequest() throws Exception { + ProxyRequest proxyRequest = testRequest(URI.create("http://foo/zone/v2/foo/bar/bla/bla/v1/something"), + "/zone/v2/"); + assertEquals("foo", proxyRequest.getEnvironment()); + assertEquals("/bla/bla/v1/something", proxyRequest.getConfigServerRequest()); + } + + @Test + public void testProxyRequestWithParameters() throws Exception { + ProxyRequest proxyRequest = testRequest(URI.create("http://foo/zone/v2/foo/bar/something?p=v&q=y"), + "/zone/v2/"); + assertEquals("foo", proxyRequest.getEnvironment()); + assertEquals("/something?p=v&q=y", proxyRequest.getConfigServerRequest()); + } + + private static ProxyRequest testRequest(URI url, String pathPrefix) throws IOException, ProxyException { + return new ProxyRequest(url, headers("controller:49152"), null, "GET", pathPrefix); + } + + private static Map<String, List<String>> headers(String hostPort) { + Map<String, List<String>> headers = new HashMap<>(); + headers.put("host", Collections.singletonList(hostPort)); + return Collections.unmodifiableMap(headers); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java new file mode 100644 index 00000000000..8dbd1c4ef61 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java @@ -0,0 +1,69 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.proxy; + +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; + +/** + * @author Haakon Dybdahl + */ +public class ProxyResponseTest { + + @Test + public void testRewriteUrl() throws Exception { + String controllerPrefix = "/zone/v2/"; + URI configServer = URI.create("http://configserver:1234"); + ProxyRequest request = new ProxyRequest(URI.create("http://foo/zone/v2/env/region/configserver"), + headers("controller:49152"), null, "GET", + controllerPrefix); + ProxyResponse proxyResponse = new ProxyResponse( + request, + "response link is http://configserver:1234/bla/bla/", + 200, + Optional.of(configServer), + "application/json"); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + proxyResponse.render(outputStream); + String document = new String(outputStream.toByteArray(),"UTF-8"); + assertEquals("response link is http://controller:49152/zone/v2/env/region/bla/bla/", document); + } + + @Test + public void testRewriteSecureUrl() throws Exception { + String controllerPrefix = "/zone/v2/"; + URI configServer = URI.create("http://configserver:1234"); + ProxyRequest request = new ProxyRequest(URI.create("https://foo/zone/v2/env/region/configserver"), + headers("controller:49152"), null, "GET", + controllerPrefix); + ProxyResponse proxyResponse = new ProxyResponse( + request, + "response link is http://configserver:1234/bla/bla/", + 200, + Optional.of(configServer), + "application/json"); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + proxyResponse.render(outputStream); + String document = new String(outputStream.toByteArray(),"UTF-8"); + assertEquals("response link is https://controller:49152/zone/v2/env/region/bla/bla/", document); + } + + private static Map<String, List<String>> headers(String hostPort) { + Map<String, List<String>> headers = new HashMap<>(); + headers.put("host", Collections.singletonList(hostPort)); + return Collections.unmodifiableMap(headers); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 45a8972eafe..6c5120df515 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -88,6 +88,12 @@ public class ContainerControllerTester { } public void notifyJobCompletion(ApplicationId applicationId, long projectId, boolean success, DeploymentJobs.JobType job) { + try { + Thread.sleep(1); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } controller().applications().notifyJobCompletion(new DeploymentJobs.JobReport(applicationId, job, projectId, 42, success ? Optional.empty() : Optional.of(DeploymentJobs.JobError.unknown) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index b55ee9a195f..c0e8b48f821 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.Supplier; import static org.junit.Assert.assertEquals; @@ -52,10 +53,18 @@ public class ContainerTester { controller.updateVersionStatus(VersionStatus.compute(controller, version)); } + public void assertResponse(Supplier<Request> request, File responseFile) throws IOException { + assertResponse(request.get(), responseFile); + } + public void assertResponse(Request request, File responseFile) throws IOException { assertResponse(request, responseFile, 200); } + public void assertResponse(Supplier<Request> request, File responseFile, int expectedStatusCode) throws IOException { + assertResponse(request.get(), responseFile, expectedStatusCode); + } + public void assertResponse(Request request, File responseFile, int expectedStatusCode) throws IOException { String expectedResponse = IOUtils.readFile(new File(responseFilePath + responseFile.toString())); expectedResponse = include(expectedResponse); @@ -72,10 +81,18 @@ public class ContainerTester { replace(new String(SlimeUtils.toJsonBytes(responseSlime), StandardCharsets.UTF_8), replaceStrings)); } + public void assertResponse(Supplier<Request> request, String expectedResponse) throws IOException { + assertResponse(request.get(), expectedResponse, 200); + } + public void assertResponse(Request request, String expectedResponse) throws IOException { assertResponse(request, expectedResponse, 200); } + public void assertResponse(Supplier<Request> request, String expectedResponse, int expectedStatusCode) throws IOException { + assertResponse(request.get(), expectedResponse, expectedStatusCode); + } + public void assertResponse(Request request, String expectedResponse, int expectedStatusCode) throws IOException { Response response = container.handleRequest(request); assertEquals(expectedResponse, response.getBodyAsString()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index e6c0ce9027d..044c5d75d12 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -43,10 +43,12 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock'/>" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService'/>" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues'/>" + + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues'/>" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.organization.MockOrganization'/>" + " <component id='com.yahoo.vespa.hosted.controller.ConfigServerClientMock'/>" + " <component id='com.yahoo.vespa.hosted.controller.ZoneRegistryMock'/>" + " <component id='com.yahoo.vespa.hosted.controller.Controller'/>" + + " <component id='com.yahoo.vespa.hosted.controller.ConfigServerProxyMock'/>" + " <component id='com.yahoo.vespa.hosted.controller.integration.MockMetricsService'/>" + " <component id='com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance'/>" + " <component id='com.yahoo.vespa.hosted.controller.maintenance.JobControl'/>" + @@ -69,6 +71,14 @@ public class ControllerContainerTest { " <handler id='com.yahoo.vespa.hosted.controller.restapi.screwdriver.ScrewdriverApiHandler'>" + " <binding>http://*/screwdriver/v1/*</binding>" + " </handler>" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v1.ZoneApiHandler'>" + + " <binding>http://*/zone/v1</binding>" + + " <binding>http://*/zone/v1/*</binding>" + + " </handler>" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v2.ZoneApiHandler'>" + + " <binding>http://*/zone/v2</binding>" + + " <binding>http://*/zone/v2/*</binding>" + + " </handler>" + "</jdisc>"; protected void assertResponse(Request request, int responseStatus, String responseMessage) throws IOException { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index e3443d6c014..bf4586f9fd0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -5,14 +5,14 @@ import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ConfigServerClientMock; -import com.yahoo.vespa.hosted.controller.LockedApplication; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockOrganization; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -47,6 +47,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Supplier; + +import static com.yahoo.application.container.handler.Request.Method.DELETE; +import static com.yahoo.application.container.handler.Request.Method.GET; +import static com.yahoo.application.container.handler.Request.Method.POST; +import static com.yahoo.application.container.handler.Request.Method.PUT; /** * @author bratseth @@ -72,63 +78,87 @@ public class ApplicationApiTest extends ControllerContainerTest { addTenantAthenzDomain(athenzUserDomain, "mytenant"); // (Necessary but not provided in this API) // GET API root - tester.assertResponse(request("/application/v4/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/", GET), new File("root.json")); // GET athens domains - tester.assertResponse(request("/application/v4/athensDomain/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/athensDomain/", GET), new File("athensDomain-list.json")); // GET OpsDB properties - tester.assertResponse(request("/application/v4/property/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/property/", GET), new File("property-list.json")); // GET cookie freshness - tester.assertResponse(request("/application/v4/cookiefreshness/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/cookiefreshness/", GET), new File("cookiefreshness.json")); // POST (add) a tenant without property ID - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), new File("tenant-without-applications.json")); // PUT (modify) a tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.PUT), + tester.assertResponse(request("/application/v4/tenant/tenant1", PUT) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), new File("tenant-without-applications.json")); // GET the authenticated user (with associated tenants) - tester.assertResponse(request("/application/v4/user", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/user", GET), new File("user.json")); // GET all tenants - tester.assertResponse(request("/application/v4/tenant/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/", GET), new File("tenant-list.json")); + + + // Add another Athens domain, so we can try to create more tenants + addTenantAthenzDomain("domain2", "mytenant"); // New domain to test tenant w/property ID + // Add property info for that property id, as well, in the mock organization. + addPropertyData((MockOrganization) controllerTester.controller().organization(), "1234"); + // POST (add) a tenant with property ID + tester.assertResponse(request("/application/v4/tenant/tenant2", POST) + .data("{\"athensDomain\":\"domain2\", \"property\":\"property2\", \"propertyId\":\"1234\"}"), + new File("tenant-without-applications-with-id.json")); + // PUT (modify) a tenant with property ID + tester.assertResponse(request("/application/v4/tenant/tenant2", PUT) + .data("{\"athensDomain\":\"domain2\", \"property\":\"property2\", \"propertyId\":\"1234\"}"), + new File("tenant-without-applications-with-id.json")); + // GET a tenant with property ID + tester.assertResponse(request("/application/v4/tenant/tenant2", GET), + new File("tenant-without-applications-with-id.json")); + + // Test legacy OpsDB tenants + // POST (add) an OpsDB tenant with property ID + tester.assertResponse(request("/application/v4/tenant/tenant3", POST) + .data("{\"userGroup\":\"group1\",\"property\":\"property1\",\"propertyId\":\"1234\"}"), + new File("opsdb-tenant-with-id-without-applications.json")); + // PUT (modify) the OpsDB tenant to set another property + tester.assertResponse(request("/application/v4/tenant/tenant3", PUT) + .data("{\"userGroup\":\"group1\",\"property\":\"property2\",\"propertyId\":\"4321\"}"), + new File("opsdb-tenant-with-new-id-without-applications.json")); + // POST (create) an application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST), new File("application-reference.json")); // GET a tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1", GET), new File("tenant-with-application.json")); // GET tenant applications - tester.assertResponse(request("/application/v4/tenant/tenant1/application/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/", GET), new File("application-list.json")); // POST triggering of a full deployment to an application (if version is omitted, current system version is used) - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", "6.1.0", Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", POST) + .data("6.1.0"), new File("application-deployment.json")); // DELETE (cancel) ongoing change - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", "", Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE), new File("application-deployment-cancelled.json")); // DELETE (cancel) again is a no-op - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", "", Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE), new File("application-deployment-cancelled-no-op.json")); // POST (deploy) an application to a zone - manual user deployment HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty()); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", - entity, - Request.Method.POST, - athenzUserDomain, "mytenant"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + .data(entity) + .domain(athenzUserDomain).user("mytenant"), new File("deploy-result.json")); // POST (deploy) an application to a zone. This simulates calls done by our tenant pipeline. @@ -138,168 +168,146 @@ public class ApplicationApiTest extends ControllerContainerTest { addScrewdriverUserToDomain("screwdriveruser1", "domain1"); // (Necessary but not provided in this API) // Trigger deployment - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", "6.1.0", Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", POST) + .data("6.1.0"), new File("application-deployment.json")); // ... systemtest - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default/", - createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId)), - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default/", POST) + .data(createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId))) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), new File("deploy-result.json")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default", - "", - Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default", DELETE), "Deactivated tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default"); controllerTester.notifyJobCompletion(id, screwdriverProjectId, true, DeploymentJobs.JobType.systemTest); // Called through the separate screwdriver/v1 API // ... staging - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default/", - createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId)), - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default/", POST) + .data(createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId))) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), new File("deploy-result.json")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default", - "", - Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default", DELETE), "Deactivated tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default"); controllerTester.notifyJobCompletion(id, screwdriverProjectId, true, DeploymentJobs.JobType.stagingTest); // ... prod zone - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/", - createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId)), - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/", POST) + .data(createApplicationDeployData(applicationPackage, Optional.of(screwdriverProjectId))) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), new File("deploy-result.json")); controllerTester.notifyJobCompletion(id, screwdriverProjectId, false, DeploymentJobs.JobType.productionCorpUsEast1); // GET tenant screwdriver projects - tester.assertResponse(request("/application/v4/tenant-pipeline/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant-pipeline/", GET), new File("tenant-pipelines.json")); + setDeploymentMaintainedInfo(controllerTester); // GET tenant application deployments - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET), new File("application.json")); // GET an application deployment - setDeploymentMaintainedInfo(controllerTester); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", GET), new File("deployment.json")); + + addIssues(controllerTester, ApplicationId.from("tenant1", "application1", "default")); + // GET at root, with "&recursive=deployment", returns info about all tenants, their applications and their deployments + tester.assertResponse(request("/application/v4/", GET) + .domain("domain1").user("mytenant") + .recursive("deployment"), + new File("recursive-root.json")); + // GET at root, with "&recursive=tenant", returns info about all tenants, with limmited info about their applications. + tester.assertResponse(request("/application/v4/", GET) + .domain("domain1").user("mytenant") + .recursive("tenant"), + new File("recursive-until-tenant-root.json")); + // GET at a tenant, with "&recursive=true", returns full info about their applications and their deployments + tester.assertResponse(request("/application/v4/tenant/tenant1/", GET) + .domain("domain1").user("mytenant") + .recursive("true"), + new File("tenant1-recursive.json")); + // GET at an application, with "&recursive=true", returns full info about its deployments + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/", GET) + .domain("domain1").user("mytenant") + .recursive("true"), + new File("application1-recursive.json")); + + // POST a 'restart application' command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart", POST), "Requested restart of tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); // POST a 'restart application' command with a host filter (other filters not supported yet) - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart?hostname=host1", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/restart?hostname=host1", POST), "Requested restart of tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); // POST a 'log' command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/log", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/log", POST), new File("log-response.json")); // Proxied to config server, not sure about the expected return format // GET (wait for) convergence - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/converge", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/converge", GET), new File("convergence.json")); // GET services - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service", GET), new File("services.json")); // GET service - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", GET), new File("service.json")); // DELETE (deactivate) a deployment - dev - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default", - "", - Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default", DELETE), "Deactivated tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default"); // DELETE (deactivate) a deployment - prod - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", - "", - Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", DELETE), "Deactivated tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); // DELETE (deactivate) a deployment is idempotent - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", - "", - Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", DELETE), "Deactivated tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default"); // DELETE an application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE), ""); // DELETE a tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE), new File("tenant-without-applications.json")); // PUT (create) the authenticated user - tester.assertResponse(request("/application/v4/user?user=newuser&domain=by", - new byte[0], - Request.Method.PUT, - athenzUserDomain, "newuser", "application/json"), + byte[] data = new byte[0]; + tester.assertResponse(request("/application/v4/user?user=newuser&domain=by", PUT) + .data(data) + .domain(athenzUserDomain).user("newuser"), new File("create-user-response.json")); // OPTIONS return 200 OK - tester.assertResponse(request("/application/v4/", "", Request.Method.OPTIONS), + tester.assertResponse(request("/application/v4/", Request.Method.OPTIONS), ""); - // Add another Athens domain, so we can try to create more tenants - addTenantAthenzDomain("domain2", "mytenant"); // New domain to test tenant w/property ID - // Add property info for that property id, as well, in the mock organization. - addPropertyData((MockOrganization) controllerTester.controller().organization(), "1234"); - // POST (add) a tenant with property ID - tester.assertResponse(request("/application/v4/tenant/tenant2", - "{\"athensDomain\":\"domain2\", \"property\":\"property2\", \"propertyId\":\"1234\"}", - Request.Method.POST), - new File("tenant-without-applications-with-id.json")); - // PUT (modify) a tenant with property ID - tester.assertResponse(request("/application/v4/tenant/tenant2", - "{\"athensDomain\":\"domain2\", \"property\":\"property2\", \"propertyId\":\"1234\"}", - Request.Method.PUT), - new File("tenant-without-applications-with-id.json")); - // GET a tenant with property ID - tester.assertResponse(request("/application/v4/tenant/tenant2", "", Request.Method.GET), - new File("tenant-without-applications-with-id.json")); - - // Test legacy OpsDB tenants - // POST (add) an OpsDB tenant with property ID - tester.assertResponse(request("/application/v4/tenant/tenant3", - "{\"userGroup\":\"group1\",\"property\":\"property1\",\"propertyId\":\"1234\"}", - Request.Method.POST), - new File("opsdb-tenant-with-id-without-applications.json")); - // PUT (modify) the OpsDB tenant to set another property - tester.assertResponse(request("/application/v4/tenant/tenant3", - "{\"userGroup\":\"group1\",\"property\":\"property2\",\"propertyId\":\"4321\"}", - Request.Method.PUT), - new File("opsdb-tenant-with-new-id-without-applications.json")); - // GET global rotation status - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation", "", Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation", GET), new File("global-rotation.json")); // GET global rotation override status - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation/override", "", Request.Method.GET), - new File("global-rotation-get.json")); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation/override", GET), + new File("global-rotation-get.json")); // SET global rotation override status - tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation/override", "{\"reason\":\"because i can\"}", Request.Method.PUT), - new File("global-rotation-put.json")); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation/override", PUT) + .data("{\"reason\":\"because i can\"}"), + new File("global-rotation-put.json")); // DELETE global rotation override status - tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation/override", "{\"reason\":\"because i can\"}", Request.Method.DELETE), - new File("global-rotation-delete.json")); + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation/override", DELETE) + .data("{\"reason\":\"because i can\"}"), + new File("global-rotation-delete.json")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/promote", "", Request.Method.POST), - "{\"message\":\"Successfully copied environment hosted-verified-prod to hosted-instance_tenant1_application1_placeholder_component_default\"}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/promote", "", Request.Method.POST), - "{\"message\":\"Successfully copied environment hosted-instance_tenant1_application1_placeholder_component_default to hosted-instance_tenant1_application1_us-west-1_prod_default\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/promote", POST), + "{\"message\":\"Successfully copied environment hosted-verified-prod to hosted-instance_tenant1_application1_placeholder_component_default\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/promote", POST), + "{\"message\":\"Successfully copied environment hosted-instance_tenant1_application1_placeholder_component_default to hosted-instance_tenant1_application1_us-west-1_prod_default\"}"); controllerTester.controller().deconstruct(); } - private void addPropertyData(MockOrganization organization, String propertyIdValue) { - PropertyId propertyId = new PropertyId(propertyIdValue); - organization.addProperty(propertyId); - organization.setContactsFor(propertyId, Arrays.asList(Collections.singletonList(User.from("alice")), - Collections.singletonList(User.from("bob")))); + private void addIssues(ContainerControllerTester tester, ApplicationId id) { + tester.controller().applications().lockedOrThrow(id, application -> + tester.controller().applications().store(application + .withDeploymentIssueId(IssueId.from("123")) + .withOwnershipIssueId(IssueId.from("321")))); } @Test @@ -312,23 +320,19 @@ public class ApplicationApiTest extends ControllerContainerTest { addScrewdriverUserToDomain("screwdriveruser1", "domain1"); // Create tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), new File("tenant-without-applications.json")); // Create application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST), new File("application-reference.json")); // POST (deploy) an application to a prod zone - allowed when project ID is not specified HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty()); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/deploy", - entity, - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/deploy", POST) + .data(entity) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), new File("deploy-result.json")); } @@ -342,15 +346,12 @@ public class ApplicationApiTest extends ControllerContainerTest { addScrewdriverUserToDomain("screwdriveruser1", "domain1"); // Create tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), new File("tenant-without-applications.json")); // Create application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST), new File("application-reference.json")); // Deploy @@ -364,10 +365,9 @@ public class ApplicationApiTest extends ControllerContainerTest { startAndTestChange(controllerTester, id, projectId, deployData); // us-east-3 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy", - deployData, - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy", POST) + .data(deployData) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), new File("deploy-result.json")); controllerTester.notifyJobCompletion(id, projectId, true, DeploymentJobs.JobType.productionUsEast3); @@ -381,22 +381,20 @@ public class ApplicationApiTest extends ControllerContainerTest { startAndTestChange(controllerTester, id, projectId, deployData); // us-west-1 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", - deployData, - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", POST) + .data(deployData) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), new File("deploy-result.json")); controllerTester.notifyJobCompletion(id, projectId, true, DeploymentJobs.JobType.productionUsWest1); // us-east-3 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy", - deployData, - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy", POST) + .data(deployData).domain(athenzScrewdriverDomain).user("screwdriveruser1"), new File("deploy-result.json")); controllerTester.notifyJobCompletion(id, projectId, true, DeploymentJobs.JobType.productionUsEast3); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.GET), + setDeploymentMaintainedInfo(controllerTester); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET), new File("application-without-change-multiple-deployments.json")); } @@ -407,63 +405,49 @@ public class ApplicationApiTest extends ControllerContainerTest { addTenantAthenzDomain("domain1", "mytenant"); // PUT (update) non-existing tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.PUT), + tester.assertResponse(request("/application/v4/tenant/tenant1", PUT) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'tenant1' does not exist\"}", 404); // GET non-existing tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", - "", - Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1", GET), "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'tenant1' does not exist\"}", 404); // GET non-existing application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET), "{\"error-code\":\"NOT_FOUND\",\"message\":\"tenant1.application1 not found\"}", 404); // GET non-existing deployment - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east/instance/default", - "", - Request.Method.GET), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east/instance/default", GET), "{\"error-code\":\"NOT_FOUND\",\"message\":\"tenant1.application1 not found\"}", 404); // POST (add) a tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), new File("tenant-without-applications.json")); // POST (add) another tenant under the same domain - tester.assertResponse(request("/application/v4/tenant/tenant2", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant2", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create tenant 'tenant2': The Athens domain 'domain1' is already connected to tenant 'tenant1'\"}", 400); // Add the same tenant again - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Tenant 'tenant1' already exists\"}", 400); // POST (create) an (empty) application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST), new File("application-reference.json")); // Create the same application again - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.POST), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"An application with id 'tenant1.application1' already exists\"}", 400); @@ -472,64 +456,56 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (deploy) an application with an invalid application package HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty()); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", - entity, - Request.Method.POST, - athenzUserDomain, "mytenant"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + .data(entity) + .domain(athenzUserDomain).user("mytenant"), new File("deploy-failure.json"), 400); // POST (deploy) an application without available capacity configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to prepare application", ConfigServerException.ErrorCode.OUT_OF_CAPACITY, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", - entity, - Request.Method.POST, - athenzUserDomain, "mytenant"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + .data(entity) + .domain(athenzUserDomain).user("mytenant"), new File("deploy-out-of-capacity.json"), 400); // POST (deploy) an application where activation fails configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to activate application", ConfigServerException.ErrorCode.ACTIVATION_CONFLICT, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", - entity, - Request.Method.POST, - athenzUserDomain, "mytenant"), - new File("deploy-activation-conflict.json"), 409); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + .data(entity) + .domain(athenzUserDomain).user("mytenant"), + new File("deploy-activation-conflict.json"), 409); // POST (deploy) an application where we get an internal server error configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Internal server error", ConfigServerException.ErrorCode.INTERNAL_SERVER_ERROR, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", - entity, - Request.Method.POST, - athenzUserDomain, "mytenant"), - new File("deploy-internal-server-error.json"), 500); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + .data(entity) + .domain(athenzUserDomain).user("mytenant"), + new File("deploy-internal-server-error.json"), 500); // DELETE tenant which has an application - tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not delete tenant 'tenant1': This tenant has active applications\"}", 400); // DELETE application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE), ""); // DELETE application again - should produce 404 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.DELETE), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete application 'tenant1.application1': Application not found\"}", + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete application 'tenant1.application1': Application not found\"}", 404); // DELETE tenant - tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE), + tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE), new File("tenant-without-applications.json")); // DELETE tenant again - should produce 404 - tester.assertResponse(request("/application/v4/tenant/tenant1", "", Request.Method.DELETE), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete tenant 'tenant1': Tenant not found\"}", + tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete tenant 'tenant1': Tenant not found\"}", 404); // Promote application chef env for nonexistent tenant/application - tester.assertResponse(request("/application/v4/tenant/dontexist/application/dontexist/environment/prod/region/us-west-1/instance/default/promote", "", Request.Method.POST), - "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"Unable to promote Chef environments for application\"}", - 500); + tester.assertResponse(request("/application/v4/tenant/dontexist/application/dontexist/environment/prod/region/us-west-1/instance/default/promote", POST), + "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"Unable to promote Chef environments for application\"}", + 500); } @Test @@ -539,102 +515,85 @@ public class ApplicationApiTest extends ControllerContainerTest { String unauthorizedUser = "othertenant"; // Mutation without an authorized user is disallowed - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST, - "domain1", null), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") + .domain("domain1").user(null), "{\"error-code\":\"FORBIDDEN\",\"message\":\"User is not authenticated\"}", 403); // ... but read methods are allowed - tester.assertResponse(request("/application/v4/tenant/", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.GET, - "domain1", null), + tester.assertResponse(request("/application/v4/tenant/", GET) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") + .domain("domain1").user(null), "[]", 200); addTenantAthenzDomain("domain1", "mytenant"); // Creating a tenant for an Athens domain the user is not admin for is disallowed - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST, - "domain1", unauthorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") + .domain("domain1").user(unauthorizedUser), "{\"error-code\":\"FORBIDDEN\",\"message\":\"The user 'othertenant' is not admin in Athenz domain 'domain1'\"}", 403); // (Create it with the right tenant id) - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.POST, - "domain1", authorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1", POST) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") + .domain("domain1").user(authorizedUser), new File("tenant-without-applications.json"), 200); // Creating an application for an Athens domain the user is not admin for is disallowed - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.POST, - "domain1", unauthorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + .domain("domain1").user(unauthorizedUser), "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}", 403); // (Create it with the right tenant id) - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.POST, - "domain1", authorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + .domain("domain1").user(authorizedUser), new File("application-reference.json"), 200); // Deploy to an authorized zone by a user tenant is disallowed HttpEntity entity = createApplicationDeployData(applicationPackage, Optional.empty()); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", - entity, - Request.Method.POST, - athenzUserDomain, "mytenant"), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", POST) + .data(entity) + .domain(athenzUserDomain).user("mytenant"), "{\"error-code\":\"FORBIDDEN\",\"message\":\"Principal 'mytenant' is not a Screwdriver principal. Excepted principal with Athenz domain 'cd.screwdriver.project', got 'domain1'.\"}", 403); // Deleting an application for an Athens domain the user is not admin for is disallowed - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.DELETE, - "domain1", unauthorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) + .domain("domain1").user(unauthorizedUser), "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}", 403); // (Deleting it with the right tenant id) - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", - "", - Request.Method.DELETE, - "domain1", authorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) + .domain("domain1").user(authorizedUser), "", 200); // Updating a tenant for an Athens domain the user is not admin for is disallowed - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}", - Request.Method.PUT, - "domain1", unauthorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1", PUT) + .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") + .domain("domain1").user(unauthorizedUser), "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}", 403); // Change Athens domain addTenantAthenzDomain("domain2", "mytenant"); - tester.assertResponse(request("/application/v4/tenant/tenant1", - "{\"athensDomain\":\"domain2\", \"property\":\"property1\"}", - Request.Method.PUT, - "domain1", authorizedUser), - "{\"type\":\"ATHENS\",\"athensDomain\":\"domain2\",\"property\":\"property1\",\"applications\":[]}", + tester.assertResponse(request("/application/v4/tenant/tenant1", PUT) + .data("{\"athensDomain\":\"domain2\", \"property\":\"property1\"}") + .domain("domain1").user(authorizedUser), + "{\"tenant\":\"tenant1\",\"type\":\"ATHENS\",\"athensDomain\":\"domain2\",\"property\":\"property1\",\"applications\":[]}", 200); // Deleting a tenant for an Athens domain the user is not admin for is disallowed - tester.assertResponse(request("/application/v4/tenant/tenant1", - "", - Request.Method.DELETE, - "domain1", unauthorizedUser), + tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE) + .domain("domain1").user(unauthorizedUser), "{\"error-code\":\"FORBIDDEN\",\"message\":\"User othertenant does not have write access to tenant tenant1\"}", 403); } @@ -669,33 +628,53 @@ public class ApplicationApiTest extends ControllerContainerTest { "}"; } - - /** Make a request with (athens) user domain1.mytenant1 */ - private Request request(String path, String data, Request.Method method) { - return request(path, data.getBytes(StandardCharsets.UTF_8), method, "domain1", "mytenant", "application/json"); - } - private Request request(String path, String data, Request.Method method, String domain, String user) { - return request(path, data.getBytes(StandardCharsets.UTF_8), method, domain, user, "application/json"); - } + private static class RequestBuilder implements Supplier<Request> { - private Request request(String path, byte[] data, Request.Method method, String domain, String user, String contentType) { - // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters - Request request = new Request("http://localhost:8080" + path + "?domain=" + domain + - (user != null ? "&user=" + user : ""), - data, method); - request.getHeaders().put("Content-Type", contentType); - return request; - } + private final String path; + private final Request.Method method; + private byte[] data = new byte[0]; + private String domain = "domain1"; + private String user = "mytenant"; + private String contentType = "application/json"; + private String recursive; - private Request request(String path, HttpEntity data, Request.Method method, String domain, String user) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - try { - data.writeTo(out); - } catch (IOException e) { - throw new UncheckedIOException(e); + private RequestBuilder(String path, Request.Method method) { + this.path = path; + this.method = method; } - return request(path, out.toByteArray(), method, domain, user, data.getContentType().getValue()); + + private RequestBuilder data(byte[] data) { this.data = data; return this; } + private RequestBuilder data(String data) { return data(data.getBytes(StandardCharsets.UTF_8)); } + private RequestBuilder data(HttpEntity data) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + data.writeTo(out); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return data(out.toByteArray()).contentType(data.getContentType().getValue()); + } + private RequestBuilder domain(String domain) { this.domain = domain; return this; } + private RequestBuilder user(String user) { this.user = user; return this; } + private RequestBuilder contentType(String contentType) { this.contentType = contentType; return this; } + private RequestBuilder recursive(String recursive) { this.recursive = recursive; return this; } + + @Override + public Request get() { + Request request = new Request("http://localhost:8080" + path + + // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters + "?domain=" + domain + (user == null ? "" : "&user=" + user) + + (recursive == null ? "" : "&recursive=" + recursive), + data, method); + request.getHeaders().put("Content-Type", contentType); + return request; + } + } + + /** Make a request with (athens) user domain1.mytenant */ + private RequestBuilder request(String path, Request.Method method) { + return new RequestBuilder(path, method); } /** @@ -734,34 +713,28 @@ public class ApplicationApiTest extends ControllerContainerTest { // system-test String testPath = String.format("/application/v4/tenant/%s/application/%s/environment/test/region/us-east-1/instance/default", application.tenant().value(), application.application().value()); - tester.assertResponse(request(testPath, - deployData, - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), - new File("deploy-result.json")); - tester.assertResponse(request(testPath, - "", - Request.Method.DELETE), + tester.assertResponse(request(testPath, POST) + .data(deployData) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), + new File("deploy-result.json")); + tester.assertResponse(request(testPath, DELETE), "Deactivated " + testPath.replaceFirst("/application/v4/", "")); controllerTester.notifyJobCompletion(application, projectId, true, DeploymentJobs.JobType.systemTest); // staging String stagingPath = String.format("/application/v4/tenant/%s/application/%s/environment/staging/region/us-east-3/instance/default", application.tenant().value(), application.application().value()); - tester.assertResponse(request(stagingPath, - deployData, - Request.Method.POST, - athenzScrewdriverDomain, "screwdriveruser1"), - new File("deploy-result.json")); - tester.assertResponse(request(stagingPath, - "", - Request.Method.DELETE), + tester.assertResponse(request(stagingPath, POST) + .data(deployData) + .domain(athenzScrewdriverDomain).user("screwdriveruser1"), + new File("deploy-result.json")); + tester.assertResponse(request(stagingPath, DELETE), "Deactivated " + stagingPath.replaceFirst("/application/v4/", "")); controllerTester.notifyJobCompletion(application, projectId, true, DeploymentJobs.JobType.stagingTest); } /** - * Cluster info, utilization and deployment metrics are maintained async by maintainers. + * Cluster info, utilization and application and deployment metrics are maintained async by maintainers. * * This sets these values as if the maintainers has been ran. * @@ -769,9 +742,9 @@ public class ApplicationApiTest extends ControllerContainerTest { */ private void setDeploymentMaintainedInfo(ContainerControllerTester controllerTester) { for (Application application : controllerTester.controller().applications().asList()) { - try (Lock lock = controllerTester.controller().applications().lock(application.id())) { - LockedApplication lockedApplication = controllerTester.controller().applications() - .require(application.id(), lock); + controllerTester.controller().applications().lockedOrThrow(application.id(), lockedApplication -> { + lockedApplication = lockedApplication.with(new ApplicationMetrics(0.5, 0.7)); + for (Deployment deployment : application.deployments().values()) { Map<ClusterSpec.Id, ClusterInfo> clusterInfo = new HashMap<>(); List<String> hostnames = new ArrayList<>(); @@ -780,13 +753,23 @@ public class ApplicationApiTest extends ControllerContainerTest { clusterInfo.put(ClusterSpec.Id.from("cluster1"), new ClusterInfo("flavor1", 37, 2, 4, 50, ClusterSpec.Type.content, hostnames)); Map<ClusterSpec.Id, ClusterUtilization> clusterUtils = new HashMap<>(); clusterUtils.put(ClusterSpec.Id.from("cluster1"), new ClusterUtilization(0.3, 0.6, 0.4, 0.3)); - deployment = deployment.withClusterInfo(clusterInfo); - deployment = deployment.withClusterUtils(clusterUtils); - deployment = deployment.withMetrics(new DeploymentMetrics(1,2,3,4,5)); - controllerTester.controller().applications().store(lockedApplication.with(deployment)); + DeploymentMetrics metrics = new DeploymentMetrics(1,2,3,4,5); + + lockedApplication = lockedApplication + .withClusterInfo(deployment.zone(), clusterInfo) + .withClusterUtilization(deployment.zone(), clusterUtils) + .with(deployment.zone(), metrics); } - } + controllerTester.controller().applications().store(lockedApplication); + }); } } + private void addPropertyData(MockOrganization organization, String propertyIdValue) { + PropertyId propertyId = new PropertyId(propertyIdValue); + organization.addProperty(propertyId); + organization.setContactsFor(propertyId, Arrays.asList(Collections.singletonList(User.from("alice")), + Collections.singletonList(User.from("bob")))); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json index fe9c373b7d5..6442ddf5c02 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json @@ -1,4 +1,6 @@ { + "application": "application1", + "instance": "default", "deploymentJobs": [ { "type": "component", @@ -30,7 +32,7 @@ "gitCommit": "commit1" } }, - "reason": "component completed successfully in build 42", + "reason": "component completed", "at": "(ignore)" }, "lastCompleted": { @@ -44,7 +46,7 @@ "gitCommit": "commit1" } }, - "reason": "component completed successfully in build 42", + "reason": "component completed", "at": "(ignore)" }, "lastSuccess": { @@ -58,7 +60,7 @@ "gitCommit": "commit1" } }, - "reason": "component completed successfully in build 42", + "reason": "component completed", "at": "(ignore)" } }, @@ -76,7 +78,7 @@ "gitCommit": "commit1" } }, - "reason":"systemTest completed successfully in build 42", + "reason":"system-test completed", "at": "(ignore)" }, "lastCompleted": { @@ -90,7 +92,7 @@ "gitCommit": "commit1" } }, - "reason":"systemTest completed successfully in build 42", + "reason":"system-test completed", "at": "(ignore)" }, "lastSuccess": { @@ -104,7 +106,7 @@ "gitCommit": "commit1" } }, - "reason":"systemTest completed successfully in build 42", + "reason":"system-test completed", "at": "(ignore)" } }, @@ -122,7 +124,7 @@ "gitCommit": "commit1" } }, - "reason":"stagingTest completed successfully in build 42", + "reason":"staging-test completed", "at": "(ignore)" }, "lastCompleted": { @@ -136,7 +138,7 @@ "gitCommit": "commit1" } }, - "reason":"stagingTest completed successfully in build 42", + "reason":"staging-test completed", "at": "(ignore)" }, "lastSuccess": { @@ -150,7 +152,7 @@ "gitCommit": "commit1" } }, - "reason":"stagingTest completed successfully in build 42", + "reason":"staging-test completed", "at": "(ignore)" } }, @@ -168,7 +170,7 @@ "gitCommit": "commit1" } }, - "reason":"productionUsWest1 completed successfully in build 42", + "reason":"production-us-west-1 completed", "at": "(ignore)" }, "lastCompleted": { @@ -182,7 +184,7 @@ "gitCommit": "commit1" } }, - "reason":"productionUsWest1 completed successfully in build 42", + "reason":"production-us-west-1 completed", "at": "(ignore)" }, "lastSuccess": { @@ -196,7 +198,7 @@ "gitCommit": "commit1" } }, - "reason":"productionUsWest1 completed successfully in build 42", + "reason":"production-us-west-1 completed", "at": "(ignore)" } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index 3dca8103ed7..fdd3dcc4d5c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -1,4 +1,6 @@ { + "application": "application1", + "instance": "default", "deploying": { "version": "(ignore)" }, @@ -63,7 +65,7 @@ "gitCommit": "commit1" } }, - "reason": "systemTest completed successfully in build 42", + "reason": "system-test completed", "at": "(ignore)" }, "lastCompleted": { @@ -77,7 +79,7 @@ "gitCommit": "commit1" } }, - "reason": "systemTest completed successfully in build 42", + "reason": "system-test completed", "at": "(ignore)" }, "lastSuccess": { @@ -91,7 +93,7 @@ "gitCommit": "commit1" } }, - "reason": "systemTest completed successfully in build 42", + "reason": "system-test completed", "at": "(ignore)" } }, @@ -109,7 +111,7 @@ "gitCommit": "commit1" } }, - "reason": "Retrying as build 42 just started failing", + "reason": "Immediate retry on failure", "at": "(ignore)" }, "lastCompleted": { @@ -123,7 +125,7 @@ "gitCommit": "commit1" } }, - "reason": "stagingTest completed successfully in build 42", + "reason": "staging-test completed", "at": "(ignore)" }, "firstFailing": { @@ -137,7 +139,7 @@ "gitCommit": "commit1" } }, - "reason": "stagingTest completed successfully in build 42", + "reason": "staging-test completed", "at": "(ignore)" } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json new file mode 100644 index 00000000000..41556c04209 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json @@ -0,0 +1,161 @@ +{ + "application": "application1", + "instance": "default", + "deploying": { + "version": "6.1" + }, + "deploymentJobs": [ + { + "type": "system-test", + "success": true, + "lastTriggered": { + "id": -1, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "", + "at": "(ignore)" + }, + "lastCompleted": { + "id": 42, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 42, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "", + "at": "(ignore)" + } + }, + { + "type": "staging-test", + "success": true, + "lastTriggered": { + "id": -1, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "system-test completed", + "at": "(ignore)" + }, + "lastCompleted": { + "id": 42, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "system-test completed", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 42, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "system-test completed", + "at": "(ignore)" + } + }, + { + "type": "production-corp-us-east-1", + "success": false, + "lastTriggered": { + "id": -1, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Immediate retry on failure", + "at": "(ignore)" + }, + "lastCompleted": { + "id": 42, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "staging-test completed", + "at": "(ignore)" + }, + "firstFailing": { + "id": 42, + "version": "6.1.0", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "staging-test completed", + "at": "(ignore)" + } + } + ], + "compileVersion": "6.1.0", + "globalRotations": [ + "http://fake-global-rotation-tenant1.application1" + ], + "instances": [ + @include(dev-us-west-1.json), + @include(prod-corp-us-east-1.json) + ], + "metrics": { + "queryServiceQuality": 0.5, + "writeServiceQuality": 0.7 + }, + "ownershipIssueId": "321", + "deploymentIssueId": "123" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json new file mode 100644 index 00000000000..062f4408518 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json @@ -0,0 +1,63 @@ +{ + "environment": "dev", + "region": "us-west-1", + "instance": "default", + "serviceUrls": [ + "http://old-endpoint.vespa.yahooapis.com:4080", + "http://qrs-endpoint.vespa.yahooapis.com:4080", + "http://feeding-endpoint.vespa.yahooapis.com:4080", + "http://global-endpoint.vespa.yahooapis.com:4080", + "http://alias-endpoint.vespa.yahooapis.com:4080" + ], + "nodes": "http://localhost:8080/zone/v2/dev/us-west-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", + "yamasUrl": "http://monitoring-system.test/?environment=dev®ion=us-west-1&application=tenant1.application1", + "version": "6.1.0", + "revision": "(ignore)", + "deployTimeEpochMs": "(ignore)", + "screwdriverId": "123", + + + "cost": { + "tco": 74, + "waste": 0, + "utilization": 2.999999999999999, + "cluster": { + "cluster1": { + "count": 2, + "resource": "cpu", + "utilization": 2.999999999999999, + "tco": 74, + "waste": 0, + "flavor": "flavor1", + "flavorCost": 37.0, + "flavorCpu": 2.0, + "flavorMem": 4.0, + "flavorDisk": 50.0, + "type": "content", + "util": { + "cpu": 2.999999999999999, + "mem": 0.4285714285714286, + "disk": 0.5714285714285715, + "diskBusy": 1.0 + }, + "usage": { + "cpu": 0.6, + "mem": 0.3, + "disk": 0.4, + "diskBusy": 0.3 + }, + "hostnames": [ + "host1", + "host2" + ] + } + } + }, + "metrics": { + "queriesPerSecond": 1.0, + "writesPerSecond": 2.0, + "documentCount": 3.0, + "queryLatencyMillis": 4.0, + "writeLatencyMillis": 5.0 + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json index 8acb4a045f3..a2e70d9c1eb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-id-without-applications.json @@ -1,4 +1,5 @@ { + "tenant": "tenant3", "type": "OPSDB", "property": "property1", "propertyId": "1234", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json index 3f4b6017971..f9161ea49b1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/opsdb-tenant-with-new-id-without-applications.json @@ -1,4 +1,5 @@ { + "tenant": "tenant3", "type": "OPSDB", "property": "property2", "propertyId": "4321", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-corp-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-corp-us-east-1.json new file mode 100644 index 00000000000..75b257da0ed --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-corp-us-east-1.json @@ -0,0 +1,68 @@ +{ + "environment": "prod", + "region": "corp-us-east-1", + "instance": "default", + "bcpStatus": { + "rotationStatus": "UNKNOWN" + }, + "serviceUrls": [ + "http://old-endpoint.vespa.yahooapis.com:4080", + "http://qrs-endpoint.vespa.yahooapis.com:4080", + "http://feeding-endpoint.vespa.yahooapis.com:4080", + "http://global-endpoint.vespa.yahooapis.com:4080", + "http://alias-endpoint.vespa.yahooapis.com:4080" + ], + "nodes": "http://localhost:8080/zone/v2/prod/corp-us-east-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", + "elkUrl": "http://log.prod.corp-us-east-1.test/#/discover?_g=()&_a=(columns:!(_source),index:'logstash-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:'HV-tenant:%22tenant1%22%20AND%20HV-application:%22application1%22%20AND%20HV-region:%22corp-us-east-1%22%20AND%20HV-instance:%22default%22%20AND%20HV-environment:%22prod%22')),sort:!('@timestamp',desc))", + "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=corp-us-east-1&application=tenant1.application1", + "version": "6.1.0", + "revision": "(ignore)", + "deployTimeEpochMs": "(ignore)", + "screwdriverId": "123", + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1", + "cost": { + "tco": 74, + "waste": 0, + "utilization": 2.999999999999999, + "cluster": { + "cluster1": { + "count": 2, + "resource": "cpu", + "utilization": 2.999999999999999, + "tco": 74, + "waste": 0, + "flavor": "flavor1", + "flavorCost": 37.0, + "flavorCpu": 2.0, + "flavorMem": 4.0, + "flavorDisk": 50.0, + "type": "content", + "util": { + "cpu": 2.999999999999999, + "mem": 0.4285714285714286, + "disk": 0.5714285714285715, + "diskBusy": 1.0 + }, + "usage": { + "cpu": 0.6, + "mem": 0.3, + "disk": 0.4, + "diskBusy": 0.3 + }, + "hostnames": [ + "host1", + "host2" + ] + } + } + }, + "metrics": { + "queriesPerSecond": 1.0, + "writesPerSecond": 2.0, + "documentCount": 3.0, + "queryLatencyMillis": 4.0, + "writeLatencyMillis": 5.0 + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json new file mode 100644 index 00000000000..a4395faede4 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-root.json @@ -0,0 +1,5 @@ +[ + @include(tenant2.json), + @include(tenant3.json), + @include(tenant1-recursive.json) +] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-until-tenant-root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-until-tenant-root.json new file mode 100644 index 00000000000..35ed8181fac --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/recursive-until-tenant-root.json @@ -0,0 +1,6 @@ +[ + @include(tenant2.json), + @include(tenant3.json), + @include(tenant-with-application.json) +] + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json index 87901218c2e..ad8e65692b4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json @@ -1,4 +1,5 @@ { + "tenant": "tenant1", "type": "ATHENS", "athensDomain": "domain1", "property": "property1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json index ede2413218d..69949c47d8c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications-with-id.json @@ -1,4 +1,5 @@ { + "tenant": "tenant2", "type": "ATHENS", "athensDomain": "domain2", "property": "property2", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json index 69669b5dfb8..3ad5a307348 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-without-applications.json @@ -1,4 +1,5 @@ { + "tenant": "tenant1", "type": "ATHENS", "athensDomain": "domain1", "property": "property1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json new file mode 100644 index 00000000000..309177e6285 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant1-recursive.json @@ -0,0 +1,9 @@ +{ + "tenant": "tenant1", + "type": "ATHENS", + "athensDomain": "domain1", + "property": "property1", + "applications": [ + @include(application1-recursive.json) + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json new file mode 100644 index 00000000000..6e66202b70d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant2.json @@ -0,0 +1,19 @@ +{ + "tenant": "tenant2", + "type": "ATHENS", + "athensDomain": "domain2", + "property": "property2", + "propertyId": "1234", + "applications": [], + "propertyUrl": "www.properties.tld/1234", + "contactsUrl": "www.contacts.tld/1234", + "issueCreationUrl": "www.issues.tld/1234", + "contacts": [ + [ + "alice" + ], + [ + "bob" + ] + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant3.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant3.json new file mode 100644 index 00000000000..fdf3ca490f4 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant3.json @@ -0,0 +1,12 @@ +{ + "tenant": "tenant3", + "type": "OPSDB", + "property": "property2", + "propertyId": "4321", + "userGroup": "group1", + "applications": [], + "propertyUrl": "www.properties.tld/4321", + "contactsUrl": "www.contacts.tld/4321", + "issueCreationUrl": "www.issues.tld/4321", + "contacts": [] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 3633860772b..354bab4379c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -1,19 +1,16 @@ { "jobs": [ { - "name": "DelayedDeployer" + "name": "ApplicationOwnershipConfirmer" }, { - "name": "BlockedChangeDeployer" - }, - { - "name": "Upgrader" + "name": "ClusterInfoMaintainer" }, { - "name": "FailureRedeployer" + "name": "ClusterUtilizationMaintainer" }, { - "name": "VersionStatusUpdater" + "name": "DeploymentExpirer" }, { "name": "DeploymentIssueReporter" @@ -22,22 +19,22 @@ "name": "DeploymentMetricsMaintainer" }, { - "name": "OutstandingChangeDeployer" + "name": "MetricsReporter" }, { - "name": "ClusterUtilizationMaintainer" + "name": "OutstandingChangeDeployer" }, { - "name": "ClusterInfoMaintainer" + "name": "ReadyJobsTrigger" }, { - "name": "DeploymentExpirer" + "name": "Upgrader" }, { - "name": "MetricsReporter" + "name": "VersionStatusUpdater" } ], "inactive": [ "DeploymentExpirer" ] -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index 7fd000b82c5..5f7fedfd75f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -1,27 +1,26 @@ { "versions":[ { - "version":"(ignore)", - "confidence":"high", - "commit":"(ignore)", - "date":0, - "controllerVersion":false, - "systemVersion":false, - "configServers":[ - - ], - "failingApplications":[ - - ], - "productionApplications":[ + "version": "(ignore)", + "confidence": "high", + "commit": "(ignore)", + "date": 0, + "controllerVersion": false, + "systemVersion": false, + "configServers": [ ], + "failingApplications": [ ], + "productionApplications": [ { - "tenant":"tenant1", - "application":"application1", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1", - "upgradePolicy":"default" + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy": "default", + "productionJobs": 1, + "productionSuccesses": 1 } - ] + ], + "deployingApplications": [ ] }, { "version":"(ignore)", @@ -40,40 +39,47 @@ ], "failingApplications":[ { - "tenant":"tenant1", - "application":"application1", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1", - "upgradePolicy":"default", - "failingSince":"(ignore)" + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy": "default", + "failing": "staging-test" } ], "productionApplications":[ { - "tenant":"tenant2", - "application":"application2", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant2/application/application2", - "upgradePolicy":"default" + "tenant": "tenant2", + "application": "application2", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant2/application/application2", + "upgradePolicy": "default", + "productionJobs": 1, + "productionSuccesses": 1 + } + ], + "deployingApplications": [ + { + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy": "default", + "running": "staging-test" } ] }, { - "version":"(ignore)", - "confidence":"normal", - "commit":"(ignore)", - "date":0, - "controllerVersion":true, - "systemVersion":false, - "configServers":[ - - ], - "failingApplications":[ - - ], - "productionApplications":[ - - ] + "version": "(ignore)", + "confidence": "normal", + "commit": "(ignore)", + "date": 0, + "controllerVersion": true, + "systemVersion": false, + "configServers": [ ], + "failingApplications": [ ], + "productionApplications": [ ], + "deployingApplications": [ ] } ] }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java index 1638a2845ed..e6b3eacd44e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiTest.java @@ -96,14 +96,14 @@ public class ScrewdriverApiTest extends ControllerContainerTest { Response response; response = container.handleRequest(new Request("http://localhost:8080/screwdriver/v1/jobsToRun", "", Request.Method.GET)); - assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.id())); - assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.id())); + assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.jobName())); + assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.jobName())); assertEquals("Response contains only two items", 2, SlimeUtils.jsonToSlime(response.getBody()).get().entries()); // Check that GET didn't affect the enqueued jobs. response = container.handleRequest(new Request("http://localhost:8080/screwdriver/v1/jobsToRun", "", Request.Method.DELETE)); - assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.id())); - assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.id())); + assertTrue("Response contains system-test", response.getBodyAsString().contains(JobType.systemTest.jobName())); + assertTrue("Response contains staging-test", response.getBodyAsString().contains(JobType.stagingTest.jobName())); assertEquals("Response contains only two items", 2, SlimeUtils.jsonToSlime(response.getBody()).get().entries()); Thread.sleep(50); @@ -148,11 +148,8 @@ public class ScrewdriverApiTest extends ControllerContainerTest { tester.containerTester().updateSystemVersion(); Application app = tester.createApplication(); - try (Lock lock = tester.controller().applications().lock(app.id())) { - tester.controller().applications().store( - tester.controller().applications().require(app.id(), lock).withProjectId(1) - ); - } + tester.controller().applications().lockedOrThrow(app.id(), application -> + tester.controller().applications().store(application.withProjectId(1))); // Unknown application assertResponse(new Request("http://localhost:8080/screwdriver/v1/trigger/tenant/foo/application/bar", @@ -163,7 +160,7 @@ public class ScrewdriverApiTest extends ControllerContainerTest { assertResponse(new Request("http://localhost:8080/screwdriver/v1/trigger/tenant/" + app.id().tenant().value() + "/application/" + app.id().application().value(), "invalid".getBytes(StandardCharsets.UTF_8), Request.Method.POST), - 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Unknown job id 'invalid'\"}"); + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Unknown job name 'invalid'\"}"); // component is triggered if no job is specified in request body assertResponse(new Request("http://localhost:8080/screwdriver/v1/trigger/tenant/" + @@ -172,7 +169,7 @@ public class ScrewdriverApiTest extends ControllerContainerTest { 200, "{\"message\":\"Triggered component for tenant1.application1\"}"); assertFalse(buildSystem.jobs().isEmpty()); - assertEquals(JobType.component.id(), buildSystem.jobs().get(0).jobName()); + assertEquals(JobType.component.jobName(), buildSystem.jobs().get(0).jobName()); assertEquals(1L, buildSystem.jobs().get(0).projectId()); buildSystem.takeJobsToRun(); @@ -182,7 +179,7 @@ public class ScrewdriverApiTest extends ControllerContainerTest { "staging-test".getBytes(StandardCharsets.UTF_8), Request.Method.POST), 200, "{\"message\":\"Triggered staging-test for tenant1.application1\"}"); assertFalse(buildSystem.jobs().isEmpty()); - assertEquals(JobType.stagingTest.id(), buildSystem.jobs().get(0).jobName()); + assertEquals(JobType.stagingTest.jobName(), buildSystem.jobs().get(0).jobName()); assertEquals(1L, buildSystem.jobs().get(0).projectId()); } @@ -197,14 +194,14 @@ public class ScrewdriverApiTest extends ControllerContainerTest { Optional<JobError> jobError) { return "{\n" + - " \"projectId\" : " + projectId + ",\n" + - " \"jobName\" :\"" + jobType.id() + "\",\n" + - " \"buildNumber\" : " + buildNumber + ",\n" + - jobError.map(message -> " \"jobError\" : \"" + message + "\",\n").orElse("") + - " \"tenant\" :\"" + applicationId.tenant().value() + "\",\n" + - " \"application\" :\"" + applicationId.application().value() + "\",\n" + - " \"instance\" :\"" + applicationId.instance().value() + "\"\n" + - "}"; + " \"projectId\" : " + projectId + ",\n" + + " \"jobName\" :\"" + jobType.jobName() + "\",\n" + + " \"buildNumber\" : " + buildNumber + ",\n" + + jobError.map(message -> " \"jobError\" : \"" + message + "\",\n").orElse("") + + " \"tenant\" :\"" + applicationId.tenant().value() + "\",\n" + + " \"application\" :\"" + applicationId.application().value() + "\",\n" + + " \"instance\" :\"" + applicationId.instance().value() + "\"\n" + + "}"; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json index e293d85b594..8ffd9511a96 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/responses/unexpected-completion.json @@ -1,4 +1,4 @@ { "error-code": "BAD_REQUEST", - "message": "Got notified about completion of job status of productionUsEast3[ last triggered: (never), last completed: (never), first failing: (not failing), lastSuccess: (never)], but that has not been triggered nor deployed" + "message": "Got notified about completion of job status of productionUsEast3[ last triggered: (never), last completed: (never), first failing: (not failing), lastSuccess: (never)], but that has neither been triggered nor deployed" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java new file mode 100644 index 00000000000..a00665b77cb --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java @@ -0,0 +1,65 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.zone.v1; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.controller.ZoneRegistryMock; +import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.Arrays; +import java.util.List; + +/** + * @author mpolden + */ +public class ZoneApiTest extends ControllerContainerTest { + + private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/"; + private static final List<Zone> zones = Arrays.asList( + new Zone(Environment.prod, RegionName.from("us-north-1")), + new Zone(Environment.dev, RegionName.from("us-north-2")), + new Zone(Environment.test, RegionName.from("us-north-3")), + new Zone(Environment.staging, RegionName.from("us-north-4")) + ); + + private ContainerControllerTester tester; + + @Before + public void before() { + ZoneRegistryMock zoneRegistry = (ZoneRegistryMock) container.components() + .getComponent(ZoneRegistryMock.class.getName()); + zoneRegistry.setDefaultRegionForEnvironment(Environment.dev, RegionName.from("us-north-2")) + .setZones(zones); + this.tester = new ContainerControllerTester(container, responseFiles); + } + + @Test + public void test_requests() throws Exception { + // GET /zone/v1 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v1"), + new File("root.json")); + + // GET /zone/v1/environment/prod + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v1/environment/prod"), + new File("prod.json")); + + // GET /zone/v1/environment/dev/default + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v1/environment/dev/default"), + new File("default-for-region.json")); + } + + @Test + public void test_invalid_requests() throws Exception { + // GET /zone/v1/environment/prod/default: No default region + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v1/environment/prod/default"), + new File("no-default-region.json"), + 400); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/default-for-region.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/default-for-region.json new file mode 100644 index 00000000000..7c4a7e2b4a5 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/default-for-region.json @@ -0,0 +1,4 @@ +{ + "name": "us-north-2", + "url": "http://localhost:8080/zone/v2/environment/dev/region/us-north-2" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/no-default-region.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/no-default-region.json new file mode 100644 index 00000000000..bdc6601a2e9 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/no-default-region.json @@ -0,0 +1,4 @@ +{ + "error-code": "BAD_REQUEST", + "message": "No default region for environment: prod" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/prod.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/prod.json new file mode 100644 index 00000000000..cebf48e6428 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/prod.json @@ -0,0 +1,6 @@ +[ + { + "name": "us-north-1", + "url": "http://localhost:8080/zone/v2/environment/prod/region/us-north-1" + } +] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/root.json new file mode 100644 index 00000000000..b3bd5247414 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/root.json @@ -0,0 +1,18 @@ +[ + { + "name": "dev", + "url": "http://localhost:8080/zone/v2/environment/dev" + }, + { + "name": "prod", + "url": "http://localhost:8080/zone/v2/environment/prod" + }, + { + "name": "staging", + "url": "http://localhost:8080/zone/v2/environment/staging" + }, + { + "name": "test", + "url": "http://localhost:8080/zone/v2/environment/test" + } +] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java new file mode 100644 index 00000000000..63899d808f9 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java @@ -0,0 +1,117 @@ +package com.yahoo.vespa.hosted.controller.restapi.zone.v2; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Request.Method; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.hosted.controller.ConfigServerProxyMock; +import com.yahoo.vespa.hosted.controller.ZoneRegistryMock; +import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author mpolden + */ +public class ZoneApiTest extends ControllerContainerTest { + + private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/"; + private static final List<Zone> zones = Arrays.asList( + new Zone(Environment.prod, RegionName.from("us-north-1")), + new Zone(Environment.dev, RegionName.from("us-north-2")), + new Zone(Environment.test, RegionName.from("us-north-3")), + new Zone(Environment.staging, RegionName.from("us-north-4")) + ); + + private ContainerControllerTester tester; + private ConfigServerProxyMock proxy; + + @Before + public void before() { + ZoneRegistryMock zoneRegistry = (ZoneRegistryMock) container.components() + .getComponent(ZoneRegistryMock.class.getName()); + zoneRegistry.setDefaultRegionForEnvironment(Environment.dev, RegionName.from("us-north-2")) + .setZones(zones); + this.tester = new ContainerControllerTester(container, responseFiles); + this.proxy = (ConfigServerProxyMock) container.components().getComponent(ConfigServerProxyMock.class.getName()); + } + + @Test + public void test_requests() throws Exception { + // GET /zone/v2 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2"), + new File("root.json")); + + // GET /zone/v2/prod/us-north-1 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2/prod/us-north-1"), + "ok"); + assertEquals("prod", proxy.lastReceived().get().getEnvironment()); + assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); + assertEquals("", proxy.lastReceived().get().getConfigServerRequest()); + assertEquals("GET", proxy.lastReceived().get().getMethod()); + + // GET /zone/v2/nodes/v2/node/?recursive=true + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/?recursive=true"), + "ok"); + + assertEquals("prod", proxy.lastReceived().get().getEnvironment()); + assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); + assertEquals("/nodes/v2/node/?recursive=true", proxy.lastReceived().get().getConfigServerRequest()); + assertEquals("GET", proxy.lastReceived().get().getMethod()); + + // POST /zone/v2/dev/us-north-2/nodes/v2/command/restart?hostname=node1 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2/dev/us-north-2/nodes/v2/command/restart?hostname=node1", + new byte[0], Method.POST), + "ok"); + assertEquals("dev", proxy.lastReceived().get().getEnvironment()); + assertEquals("us-north-2", proxy.lastReceived().get().getRegion()); + assertEquals("/nodes/v2/command/restart?hostname=node1", proxy.lastReceived().get().getConfigServerRequest()); + assertEquals("POST", proxy.lastReceived().get().getMethod()); + + // PUT /zone/v2/prod/us-north-1/nodes/v2/state/dirty/node1 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/state/dirty/node1", + new byte[0], Method.PUT), "ok"); + assertEquals("prod", proxy.lastReceived().get().getEnvironment()); + assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); + assertEquals("/nodes/v2/state/dirty/node1", proxy.lastReceived().get().getConfigServerRequest()); + assertEquals("PUT", proxy.lastReceived().get().getMethod()); + + // DELETE /zone/v2/prod/us-north-1/nodes/v2/node/node1 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/node1", + new byte[0], Method.DELETE), "ok"); + assertEquals("prod", proxy.lastReceived().get().getEnvironment()); + assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); + assertEquals("/nodes/v2/node/node1", proxy.lastReceived().get().getConfigServerRequest()); + assertEquals("DELETE", proxy.lastReceived().get().getMethod()); + + // PATCH /zone/v2/prod/us-north-1/nodes/v2/node/node1 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/node1", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), + Method.PATCH), "ok"); + assertEquals("prod", proxy.lastReceived().get().getEnvironment()); + assertEquals("us-north-1", proxy.lastReceived().get().getRegion()); + assertEquals("/nodes/v2/node/node1", proxy.lastReceived().get().getConfigServerRequest()); + assertEquals("PATCH", proxy.lastReceived().get().getMethod()); + assertEquals("{\"currentRestartGeneration\": 1}", proxy.lastRequestBody().get()); + } + + @Test + public void test_invalid_requests() throws Exception { + // GET /zone/v2/prod/us-north-34/nodes/v2 + tester.containerTester().assertResponse(new Request("http://localhost:8080/zone/v2/prod/us-north-42/nodes/v2", + new byte[0], Method.POST), + new File("unknown-zone.json"), 400); + assertFalse(proxy.lastReceived().isPresent()); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/root.json new file mode 100644 index 00000000000..ab168854267 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/root.json @@ -0,0 +1,26 @@ +{ + "uris": [ + "http://localhost:8080/zone/v2/prod/us-north-1", + "http://localhost:8080/zone/v2/dev/us-north-2", + "http://localhost:8080/zone/v2/test/us-north-3", + "http://localhost:8080/zone/v2/staging/us-north-4" + ], + "zones": [ + { + "environment": "prod", + "region": "us-north-1" + }, + { + "environment": "dev", + "region": "us-north-2" + }, + { + "environment": "test", + "region": "us-north-3" + }, + { + "environment": "staging", + "region": "us-north-4" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/unknown-zone.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/unknown-zone.json new file mode 100644 index 00000000000..c7d6e4b8400 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/unknown-zone.json @@ -0,0 +1,4 @@ +{ + "error-code": "BAD_REQUEST", + "message": "No such zone: prod.us-north-42" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 519c457e73b..4f97c078c9b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; +import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.provision.Environment; @@ -19,6 +20,8 @@ import org.junit.Test; import java.net.URI; import java.net.URISyntaxException; +import java.time.Duration; +import java.util.Collections; import java.util.List; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; @@ -28,6 +31,7 @@ import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobTy import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -95,23 +99,20 @@ public class VersionStatusTest { List<VespaVersion> versions = tester.controller().versionStatus().versions(); assertEquals("The two versions above exist", 2, versions.size()); + System.err.println(tester.controller().applications().deploymentTrigger().jobTimeoutLimit()); + VespaVersion v1 = versions.get(0); assertEquals(version1, v1.versionNumber()); - assertEquals(0, v1.statistics().failing().size()); - // All applications are on v1 in at least one zone - assertEquals(3, v1.statistics().production().size()); - assertTrue(v1.statistics().production().contains(app2.id())); - assertTrue(v1.statistics().production().contains(app1.id())); + assertEquals("No applications are failing on version1.", ImmutableSet.of(), v1.statistics().failing()); + assertEquals("All applications have at least one active production deployment on version 1.", ImmutableSet.of(app1.id(), app2.id(), app3.id()), v1.statistics().production()); + assertEquals("No applications have active deployment jobs on version1.", ImmutableSet.of(), v1.statistics().deploying()); VespaVersion v2 = versions.get(1); assertEquals(version2, v2.versionNumber()); - // All applications have failed on v2 in at least one zone - assertEquals(3, v2.statistics().failing().size()); - assertTrue(v2.statistics().failing().contains(app1.id())); - assertTrue(v2.statistics().failing().contains(app3.id())); - // Only one application is on v2 in at least one zone - assertEquals(1, v2.statistics().production().size()); - assertTrue(v2.statistics().production().contains(app2.id())); + assertEquals("All applications have failed on version2 in at least one zone.", ImmutableSet.of(app1.id(), app2.id(), app3.id()), v2.statistics().failing()); + assertEquals("Only app2 has successfully deployed to production on version2.", ImmutableSet.of(app2.id()), v2.statistics().production()); + // Should test the below, but can't easily be done with current test framework. This test passes in DeploymentApiTest. + // assertEquals("All applications are being retried on version2.", ImmutableSet.of(app1.id(), app2.id(), app3.id()), v2.statistics().deploying()); } @Test @@ -161,6 +162,12 @@ public class VersionStatusTest { assertEquals("One canary failed: Broken", Confidence.broken, confidence(tester.controller(), version1)); + // Finish running jobs + tester.deployAndNotify(canary2, DeploymentTester.applicationPackage("canary"), false, systemTest); + tester.clock().advance(Duration.ofHours(1)); + tester.deployAndNotify(canary1, DeploymentTester.applicationPackage("canary"), false, productionUsWest1); + tester.deployAndNotify(canary2, DeploymentTester.applicationPackage("canary"), false, systemTest); + // New version is released Version version2 = new Version("5.2"); tester.upgradeSystem(version2); @@ -204,6 +211,7 @@ public class VersionStatusTest { // Another default application upgrades, raising confidence to high tester.completeUpgrade(default8, version2, "default"); + tester.completeUpgrade(default9, version2, "default"); tester.updateVersionStatus(); assertEquals("Confidence remains unchanged for version0: High", @@ -241,7 +249,7 @@ public class VersionStatusTest { } @Test - public void testIgnoreConfigdeince() { + public void testIgnoreConfidence() { DeploymentTester tester = new DeploymentTester(); Version version0 = new Version("5.0"); @@ -270,7 +278,6 @@ public class VersionStatusTest { tester.completeUpgradeWithError(default3, version1, "default", stagingTest); tester.completeUpgradeWithError(default4, version1, "default", stagingTest); tester.updateVersionStatus(); - assertEquals("Canaries have upgraded, 1 of 4 default apps failing: Broken", Confidence.broken, confidence(tester.controller(), version1)); @@ -295,8 +302,9 @@ public class VersionStatusTest { Version versionWithUnknownTag = new Version("6.1.2"); Application app = tester.createAndDeploy("tenant1", "domain1","application1", Environment.test, 11); - applications.notifyJobCompletion(mockReport(app, component, true)); - applications.notifyJobCompletion(mockReport(app, systemTest, true)); + tester.clock().advance(Duration.ofMillis(1)); + applications.notifyJobCompletion(DeploymentTester.jobReport(app, component, true)); + applications.notifyJobCompletion(DeploymentTester.jobReport(app, systemTest, true)); List<VespaVersion> vespaVersions = VersionStatus.compute(tester.controller()).versions(); @@ -313,14 +321,4 @@ public class VersionStatusTest { .orElseThrow(() -> new IllegalArgumentException("Expected to find version: " + version)); } - private DeploymentJobs.JobReport mockReport(Application application, DeploymentJobs.JobType jobType, boolean success) { - return new DeploymentJobs.JobReport( - application.id(), - jobType, - application.deploymentJobs().projectId().get(), - 42, - JobError.from(success) - ); - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java index 561799529f9..b4074fc1944 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepositoryTest.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.rotation; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; @@ -22,6 +22,10 @@ import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; /** * @author Oyvind Gronnesby @@ -100,12 +104,13 @@ public class ControllerRotationRepositoryTest { private ControllerRotationRepository repository; private ControllerRotationRepository repositoryWhitespaces; - + private Metric metric; @Before public void setup_repository() { - repository = new ControllerRotationRepository(rotationsConfig, controllerDb, MetricReceiver.nullImplementation); - repositoryWhitespaces = new ControllerRotationRepository(rotationsConfigWhitespaces, controllerDb, MetricReceiver.nullImplementation); + metric = mock(Metric.class); + repository = new ControllerRotationRepository(rotationsConfig, controllerDb, metric); + repositoryWhitespaces = new ControllerRotationRepository(rotationsConfigWhitespaces, controllerDb, metric); controllerDb.assignRotation(new RotationId("foo-1"), applicationId); } @@ -129,6 +134,7 @@ public class ControllerRotationRepositoryTest { Set<Rotation> rotations = repository.getOrAssignRotation(other, deploymentSpec); Rotation assignedRotation = new Rotation(new RotationId("foo-2"), "foo-2.com"); assertContainsOnly(assignedRotation, rotations); + verify(metric).set(eq(ControllerRotationRepository.REMAINING_ROTATIONS_METRIC_NAME), eq(1), any()); } @Test @@ -140,6 +146,7 @@ public class ControllerRotationRepositoryTest { thrown.expectMessage("no rotations available"); repository.getOrAssignRotation(third, deploymentSpec); + verify(metric).set(eq(ControllerRotationRepository.REMAINING_ROTATIONS_METRIC_NAME), eq(0), any()); } @Test |