diff options
Diffstat (limited to 'controller-server/src/test/java/com')
132 files changed, 3158 insertions, 3146 deletions
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 02c86f78ec0..0ff14e01874 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.google.common.collect.Sets; @@ -6,30 +6,37 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.path.Path; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; import org.junit.Test; @@ -38,7 +45,6 @@ import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -176,51 +182,34 @@ public class ControllerTest { @Test public void testGlobalRotations() { - // Setup - ControllerTester tester = this.tester.controllerTester(); - ZoneId zone = ZoneId.from(Environment.defaultEnvironment(), RegionName.defaultName()); - ApplicationId app = ApplicationId.from("tenant", "app1", "default"); - DeploymentId deployment = new DeploymentId(app, zone); - tester.serviceRegistry().routingGeneratorMock().putEndpoints(deployment, List.of( - new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream2"), - new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"), - new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream3"), - new RoutingEndpoint("http://global-endpoint-2.vespa.yahooapis.com:4080", "host2", true, "upstream4"), - new RoutingEndpoint("http://global-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1"), - new RoutingEndpoint("http://alias-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1") - )); - - Supplier<Map<RoutingEndpoint, EndpointStatus>> globalRotationStatus = () -> tester.controller().applications().globalRotationStatus(deployment); - Supplier<List<EndpointStatus>> upstreamOneEndpoints = () -> { - return globalRotationStatus.get() - .entrySet().stream() - .filter(kv -> kv.getKey().upstreamName().equals("upstream1")) - .map(Map.Entry::getValue) - .collect(Collectors.toList()); - }; + var context = tester.newDeploymentContext(); + var zone1 = ZoneId.from("prod", "us-west-1"); + var zone2 = ZoneId.from("prod", "us-east-3"); + var applicationPackage = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("default", "default", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage).deploy(); // Check initial rotation status - assertEquals(3, globalRotationStatus.get().size()); - assertEquals(2, upstreamOneEndpoints.get().size()); - assertTrue("All upstreams are in", upstreamOneEndpoints.get().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in)); - - // Set the global rotations out of service - EndpointStatus status = new EndpointStatus(EndpointStatus.Status.out, "unit-test", "Test", tester.clock().instant().getEpochSecond()); - tester.controller().applications().setGlobalRotationStatus(deployment, status); - assertEquals(2, upstreamOneEndpoints.get().size()); - assertTrue("All upstreams are out", upstreamOneEndpoints.get().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.out)); - assertTrue("Reason is set", upstreamOneEndpoints.get().stream().allMatch(es -> es.getReason().equals("unit-test"))); - - // Deployment without a global endpoint - tester.serviceRegistry().routingGeneratorMock().putEndpoints(deployment, List.of( - new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream2"), - new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"), - new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream3") - )); - try { - tester.controller().applications().setGlobalRotationStatus(deployment, status); - fail("Expected exception"); - } catch (IllegalArgumentException ignored) {} + var deployment1 = context.deploymentIdIn(zone1); + var status1 = tester.controller().routing().globalRotationStatus(deployment1); + assertEquals(1, status1.size()); + assertTrue("All upstreams are in", status1.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in)); + + // Set the deployment out of service in the global rotation + var newStatus = new EndpointStatus(EndpointStatus.Status.out, "unit-test", ControllerTest.class.getSimpleName(), tester.clock().instant().getEpochSecond()); + tester.controller().routing().setGlobalRotationStatus(deployment1, newStatus); + status1 = tester.controller().routing().globalRotationStatus(deployment1); + assertEquals(1, status1.size()); + assertTrue("All upstreams are out", status1.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.out)); + assertTrue("Reason is set", status1.values().stream().allMatch(es -> es.getReason().equals("unit-test"))); + + // Other deployment remains in + var status2 = tester.controller().routing().globalRotationStatus(context.deploymentIdIn(zone2)); + assertEquals(1, status2.size()); + assertTrue("All upstreams are in", status2.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in)); } @Test @@ -297,10 +286,10 @@ public class ControllerTest { var context = tester.newDeploymentContext("tenant1", "app1", "default"); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) - .endpoint("foobar", "qrs", "us-west-1", "us-central-1") - .endpoint("default", "qrs", "us-west-1", "us-central-1") - .endpoint("all", "qrs") - .endpoint("west", "qrs", "us-west-1") + .endpoint("foobar", "qrs", "us-west-1", "us-central-1") // Rotation 01 + .endpoint("default", "qrs", "us-west-1", "us-central-1") // Rotation 02 + .endpoint("all", "qrs") // Rotation 03 + .endpoint("west", "qrs", "us-west-1") // Rotation 04 .region("us-west-1") .region("us-central-1") .build(); @@ -312,9 +301,9 @@ public class ControllerTest { var notWest = Set.of( "rotation-id-01", "foobar--app1--tenant1.global.vespa.oath.cloud", "rotation-id-02", "app1--tenant1.global.vespa.oath.cloud", - "rotation-id-04", "all--app1--tenant1.global.vespa.oath.cloud" + "rotation-id-03", "all--app1--tenant1.global.vespa.oath.cloud" ); - var west = Sets.union(notWest, Set.of("rotation-id-03", "west--app1--tenant1.global.vespa.oath.cloud")); + var west = Sets.union(notWest, Set.of("rotation-id-04", "west--app1--tenant1.global.vespa.oath.cloud")); for (Deployment deployment : deployments) { assertEquals("Rotation names are passed to config server in " + deployment.zone(), @@ -328,7 +317,7 @@ public class ControllerTest { var record1 = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); assertTrue(record1.isPresent()); assertEquals("app1--tenant1.global.vespa.oath.cloud", record1.get().name().asString()); - assertEquals("rotation-fqdn-04.", record1.get().data().asString()); + assertEquals("rotation-fqdn-02.", record1.get().data().asString()); var record2 = tester.controllerTester().findCname("foobar--app1--tenant1.global.vespa.oath.cloud"); assertTrue(record2.isPresent()); @@ -338,12 +327,12 @@ public class ControllerTest { var record3 = tester.controllerTester().findCname("all--app1--tenant1.global.vespa.oath.cloud"); assertTrue(record3.isPresent()); assertEquals("all--app1--tenant1.global.vespa.oath.cloud", record3.get().name().asString()); - assertEquals("rotation-fqdn-02.", record3.get().data().asString()); + assertEquals("rotation-fqdn-03.", record3.get().data().asString()); var record4 = tester.controllerTester().findCname("west--app1--tenant1.global.vespa.oath.cloud"); assertTrue(record4.isPresent()); assertEquals("west--app1--tenant1.global.vespa.oath.cloud", record4.get().name().asString()); - assertEquals("rotation-fqdn-03.", record4.get().data().asString()); + assertEquals("rotation-fqdn-04.", record4.get().data().asString()); } @Test @@ -475,23 +464,20 @@ public class ControllerTest { .environment(Environment.prod) .endpoint("default", "qrs", "us-west-1", "us-central-1") .region("us-west-1") - .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .region("us-central-1") .build(); context.submit(applicationPackage).deploy(); ApplicationPackage applicationPackage2 = new ApplicationPackageBuilder() .environment(Environment.prod) .region("us-west-1") - .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .region("us-central-1") .allow(ValidationId.globalEndpointChange) .build(); context.submit(applicationPackage2).deploy(); - assertEquals( - List.of(AssignedRotation.fromStrings("qrs", "default", "rotation-id-01", Set.of())), - context.instance().rotations() - ); + assertEquals(List.of(), context.instance().rotations()); assertEquals( Set.of(), @@ -528,9 +514,9 @@ public class ControllerTest { context.submit(applicationPackage); tester.applications().deleteApplication(context.application().id(), tester.controllerTester().credentialsFor(context.application().id().tenant())); - try (RotationLock lock = tester.applications().rotationRepository().lock()) { + try (RotationLock lock = tester.controller().routing().rotations().lock()) { assertTrue("Rotation is unassigned", - tester.applications().rotationRepository().availableRotations(lock) + tester.controller().routing().rotations().availableRotations(lock) .containsKey(new RotationId("rotation-id-01"))); } context.flushDnsUpdates(); @@ -655,8 +641,6 @@ public class ControllerTest { .region("us-west-1") .region("us-east-3") .build(); - SourceRevision source = new SourceRevision("repo", "master", "commit1"); - context.submit(applicationPackage).deploy(); DeploymentId deployment1 = context.deploymentIdIn(ZoneId.from(Environment.prod, RegionName.from("us-west-1"))); @@ -715,8 +699,12 @@ public class ControllerTest { // Create app1 var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); - var applicationPackage = new ApplicationPackageBuilder().environment(Environment.prod) - .region("us-west-1") + var prodZone = ZoneId.from("prod", "us-west-1"); + tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(prodZone)); + var applicationPackage = new ApplicationPackageBuilder().athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) + .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION) + .environment(prodZone.environment()) + .region(prodZone.region()) .build(); // Deploy app1 in production context1.submit(applicationPackage).deploy(); @@ -725,13 +713,13 @@ public class ControllerTest { assertEquals(Stream.concat(Stream.of("vznqtz7a5ygwjkbhhj7ymxvlrekgt4l6g.vespa.oath.cloud", "app1.tenant1.global.vespa.oath.cloud", "*.app1.tenant1.global.vespa.oath.cloud"), - tester.controller().zoneRegistry().zones().all().ids().stream() + tester.controller().zoneRegistry().zones().controllerUpgraded().ids().stream() .flatMap(zone -> Stream.of("", "*.") .map(prefix -> prefix + "app1.tenant1." + zone.region().value() + (zone.environment() == Environment.prod ? "" : "." + zone.environment().value()) + ".vespa.oath.cloud"))) .collect(Collectors.toUnmodifiableList()), - tester.controllerTester().serviceRegistry().applicationCertificateMock().dnsNamesOf(context1.instanceId())); + tester.controllerTester().serviceRegistry().endpointCertificateMock().dnsNamesOf(context1.instanceId())); // Next deployment reuses certificate context1.submit(applicationPackage).deploy(); @@ -739,13 +727,12 @@ public class ControllerTest { // Create app2 var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); - ZoneId zone = ZoneId.from("dev", "us-east-1"); + var devZone = ZoneId.from("dev", "us-east-1"); - // Deploy app2, after "removing" direct routing everywhere - tester.controllerTester().zoneRegistry().setDirectlyRouted(); - tester.controller().applications().deploy(context2.instanceId(), zone, Optional.of(applicationPackage), DeployOptions.none()); + // Deploy app2 in a zone with shared routing + tester.controller().applications().deploy(context2.instanceId(), devZone, Optional.of(applicationPackage), DeployOptions.none()); assertTrue("Application deployed and activated", - tester.configServer().application(context2.instanceId(), zone).get().activated()); + tester.configServer().application(context2.instanceId(), devZone).get().activated()); assertFalse("Does not provision certificate in zones with routing layer", certificate.apply(context2.instance()).isPresent()); } @@ -783,4 +770,195 @@ public class ControllerTest { } } + @Test + public void testDeployWithoutSourceRevision() { + var context = tester.newDeploymentContext(); + var applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("default") + .environment(Environment.prod) + .region("us-west-1") + .build(); + + // Submit without source revision + context.submit(applicationPackage, Optional.empty()) + .deploy(); + assertEquals("Deployed application", 1, context.instance().deployments().size()); + } + + @Test + public void testDeployWithGlobalEndpointsAndMultipleRoutingMethods() { + var context = tester.newDeploymentContext(); + var zone1 = ZoneId.from("prod", "us-west-1"); + var zone2 = ZoneId.from("prod", "us-east-3"); + var applicationPackage = new ApplicationPackageBuilder() + .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) + .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION) + .endpoint("default", "default", zone1.region().value(), zone2.region().value()) + .endpoint("east", "default", zone2.region().value()) + .region(zone1.region()) + .region(zone2.region()) + .build(); + + // Zone 1 supports shared and sharedLayer4 + tester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone1), RoutingMethod.shared, + RoutingMethod.sharedLayer4); + // Zone 2 supports shared and exclusive + tester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone2), RoutingMethod.shared, + RoutingMethod.exclusive); + + context.submit(applicationPackage).deploy(); + var expectedRecords = List.of( + // The 'east' global endpoint, pointing to zone 2 with exclusive routing + new Record(Record.Type.ALIAS, + RecordName.from("east.application.tenant.global.vespa.oath.cloud"), + RecordData.from("lb-0--tenant:application:default--prod.us-east-3/dns-zone-1/prod.us-east-3")), + + // The 'default' global endpoint, pointing to both zones with shared routing, via rotation + new Record(Record.Type.CNAME, + RecordName.from("application--tenant.global.vespa.oath.cloud"), + RecordData.from("rotation-fqdn-01.")), + + // The zone-scoped endpoint pointing to zone 2 with exclusive routing + new Record(Record.Type.CNAME, + RecordName.from("application.tenant.us-east-3.vespa.oath.cloud"), + RecordData.from("lb-0--tenant:application:default--prod.us-east-3.")), + + // The 'east' global endpoint, pointing to zone 2 with shared routing, via rotation + new Record(Record.Type.CNAME, + RecordName.from("east--application--tenant.global.vespa.oath.cloud"), + RecordData.from("rotation-fqdn-02."))); + assertEquals(expectedRecords, List.copyOf(tester.controllerTester().nameService().records())); + } + + @Test + public void testDirectRoutingSupport() { + var context = tester.newDeploymentContext(); + var zone1 = ZoneId.from("prod", "us-west-1"); + var zone2 = ZoneId.from("prod", "us-east-3"); + var applicationPackageBuilder = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()); + tester.controllerTester().zoneRegistry() + .setRoutingMethod(ZoneApiMock.from(zone1), RoutingMethod.shared, RoutingMethod.sharedLayer4) + .setRoutingMethod(ZoneApiMock.from(zone2), RoutingMethod.shared, RoutingMethod.sharedLayer4); + Supplier<Set<RoutingMethod>> routingMethods = () -> tester.controller().routing().endpointsOf(context.deploymentIdIn(zone1)) + .asList() + .stream() + .map(Endpoint::routingMethod) + .collect(Collectors.toSet()); + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.ALLOW_DIRECT_ROUTING.id(), false); + + // Without everything + context.submit(applicationPackageBuilder.build()).deploy(); + assertEquals(Set.of(RoutingMethod.shared), routingMethods.get()); + + // Without Athenz service + context.submit(applicationPackageBuilder.compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION).build()) + .deploy(); + assertEquals(Set.of(RoutingMethod.shared), routingMethods.get()); + + // Without feature flag + applicationPackageBuilder = applicationPackageBuilder.compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION) + .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")); + context.submit(applicationPackageBuilder.build()).deploy(); + assertEquals(Set.of(RoutingMethod.shared), routingMethods.get()); + + // With everything required + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.ALLOW_DIRECT_ROUTING.id(), true); + context.submit(applicationPackageBuilder.build()).deploy(); + assertEquals(Set.of(RoutingMethod.shared, RoutingMethod.sharedLayer4), routingMethods.get()); + + // Global endpoint is configured and includes directly routed endpoint name + applicationPackageBuilder = applicationPackageBuilder.endpoint("default", "default"); + context.submit(applicationPackageBuilder.build()).deploy(); + for (var zone : List.of(zone1, zone2)) { + assertEquals(Set.of("rotation-id-01", + "application.tenant.global.vespa.oath.cloud", + "application--tenant.global.vespa.oath.cloud"), + tester.configServer().rotationNames().get(context.deploymentIdIn(zone))); + } + } + + @Test + public void testChangeEndpointCluster() { + var context = tester.newDeploymentContext(); + var west = ZoneId.from("prod", "us-west-1"); + var east = ZoneId.from("prod", "us-east-3"); + + // Deploy application + var applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .endpoint("default", "foo") + .region(west.region().value()) + .region(east.region().value()) + .build(); + context.submit(applicationPackage).deploy(); + assertEquals(ClusterSpec.Id.from("foo"), tester.applications().requireInstance(context.instanceId()) + .rotations().get(0).clusterId()); + + // Redeploy with endpoint cluster changed needs override + applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .endpoint("default", "bar") + .region(west.region().value()) + .region(east.region().value()) + .build(); + try { + context.submit(applicationPackage).deploy(); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("global-endpoint-change: application 'tenant.application' has endpoints [endpoint " + + "'default' (cluster foo) -> us-east-3, us-west-1], but does not include all of these in " + + "deployment.xml. Deploying given deployment.xml will remove " + + "[endpoint 'default' (cluster foo) -> us-east-3, us-west-1] and add " + + "[endpoint 'default' (cluster bar) -> us-east-3, us-west-1]. To allow this add " + + "<allow until='yyyy-mm-dd'>global-endpoint-change</allow> to validation-overrides.xml, see " + + "https://docs.vespa.ai/documentation/reference/validation-overrides.html", e.getMessage()); + } + + // Redeploy with override succeeds + applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .endpoint("default", "bar") + .region(west.region().value()) + .region(east.region().value()) + .allow(ValidationId.globalEndpointChange) + .build(); + context.submit(applicationPackage).deploy(); + assertEquals(ClusterSpec.Id.from("bar"), tester.applications().requireInstance(context.instanceId()) + .rotations().get(0).clusterId()); + } + + @Test + public void testReadableApplications() { + var db = new MockCuratorDb(); + var tester = new DeploymentTester(new ControllerTester(db)); + + // Create and deploy two applications + var app1 = tester.newDeploymentContext("t1", "a1", "default") + .submit() + .deploy(); + var app2 = tester.newDeploymentContext("t2", "a2", "default") + .submit() + .deploy(); + assertEquals(2, tester.applications().readable().size()); + + // Write invalid data to one application + db.curator().set(Path.fromString("/controller/v1/applications/" + app2.application().id().serialized()), + new byte[]{(byte) 0xDE, (byte) 0xAD}); + + // Can read the remaining readable + assertEquals(1, tester.applications().readable().size()); + + // Unconditionally reading all applications fails + try { + tester.applications().asList(); + fail("Expected exception"); + } catch (Exception ignored) { + } + + // Deployment for readable application still succeeds + app1.submit().deploy(); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index c83463bc1ea..775eb2a4d75 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -33,6 +33,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; +import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -59,7 +60,6 @@ import java.util.OptionalLong; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; -import java.util.function.Supplier; import java.util.logging.Handler; import java.util.logging.Logger; @@ -96,6 +96,10 @@ public final class ControllerTester { new ServiceRegistryMock()); } + public ControllerTester(ServiceRegistryMock serviceRegistryMock) { + this(new AthenzDbMock(), new MockCuratorDb(), defaultRotationsConfig(), serviceRegistryMock); + } + public ControllerTester(RotationsConfig rotationsConfig) { this(rotationsConfig, new MockCuratorDb()); } @@ -160,7 +164,9 @@ public final class ControllerTester { public AthenzDbMock athenzDb() { return athenzDb; } - public MemoryNameService nameService() { return serviceRegistry.nameServiceMock(); } + public MemoryNameService nameService() { + return serviceRegistry.nameService(); + } public ZoneRegistryMock zoneRegistry() { return serviceRegistry.zoneRegistry(); } @@ -188,27 +194,6 @@ public final class ControllerTester { controller = createController(curator, rotationsConfig, athenzDb, serviceRegistry); } - /** Creates the given tenant and application and deploys it */ - public void createAndDeploy(String tenantName, String domainName, String applicationName, Environment environment, long projectId, Long propertyId) { - createAndDeploy(tenantName, domainName, applicationName, toZone(environment), projectId, propertyId); - } - - /** Creates the given tenant and application and deploys it */ - public void createAndDeploy(String tenantName, String domainName, String applicationName, - String instanceName, ZoneId zone, long projectId, Long propertyId) { - throw new AssertionError("Not supposed to use this"); - } - - /** Creates the given tenant and application and deploys it */ - public void createAndDeploy(String tenantName, String domainName, String applicationName, ZoneId zone, long projectId, Long propertyId) { - createAndDeploy(tenantName, domainName, applicationName, "default", zone, projectId, propertyId); - } - - /** Creates the given tenant and application and deploys it */ - public void createAndDeploy(String tenantName, String domainName, String applicationName, Environment environment, long projectId) { - createAndDeploy(tenantName, domainName, applicationName, environment, projectId, null); - } - /** Upgrade controller to given version */ public void upgradeController(Version version, String commitSha, Instant commitDate) { for (var hostname : controller().curator().cluster()) { @@ -308,9 +293,8 @@ public final class ControllerTester { AthenzCredentials credentials = new AthenzCredentials( new AthenzPrincipal(user), domain, new OktaIdentityToken("okta-identity-token"), new OktaAccessToken("okta-access-token")); controller().tenants().create(tenantSpec, credentials); - if (contact.isPresent()) - controller().tenants().lockOrThrow(name, LockedTenant.Athenz.class, tenant -> - controller().tenants().store(tenant.with(contact.get()))); + contact.ifPresent(value -> controller().tenants().lockOrThrow(name, LockedTenant.Athenz.class, tenant -> + controller().tenants().store(tenant.with(value)))); assertNotNull(controller().tenants().get(name)); return name; } @@ -322,20 +306,20 @@ public final class ControllerTester { return tenant; } - public Optional<Credentials> credentialsFor(TenantName tenantName) { + public Credentials credentialsFor(TenantName tenantName) { Tenant tenant = controller().tenants().require(tenantName); switch (tenant.type()) { case athenz: - return Optional.of(new AthenzCredentials(new AthenzPrincipal(new AthenzUser("user")), + return new AthenzCredentials(new AthenzPrincipal(new AthenzUser("user")), ((AthenzTenant) tenant).domain(), new OktaIdentityToken("okta-identity-token"), - new OktaAccessToken("okta-access-token"))); + new OktaAccessToken("okta-access-token")); case cloud: - return Optional.of(new Credentials(new SimplePrincipal("dev"))); + return new Credentials(new SimplePrincipal("dev")); default: - return Optional.empty(); + throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'"); } } @@ -356,14 +340,6 @@ public final class ControllerTester { return application; } - public void deploy(ApplicationId id, ZoneId zone) { - deploy(id, zone, new ApplicationPackage(new byte[0])); - } - - public void deploy(ApplicationId id, ZoneId zone, ApplicationPackage applicationPackage) { - deploy(id, zone, applicationPackage, false); - } - public void deploy(ApplicationId id, ZoneId zone, ApplicationPackage applicationPackage, boolean deployCurrentVersion) { deploy(id, zone, Optional.of(applicationPackage), deployCurrentVersion); } @@ -379,10 +355,6 @@ public final class ControllerTester { new DeployOptions(false, version, false, deployCurrentVersion)); } - public Supplier<Instance> application(ApplicationId application) { - return () -> controller().applications().requireInstance(application); - } - private static Controller createController(CuratorDb curator, RotationsConfig rotationsConfig, AthenzDbMock athensDb, ServiceRegistryMock serviceRegistry) { @@ -393,7 +365,7 @@ public final class ControllerTester { new InMemoryFlagSource(), new MockMavenRepository(), serviceRegistry, - new MetricsMock()); + new MetricsMock(), new SecretStoreMock()); // Calculate initial versions controller.updateVersionStatus(VersionStatus.compute(controller)); return controller; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterCostTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterCostTest.java deleted file mode 100644 index 6df2d55d52b..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterCostTest.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.provision.ClusterSpec; -import org.junit.Assert; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -/** - * @author smorgrav - */ -public class ClusterCostTest { - - @Test - public void clusterCost() { - List<String> hostnames = new ArrayList<>(); - hostnames.add("host1"); - hostnames.add("host2"); - ClusterInfo info = new ClusterInfo("test", 100, 10, 10, 10, ClusterSpec.Type.container, hostnames); - ClusterUtilization util = new ClusterUtilization(0.3, 0.2, 0.5, 0.1); - ClusterCost cost = new ClusterCost(info, util); - - // CPU is fully utilized - Assert.assertEquals(200, cost.getTco(), Double.MIN_VALUE); - Assert.assertEquals(0, cost.getWaste(), Double.MIN_VALUE); - - // Set Disk as the most utilized resource - util = new ClusterUtilization(0.3, 0.1, 0.5, 0.1); - cost = new ClusterCost(info, util); - Assert.assertEquals(200, cost.getTco(), Double.MIN_NORMAL); // TCO is independent of utilization - Assert.assertEquals(57.1428571429, cost.getWaste(), 0.001); // Waste is not independent - } -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilizationTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilizationTest.java deleted file mode 100644 index c930978eb1e..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilizationTest.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import org.junit.Assert; -import org.junit.Test; - -/** - * @author smorgrav - */ -public class ClusterUtilizationTest { - - private static final double delta = Double.MIN_NORMAL; - - @Test - public void getMaxUtilization() { - ClusterUtilization resources = new ClusterUtilization(0.3, 0.1, 0.4, 0.5); - Assert.assertEquals(0.5, resources.getMaxUtilization(), delta); - - resources = new ClusterUtilization(0.3, 0.1, 0.4, 0.0); - Assert.assertEquals(0.4, resources.getMaxUtilization(), delta); - - resources = new ClusterUtilization(0.4, 0.3, 0.3, 0.0); - Assert.assertEquals(0.4, resources.getMaxUtilization(), delta); - - resources = new ClusterUtilization(0.1, 0.3, 0.3, 0.0); - Assert.assertEquals(0.3, resources.getMaxUtilization(), delta); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentCostTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentCostTest.java deleted file mode 100644 index 2e58253d768..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentCostTest.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.provision.ClusterSpec; -import org.junit.Assert; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author smorgrav - */ -public class DeploymentCostTest { - - @Test - public void deploymentCost() { - Map<String, ClusterCost> clusters = new HashMap<>(); - clusters.put("cluster1", createClusterCost(100, 0.2)); - clusters.put("cluster2", createClusterCost(50, 0.1)); - - DeploymentCost cost = new DeploymentCost(clusters); - Assert.assertEquals(300, cost.getTco(), Double.MIN_VALUE); // 2*100 + 2*50 - Assert.assertEquals(28.5714285714, cost.getWaste(), 0.001); // from cluster2 - Assert.assertEquals(0.7142857142857143, cost.getUtilization(), Double.MIN_VALUE); // from cluster2 - } - - private ClusterCost createClusterCost(int flavorCost, double cpuUtil) { - List<String> hostnames = new ArrayList<>(); - hostnames.add("host1"); - hostnames.add("host2"); - ClusterInfo info = new ClusterInfo("test", flavorCost, 10, 10, 10, ClusterSpec.Type.container, hostnames); - ClusterUtilization util = new ClusterUtilization(0.3, cpuUtil, 0.5, 0.1); - return new ClusterCost(info, util); - } -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index ea97e3e6c71..f968b8c76d7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -4,7 +4,9 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import org.junit.Test; @@ -21,7 +23,7 @@ public class EndpointTest { private static final ApplicationId app2 = ApplicationId.from("t2", "a2", "i2"); @Test - public void test_global_endpoints() { + public void global_endpoints() { EndpointId endpointId = EndpointId.defaultId(); Map<String, Endpoint> tests = Map.of( @@ -47,29 +49,29 @@ public class EndpointTest { // Main endpoint with direct routing and default TLS port "https://a1.t1.global.vespa.oath.cloud/", - Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint with custom rotation name "https://r1.a1.t1.global.vespa.oath.cloud/", - Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint for custom instance in default rotation "https://i2.a2.t2.global.vespa.oath.cloud/", - Endpoint.of(app2).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app2).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint for custom instance with custom rotation name "https://r2.i2.a2.t2.global.vespa.oath.cloud/", - Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint in public system "https://a1.t1.global.public.vespa.oath.cloud/", - Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.Public) + Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); } @Test - public void test_global_endpoints_with_endpoint_id() { + public void global_endpoints_with_endpoint_id() { var endpointId = EndpointId.defaultId(); Map<String, Endpoint> tests = Map.of( @@ -95,29 +97,29 @@ public class EndpointTest { // Main endpoint with direct routing and default TLS port "https://a1.t1.global.vespa.oath.cloud/", - Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint with custom rotation name "https://r1.a1.t1.global.vespa.oath.cloud/", - Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint for custom instance in default rotation "https://i2.a2.t2.global.vespa.oath.cloud/", - Endpoint.of(app2).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app2).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint for custom instance with custom rotation name "https://r2.i2.a2.t2.global.vespa.oath.cloud/", - Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).directRouting().in(SystemName.main), + Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main), // Main endpoint in public system "https://a1.t1.global.public.vespa.oath.cloud/", - Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.Public) + Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); } @Test - public void test_zone_endpoints() { + public void zone_endpoints() { var cluster = ClusterSpec.Id.from("default"); // Always default for non-direct routing var prodZone = ZoneId.from("prod", "us-north-1"); var testZone = ZoneId.from("test", "us-north-2"); @@ -153,17 +155,21 @@ public class EndpointTest { // Non-default cluster in public "https://c1.a1.t1.us-north-1.public.vespa.oath.cloud/", - Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).directRouting().in(SystemName.Public), + Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public), // Non-default cluster and instance in public "https://c2.i2.a2.t2.us-north-1.public.vespa.oath.cloud/", - Endpoint.of(app2).target(ClusterSpec.Id.from("c2"), prodZone).on(Port.tls()).directRouting().in(SystemName.Public) + Endpoint.of(app2).target(ClusterSpec.Id.from("c2"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public), + + // Endpoint in main using shared layer 4 + "https://a1.t1.us-north-1.vespa.oath.cloud/", + Endpoint.of(app1).target(cluster, prodZone).on(Port.tls()).routingMethod(RoutingMethod.sharedLayer4).in(SystemName.main) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); } @Test - public void test_wildcard_endpoints() { + public void wildcard_endpoints() { var defaultCluster = ClusterSpec.Id.from("default"); var prodZone = ZoneId.from("prod", "us-north-1"); var testZone = ZoneId.from("test", "us-north-2"); @@ -173,7 +179,7 @@ public class EndpointTest { "https://a1.t1.global.public.vespa.oath.cloud/", Endpoint.of(app1) .named(EndpointId.defaultId()) - .directRouting() + .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public), @@ -181,7 +187,7 @@ public class EndpointTest { "https://*.a1.t1.global.public.vespa.oath.cloud/", Endpoint.of(app1) .wildcard() - .directRouting() + .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public), @@ -189,7 +195,7 @@ public class EndpointTest { "https://a1.t1.us-north-1.public.vespa.oath.cloud/", Endpoint.of(app1) .target(defaultCluster, prodZone) - .directRouting() + .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public), @@ -197,7 +203,7 @@ public class EndpointTest { "https://*.a1.t1.us-north-1.public.vespa.oath.cloud/", Endpoint.of(app1) .wildcard(prodZone) - .directRouting() + .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public), @@ -205,7 +211,7 @@ public class EndpointTest { "https://a1.t1.us-north-2.test.public.vespa.oath.cloud/", Endpoint.of(app1) .target(defaultCluster, testZone) - .directRouting() + .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public), @@ -213,11 +219,37 @@ public class EndpointTest { "https://*.a1.t1.us-north-2.test.public.vespa.oath.cloud/", Endpoint.of(app1) .wildcard(testZone) - .directRouting() + .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.Public) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); } + + @Test + public void upstream_name() { + var zone = ZoneId.from("prod", "us-north-1"); + var tests1 = Map.of( + // With default cluster + "a1.t1.us-north-1.prod", + Endpoint.of(app1).named(EndpointId.defaultId()).on(Port.tls(4443)).in(SystemName.main), + + // With non-default cluster + "c1.a1.t1.us-north-1.prod", + Endpoint.of(app1).named(EndpointId.of("c1")).on(Port.tls(4443)).in(SystemName.main) + ); + var tests2 = Map.of( + // With non-default instance + "i2.a2.t2.us-north-1.prod", + Endpoint.of(app2).named(EndpointId.defaultId()).on(Port.tls(4443)).in(SystemName.main), + + // With non-default instance and cluster + "c2.i2.a2.t2.us-north-1.prod", + Endpoint.of(app2).named(EndpointId.of("c2")).on(Port.tls(4443)).in(SystemName.main) + ); + tests1.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamIdOf(new DeploymentId(app1, zone)))); + tests2.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamIdOf(new DeploymentId(app2, zone)))); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java new file mode 100644 index 00000000000..d7bc73adf37 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java @@ -0,0 +1,124 @@ +package com.yahoo.vespa.hosted.controller.certificate; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.security.KeyAlgorithm; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Before; +import org.junit.Test; + +import javax.security.auth.x500.X500Principal; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.time.Clock; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author andreer + */ +public class EndpointCertificateManagerTest { + + private final SecretStoreMock secretStore = new SecretStoreMock(); + private final ZoneRegistryMock zoneRegistryMock = new ZoneRegistryMock(SystemName.main); + private final MockCuratorDb mockCuratorDb = new MockCuratorDb(); + private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(); + private final InMemoryFlagSource inMemoryFlagSource = new InMemoryFlagSource(); + private final Clock clock = Clock.systemUTC(); + private final EndpointCertificateManager endpointCertificateManager = + new EndpointCertificateManager(zoneRegistryMock, mockCuratorDb, secretStore, endpointCertificateMock, clock, inMemoryFlagSource); + + private static final KeyPair testKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 192); + private static final X509Certificate testCertificate = X509CertificateBuilder + .fromKeypair( + testKeyPair, + new X500Principal("CN=test"), + Instant.now(), Instant.now().plus(5, ChronoUnit.MINUTES), + SignatureAlgorithm.SHA256_WITH_ECDSA, + X509CertificateBuilder.generateRandomSerialNumber()) + .addSubjectAlternativeName("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud") + .addSubjectAlternativeName("default.default.global.vespa.oath.cloud") + .addSubjectAlternativeName("*.default.default.global.vespa.oath.cloud") + .addSubjectAlternativeName("default.default.us-east-1.test.vespa.oath.cloud") + .addSubjectAlternativeName("*.default.default.us-east-1.test.vespa.oath.cloud") + .build(); + + private final Instance testInstance = new Instance(ApplicationId.defaultId()); + private final String testKeyName = "testKeyName"; + private final String testCertName = "testCertName"; + private ZoneId testZone; + + @Before + public void setUp() { + zoneRegistryMock.exclusiveRoutingIn(zoneRegistryMock.zones().all().zones()); + testZone = zoneRegistryMock.zones().directlyRouted().zones().stream().findFirst().orElseThrow().getId(); + inMemoryFlagSource.withBooleanFlag(Flags.VALIDATE_ENDPOINT_CERTIFICATES.id(), true); + } + + @Test + public void provisions_new_certificate() { + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); + assertTrue(endpointCertificateMetadata.isPresent()); + assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key")); + assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert")); + assertEquals(0, endpointCertificateMetadata.get().version()); + } + + @Test + public void reuses_stored_certificate_metadata() { + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7)); + secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 7); + secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 7); + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); + assertTrue(endpointCertificateMetadata.isPresent()); + assertEquals(testKeyName, endpointCertificateMetadata.get().keyName()); + assertEquals(testCertName, endpointCertificateMetadata.get().certName()); + assertEquals(7, endpointCertificateMetadata.get().version()); + } + + @Test + public void uses_refreshed_certificate_when_available_and_valid() { + inMemoryFlagSource.withBooleanFlag(Flags.USE_REFRESHED_ENDPOINT_CERTIFICATE.id(), true); + + secretStore.setSecret(testKeyName, "secret-key", 7); + secretStore.setSecret(testCertName, "cert", 7); + secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 8); + secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 9); + secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 8); + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7)); + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); + assertTrue(endpointCertificateMetadata.isPresent()); + assertEquals(testKeyName, endpointCertificateMetadata.get().keyName()); + assertEquals(testCertName, endpointCertificateMetadata.get().certName()); + assertEquals(8, endpointCertificateMetadata.get().version()); + } + + @Test + public void reprovisions_certificate_when_necessary() { + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, Optional.of("uuid"), Optional.of(List.of()), Optional.empty())); + secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0); + secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 0); + Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone); + assertTrue(endpointCertificateMetadata.isPresent()); + assertEquals(0, endpointCertificateMetadata.get().version()); + assertEquals(endpointCertificateMetadata, mockCuratorDb.readEndpointCertificateMetadata(testInstance.id())); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 9b0706d184f..2f2b1fbb364 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -1,17 +1,23 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; +import com.yahoo.security.SignatureAlgorithm; +import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import javax.security.auth.x500.X500Principal; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.UncheckedIOException; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; import java.security.cert.X509Certificate; import java.text.SimpleDateFormat; import java.time.Duration; @@ -52,6 +58,7 @@ public class ApplicationPackageBuilder { private String searchDefinition = "search test { }"; private boolean explicitSystemTest = false; private boolean explicitStagingTest = false; + private Version compileVersion = Version.fromString("6.1"); public ApplicationPackageBuilder majorVersion(int majorVersion) { this.majorVersion = OptionalInt.of(majorVersion); @@ -159,6 +166,11 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder compileVersion(Version version) { + compileVersion = version; + return this; + } + public ApplicationPackageBuilder athenzIdentity(AthenzDomain domain, AthenzService service) { this.athenzIdentityAttributes = String.format("athenz-domain='%s' athenz-service='%s'", domain.value(), service.value()); @@ -186,6 +198,24 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder trustDefaultCertificate() { + try { + var generator = KeyPairGenerator.getInstance("RSA"); + var builder = X509CertificateBuilder.fromKeypair( + generator.generateKeyPair(), + new X500Principal("CN=name"), + Instant.now(), + Instant.now().plusMillis(300_000), + SignatureAlgorithm.SHA256_WITH_RSA, + X509CertificateBuilder.generateRandomSerialNumber() + ); + this.trustedCertificates.add(builder.build()); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + return this; + } + private byte[] deploymentSpec() { StringBuilder xml = new StringBuilder(); xml.append("<deployment version='1.0' "); @@ -201,11 +231,11 @@ public class ApplicationPackageBuilder { xml.append("'/>\n"); } xml.append(notifications); - xml.append(blockChange); if (explicitSystemTest) xml.append(" <test />\n"); if (explicitStagingTest) xml.append(" <staging />\n"); + xml.append(blockChange); xml.append(" <"); xml.append(environment.value()); if (globalServiceId != null) { @@ -237,8 +267,8 @@ public class ApplicationPackageBuilder { return searchDefinition.getBytes(UTF_8); } - private static byte[] buildMeta() { - return "{\"compileVersion\":\"6.1\",\"buildTime\":1000}".getBytes(UTF_8); + private static byte[] buildMeta(Version compileVersion) { + return ("{\"compileVersion\":\"" + compileVersion.toFullString() + "\",\"buildTime\":1000}").getBytes(UTF_8); } public ApplicationPackage build() { @@ -262,7 +292,7 @@ public class ApplicationPackageBuilder { out.write(searchDefinition()); out.closeEntry(); out.putNextEntry(new ZipEntry(dir + "build-meta.json")); - out.write(buildMeta()); + out.write(buildMeta(compileVersion)); out.closeEntry(); out.putNextEntry(new ZipEntry(dir + "security/clients.pem")); out.write(X509CertificateUtils.toPem(trustedCertificates).getBytes(UTF_8)); @@ -284,7 +314,7 @@ public class ApplicationPackageBuilder { out.write(deploymentXml.getBytes(UTF_8)); out.closeEntry(); out.putNextEntry(new ZipEntry("build-meta.json")); - out.write(buildMeta()); + out.write(buildMeta(Version.fromString("6.1"))); out.closeEntry(); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 2792a59b523..a04dc6fb579 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -6,12 +6,15 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; @@ -24,14 +27,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; +import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; +import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; @@ -42,8 +45,8 @@ import java.math.BigInteger; import java.net.URI; import java.security.KeyPair; import java.security.cert.X509Certificate; +import java.time.Duration; import java.time.Instant; -import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -90,13 +93,13 @@ public class DeploymentContext { .emailAddress("b@a") .trust(generateCertificate()) .build(); + public static final SourceRevision defaultSourceRevision = new SourceRevision("repository1", "master", "commit1"); private final TenantAndApplicationId applicationId; private final ApplicationId instanceId; private final TesterId testerId; private final JobController jobs; - private final RoutingGeneratorMock routing; private final JobRunner runner; private final DeploymentTester tester; @@ -104,15 +107,14 @@ public class DeploymentContext { private boolean deferDnsUpdates = false; public DeploymentContext(ApplicationId instanceId, DeploymentTester tester) { - this.applicationId = TenantAndApplicationId.from(instanceId); this.instanceId = instanceId; this.testerId = TesterId.of(instanceId); this.jobs = tester.controller().jobController(); this.runner = tester.runner(); this.tester = tester; - this.routing = tester.controllerTester().serviceRegistry().routingGeneratorMock(); createTenantAndApplication(); + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.ALLOW_DIRECT_ROUTING.id(), true); } private void createTenantAndApplication() { @@ -190,6 +192,15 @@ public class DeploymentContext { return this; } + /** Defer provisioning of load balancers in zones in given environment */ + public DeploymentContext deferLoadBalancerProvisioningIn(Environment... environment) { + return deferLoadBalancerProvisioningIn(Set.of(environment)); + } + + public DeploymentContext deferLoadBalancerProvisioningIn(Set<Environment> environments) { + configServer().deferLoadBalancerProvisioningIn(environments); + return this; + } /** Defer DNS updates */ public DeploymentContext deferDnsUpdates() { @@ -199,46 +210,45 @@ public class DeploymentContext { /** Flush all pending DNS updates */ public DeploymentContext flushDnsUpdates() { - tester.nameServiceDispatcher().run(); + flushDnsUpdates(Integer.MAX_VALUE); assertTrue("All name service requests dispatched", tester.controller().curator().readNameServiceQueue().requests().isEmpty()); return this; } - /** Add a routing policy for this in given zone, with status set to active */ - public DeploymentContext addRoutingPolicy(ZoneId zone, boolean active) { - return addRoutingPolicy(instanceId, zone, active); - } - - /** Add a routing policy for tester instance of this in given zone, with status set to active */ - public DeploymentContext addTesterRoutingPolicy(ZoneId zone, boolean active) { - return addRoutingPolicy(testerId.id(), zone, active); + /** Flush count pending DNS updates */ + public DeploymentContext flushDnsUpdates(int count) { + var dispatcher = new NameServiceDispatcher(tester.controller(), Duration.ofDays(1), + new JobControl(tester.controller().curator()), count); + dispatcher.run(); + return this; } - private DeploymentContext addRoutingPolicy(ApplicationId instance, ZoneId zone, boolean active) { - var clusterId = "default" + (!active ? "-inactive" : ""); - var id = new RoutingPolicyId(instance, ClusterSpec.Id.from(clusterId), zone); - var policies = new LinkedHashMap<>(tester.controller().curator().readRoutingPolicies(instance)); + /** Add a routing policy for this in given zone, with status set to inactive */ + public DeploymentContext addInactiveRoutingPolicy(ZoneId zone) { + var clusterId = "default-inactive"; + var id = new RoutingPolicyId(instanceId, ClusterSpec.Id.from(clusterId), zone); + var policies = new LinkedHashMap<>(tester.controller().curator().readRoutingPolicies(instanceId)); policies.put(id, new RoutingPolicy(id, HostName.from("lb-host"), Optional.empty(), - Set.of(EndpointId.of("c0")), - new Status(active, GlobalRouting.DEFAULT_STATUS))); - tester.controller().curator().writeRoutingPolicies(instance, policies); + Set.of(EndpointId.of("default")), + new Status(false, GlobalRouting.DEFAULT_STATUS))); + tester.controller().curator().writeRoutingPolicies(instanceId, policies); return this; } /** Submit given application package for deployment */ public DeploymentContext submit(ApplicationPackage applicationPackage) { - return submit(applicationPackage, defaultSourceRevision); + return submit(applicationPackage, Optional.of(defaultSourceRevision)); } /** Submit given application package for deployment */ - public DeploymentContext submit(ApplicationPackage applicationPackage, SourceRevision sourceRevision) { + public DeploymentContext submit(ApplicationPackage applicationPackage, Optional<SourceRevision> sourceRevision) { var projectId = tester.controller().applications() .requireApplication(applicationId) .projectId() .orElse(1000); // These are really set through submission, so just pick one if it hasn't been set. - lastSubmission = jobs.submit(applicationId, Optional.of(sourceRevision), Optional.of("a@b"), Optional.empty(), + lastSubmission = jobs.submit(applicationId, sourceRevision, Optional.of("a@b"), Optional.empty(), Optional.empty(), projectId, applicationPackage, new byte[0]); return this; } @@ -279,7 +289,6 @@ public class DeploymentContext { runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasFailed()); assertTrue(jobs.run(id).get().hasEnded()); - doTeardown(job); return this; } @@ -303,9 +312,6 @@ public class DeploymentContext { throw new AssertionError("Job '" + run.id() + "' was run twice"); assertFalse("Change should have no targets, but was " + instance().change(), instance().change().hasTargets()); - if (!deferDnsUpdates) { - flushDnsUpdates(); - } return this; } @@ -331,8 +337,8 @@ public class DeploymentContext { if (job.type().environment().isManuallyDeployed()) return this; } - doTests(job); - doTeardown(job); + if (job.type().isTest()) + doTests(job); return this; } @@ -361,11 +367,10 @@ public class DeploymentContext { triggerJobs(); RunId id = currentRun(job).id(); doDeploy(job); - tester.clock().advance(InternalStepRunner.installationTimeout.plusSeconds(1)); + tester.clock().advance(InternalStepRunner.Timeouts.of(tester.controller().system()).noNodesDown().plusSeconds(1)); runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasFailed()); assertTrue(jobs.run(id).get().hasEnded()); - doTeardown(job); return this; } @@ -376,19 +381,13 @@ public class DeploymentContext { RunId id = currentRun(job).id(); doDeploy(job); doUpgrade(job); - tester.clock().advance(InternalStepRunner.installationTimeout.plusSeconds(1)); + tester.clock().advance(InternalStepRunner.Timeouts.of(tester.controller().system()).noNodesDown().plusSeconds(1)); runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasFailed()); assertTrue(jobs.run(id).get().hasEnded()); - doTeardown(job); return this; } - /** Sets a single endpoint in the routing layer for the instance in this */ - public DeploymentContext setEndpoints(ZoneId zone) { - return setEndpoints(zone, false); - } - /** Deploy default application package, start a run for that change and return its ID */ public RunId newRun(JobType type) { submit(); @@ -409,12 +408,13 @@ public class DeploymentContext { /** Start tests in system test stage */ public RunId startSystemTestTests() { - RunId id = newRun(JobType.systemTest); + var id = newRun(JobType.systemTest); + var testZone = JobType.systemTest.zone(tester.controller().system()); runner.run(); - configServer().convergeServices(instanceId, JobType.systemTest.zone(tester.controller().system())); - configServer().convergeServices(testerId.id(), JobType.systemTest.zone(tester.controller().system())); - setEndpoints(JobType.systemTest.zone(tester.controller().system())); - setTesterEndpoints(JobType.systemTest.zone(tester.controller().system())); + if ( ! deferDnsUpdates) + flushDnsUpdates(); + configServer().convergeServices(instanceId, testZone); + configServer().convergeServices(testerId.id(), testZone); runner.run(); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.endTests)); assertTrue(jobs.run(id).get().steps().get(Step.endTests).startTime().isPresent()); @@ -440,7 +440,10 @@ public class DeploymentContext { // First step is always a deployment. runner.advance(currentRun(job)); - if ( ! job.type().environment().isManuallyDeployed()) + if ( ! deferDnsUpdates) + flushDnsUpdates(); + + if (job.type().isTest()) doInstallTester(job); if (job.type() == JobType.stagingTest) { // Do the initial deployment and installation of the real application. @@ -448,13 +451,11 @@ public class DeploymentContext { Versions versions = currentRun(job).versions(); tester.configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), versions.sourcePlatform().orElse(versions.targetPlatform())); configServer().convergeServices(id.application(), zone); - setEndpoints(zone); runner.advance(currentRun(job)); assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); // All installation is complete and endpoints are ready, so setup may begin. - if (job.type().isDeployment()) - assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); + assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester)); assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.startStagingSetup)); @@ -486,34 +487,6 @@ public class DeploymentContext { return run; } - /** Sets a single endpoint in the routing layer for the tester instance in this */ - private DeploymentContext setTesterEndpoints(ZoneId zone) { - return setEndpoints(zone, true); - } - - /** Sets a single endpoint in the routing layer; this matches that required for the tester */ - private DeploymentContext setEndpoints(ZoneId zone, boolean tester) { - var id = instanceId; - if (tester) { - id = testerId.id(); - } - routing.putEndpoints(new DeploymentId(id, zone), - Collections.singletonList(new RoutingEndpoint(String.format("https://%s--%s--%s.%s.%s.vespa:43", - id.instance().value(), - id.application().value(), - id.tenant().value(), - zone.region().value(), - zone.environment().value()), - "host1", - true, - String.format("cluster1.%s.%s.%s.%s", - id.application().value(), - id.tenant().value(), - zone.region().value(), - zone.environment().value())))); - return this; - } - /** Lets nodes converge on new application version. */ private void doConverge(JobId job) { RunId id = currentRun(job).id(); @@ -521,14 +494,13 @@ public class DeploymentContext { assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installReal)); configServer().convergeServices(id.application(), zone); - setEndpoints(zone); runner.advance(currentRun(job)); if (job.type().environment().isManuallyDeployed()) { assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); assertTrue(jobs.run(id).get().hasEnded()); return; } - assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); + assertEquals("Status of " + id, Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal)); } /** Installs tester and starts tests. */ @@ -542,8 +514,7 @@ public class DeploymentContext { assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); configServer().convergeServices(TesterId.of(id.application()).id(), zone); runner.advance(currentRun(job)); - assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); - setTesterEndpoints(zone); + assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester)); runner.advance(currentRun(job)); } @@ -567,16 +538,6 @@ public class DeploymentContext { assertTrue(configServer().nodeRepository().list(zone, TesterId.of(id.application()).id()).isEmpty()); } - /** Removes endpoints from routing layer — always call this. */ - private void doTeardown(JobId job) { - ZoneId zone = zone(job); - DeploymentId deployment = new DeploymentId(job.application(), zone); - - if ( ! instance().deployments().containsKey(zone)) - routing.removeEndpoints(deployment); - routing.removeEndpoints(new DeploymentId(TesterId.of(job.application()).id(), zone)); - } - private JobId jobId(JobType type) { return new JobId(instanceId, type); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 5b5c6b61357..589229b32d4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.log.LogLevel; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.Application; @@ -10,17 +9,14 @@ import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; +import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.maintenance.JobRunnerTest; -import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; import com.yahoo.vespa.hosted.controller.maintenance.OutstandingChangeDeployer; import com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; @@ -30,7 +26,6 @@ import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; import java.time.temporal.TemporalAdjusters; -import java.util.Collections; import java.util.logging.Logger; import static org.junit.Assert.assertTrue; @@ -48,20 +43,16 @@ public class DeploymentTester { public static final TenantAndApplicationId appId = TenantAndApplicationId.from("tenant", "application"); public static final ApplicationId instanceId = appId.defaultInstance(); - public static final TesterId testerId = TesterId.of(instanceId); private final ControllerTester tester; private final JobController jobs; - private final RoutingGeneratorMock routing; private final MockTesterCloud cloud; private final JobRunner runner; private final Upgrader upgrader; private final ReadyJobsTrigger readyJobsTrigger; private final OutstandingChangeDeployer outstandingChangeDeployer; - private final NameServiceDispatcher nameServiceDispatcher; public JobController jobs() { return jobs; } - public RoutingGeneratorMock routing() { return routing; } public MockTesterCloud cloud() { return cloud; } public JobRunner runner() { return runner; } public ConfigServerMock configServer() { return tester.configServer(); } @@ -75,6 +66,7 @@ public class DeploymentTester { public Application application(TenantAndApplicationId id ) { return applications().requireApplication(id); } public Instance instance() { return instance(instanceId); } public Instance instance(ApplicationId id) { return applications().requireInstance(id); } + public DeploymentStatusList deploymentStatuses() { return jobs.deploymentStatuses(ApplicationList.from(applications().asList())); } public DeploymentTester() { this(new ControllerTester()); @@ -83,7 +75,6 @@ public class DeploymentTester { public DeploymentTester(ControllerTester controllerTester) { tester = controllerTester; jobs = tester.controller().jobController(); - routing = tester.serviceRegistry().routingGeneratorMock(); cloud = (MockTesterCloud) tester.controller().jobController().cloud(); var jobControl = new JobControl(tester.controller().curator()); runner = new JobRunner(tester.controller(), Duration.ofDays(1), jobControl, @@ -92,9 +83,6 @@ public class DeploymentTester { upgrader.setUpgradesPerMinute(1); // Anything that makes it at least one for any maintenance period is fine. readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), maintenanceInterval, jobControl); outstandingChangeDeployer = new OutstandingChangeDeployer(tester.controller(), maintenanceInterval, jobControl); - nameServiceDispatcher = new NameServiceDispatcher(tester.controller(), maintenanceInterval, jobControl, - Integer.MAX_VALUE); - routing.putEndpoints(new DeploymentId(null, null), Collections.emptyList()); // Turn off default behaviour for the mock. // Get deployment job logs to stderr. Logger.getLogger("").setLevel(LogLevel.DEBUG); @@ -112,10 +100,6 @@ public class DeploymentTester { public OutstandingChangeDeployer outstandingChangeDeployer() { return outstandingChangeDeployer; } - public NameServiceDispatcher nameServiceDispatcher() { - return nameServiceDispatcher; - } - public DeploymentTester atMondayMorning() { return at(tester.clock().instant().atZone(ZoneOffset.UTC) .with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)) @@ -148,13 +132,6 @@ public class DeploymentTester { return newDeploymentContext(tenantName, applicationName, instanceName).application(); } - /** - * Sets a single endpoint in the routing mock; this matches that required for the tester. - */ - public void setEndpoints(ApplicationId id, ZoneId zone) { - newDeploymentContext(id).setEndpoints(zone); - } - /** Aborts and finishes all running jobs. */ public void abortAll() { triggerJobs(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 656afda149d..2b5e8b3e91c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -3,13 +3,19 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Test; @@ -27,6 +33,8 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApNortheast2; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApSoutheast1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionAwsUsEast1a; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdAwsUsEast1a; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdUsCentral1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionEuWest1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; @@ -57,7 +65,7 @@ import static org.junit.Assert.assertTrue; */ public class DeploymentTriggerTest { - private final DeploymentTester tester = new DeploymentTester(); + private DeploymentTester tester = new DeploymentTester(); @Test public void testTriggerFailing() { @@ -965,7 +973,7 @@ public class DeploymentTriggerTest { tester.triggerJobs(); assertEquals(List.of(), tester.jobs().active()); - tester.atMondayMorning().clock().advance(Duration.ofDays(5)); // Inside block window for second instance. + tester.atMondayMorning().clock().advance(Duration.ofDays(5)); // Inside revision block window for second, conservative instance. Version version = Version.fromString("8.1"); tester.controllerTester().upgradeSystem(version); tester.upgrader().maintain(); @@ -973,16 +981,18 @@ public class DeploymentTriggerTest { assertEquals(Change.empty(), app2.instance().change()); assertEquals(Change.empty(), app3.instance().change()); - // Upgrade instance 1; a failure allows an application change to accompany the upgrade. + // Upgrade instance 1; a failure in any instance allows an application change to accompany the upgrade. // The new platform won't roll out to the conservative instance until the normal one is upgraded. - app1.failDeployment(systemTest); + app2.failDeployment(systemTest); app1.submit(applicationPackage); assertEquals(Change.of(version).with(app1.application().latestVersion().get()), app1.instance().change()); - app1.runJob(systemTest) - .jobAborted(stagingTest) + app2.runJob(systemTest); + app1.jobAborted(stagingTest) .runJob(stagingTest) .runJob(productionUsWest1) .runJob(productionUsEast3); + app1.runJob(stagingTest); // Tests with only the outstanding application change. + app2.runJob(systemTest); // Tests with only the outstanding application change. tester.clock().advance(Duration.ofHours(2)); app1.runJob(productionEuWest1); tester.clock().advance(Duration.ofHours(1)); @@ -997,9 +1007,7 @@ public class DeploymentTriggerTest { app1.runJob(testUsEast3); app1.runJob(productionApSoutheast1); - app2.runJob(systemTest) // Testing outstanding revision. - .runJob(stagingTest); // Testing outstanding revision. - + // Confidence rises to high, for the new version, and instance 2 starts to upgrade. tester.controllerTester().computeVersionStatus(); tester.upgrader().maintain(); tester.outstandingChangeDeployer().run(); @@ -1009,14 +1017,13 @@ public class DeploymentTriggerTest { assertEquals(Change.of(version), app2.instance().change()); assertEquals(Change.empty(), app3.instance().change()); - app2.runJob(systemTest) // Explicitly defined for this instance. - .runJob(stagingTest) // Never completed successfully with just the upgrade. + app1.runJob(stagingTest); // Never completed successfully with just the upgrade. + app2.runJob(systemTest) // Never completed successfully with just the upgrade. .runJob(productionEuWest1) - .failDeployment(testEuWest1) - .runJob(systemTest); // Testing outstanding revision with currently deployed (upgraded) platform. + .failDeployment(testEuWest1); + // Instance 2 failed the last job, and now exist block window, letting application change roll out with the upgrade. tester.clock().advance(Duration.ofDays(1)); // Leave block window for revisions. - app2.abortJob(testEuWest1); tester.upgrader().maintain(); tester.outstandingChangeDeployer().run(); assertEquals(0, tester.jobs().active().size()); @@ -1067,4 +1074,58 @@ public class DeploymentTriggerTest { assertEquals(Change.of(app.lastSubmission().get()), app.instance().change()); } + @Test + public void mixedDirectAndPipelineJobsInProduction() { + ApplicationPackage cdPackage = new ApplicationPackageBuilder().region("cd-us-central-1") + .region("cd-aws-us-east-1a") + .build(); + ServiceRegistryMock services = new ServiceRegistryMock(); + var zones = List.of(ZoneApiMock.fromId("test.cd-us-central-1"), + ZoneApiMock.fromId("staging.cd-us-central-1"), + ZoneApiMock.fromId("prod.cd-us-central-1"), + ZoneApiMock.fromId("prod.cd-aws-us-east-1a")); + services.zoneRegistry() + .setSystemName(SystemName.cd) + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.shared); + tester = new DeploymentTester(new ControllerTester(services)); + tester.configServer().bootstrap(services.zoneRegistry().zones().all().ids(), SystemApplication.values()); + tester.controllerTester().upgradeSystem(Version.fromString("6.1")); + tester.controllerTester().computeVersionStatus(); + var app = tester.newDeploymentContext(); + + app.runJob(productionCdUsCentral1, cdPackage); + app.submit(cdPackage); + app.runJob(systemTest); + // Staging test requires unknown initial version, and is broken. + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false); + app.runJob(productionCdUsCentral1) + .abortJob(stagingTest) // Complete failing run. + .runJob(stagingTest) + .runJob(productionCdAwsUsEast1a); + + app.runJob(productionCdUsCentral1, cdPackage); + var version = new Version("7.1"); + tester.controllerTester().upgradeSystem(version); + tester.upgrader().maintain(); + // System and staging tests both require unknown versions, and are broken. + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false); + app.runJob(productionCdUsCentral1) + .abortJob(systemTest) + .abortJob(stagingTest) + .runJob(systemTest) + .runJob(stagingTest) + .runJob(productionCdAwsUsEast1a); + + app.runJob(productionCdUsCentral1, cdPackage); + app.submit(cdPackage); + app.runJob(systemTest); + // Staging test requires unknown initial version, and is broken. + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false); + app.runJob(productionCdUsCentral1) + .jobAborted(stagingTest) + .runJob(stagingTest) + .runJob(productionCdAwsUsEast1a); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index ff66ab38d32..99325ff909d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -5,18 +5,19 @@ import com.google.common.collect.ImmutableList; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.controller.RoutingController; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -54,13 +55,13 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.app import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.publicCdApplicationPackage; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.instanceId; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * @author jonmv @@ -140,9 +141,6 @@ public class InternalStepRunnerTest { ZoneId zone = id.type().zone(system()); HostName host = tester.configServer().hostFor(instanceId, zone); - tester.setEndpoints(app.testerId().id(), JobType.productionUsCentral1.zone(system())); - tester.runner().run(); - tester.configServer().setConfigChangeActions(new ConfigChangeActions(singletonList(new RestartAction("cluster", "container", "search", @@ -163,18 +161,22 @@ public class InternalStepRunnerTest { tester.runner().run(); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal)); - tester.clock().advance(InternalStepRunner.installationTimeout.plus(Duration.ofSeconds(1))); + tester.clock().advance(InternalStepRunner.Timeouts.of(system()).noNodesDown().plus(Duration.ofSeconds(1))); tester.runner().run(); - assertEquals(RunStatus.error, tester.jobs().run(id).get().status()); + assertEquals(installationFailed, tester.jobs().run(id).get().status()); } @Test public void waitsForEndpointsAndTimesOut() { app.newRun(JobType.systemTest); - // Tester fails to show up for staging tests, and the real deployment for system tests. - tester.setEndpoints(app.testerId().id(), JobType.systemTest.zone(system())); - tester.setEndpoints(app.instanceId(), JobType.stagingTest.zone(system())); + // Tester endpoint fails to show up for staging tests, and the real deployment for system tests. + var testZone = JobType.systemTest.zone(system()); + var stagingZone = JobType.stagingTest.zone(system()); + tester.newDeploymentContext(app.testerId().id()) + .deferLoadBalancerProvisioningIn(testZone.environment()); + tester.newDeploymentContext(app.instanceId()) + .deferLoadBalancerProvisioningIn(stagingZone.environment()); tester.runner().run(); tester.configServer().convergeServices(app.instanceId(), JobType.stagingTest.zone(system())); @@ -185,10 +187,9 @@ public class InternalStepRunnerTest { tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system())); tester.runner().run(); - tester.clock().advance(InternalStepRunner.endpointTimeout.plus(Duration.ofSeconds(1))); + tester.clock().advance(InternalStepRunner.Timeouts.of(system()).endpoint().plus(Duration.ofSeconds(1))); tester.runner().run(); assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installTester)); } @Test @@ -200,21 +201,17 @@ public class InternalStepRunnerTest { // Node is down too long in system test, and no nodes go down in staging. tester.runner().run(); - tester.setEndpoints(app.testerId().id(), JobType.systemTest.zone(system())); tester.configServer().setVersion(app.testerId().id(), JobType.systemTest.zone(system()), tester.controller().systemVersion()); tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); - tester.setEndpoints(app.instanceId(), JobType.systemTest.zone(system())); - tester.setEndpoints(app.testerId().id(), JobType.stagingTest.zone(system())); tester.configServer().setVersion(app.testerId().id(), JobType.stagingTest.zone(system()), tester.controller().systemVersion()); tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system())); - tester.setEndpoints(app.instanceId(), JobType.stagingTest.zone(system())); tester.runner().run(); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installTester)); Node systemTestNode = tester.configServer().nodeRepository().list(JobType.systemTest.zone(system()), app.instanceId()).iterator().next(); - tester.clock().advance(InternalStepRunner.installationTimeout.minus(Duration.ofSeconds(1))); + tester.clock().advance(InternalStepRunner.Timeouts.of(system()).noNodesDown().minus(Duration.ofSeconds(1))); tester.configServer().nodeRepository().putByHostname(JobType.systemTest.zone(system()), new Node.Builder(systemTestNode) .serviceState(Node.ServiceState.allowedDown) @@ -229,7 +226,7 @@ public class InternalStepRunnerTest { assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal)); - tester.clock().advance(InternalStepRunner.installationTimeout.minus(Duration.ofSeconds(3))); + tester.clock().advance(InternalStepRunner.Timeouts.of(system()).nodesDown().minus(Duration.ofSeconds(3))); tester.runner().run(); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); @@ -243,13 +240,11 @@ public class InternalStepRunnerTest { app.newRun(JobType.systemTest); tester.runner().run(); tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); - tester.setEndpoints(app.instanceId(), JobType.systemTest.zone(system())); tester.runner().run(); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); tester.applications().deactivate(app.instanceId(), JobType.systemTest.zone(system())); - tester.setEndpoints(app.testerId().id(), JobType.systemTest.zone(system())); tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); tester.runner().run(); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); @@ -260,15 +255,27 @@ public class InternalStepRunnerTest { @Test public void alternativeEndpointsAreDetected() { - app.newRun(JobType.systemTest); - tester.runner().run();; - tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); + var systemTestZone = JobType.systemTest.zone(system()); + var stagingZone = JobType.stagingTest.zone(system()); + tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(systemTestZone), ZoneApiMock.from(stagingZone)); + var applicationPackage = new ApplicationPackageBuilder() + .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) + .upgradePolicy("default") + .region("us-central-1") + .parallel("us-west-1", "us-east-3") + .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION) + .build(); + app.submit(applicationPackage) + .triggerJobs(); + + tester.runner().run(); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); - app.addRoutingPolicy(JobType.systemTest.zone(system()), true); - app.addTesterRoutingPolicy(JobType.systemTest.zone(system()), true); - tester.runner().run();; + + app.flushDnsUpdates(); + tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); + tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); + tester.runner().run(); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); } @@ -314,15 +321,13 @@ public class InternalStepRunnerTest { RunId id = app.startSystemTestTests(); tester.runner().run(); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); - assertEquals(URI.create(tester.routing().endpoints(new DeploymentId(app.testerId().id(), JobType.systemTest.zone(system()))).get(0).endpoint()), - tester.cloud().testerUrl()); + var testZone = JobType.systemTest.zone(system()); Inspector configObject = SlimeUtils.jsonToSlime(tester.cloud().config()).get(); assertEquals(app.instanceId().serializedForm(), configObject.field("application").asString()); - assertEquals(JobType.systemTest.zone(system()).value(), configObject.field("zone").asString()); + assertEquals(testZone.value(), configObject.field("zone").asString()); assertEquals(system().value(), configObject.field("system").asString()); - assertEquals(1, configObject.field("endpoints").children()); - assertEquals(1, configObject.field("endpoints").field(JobType.systemTest.zone(system()).value()).entries()); - configObject.field("endpoints").field(JobType.systemTest.zone(system()).value()).traverse((ArrayTraverser) (__, endpoint) -> assertEquals(tester.routing().endpoints(new DeploymentId(instanceId, JobType.systemTest.zone(system()))).get(0).endpoint(), endpoint.asString())); + assertEquals(1, configObject.field("zoneEndpoints").children()); + assertEquals(1, configObject.field("zoneEndpoints").field(testZone.value()).children()); long lastId = tester.jobs().details(id).get().lastId().getAsLong(); tester.cloud().add(new LogEntry(0, Instant.ofEpochMilli(123), info, "Ready!")); @@ -368,7 +373,6 @@ public class InternalStepRunnerTest { tester.runner().run(); // Job run order determined by JobType enum order per application. tester.configServer().convergeServices(app.instanceId(), zone); - tester.setEndpoints(app.instanceId(), zone); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal)); assertEquals(applicationPackage.hash(), tester.configServer().application(app.instanceId(), zone).get().applicationPackage().hash()); assertEquals(otherPackage.hash(), tester.configServer().application(app.instanceId(), JobType.perfUsEast3.zone(system())).get().applicationPackage().hash()); @@ -377,12 +381,6 @@ public class InternalStepRunnerTest { tester.runner().run(); assertEquals(1, tester.jobs().active().size()); assertEquals(version, tester.instance(app.instanceId()).deployments().get(zone).version()); - - try { - tester.jobs().deploy(app.instanceId(), JobType.productionApNortheast1, Optional.empty(), applicationPackage); - fail("Deployments outside dev should not be allowed."); - } - catch (IllegalArgumentException expected) { } } @Test @@ -424,10 +422,13 @@ public class InternalStepRunnerTest { @Test public void certificateTimeoutAbortsJob() { - tester.controllerTester().zoneRegistry().setSystemName(SystemName.PublicCd); - tester.controllerTester().zoneRegistry().setZones(ZoneApiMock.fromId("test.aws-us-east-1c"), - ZoneApiMock.fromId("staging.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-us-east-1c")); + tester.controllerTester().zoneRegistry().setSystemName(SystemName.Public); + var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), + ZoneApiMock.fromId("staging.aws-us-east-1c"), + ZoneApiMock.fromId("prod.aws-us-east-1c")); + tester.controllerTester().zoneRegistry() + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.exclusive); tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.values()); RunId id = app.startSystemTestTests(); @@ -435,7 +436,7 @@ public class InternalStepRunnerTest { trusted.add(tester.jobs().run(id).get().testerCertificate().get()); assertEquals(trusted, tester.configServer().application(app.instanceId(), id.type().zone(system())).get().applicationPackage().trustedCertificates()); - tester.clock().advance(InternalStepRunner.certificateTimeout.plus(Duration.ofSeconds(1))); + tester.clock().advance(InternalStepRunner.Timeouts.of(system()).testerCertificate().plus(Duration.ofSeconds(1))); tester.runner().run(); assertEquals(RunStatus.aborted, tester.jobs().run(id).get().status()); } @@ -451,11 +452,45 @@ public class InternalStepRunnerTest { "3554970337.947845\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)"; @Test + public void generates_correct_tester_flavor() { + DeploymentSpec spec = DeploymentSpec.fromXml("<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" + + " <instance id='first'>\n" + + " <test tester-flavor=\"d-6-16-100\" />\n" + + " <prod>\n" + + " <region active=\"true\">us-west-1</region>\n" + + " <test>us-west-1</test>\n" + + " </prod>\n" + + " </instance>\n" + + " <instance id='second'>\n" + + " <test />\n" + + " <staging />\n" + + " <prod tester-flavor=\"d-6-16-100\">\n" + + " <parallel>\n" + + " <region active=\"true\">us-east-3</region>\n" + + " <region active=\"true\">us-central-1</region>\n" + + " </parallel>\n" + + " <region active=\"true\">us-west-1</region>\n" + + " <test>us-west-1</test>\n" + + " </prod>\n" + + " </instance>\n" + + "</deployment>\n"); + + NodeResources firstResources = InternalStepRunner.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("first")); + assertEquals(InternalStepRunner.DEFAULT_TESTER_RESOURCES, firstResources); + + NodeResources secondResources = InternalStepRunner.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("second")); + assertEquals(6, secondResources.vcpu(), 1e-9); + assertEquals(16, secondResources.memoryGb(), 1e-9); + assertEquals(100, secondResources.diskGb(), 1e-9); + } + + @Test public void generates_correct_services_xml_test() { - assertFile("test_runner_services.xml-cd", new String(InternalStepRunner.servicesXml(AthenzDomain.from("vespa.vespa.cd"), - true, - false, - new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local)))); + assertFile("test_runner_services.xml-cd", + new String(InternalStepRunner.servicesXml( + true, + false, + new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local)))); } private void assertFile(String resourceName, String actualContent) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java index ce22e0772ff..6a12c4457db 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java @@ -1,11 +1,14 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import org.junit.Test; import java.io.IOException; @@ -29,8 +32,10 @@ public class TestConfigSerializerTest { byte[] json = new TestConfigSerializer(SystemName.PublicCd).configJson(instanceId, JobType.systemTest, true, - Map.of(zone, Map.of(ClusterSpec.Id.from("ai"), - URI.create("https://server/"))), + Map.of(zone, List.of(Endpoint.of(ApplicationId.defaultId()) + .named(EndpointId.of("ai")) + .on(Endpoint.Port.tls()) + .in(SystemName.main))), Map.of(zone, List.of("facts"))); byte[] expected = Files.readAllBytes(Paths.get("src/test/resources/testConfig.json")); assertEquals(new String(SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlime(expected))), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java index a7b9cbd1e7e..23d21d2b51a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java @@ -55,7 +55,7 @@ public class ArtifactRepositoryMock extends AbstractComponent implements Artifac return Objects.hash(applicationId, applicationVersion); } - private class Artifact { + private static class Artifact { private final byte[] data; private int hits = 0; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 66c7334dc2d..9e9d27fd744 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -5,20 +5,23 @@ import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; @@ -44,10 +47,12 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -68,8 +73,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final Map<DeploymentId, ServiceConvergence> serviceStatus = new HashMap<>(); private final Set<ApplicationId> disallowConvergenceCheckApplications = new HashSet<>(); private final Version initialVersion = new Version(6, 1, 0); + private final DockerImage initialDockerImage = DockerImage.fromString("dockerImage:6.1.0"); private final Set<DeploymentId> suspendedApplications = new HashSet<>(); - private final Map<ZoneId, List<LoadBalancer>> loadBalancers = new HashMap<>(); + private final Map<ZoneId, Set<LoadBalancer>> loadBalancers = new HashMap<>(); + private final Set<Environment> deferLoadBalancerProvisioning = new HashSet<>(); private final Map<DeploymentId, List<Log>> warnings = new HashMap<>(); private final Map<DeploymentId, Set<String>> rotationNames = new HashMap<>(); private final Map<DeploymentId, List<ClusterMetrics>> clusterMetrics = new HashMap<>(); @@ -100,6 +107,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer .parentHostname(parent.hostname()) .currentVersion(initialVersion) .wantedVersion(initialVersion) + .currentDockerImage(initialDockerImage) + .wantedDockerImage(initialDockerImage) .currentOsVersion(Version.emptyVersion) .wantedOsVersion(Version.emptyVersion) .resources(new NodeResources(2, 8, 50, 1, slow, remote)) @@ -243,6 +252,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer this.clusterMetrics.put(deployment, clusterMetrics); } + public void deferLoadBalancerProvisioningIn(Set<Environment> environments) { + deferLoadBalancerProvisioning.addAll(environments); + } + @Override public NodeRepositoryMock nodeRepository() { return nodeRepository; @@ -260,8 +273,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer disallowConvergenceCheckApplications.add(applicationId); } - private List<LoadBalancer> getLoadBalancers(ZoneId zone) { - return loadBalancers.getOrDefault(zone, Collections.emptyList()); + private Set<LoadBalancer> getLoadBalancers(ZoneId zone) { + return loadBalancers.getOrDefault(zone, new LinkedHashSet<>()); } @Override @@ -281,8 +294,24 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return TesterCloud.Status.SUCCESS; } - public void addLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) { - this.loadBalancers.putIfAbsent(zone, new ArrayList<>()); + @Override + public String startTests(DeploymentId deployment, TesterCloud.Suite suite, byte[] config) { + return "Tests started"; + } + + @Override + public List<LogEntry> getTesterLog(DeploymentId deployment, long after) { + return List.of(); + } + + @Override + public boolean isTesterReady(DeploymentId deployment) { + return false; + } + + /** Add any of given loadBalancers that do not already exist to the load balancers in zone */ + public void putLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) { + this.loadBalancers.putIfAbsent(zone, new LinkedHashSet<>()); this.loadBalancers.get(zone).addAll(loadBalancers); } @@ -291,43 +320,51 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, - Set<ContainerEndpoint> containerEndpoints, - Optional<EndpointCertificateMetadata> endpointCertificateMetadata, byte[] content) { - lastPrepareVersion = deployOptions.vespaVersion.map(Version::fromString).orElse(null); + public PreparedApplication deploy(DeploymentData deployment) { + lastPrepareVersion = deployment.platform(); if (prepareException != null) { RuntimeException prepareException = this.prepareException; this.prepareException = null; throw prepareException; } - applications.put(deployment, new Application(deployment.applicationId(), lastPrepareVersion, new ApplicationPackage(content))); + DeploymentId id = new DeploymentId(deployment.instance(), deployment.zone()); + applications.put(id, new Application(id.applicationId(), lastPrepareVersion, new ApplicationPackage(deployment.applicationPackage()))); - if (nodeRepository().list(deployment.zoneId(), deployment.applicationId()).isEmpty()) - provision(deployment.zoneId(), deployment.applicationId()); + if (nodeRepository().list(id.zoneId(), id.applicationId()).isEmpty()) + provision(id.zoneId(), id.applicationId()); this.rotationNames.put( - deployment, - containerEndpoints.stream() - .map(ContainerEndpoint::names) - .flatMap(Collection::stream) - .collect(Collectors.toSet()) + id, + deployment.containerEndpoints().stream() + .map(ContainerEndpoint::names) + .flatMap(Collection::stream) + .collect(Collectors.toSet()) ); + if (!deferLoadBalancerProvisioning.contains(id.zoneId().environment())) { + putLoadBalancers(id.zoneId(), List.of(new LoadBalancer(UUID.randomUUID().toString(), + id.applicationId(), + ClusterSpec.Id.from("default"), + HostName.from("lb-0--" + id.applicationId().serializedForm() + "--" + id.zoneId().toString()), + LoadBalancer.State.active, + Optional.of("dns-zone-1")))); + } + return () -> { - Application application = applications.get(deployment); + Application application = applications.get(id); application.activate(); - List<Node> nodes = nodeRepository.list(deployment.zoneId(), deployment.applicationId()); + List<Node> nodes = nodeRepository.list(id.zoneId(), id.applicationId()); for (Node node : nodes) { - nodeRepository.putByHostname(deployment.zoneId(), new Node.Builder(node) + nodeRepository.putByHostname(id.zoneId(), new Node.Builder(node) .state(Node.State.active) .wantedVersion(application.version().get()) .build()); } - serviceStatus.put(deployment, new ServiceConvergence(deployment.applicationId(), - deployment.zoneId(), - false, - 2, - nodes.stream() + serviceStatus.put(id, new ServiceConvergence(id.applicationId(), + id.zoneId(), + false, + 2, + nodes.stream() .map(node -> new ServiceConvergence.Status(node.hostname(), 43, "container", @@ -342,7 +379,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer Collections.emptyList()); setConfigChangeActions(null); prepareResponse.tenant = new TenantId("tenant"); - prepareResponse.log = warnings.getOrDefault(deployment, Collections.emptyList()); + prepareResponse.log = warnings.getOrDefault(id, Collections.emptyList()); return prepareResponse; }; } @@ -366,6 +403,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer throw new NotFoundException("No application with id " + applicationId + " exists, cannot deactivate"); applications.remove(deployment); serviceStatus.remove(deployment); + removeLoadBalancers(deployment.applicationId(), deployment.zoneId()); } // Returns a canned example response @@ -401,8 +439,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer // Returns a canned example response @Override - public Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, - String environment, String region, String serviceName, String restPath) { + public Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath) { Map<String,List<?>> root = new HashMap<>(); List<Map<?,?>> resources = new ArrayList<>(); Map<String,String> resource = new HashMap<>(); @@ -413,6 +450,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override + public String getClusterControllerStatus(DeploymentId deployment, String restPath) { + return "<h1>OK</h1>"; + } + + @Override public void setGlobalRotationStatus(DeploymentId deployment, String upstreamName, EndpointStatus status) { endpoints.put(upstreamName, status); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsMock.java index 3dd6689dfdf..e2f598340be 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsMock.java @@ -21,7 +21,8 @@ public class MetricsMock implements Metric { @Override public void set(String key, Number val, Context ctx) { - Map<String, Number> metricsMap = metrics.getOrDefault(ctx, new HashMap<>()); + metrics.putIfAbsent(ctx, new HashMap<>()); + Map<String, Number> metricsMap = metrics.get(ctx); metricsMap.put(key, val); } @@ -43,10 +44,6 @@ public class MetricsMock implements Metric { return ctx; } - public Map<Context, Map<String, Number>> getMetrics() { - return metrics; - } - /** Returns a zero-context metric by name, or null if it is not present */ public Number getMetric(String name) { Map<String, Number> valuesForEmptyContext = metrics.get(createContext(Collections.emptyMap())); @@ -64,9 +61,8 @@ public class MetricsMock implements Metric { /** Returns metric filtered by dimension and name */ public Optional<Number> getMetric(Predicate<Map<String, String>> dimensionMatcher, String name) { - Map<String, Number> metrics = getMetrics(dimensionMatcher).entrySet() + Map<String, Number> metrics = getMetrics(dimensionMatcher).values() .stream() - .map(Map.Entry::getValue) .findFirst() .orElseGet(Collections::emptyMap); return Optional.ofNullable(metrics.get(name)); @@ -81,13 +77,16 @@ public class MetricsMock implements Metric { } @Override - public boolean equals(Object obj) { - return Objects.deepEquals(obj, dimensions); + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MapContext that = (MapContext) o; + return dimensions.equals(that.dimensions); } @Override public int hashCode() { - return Objects.toString(dimensions).hashCode(); + return Objects.hash(dimensions); } public Map<String, String> getDimensions() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 632b8499e11..4aab21a44fe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -208,7 +208,10 @@ public class NodeRepositoryMock implements NodeRepository { public void doUpgrade(DeploymentId deployment, Optional<HostName> hostName, Version version) { modifyNodes(deployment, hostName, node -> { assert node.wantedVersion().equals(version); - return new Node.Builder(node).currentVersion(version).build(); + return new Node.Builder(node) + .currentVersion(version) + .currentDockerImage(node.wantedDockerImage()) + .build(); }); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java index a14b7b82d67..c088965c2ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java @@ -5,6 +5,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.secretstore.SecretStore; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -45,4 +46,8 @@ public class SecretStoreMock extends AbstractComponent implements SecretStore { return secrets.getOrDefault(key, new TreeMap<>()).get(version); } + @Override + public List<Integer> listSecretVersions(String key) { + return List.copyOf(secrets.getOrDefault(key, new TreeMap<>()).keySet()); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index bd82807342e..98178f2a19f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.google.inject.Inject; @@ -10,7 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; -import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateMock; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService; @@ -21,8 +21,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportCons import com.yahoo.vespa.hosted.controller.api.integration.resource.MockTenantCost; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; @@ -42,9 +40,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final ConfigServerMock configServerMock; private final MemoryNameService memoryNameService = new MemoryNameService(); private final MemoryGlobalRoutingService memoryGlobalRoutingService = new MemoryGlobalRoutingService(); - private final RoutingGeneratorMock routingGeneratorMock = new RoutingGeneratorMock(RoutingGeneratorMock.TEST_ENDPOINTS); private final MockMailer mockMailer = new MockMailer(); - private final ApplicationCertificateMock applicationCertificateMock = new ApplicationCertificateMock(); + private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(); private final MockMeteringClient mockMeteringClient = new MockMeteringClient(); private final MockContactRetriever mockContactRetriever = new MockContactRetriever(); private final MockIssueHandler mockIssueHandler = new MockIssueHandler(); @@ -55,7 +52,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final MockBilling mockBilling = new MockBilling(); private final MockAwsEventFetcher mockAwsEventFetcher = new MockAwsEventFetcher(); private final ArtifactRepositoryMock artifactRepositoryMock = new ArtifactRepositoryMock(); - private final MockTesterCloud mockTesterCloud = new MockTesterCloud(); + private final MockTesterCloud mockTesterCloud; private final ApplicationStoreMock applicationStoreMock = new ApplicationStoreMock(); private final MockRunDataStore mockRunDataStore = new MockRunDataStore(); private final MockTenantCost mockTenantCost = new MockTenantCost(); @@ -64,6 +61,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); this.configServerMock = new ConfigServerMock(zoneRegistryMock); + this.mockTesterCloud = new MockTesterCloud(nameService()); } @Inject @@ -91,18 +89,13 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override - public RoutingGenerator routingGenerator() { - return routingGeneratorMock; - } - - @Override public MockMailer mailer() { return mockMailer; } @Override - public ApplicationCertificateMock applicationCertificateProvider() { - return applicationCertificateMock; + public EndpointCertificateMock endpointCertificateProvider() { + return endpointCertificateMock; } @Override @@ -192,28 +185,16 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return configServerMock; } - public MemoryNameService nameServiceMock() { - return memoryNameService; - } - public MemoryGlobalRoutingService globalRoutingServiceMock() { return memoryGlobalRoutingService; } - public RoutingGeneratorMock routingGeneratorMock() { - return routingGeneratorMock; - } - public MockContactRetriever contactRetrieverMock() { return mockContactRetriever; } - public ArtifactRepositoryMock artifactRepositoryMock() { - return artifactRepositoryMock; - } - - public ApplicationCertificateMock applicationCertificateMock() { - return applicationCertificateMock; + public EndpointCertificateMock endpointCertificateMock() { + return endpointCertificateMock; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java index 269bdcc5dca..af0e3d5807d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java @@ -8,6 +8,8 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; +import java.util.Objects; + /** * @author hakonhall */ @@ -34,6 +36,10 @@ public class ZoneApiMock implements ZoneApi { return newBuilder().with(ZoneId.from(environment, region)).build(); } + public static ZoneApiMock from(ZoneId zone) { + return newBuilder().with(zone).build(); + } + @Override public SystemName getSystemName() { return systemName; } @@ -46,8 +52,21 @@ public class ZoneApiMock implements ZoneApi { @Override public String getCloudNativeRegionName() { return cloudNativeRegionName; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ZoneApiMock that = (ZoneApiMock) o; + return id.equals(that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + public static class Builder { - private SystemName systemName = SystemName.defaultSystem(); + private final SystemName systemName = SystemName.defaultSystem(); private ZoneId id = ZoneId.defaultId(); private CloudName cloudName = CloudName.defaultName(); private String cloudNativeRegionName = id.region().value(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java index 4cd50625ca3..23f59683f4b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java @@ -1,9 +1,10 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneFilter; import com.yahoo.config.provision.zone.ZoneId; @@ -11,6 +12,7 @@ import com.yahoo.config.provision.zone.ZoneList; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -23,22 +25,22 @@ import java.util.stream.Collectors; public class ZoneFilterMock implements ZoneList { private final List<ZoneApi> zones; - private final Set<ZoneApi> directlyRouted; + private final Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods; private final boolean negate; - private ZoneFilterMock(List<ZoneApi> zones, Set<ZoneApi> directlyRouted, boolean negate) { + private ZoneFilterMock(List<ZoneApi> zones, Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods, boolean negate) { this.zones = zones; - this.directlyRouted = directlyRouted; + this.zoneRoutingMethods = zoneRoutingMethods; this.negate = negate; } - public static ZoneFilter from(Collection<ZoneApi> zones, Set<ZoneApi> directlyRouted) { - return new ZoneFilterMock(List.copyOf(zones), Set.copyOf(directlyRouted), false); + public static ZoneFilter from(Collection<? extends ZoneApi> zones, Map<ZoneApi, List<RoutingMethod>> routingMethods) { + return new ZoneFilterMock(List.copyOf(zones), Map.copyOf(routingMethods), false); } @Override public ZoneList not() { - return new ZoneFilterMock(zones, directlyRouted, ! negate); + return new ZoneFilterMock(zones, zoneRoutingMethods, ! negate); } @Override @@ -53,7 +55,12 @@ public class ZoneFilterMock implements ZoneList { @Override public ZoneList directlyRouted() { - return filter(directlyRouted::contains); + return routingMethod(RoutingMethod.exclusive); + } + + @Override + public ZoneList routingMethod(RoutingMethod method) { + return filter(zone -> zoneRoutingMethods.getOrDefault(zone, List.of()).contains(method)); } @Override @@ -93,7 +100,7 @@ public class ZoneFilterMock implements ZoneList { condition.negate().test(zone) : condition.test(zone)) .collect(Collectors.toList()), - directlyRouted, false); + zoneRoutingMethods, false); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 6f8ff39456f..efc875b06f5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.component.AbstractComponent; @@ -9,6 +9,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneFilter; @@ -25,7 +26,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; /** * @author mpolden @@ -34,11 +34,12 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry private final Map<ZoneId, Duration> deploymentTimeToLive = new HashMap<>(); private final Map<Environment, RegionName> defaultRegionForEnvironment = new HashMap<>(); - private List<ZoneApi> zones = List.of(); + private final Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>(); + private final Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods = new HashMap<>(); + + private List<? extends ZoneApi> zones; private SystemName system; private UpgradePolicy upgradePolicy = null; - private Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>(); - private Set<ZoneApi> directlyRouted = Set.of(); /** * This sets the default list of zones contained in this. If your test need a particular set of zones, use @@ -46,21 +47,21 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry */ public ZoneRegistryMock(SystemName system) { this.system = system; - setZones(List.of( - ZoneApiMock.fromId("test.us-east-1"), - ZoneApiMock.fromId("staging.us-east-3"), - ZoneApiMock.fromId("dev.us-east-1"), - ZoneApiMock.fromId("dev.aws-us-east-2a"), - ZoneApiMock.fromId("perf.us-east-3"), - ZoneApiMock.fromId("prod.aws-us-east-1a"), - ZoneApiMock.fromId("prod.ap-northeast-1"), - ZoneApiMock.fromId("prod.ap-northeast-2"), - ZoneApiMock.fromId("prod.ap-southeast-1"), - ZoneApiMock.fromId("prod.us-east-3"), - ZoneApiMock.fromId("prod.us-west-1"), - ZoneApiMock.fromId("prod.us-central-1"), - ZoneApiMock.fromId("prod.eu-west-1"))); - setDirectlyRouted(Set.copyOf(this.zones)); + this.zones = List.of(ZoneApiMock.fromId("test.us-east-1"), + ZoneApiMock.fromId("staging.us-east-3"), + ZoneApiMock.fromId("dev.us-east-1"), + ZoneApiMock.fromId("dev.aws-us-east-2a"), + ZoneApiMock.fromId("perf.us-east-3"), + ZoneApiMock.fromId("prod.aws-us-east-1a"), + ZoneApiMock.fromId("prod.ap-northeast-1"), + ZoneApiMock.fromId("prod.ap-northeast-2"), + ZoneApiMock.fromId("prod.ap-southeast-1"), + ZoneApiMock.fromId("prod.us-east-3"), + ZoneApiMock.fromId("prod.us-west-1"), + ZoneApiMock.fromId("prod.us-central-1"), + ZoneApiMock.fromId("prod.eu-west-1")); + // All zones use a shared routing method by default + setRoutingMethod(this.zones, RoutingMethod.shared); } public ZoneRegistryMock setDeploymentTimeToLive(ZoneId zone, Duration duration) { @@ -73,7 +74,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return this; } - public ZoneRegistryMock setZones(List<ZoneApi> zones) { + public ZoneRegistryMock setZones(List<? extends ZoneApi> zones) { this.zones = zones; return this; } @@ -97,12 +98,28 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return this; } - public ZoneRegistryMock setDirectlyRouted(ZoneApi... zones) { - return setDirectlyRouted(Set.of(zones)); + public ZoneRegistryMock exclusiveRoutingIn(ZoneApi... zones) { + return exclusiveRoutingIn(List.of(zones)); + } + + public ZoneRegistryMock exclusiveRoutingIn(List<? extends ZoneApi> zones) { + return setRoutingMethod(zones, RoutingMethod.exclusive); } - public ZoneRegistryMock setDirectlyRouted(Set<ZoneApi> zones) { - directlyRouted = zones; + public ZoneRegistryMock setRoutingMethod(ZoneApi zone, RoutingMethod... routingMethods) { + return setRoutingMethod(zone, List.of(routingMethods)); + } + + public ZoneRegistryMock setRoutingMethod(List<? extends ZoneApi> zones, RoutingMethod... routingMethods) { + zones.forEach(zone -> setRoutingMethod(zone, List.of(routingMethods))); + return this; + } + + public ZoneRegistryMock setRoutingMethod(ZoneApi zone, List<RoutingMethod> routingMethods) { + if (routingMethods.stream().distinct().count() != routingMethods.size()) { + throw new IllegalArgumentException("Routing methods must be distinct"); + } + this.zoneRoutingMethods.put(zone, List.copyOf(routingMethods)); return this; } @@ -113,7 +130,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry @Override public ZoneFilter zones() { - return ZoneFilterMock.from(zones, directlyRouted); + return ZoneFilterMock.from(zones, zoneRoutingMethods); } @Override @@ -147,6 +164,11 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override + public List<RoutingMethod> routingMethods(ZoneId zone) { + return List.copyOf(zoneRoutingMethods.getOrDefault(ZoneApiMock.from(zone), List.of())); + } + + @Override public URI dashboardUrl() { return URI.create("https://dashboard.tld"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java index 8997f34fb98..299c81fdb3c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipI import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import org.junit.Before; import org.junit.Test; @@ -42,70 +41,69 @@ public class ApplicationOwnershipConfirmerTest { @Test public void testConfirmation() { Optional<Contact> contact = Optional.of(tester.controllerTester().serviceRegistry().contactRetrieverMock().contact()); - var propertyApp = tester.newDeploymentContext(); + var app = tester.newDeploymentContext(); tester.controller().tenants().lockOrThrow(appId.tenant(), LockedTenant.Athenz.class, tenant -> tester.controller().tenants().store(tenant.with(contact.get()))); - propertyApp.submit().deploy(); + app.submit().deploy(); - UserTenant user = UserTenant.create("by-user", contact); - tester.controller().tenants().createUser(user); - var userApp = tester.newDeploymentContext("by-user", "application", "default"); - userApp.submit().deploy(); + var appWithoutContact = tester.newDeploymentContext("other", "application", "default"); + appWithoutContact.submit().deploy(); - assertFalse("No issue is initially stored for a new application.", propertyApp.application().ownershipIssueId().isPresent()); - assertFalse("No issue is initially stored for a new application.", userApp.application().ownershipIssueId().isPresent()); - assertFalse("No escalation has been attempted for a new application", issues.escalatedToContact || issues.escalatedToTerminator); + assertFalse("No issue is initially stored for a new application.", app.application().ownershipIssueId().isPresent()); + assertFalse("No issue is initially stored for a new application.", appWithoutContact.application().ownershipIssueId().isPresent()); + assertFalse("No escalation has been attempted for a new application", issues.escalated); // Set response from the issue mock, which will be obtained by the maintainer on issue filing. Optional<IssueId> issueId = Optional.of(IssueId.from("1")); issues.response = issueId; confirmer.maintain(); - assertFalse("No issue is stored for an application newer than 3 months.", propertyApp.application().ownershipIssueId().isPresent()); - assertFalse("No issue is stored for an application newer than 3 months.", userApp.application().ownershipIssueId().isPresent()); + assertFalse("No issue is stored for an application newer than 3 months.", app.application().ownershipIssueId().isPresent()); + assertFalse("No issue is stored for an application newer than 3 months.", appWithoutContact.application().ownershipIssueId().isPresent()); tester.clock().advance(Duration.ofDays(91)); confirmer.maintain(); - assertEquals("Confirmation issue has been filed for property owned application.", issueId, propertyApp.application().ownershipIssueId()); - assertEquals("Confirmation issue has been filed for user owned application.", issueId, userApp.application().ownershipIssueId()); - assertTrue(issues.escalatedToTerminator); - assertTrue("Both applications have had their responses ensured.", issues.escalatedToContact); + assertEquals("Confirmation issue has been filed for application with contact.", issueId, app.application().ownershipIssueId()); + assertTrue("The confirmation issue response has been ensured.", issues.escalated); + assertEquals("No confirmation issue has been filed for application without contact.", Optional.empty(), appWithoutContact.application().ownershipIssueId()); // No new issue is created, so return empty now. issues.response = Optional.empty(); confirmer.maintain(); - assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, propertyApp.application().ownershipIssueId()); - assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, userApp.application().ownershipIssueId()); - - // The user deletes all production deployments — see that the issue is forgotten. - assertEquals("Confirmation issue for user is still open.", issueId, userApp.application().ownershipIssueId()); - userApp.application().productionDeployments().values().stream().flatMap(List::stream) - .forEach(deployment -> tester.controller().applications().deactivate(userApp.instanceId(), deployment.zone())); - assertTrue("No production deployments are listed for user.", userApp.application().require(InstanceName.defaultName()).productionDeployments().isEmpty()); - confirmer.maintain(); + assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, app.application().ownershipIssueId()); // Time has passed, and a new confirmation issue is in order for the property which is still in production. Optional<IssueId> issueId2 = Optional.of(IssueId.from("2")); issues.response = issueId2; confirmer.maintain(); - assertEquals("A new confirmation issue id is stored when something is returned to the maintainer.", issueId2, propertyApp.application().ownershipIssueId()); - assertEquals("Confirmation issue for application without production deployments has not been filed.", issueId, userApp.application().ownershipIssueId()); + assertEquals("A new confirmation issue id is stored when something is returned to the maintainer.", issueId2, app.application().ownershipIssueId()); - assertFalse("No owner is stored for application", propertyApp.application().owner().isPresent()); + assertFalse("No owner is stored for application", app.application().owner().isPresent()); issues.owner = Optional.of(User.from("username")); confirmer.maintain(); - assertEquals("Owner has been added to application", propertyApp.application().owner().get().username(), "username"); + assertEquals("Owner has been added to application", app.application().owner().get().username(), "username"); + + // The app deletes all production deployments — see that the issue is forgotten. + assertEquals("Confirmation issue for application is still open.", issueId2, app.application().ownershipIssueId()); + app.application().productionDeployments().values().stream().flatMap(List::stream) + .forEach(deployment -> tester.controller().applications().deactivate(app.instanceId(), deployment.zone())); + assertTrue("No production deployments are listed for user.", app.application().require(InstanceName.defaultName()).productionDeployments().isEmpty()); + confirmer.maintain(); + + // Time has passed, and a new confirmation issue is in order for the property which is still in production. + issues.response = Optional.of(IssueId.from("3")); + confirmer.maintain(); + assertEquals("Confirmation issue for application without production deployments has not been filed.", issueId2, app.application().ownershipIssueId()); } - private class MockOwnershipIssues implements OwnershipIssues { + private static class MockOwnershipIssues implements OwnershipIssues { private Optional<IssueId> response; - private boolean escalatedToContact = false; - private boolean escalatedToTerminator = false; + private boolean escalated = false; private Optional<User> owner = Optional.empty(); @Override @@ -115,8 +113,7 @@ public class ApplicationOwnershipConfirmerTest { @Override public void ensureResponse(IssueId issueId, Optional<Contact> contact) { - if (contact.isPresent()) escalatedToContact = true; - else escalatedToTerminator = true; + escalated = true; } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java index cd2a4fd8453..1c66a01db8e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java @@ -27,10 +27,10 @@ import static org.junit.Assert.*; */ public class CloudEventReporterTest { - private ControllerTester tester = new ControllerTester(); - private ZoneApiMock nonAwsZone = createZone("prod.zone3", "region-1", "other"); - private ZoneApiMock awsZone1 = createZone("prod.zone1", "region-1", "aws"); - private ZoneApiMock awsZone2 = createZone("prod.zone2", "region-2", "aws"); + private final ControllerTester tester = new ControllerTester(); + private final ZoneApiMock nonAwsZone = createZone("prod.zone3", "region-1", "other"); + private final ZoneApiMock awsZone1 = createZone("prod.zone1", "region-1", "aws"); + private final ZoneApiMock awsZone2 = createZone("prod.zone2", "region-2", "aws"); /** @@ -153,4 +153,4 @@ public class CloudEventReporterTest { .build(); } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java deleted file mode 100644 index ff72a2f7231..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import org.junit.Test; - -import java.time.Duration; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author smorgrav - */ -public class ClusterInfoMaintainerTest { - - private final ControllerTester tester = new ControllerTester(); - - @Test - public void maintain() { - tester.createTenant("tenant1", "domain123", 321L); - ApplicationId app = tester.createApplication("tenant1", "app1", "default").id().defaultInstance(); - ZoneId zone = ZoneId.from("dev", "us-east-1"); - tester.deploy(app, zone); - - // Precondition: no cluster info attached to the deployments - Deployment deployment = tester.controller().applications().getInstance(app).get().deployments().values().stream() - .findFirst() - .get(); - assertEquals(0, deployment.clusterInfo().size()); - - addNodes(zone); - ClusterInfoMaintainer maintainer = new ClusterInfoMaintainer(tester.controller(), Duration.ofHours(1), - new JobControl(new MockCuratorDb())); - maintainer.maintain(); - - deployment = tester.controller().applications().getInstance(app).get().deployments().values().stream() - .findFirst() - .get(); - assertEquals(2, deployment.clusterInfo().size()); - assertEquals(10, deployment.clusterInfo().get(ClusterSpec.Id.from("clusterA")).getFlavorCost()); - } - - private void addNodes(ZoneId zone) { - var nodeA = new Node.Builder() - .hostname(HostName.from("hostA")) - .parentHostname(HostName.from("parentHostA")) - .state(Node.State.active) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(1, 1, 1, 1)) - .cost(10) - .clusterId("clusterA") - .clusterType(Node.ClusterType.container) - .build(); - var nodeB = new Node.Builder() - .hostname(HostName.from("hostB")) - .parentHostname(HostName.from("parentHostB")) - .state(Node.State.active) - .type(NodeType.tenant) - .owner(ApplicationId.from("tenant1", "app1", "default")) - .currentVersion(Version.fromString("7.42")) - .wantedVersion(Version.fromString("7.42")) - .currentOsVersion(Version.fromString("7.6")) - .wantedOsVersion(Version.fromString("7.6")) - .serviceState(Node.ServiceState.expectedUp) - .resources(new NodeResources(1, 1, 1, 1)) - .cost(20) - .clusterId("clusterB") - .clusterType(Node.ClusterType.container) - .build(); - tester.configServer().nodeRepository().addNodes(zone, List.of(nodeA, nodeB)); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java index 0db718080c2..32d2e2f35f4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java @@ -37,22 +37,22 @@ public class ContactInformationMaintainerTest { @Test public void updates_contact_information() { - long propertyId = 1; - TenantName name = tester.createTenant("tenant1", "domain1", propertyId); - Supplier<AthenzTenant> tenant = () -> (AthenzTenant) tester.controller().tenants().require(name); - assertFalse("No contact information initially", tenant.get().contact().isPresent()); + PropertyId propertyId1 = new PropertyId("1"); + PropertyId propertyId2 = new PropertyId("2"); + TenantName name1 = tester.createTenant("tenant1", "domain1", 1L); + TenantName name2 = tester.createTenant("zenant1", "domain2", 2L); + Supplier<AthenzTenant> tenant1 = () -> (AthenzTenant) tester.controller().tenants().require(name1); + Supplier<AthenzTenant> tenant2 = () -> (AthenzTenant) tester.controller().tenants().require(name2); + assertFalse("No contact information initially", tenant1.get().contact().isPresent()); + assertFalse("No contact information initially", tenant2.get().contact().isPresent()); Contact contact = testContact(); - registerContact(propertyId, contact); - maintainer.run(); + tester.serviceRegistry().contactRetriever().addContact(propertyId1, () -> { throw new RuntimeException("ERROR"); }); + tester.serviceRegistry().contactRetriever().addContact(propertyId2, () -> contact); + maintainer.maintain(); - assertTrue("Contact information added", tenant.get().contact().isPresent()); - assertEquals(contact, tenant.get().contact().get()); - } - - private void registerContact(long propertyId, Contact contact) { - PropertyId p = new PropertyId(String.valueOf(propertyId)); - tester.serviceRegistry().contactRetrieverMock().addContact(p, contact); + assertEquals("No contact information added due to error", Optional.empty(), tenant1.get().contact()); + assertEquals("Contact information added", Optional.of(contact), tenant2.get().contact()); } private static Contact testContact() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index 258dad3b6d6..bef04b338e6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java index ea946e132a9..ee5639dbfec 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; @@ -13,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java index ed8918786c5..931f22dd7f9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -40,7 +39,6 @@ public class DeploymentMetricsMaintainerTest { DeploymentMetricsMaintainer maintainer = maintainer(tester.controller()); Supplier<Application> app = application::application; - Supplier<Instance> instance = application::instance; Supplier<Deployment> deployment = () -> application.deployment(ZoneId.from("dev", "us-east-1")); // No metrics gathered yet diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 9de0020ce4a..d2946651619 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -62,6 +62,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -98,7 +99,7 @@ public class JobRunnerTest { jobs.start(id, systemTest, versions); fail("Job is already running, so this should not be allowed!"); } - catch (IllegalStateException e) { } + catch (IllegalStateException ignored) { } jobs.start(id, stagingTest, versions); assertTrue(jobs.last(id, systemTest).get().stepStatuses().values().stream().allMatch(unfinished::equals)); @@ -184,7 +185,7 @@ public class JobRunnerTest { runner.maintain(); assertTrue(run.get().hasFailed()); assertTrue(run.get().hasEnded()); - assertTrue(run.get().status() == aborted); + assertSame(aborted, run.get().status()); // A new run is attempted. jobs.start(id, systemTest, versions); @@ -195,7 +196,7 @@ public class JobRunnerTest { runner.maintain(); assertTrue(run.get().hasEnded()); assertTrue(run.get().hasFailed()); - assertFalse(run.get().status() == aborted); + assertNotSame(aborted, run.get().status()); assertEquals(failed, run.get().stepStatuses().get(deployTester)); assertEquals(unfinished, run.get().stepStatuses().get(installTester)); assertEquals(succeeded, run.get().stepStatuses().get(report)); @@ -241,7 +242,7 @@ public class JobRunnerTest { jobs.locked(id, systemTest, deactivateTester, step -> { }); fail("deployTester step should still be locked!"); } - catch (TimeoutException e) { } + catch (TimeoutException ignored) { } // Thread is still trying to deploy tester -- delete application, and see all data is garbage collected. assertEquals(Collections.singletonList(runId), jobs.active().stream().map(run -> run.id()).collect(Collectors.toList())); @@ -338,6 +339,25 @@ public class JobRunnerTest { } @Test + public void onlySuccessfulRunExpiresThenAnotherFails() { + DeploymentTester tester = new DeploymentTester(); + JobController jobs = tester.controller().jobController(); + var app = tester.newDeploymentContext().submit(); + JobId jobId = new JobId(app.instanceId(), systemTest); + assertFalse(jobs.lastSuccess(jobId).isPresent()); + + app.runJob(systemTest); + assertTrue(jobs.lastSuccess(jobId).isPresent()); + assertEquals(1, jobs.runs(jobId).size()); + + tester.clock().advance(JobController.maxHistoryAge.plusSeconds(1)); + app.submit(); + app.failDeployment(systemTest); + assertFalse(jobs.lastSuccess(jobId).isPresent()); + assertEquals(1, jobs.runs(jobId).size()); + } + + @Test public void timeout() { DeploymentTester tester = new DeploymentTester(); JobController jobs = tester.controller().jobController(); @@ -375,12 +395,11 @@ public class JobRunnerTest { jobs.finish(jobs.last(id, systemTest).get().id()); } - Map<String, String> context = Map.of("tenant", "tenant", - "application", "real", - "instance", "default", - "job", "system-test", - "environment", "test", - "region", "us-east-1"); + Map<String, String> context = Map.of("applicationId", "tenant.real.default", + "tenantName", "tenant", + "app", "real.default", + "test", "true", + "zone", "test.us-east-1"); MetricsMock metric = ((MetricsMock) tester.controller().metric()); assertEquals(RunStatus.values().length - 1, metric.getMetric(context::equals, JobMetrics.start).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.abort).get().intValue()); @@ -389,12 +408,13 @@ public class JobRunnerTest { assertEquals(1, metric.getMetric(context::equals, JobMetrics.convergenceFailure).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.deploymentFailure).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.outOfCapacity).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.endpointCertificateTimeout).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.testFailure).get().intValue()); } public static ExecutorService inThreadExecutor() { return new AbstractExecutorService() { - AtomicBoolean shutDown = new AtomicBoolean(false); + final AtomicBoolean shutDown = new AtomicBoolean(false); @Override public void shutdown() { shutDown.set(true); } @Override public List<Runnable> shutdownNow() { shutDown.set(true); return Collections.emptyList(); } @Override public boolean isShutdown() { return shutDown.get(); } @@ -406,7 +426,7 @@ public class JobRunnerTest { private static ExecutorService phasedExecutor(Phaser phaser) { return new AbstractExecutorService() { - ExecutorService delegate = Executors.newFixedThreadPool(32); + final ExecutorService delegate = Executors.newFixedThreadPool(32); @Override public void shutdown() { delegate.shutdown(); } @Override public List<Runnable> shutdownNow() { return delegate.shutdownNow(); } @Override public boolean isShutdown() { return delegate.isShutdown(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java index 3aa1f2b5af2..efd5d61ce56 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java @@ -38,17 +38,22 @@ public class MaintainerTest { @Test public void staggering() { List<HostName> cluster = List.of(HostName.from("cfg1"), HostName.from("cfg2"), HostName.from("cfg3")); - Instant now = Instant.ofEpochMilli(1001); Duration interval = Duration.ofMillis(300); - assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval)); - assertEquals(399, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval)); - assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval)); + Instant now = Instant.ofEpochMilli(1000); + assertEquals(200, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval)); + assertEquals( 0, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval)); + assertEquals(100, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval)); - now = Instant.ofEpochMilli(1101); + now = Instant.ofEpochMilli(1001); assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval)); assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval)); - assertEquals(399, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval)); + assertEquals( 99, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval)); + + now = Instant.ofEpochMilli(1101); + assertEquals( 99, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval)); + assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval)); + assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval)); assertEquals(300, Maintainer.staggeredDelay(cluster, HostName.from("cfg0"), now, interval)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index e0fb2aa0ee1..c00705149e9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -256,7 +256,7 @@ public class MetricsReporterTest { tester.configServer().nodeRepository().list(zone1.getId(), SystemApplication.configServer.id()).stream() .map(Node::wantedVersion).min(Comparator.naturalOrder()).get()); tester.configServer().setVersion(SystemApplication.configServer.id(), zone1.getId(), version, 1); - tester.clock().advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); + tester.clock().advance(Duration.ofMinutes(60).plus(Duration.ofSeconds(1))); tester.computeVersionStatus(); reporter.maintain(); assertEquals(2, getNodesFailingUpgrade()); @@ -278,7 +278,7 @@ public class MetricsReporterTest { var cloud = CloudName.defaultName(); tester.zoneRegistry().setOsUpgradePolicy(cloud, UpgradePolicy.create().upgrade(zone)); var osUpgrader = new OsUpgrader(tester.controller(), Duration.ofDays(1), - new JobControl(tester.curator()), CloudName.defaultName());; + new JobControl(tester.curator()), CloudName.defaultName()); var statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator())); tester.configServer().bootstrap(List.of(zone.getId()), SystemApplication.configServerHost, SystemApplication.tenantHost); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 442a2fd1853..314901d5e4b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -2,9 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -17,6 +15,7 @@ import org.junit.Test; import java.time.Duration; import java.util.List; +import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -47,7 +46,7 @@ public class OutstandingChangeDeployerTest { assertFalse(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets()); assertEquals(1, app1.application().latestVersion().get().buildNumber().getAsLong()); - app1.submit(applicationPackage, new SourceRevision("repository1", "master", "cafed00d")); + app1.submit(applicationPackage, Optional.of(new SourceRevision("repository1", "master", "cafed00d"))); assertTrue(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets()); assertEquals("1.0.2-cafed00d", app1.deploymentStatus().outstandingChange(app1.instance().name()).application().get().id()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index 26012443e3e..e0a97d32597 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -28,9 +28,8 @@ import static org.junit.Assert.assertEquals; public class ResourceMeterMaintainerTest { private final ControllerTester tester = new ControllerTester(); - private final double DELTA = Double.MIN_VALUE; - private MockMeteringClient snapshotConsumer = new MockMeteringClient(); - private MetricsMock metrics = new MetricsMock(); + private final MockMeteringClient snapshotConsumer = new MockMeteringClient(); + private final MetricsMock metrics = new MetricsMock(); @Test public void testMaintainer() { @@ -45,16 +44,16 @@ public class ResourceMeterMaintainerTest { ResourceSnapshot app1 = consumedResources.stream().filter(snapshot -> snapshot.getApplicationId().equals(ApplicationId.from("tenant1", "app1", "default"))).findFirst().orElseThrow(); ResourceSnapshot app2 = consumedResources.stream().filter(snapshot -> snapshot.getApplicationId().equals(ApplicationId.from("tenant2", "app2", "default"))).findFirst().orElseThrow(); - assertEquals(24, app1.getCpuCores(), DELTA); - assertEquals(24, app1.getMemoryGb(), DELTA); - assertEquals(500, app1.getDiskGb(), DELTA); + assertEquals(24, app1.getCpuCores(), Double.MIN_VALUE); + assertEquals(24, app1.getMemoryGb(), Double.MIN_VALUE); + assertEquals(500, app1.getDiskGb(), Double.MIN_VALUE); - assertEquals(40, app2.getCpuCores(), DELTA); - assertEquals(24, app2.getMemoryGb(), DELTA); - assertEquals(500, app2.getDiskGb(), DELTA); + assertEquals(40, app2.getCpuCores(), Double.MIN_VALUE); + assertEquals(24, app2.getMemoryGb(), Double.MIN_VALUE); + assertEquals(500, app2.getDiskGb(), Double.MIN_VALUE); assertEquals(tester.clock().millis()/1000, metrics.getMetric("metering_last_reported")); - assertEquals(2224.0d, (Double) metrics.getMetric("metering_total_reported"), DELTA); + assertEquals(2224.0d, (Double) metrics.getMetric("metering_total_reported"), Double.MIN_VALUE); } private void setUpZones() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java index 0a0aa7c9ded..653a4c5394b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java @@ -19,7 +19,7 @@ import static org.junit.Assert.*; */ public class ResourceTagMaintainerTest { - ControllerTester tester = new ControllerTester(); + final ControllerTester tester = new ControllerTester(); @Test public void maintain() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdaterTest.java index d7a28708049..77567a58ed2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdaterTest.java @@ -65,7 +65,7 @@ public class RotationStatusUpdaterTest { .region(zone1.region().value()) .region(zone2.region().value()) .region(zone3.region().value()) - .endpoint("default", "default", "us-east-3", "us-west-1") + .endpoint("default", "foo", "us-east-3", "us-west-1") .endpoint("eu", "default", "eu-west-1") .build(); context.submit(applicationPackage) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 249c2644170..78f198bf05f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -136,7 +136,7 @@ public class UpgraderTest { canary1.deployPlatform(version3); - tester.controllerTester().computeVersionStatus();; + tester.controllerTester().computeVersionStatus(); assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); tester.upgrader().maintain(); tester.triggerJobs(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json deleted file mode 100644 index 31949cce282..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "id": "tenant1:app1:default", - "deploymentSpecField": "<deployment version='1.0'>\n <test />\n <staging />\n <prod>\n <region active=\"true\">us-central-1</region>\n <region active=\"true\">us-west-1</region>\n </prod>\n</deployment>\n", - "validationOverrides": "<deployment version='1.0'/>", - "deployments": [], - "deploymentJobs": { - "jobStatus": [ - { - "jobType": "system-test", - "lastTriggered": { - "version": "6.42.1", - "upgrade": false, - "at": 1506330088050 - } - } - ] - }, - "outstandingChangeField": false -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index c4cb01f8164..a73445dfa5a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -5,10 +5,9 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyUtils; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; @@ -18,7 +17,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; @@ -36,7 +34,6 @@ import java.security.PublicKey; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -99,7 +96,6 @@ public class ApplicationSerializerTest { Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z"); deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), - createClusterInfo(3, 4), new DeploymentMetrics(2, 3, 4, 5, 6, Optional.of(Instant.now().truncatedTo(ChronoUnit.MILLIS)), Map.of(DeploymentMetrics.Warning.all, 3)), @@ -187,16 +183,6 @@ public class ApplicationSerializerTest { assertEquals(original.require(id1.instance()).change(), serialized.require(id1.instance()).change()); assertEquals(original.require(id3.instance()).change(), serialized.require(id3.instance()).change()); - // Test cluster info - assertEquals(3, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().size()); - assertEquals(10, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCost()); - assertEquals(ClusterSpec.Type.content, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getClusterType()); - assertEquals("flavor2", serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavor()); - assertEquals(4, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getHostnames().size()); - assertEquals(2, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCPU(), Double.MIN_VALUE); - assertEquals(4, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorMem(), Double.MIN_VALUE); - assertEquals(50, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorDisk(), Double.MIN_VALUE); - // Test metrics assertEquals(original.metrics().queryServiceQuality(), serialized.metrics().queryServiceQuality(), Double.MIN_VALUE); assertEquals(original.metrics().writeServiceQuality(), serialized.metrics().writeServiceQuality(), Double.MIN_VALUE); @@ -209,21 +195,6 @@ public class ApplicationSerializerTest { assertEquals(original.require(id1.instance()).deployments().get(zone2).metrics().warnings(), serialized.require(id1.instance()).deployments().get(zone2).metrics().warnings()); } - private Map<ClusterSpec.Id, ClusterInfo> createClusterInfo(int clusters, int hosts) { - Map<ClusterSpec.Id, ClusterInfo> result = new HashMap<>(); - - for (int cluster = 0; cluster < clusters; cluster++) { - List<String> hostnames = new ArrayList<>(); - for (int host = 0; host < hosts; host++) { - hostnames.add("hostname" + cluster*host + host); - } - - result.put(ClusterSpec.Id.from("id" + cluster), new ClusterInfo("flavor" + cluster, 10, - 2, 4, 50, ClusterSpec.Type.content, hostnames)); - } - return result; - } - @Test public void testCompleteApplicationDeserialization() throws Exception { byte[] applicationJson = Files.readAllBytes(testData.resolve("complete-application.json")); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java new file mode 100644 index 00000000000..3c80245c025 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java @@ -0,0 +1,47 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import org.junit.Test; + +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.*; + +public class EndpointCertificateMetadataSerializerTest { + + private final EndpointCertificateMetadata sample = + new EndpointCertificateMetadata("keyName", "certName", 1); + private final EndpointCertificateMetadata sampleWithRequestMetadata = + new EndpointCertificateMetadata("keyName", "certName", 1, Optional.of("requestId"), Optional.of(List.of("SAN1", "SAN2")), Optional.of("issuer")); + + @Test + public void serialize() { + assertEquals( + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1}", + EndpointCertificateMetadataSerializer.toSlime(sample).toString()); + } + + @Test + public void serializeWithRequestMetadata() { + assertEquals( + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\"}", + EndpointCertificateMetadataSerializer.toSlime(sampleWithRequestMetadata).toString()); + } + + @Test + public void deserializeFromJson() { + assertEquals( + sample, + EndpointCertificateMetadataSerializer.fromJsonString( + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1}")); + } + + @Test + public void deserializeFromJsonWithRequestMetadata() { + assertEquals( + sampleWithRequestMetadata, + EndpointCertificateMetadataSerializer.fromJsonString( + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\"}")); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index c9ec5adc98c..7ca964e06dd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index e5757604caf..b0a0cdf6293 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -5,12 +5,13 @@ import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; +import com.yahoo.vespa.hosted.controller.deployment.JobProfile; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Step; @@ -54,8 +55,8 @@ public class RunSerializerTest { private static final RunSerializer serializer = new RunSerializer(); private static final Path runFile = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json"); private static final RunId id = new RunId(ApplicationId.from("tenant", "application", "default"), - JobType.productionUsEast3, - (long) 112358); + JobType.productionUsEast3, + 112358); private static final Instant start = Instant.parse("2007-12-03T10:15:30.00Z"); @Test @@ -148,7 +149,7 @@ public class RunSerializerTest { assertEquals(run.versions(), phoenix.versions()); assertEquals(run.steps(), phoenix.steps()); - Run initial = Run.initial(id, run.versions(), run.start()); + Run initial = Run.initial(id, run.versions(), run.start(), JobProfile.production); assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index ff1c952c2a5..9c085bb72f7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; -import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import org.junit.Test; import java.net.URI; @@ -77,14 +76,6 @@ public class TenantSerializerTest { } @Test - public void user_tenant() { - UserTenant tenant = UserTenant.create("by-foo", Optional.of(contact())); - UserTenant serialized = (UserTenant) serializer.tenantFrom(serializer.toSlime(tenant)); - assertEquals(tenant.name(), serialized.name()); - assertEquals(contact(), serialized.contact().get()); - } - - @Test public void cloud_tenant() { CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"), new BillingInfo("old cat lady", "vespa"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index c224e24618e..0c1f94d90cf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -28,18 +28,11 @@ public class VersionStatusSerializerTest { @Test public void testSerialization() { List<VespaVersion> vespaVersions = new ArrayList<>(); - DeploymentStatistics statistics = new DeploymentStatistics( - Version.fromString("5.0"), - Collections.singletonList(ApplicationId.from("tenant1", "failing1", "default")), - List.of(ApplicationId.from("tenant2", "success1", "default"), - ApplicationId.from("tenant2", "success2", "default")), - List.of(ApplicationId.from("tenant1", "failing1", "default"), - ApplicationId.from("tenant2", "success2", "default")) - ); - vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, false, + Version version = Version.fromString("5.0"); + vespaVersions.add(new VespaVersion(version, "dead", Instant.now(), false, false, true, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"), Instant.ofEpochMilli(123), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); - vespaVersions.add(new VespaVersion(statistics, "cafe", Instant.now(), true, true, + vespaVersions.add(new VespaVersion(version, "cafe", Instant.now(), true, true, false, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"), Instant.ofEpochMilli(456), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); VersionStatus status = new VersionStatus(vespaVersions); @@ -55,7 +48,7 @@ public class VersionStatusSerializerTest { assertEquals(a.isControllerVersion(), b.isControllerVersion()); assertEquals(a.isSystemVersion(), b.isSystemVersion()); assertEquals(a.isReleased(), b.isReleased()); - assertEquals(a.statistics(), b.statistics()); + assertEquals(a.versionNumber(), b.versionNumber()); assertEquals(a.nodeVersions(), b.nodeVersions()); assertEquals(a.confidence(), b.confidence()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index df8787a2a4b..62b52d0d087 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -5,24 +5,16 @@ import com.yahoo.application.container.JDisc; import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; import com.yahoo.component.ComponentSpecification; -import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.container.http.filter.FilterChainRepository; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; import com.yahoo.jdisc.http.filter.SecurityRequestFilterChain; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; -import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; -import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import org.junit.ComparisonFailure; import java.io.File; @@ -32,7 +24,6 @@ import java.nio.charset.CharacterCodingException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; -import java.time.Instant; import java.util.Optional; import java.util.function.Consumer; import java.util.function.Supplier; @@ -54,17 +45,11 @@ public class ContainerTester { this.container = container; this.responseFilePath = responseFilePath; } - - public JDisc container() { return container; } public Controller controller() { return (Controller) container.components().getComponent(Controller.class.getName()); } - public ConfigServerMock configServer() { - return serviceRegistry().configServerMock(); - } - public AthenzClientFactoryMock athenzClientFactory() { return (AthenzClientFactoryMock) container.components().getComponent(AthenzClientFactoryMock.class.getName()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java index 16447f47ee8..8dd1f9ad10a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java @@ -1,12 +1,14 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi; +import ai.vespa.hosted.api.MultiPartStreamer; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.api.integration.user.User; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; +import com.yahoo.yolean.Exceptions; import java.nio.charset.StandardCharsets; import java.security.Principal; @@ -61,19 +63,24 @@ public class ControllerContainerCloudTest extends ControllerContainerTest { protected RequestBuilder request(String path) { return new RequestBuilder(path, Request.Method.GET); } protected RequestBuilder request(String path, Request.Method method) { return new RequestBuilder(path, method); } - protected class RequestBuilder implements Supplier<Request> { + protected static class RequestBuilder implements Supplier<Request> { private final String path; private final Request.Method method; private byte[] data = new byte[0]; private Principal principal = () -> "user@test"; private User user; private Set<Role> roles = Set.of(Role.everyone()); + private String contentType; private RequestBuilder(String path, Request.Method method) { this.path = path; this.method = method; } + public RequestBuilder contentType(String contentType) { this.contentType = contentType; return this; } + public RequestBuilder data(MultiPartStreamer streamer) { + return Exceptions.uncheck(() -> data(streamer.data().readAllBytes()).contentType(streamer.contentType())); + } public RequestBuilder data(byte[] data) { this.data = data; return this; } public RequestBuilder data(String data) { this.data = data.getBytes(StandardCharsets.UTF_8); return this; } public RequestBuilder principal(String principal) { this.principal = new SimplePrincipal(principal); return this; } @@ -85,7 +92,7 @@ public class ControllerContainerCloudTest extends ControllerContainerTest { Request request = new Request("http://localhost:8080" + path, data, method, principal); request.getAttributes().put(SecurityContext.ATTRIBUTE_NAME, new SecurityContext(principal, roles)); if (user != null) request.getAttributes().put(User.ATTRIBUTE_NAME, user); - request.getHeaders().put("Content-Type", "application/json"); + request.getHeaders().put("Content-Type", contentType); return request; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index b7adc3064fa..ad8a3409eef 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.restapi; import com.yahoo.application.Networking; import com.yahoo.application.container.JDisc; import com.yahoo.application.container.handler.Request; -import com.yahoo.application.container.handler.Response; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; @@ -14,13 +13,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFact import org.junit.After; import org.junit.Before; -import java.io.UncheckedIOException; -import java.nio.charset.CharacterCodingException; - import static com.yahoo.vespa.hosted.controller.integration.AthenzFilterMock.IDENTITY_HEADER_NAME; import static com.yahoo.vespa.hosted.controller.integration.AthenzFilterMock.OKTA_ACCESS_TOKEN_HEADER_NAME; import static com.yahoo.vespa.hosted.controller.integration.AthenzFilterMock.OKTA_IDENTITY_TOKEN_HEADER_NAME; -import static org.junit.Assert.assertEquals; /** * Superclass of REST API tests which needs to set up a functional container instance. @@ -65,6 +60,8 @@ public class ControllerContainerTest { " <item key=\"rotation-id-5\">rotation-fqdn-5</item>\n" + " </rotations>\n" + " </config>\n" + + " " + + "<accesslog type='disabled'/>\n" + " <component id='com.yahoo.vespa.flags.InMemoryFlagSource'/>\n" + " <component id='com.yahoo.vespa.configserver.flags.db.FlagsDbImpl'/>\n" + " <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" + @@ -90,9 +87,6 @@ public class ControllerContainerTest { " <handler id='com.yahoo.vespa.hosted.controller.restapi.os.OsApiHandler'>\n" + " <binding>http://*/os/v1/*</binding>\n" + " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.cost.CostApiHandler'>\n" + - " <binding>http://*/cost/v1/*</binding>\n" + - " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v2.ZoneApiHandler'>\n" + " <binding>http://*/zone/v2</binding>\n" + " <binding>http://*/zone/v2/*</binding>\n" + @@ -151,17 +145,6 @@ public class ControllerContainerTest { " </http>\n"; } - protected void assertResponse(Request request, int responseStatus, String responseMessage) { - Response response = container.handleRequest(request); - // Compare both status and message at once for easier diagnosis - try { - assertEquals("status: " + responseStatus + "\nmessage: " + responseMessage, - "status: " + response.getStatus() + "\nmessage: " + response.getBodyAsString()); - } catch (CharacterCodingException e) { - throw new UncheckedIOException(e); - } - } - protected static Request authenticatedRequest(String uri) { return addIdentityToRequest(new Request(uri), defaultUser); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ResponseHandlerToApplicationResponseWrapper.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ResponseHandlerToApplicationResponseWrapper.java index 55cfc075c7c..1af5de3e4ea 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ResponseHandlerToApplicationResponseWrapper.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ResponseHandlerToApplicationResponseWrapper.java @@ -39,7 +39,7 @@ public class ResponseHandlerToApplicationResponseWrapper implements ResponseHand }); } - private class SimpleContentChannel implements ContentChannel { + private static class SimpleContentChannel implements ContentChannel { private final Queue<ByteBuffer> buffers = new ConcurrentLinkedQueue<>(); private final AtomicBoolean closed = new AtomicBoolean(false); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java new file mode 100644 index 00000000000..5c5dc4b5fe6 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -0,0 +1,78 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.integration.user.User; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; +import com.yahoo.vespa.hosted.controller.security.CloudTenantSpec; +import com.yahoo.vespa.hosted.controller.security.Credentials; +import org.junit.Before; +import org.junit.Test; + +import java.util.Set; + +import static com.yahoo.application.container.handler.Request.Method.POST; +import static com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiTest.createApplicationSubmissionData; + +public class ApplicationApiCloudTest extends ControllerContainerCloudTest { + + private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/"; + + private ContainerTester tester; + private DeploymentTester deploymentTester; + + private static final TenantName tenantName = TenantName.from("scoober"); + private static final ApplicationName applicationName = ApplicationName.from("albums"); + private static final User developerUser = new User("developer", "Joe Developer", "", ""); + + @Before + public void before() { + tester = new ContainerTester(container, responseFiles); + deploymentTester = new DeploymentTester(new ControllerTester(tester)); + deploymentTester.controllerTester().computeVersionStatus(); + } + + @Test + public void test_missing_security_clients_pem() { + setupTenantAndApplication(); + + var application = prodBuilder().build(); + + var deployRequest = request("/application/v4/tenant/scoober/application/albums/submit", POST) + .data(createApplicationSubmissionData(application, 0)) + .roles(Set.of(Role.developer(tenantName))); + + tester.assertResponse( + deployRequest, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Missing required file 'security/clients.pem'\"}", + 400); + } + + + private ApplicationPackageBuilder prodBuilder() { + return new ApplicationPackageBuilder() + .instances("default") + .environment(Environment.prod) + .region("aws-us-east-1a"); + } + + private void setupTenantAndApplication() { + var tenantSpec = new CloudTenantSpec(tenantName, ""); + tester.controller().tenants().create(tenantSpec, credentials("developer@scoober")); + + var appId = TenantAndApplicationId.from(tenantName, applicationName); + tester.controller().applications().createApplication(appId, credentials("developer@scoober")); + } + + private static Credentials credentials(String name) { + return new Credentials(() -> name); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index e96d25c2cab..2752ba64b61 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -9,10 +9,10 @@ import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AthenzService; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; @@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.LockedTenant; +import com.yahoo.vespa.hosted.controller.RoutingController; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -40,15 +41,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostInfo; -import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringInfo; +import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData; import com.yahoo.vespa.hosted.controller.api.integration.resource.MockTenantCost; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -58,6 +57,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.RotationStatusUpdater; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; @@ -75,7 +75,6 @@ import org.junit.Test; import java.io.File; import java.math.BigDecimal; import java.net.URI; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; import java.time.YearMonth; @@ -96,6 +95,9 @@ import static com.yahoo.application.container.handler.Request.Method.GET; import static com.yahoo.application.container.handler.Request.Method.PATCH; import static com.yahoo.application.container.handler.Request.Method.POST; import static com.yahoo.application.container.handler.Request.Method.PUT; +import static java.net.URLEncoder.encode; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.joining; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -174,21 +176,6 @@ public class ApplicationApiTest extends ControllerContainerTest { .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"), new File("tenant-without-applications.json")); - // GET the authenticated user (with associated tenants) - tester.assertResponse(request("/application/v4/user", GET).userIdentity(USER_ID), - new File("user.json")); - // PUT a user tenant - tester.assertResponse(request("/application/v4/user", PUT).userIdentity(USER_ID), - "{\"message\":\"Created user 'by-myuser'\"}"); - // GET the authenticated user which now exists (with associated tenants) - tester.assertResponse(request("/application/v4/user", GET).userIdentity(USER_ID), - new File("user-which-exists.json")); - // DELETE the user - tester.assertResponse(request("/application/v4/tenant/by-myuser", DELETE).userIdentity(USER_ID), - "{\"tenant\":\"by-myuser\",\"type\":\"USER\",\"applications\":[]}"); - // GET all tenants - tester.assertResponse(request("/application/v4/tenant/", GET).userIdentity(USER_ID), - new File("tenant-list.json")); // GET list of months for a tenant tester.assertResponse(request("/application/v4/tenant/tenant1/cost", GET).userIdentity(USER_ID).oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), @@ -231,24 +218,59 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET tenant applications tester.assertResponse(request("/application/v4/tenant/tenant1/application/", GET).userIdentity(USER_ID), - new File("instance-list.json")); + new File("application-list.json")); // GET tenant applications (instances of "application1" only) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/", GET).userIdentity(USER_ID), - new File("instance-list.json")); + new File("application-list.json")); + // GET at a tenant, with "&recursive=true&production=true", recurses over no instances yet, as they are not in deployment spec. + tester.assertResponse(request("/application/v4/tenant/tenant1/", GET) + .userIdentity(USER_ID) + .properties(Map.of("recursive", "true", + "production", "true")), + new File("tenant-without-applications.json")); + // GET at an application, with "&recursive=true&production=true", recurses over no instances yet, as they are not in deployment spec. + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET) + .userIdentity(USER_ID) + .properties(Map.of("recursive", "true", + "production", "true")), + new File("application-without-instances.json")); addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); ApplicationId id = ApplicationId.from("tenant1", "application1", "instance1"); var app1 = deploymentTester.newDeploymentContext(id); - // POST (deploy) an application to start a manual deployment to dev + // POST (deploy) an application to start a manual deployment in prod is not allowed MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/production-us-east-3/", POST) + .data(entity) + .userIdentity(USER_ID), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Direct deployments are only allowed to manually deployed environments.\"}", 400); + + // POST (deploy) an application to start a manual deployment in prod is allowed for operators + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/production-us-east-3/", POST) + .data(entity) + .userIdentity(HOSTED_VESPA_OPERATOR), + "{\"message\":\"Deployment started in run 1 of production-us-east-3 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); + app1.runJob(JobType.productionUsEast3); + tester.controller().applications().deactivate(app1.instanceId(), ZoneId.from("prod", "us-east-3")); + + // POST (deploy) an application to start a manual deployment to dev tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/dev-us-east-1/", POST) .data(entity) .userIdentity(USER_ID), "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); app1.runJob(JobType.devUsEast1); + // GET dev application package + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1/package", GET) + .userIdentity(USER_ID), + (response) -> { + assertEquals("attachment; filename=\"tenant1.application1.instance1.dev.us-east-1.zip\"", response.getHeaders().getFirst("Content-Disposition")); + assertArrayEquals(applicationPackageInstance1.zippedContent(), response.getBody()); + }, + 200); + // POST an application package is not generally allowed under user instance tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/otheruser/deploy/dev-us-east-1", POST) .userIdentity(OTHER_USER_ID) @@ -355,6 +377,18 @@ public class ApplicationApiTest extends ControllerContainerTest { .data("{\"skipTests\":true}") .userIdentity(USER_ID), "{\"message\":\"Triggered production-us-west-1 for tenant2.application2.instance1\"}"); + app2.runJob(JobType.productionUsWest1); + + // POST a re-triggering to force a production job to start with previous parameters + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/instance1/job/production-us-west-1", POST) + .data("{\"reTrigger\":true}") + .userIdentity(USER_ID), + "{\"message\":\"Triggered production-us-west-1 for tenant2.application2.instance1\"}"); + + // DELETE manually deployed prod deployment again + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/instance1/environment/prod/region/us-west-1", DELETE) + .userIdentity(HOSTED_VESPA_OPERATOR), + "{\"message\":\"Deactivated tenant2.application2.instance1 in prod.us-west-1\"}"); // GET application having both change and outstanding change tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", GET) @@ -395,6 +429,11 @@ public class ApplicationApiTest extends ControllerContainerTest { .data("{\"majorVersion\":null}"), "{\"message\":\"Set major version to empty\"}"); + // GET compile version for an application + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/compile-version", GET) + .userIdentity(USER_ID), + "{\"compileVersion\":\"6.1.0\"}"); + // DELETE the pem deploy key tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/key", DELETE) .userIdentity(USER_ID) @@ -482,7 +521,7 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"Triggered pin to 6.1 for tenant1.application1.instance1\"}"); assertTrue("Action is logged to audit log", tester.controller().auditLogger().readLog().entries().stream() - .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin"))); + .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin?"))); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", GET) @@ -539,9 +578,11 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST a 'restart application' command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) - .screwdriverIdentity(SCREWDRIVER_ID), + .userIdentity(HOSTED_VESPA_OPERATOR), "{\"message\":\"Requested restart of tenant1.application1.instance1 in prod.us-central-1\"}"); + addUserToHostedOperatorRole(HostedAthenzIdentities.from(SCREWDRIVER_ID)); + // POST a 'restart application' in staging environment command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), @@ -558,7 +599,8 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"Requested restart of tenant1.application1.instance1 in dev.us-central-1\"}"); // POST a 'restart application' command with a host filter (other filters not supported yet) - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart?hostname=node-1-tenant-host-prod.us-central-1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) + .properties(Map.of("hostname", "node-1-tenant-host-prod.us-central-1")) .screwdriverIdentity(SCREWDRIVER_ID), "{\"message\":\"Requested restart of tenant1.application1.instance1 in prod.us-central-1\"}", 200); @@ -607,10 +649,7 @@ public class ApplicationApiTest extends ControllerContainerTest { ZoneId.from("dev", "us-east-1"), Optional.of(applicationPackageDefault), new DeployOptions(false, Optional.empty(), false, false)); - tester.serviceRegistry().routingGeneratorMock().putEndpoints(new DeploymentId(ApplicationId.from("tenant1", "application1", "default"), ZoneId.from("prod", "us-central-1")), - List.of(new RoutingEndpoint("https://us-central-1.prod.default", "host", false, "upstream"))); - tester.serviceRegistry().routingGeneratorMock().putEndpoints(new DeploymentId(ApplicationId.from("tenant1", "application1", "my-user"), ZoneId.from("dev", "us-east-1")), - List.of(new RoutingEndpoint("https://us-east-1.dev.my-user", "host", false, "upstream"))); + // GET test-config for local tests against a dev deployment tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/my-user/job/dev-us-east-1/test-config", GET) .userIdentity(USER_ID), @@ -662,7 +701,9 @@ public class ApplicationApiTest extends ControllerContainerTest { 200); // GET application package for previous build - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package?build=1", GET).userIdentity(HOSTED_VESPA_OPERATOR), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET) + .properties(Map.of("build", "1")) + .userIdentity(HOSTED_VESPA_OPERATOR), (response) -> { assertEquals("attachment; filename=\"tenant1.application1-build1.zip\"", response.getHeaders().getFirst("Content-Disposition")); assertArrayEquals(applicationPackageInstance1.zippedContent(), response.getBody()); @@ -724,18 +765,6 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), "{\"message\":\"Aborting run 2 of staging-test for tenant1.application1.instance1\"}"); - // PUT (create) the authenticated user - byte[] data = new byte[0]; - tester.assertResponse(request("/application/v4/user?user=new_user&domain=by", PUT) - .data(data) - .userIdentity(new UserId("new_user")), // Normalized to by-new-user by API - new File("create-user-response.json")); - - // GET user lists only tenants for the authenticated user - tester.assertResponse(request("/application/v4/user", GET) - .userIdentity(new UserId("other_user")), - "{\"user\":\"other_user\",\"tenants\":[],\"tenantExists\":false}"); - // OPTIONS return 200 OK tester.assertResponse(request("/application/v4/", Request.Method.OPTIONS) .userIdentity(USER_ID), @@ -776,8 +805,6 @@ public class ApplicationApiTest extends ControllerContainerTest { // Create tenant and deploy var app = deploymentTester.newDeploymentContext(createTenantAndApplication()); app.submit(applicationPackage).deploy(); - app.addRoutingPolicy(westZone, true); - app.addRoutingPolicy(eastZone, true); // Invalid application fails tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation", GET) @@ -862,19 +889,22 @@ public class ApplicationApiTest extends ControllerContainerTest { 400); // GET global rotation status for us-west-1 in default endpoint - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation?endpointId=default", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET) + .properties(Map.of("endpointId", "default")) .userIdentity(USER_ID), "{\"bcpStatus\":{\"rotationStatus\":\"IN\"}}", 200); // GET global rotation status for us-west-1 in eu endpoint - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation?endpointId=eu", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET) + .properties(Map.of("endpointId", "eu")) .userIdentity(USER_ID), "{\"bcpStatus\":{\"rotationStatus\":\"UNKNOWN\"}}", 200); // GET global rotation status for eu-west-1 in eu endpoint - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/eu-west-1/global-rotation?endpointId=eu", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/eu-west-1/global-rotation", GET) + .properties(Map.of("endpointId", "eu")) .userIdentity(USER_ID), "{\"bcpStatus\":{\"rotationStatus\":\"IN\"}}", 200); @@ -898,10 +928,8 @@ public class ApplicationApiTest extends ControllerContainerTest { .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), new File("instance-reference.json")); - // Grant deploy access - addScrewdriverUserToDeployRole(SCREWDRIVER_ID, - ATHENZ_TENANT_DOMAIN, - ApplicationName.from("application1")); + // Add build service to operator role + addUserToHostedOperatorRole(HostedAthenzIdentities.from(SCREWDRIVER_ID)); // POST (deploy) an application to a prod zone - allowed when project ID is not specified MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true); @@ -944,7 +972,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(246), ZoneId.defaultId()), new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(492), ZoneId.defaultId()))); - mockMeteringClient.setMeteringInfo(new MeteringInfo(thisMonth, lastMonth, currentSnapshot, snapshotHistory)); + mockMeteringClient.setMeteringData(new MeteringData(thisMonth, lastMonth, currentSnapshot, snapshotHistory)); tester.assertResponse(request("/application/v4/tenant/doesnotexist/application/doesnotexist/metering", GET) .userIdentity(USER_ID) @@ -1008,6 +1036,12 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'tenant1' does not exist\"}", 404); + // GET non-existing tenant's applications + tester.assertResponse(request("/application/v4/tenant/tenant1/application", GET) + .userIdentity(USER_ID), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'tenant1' does not exist\"}", + 404); + // GET non-existing application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET) .userIdentity(USER_ID), @@ -1051,14 +1085,6 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"error-code\":\"BAD_REQUEST\",\"message\":\"New tenant or application names must start with a letter, may contain no more than 20 characters, and may only contain lowercase letters, digits or dashes, but no double-dashes.\"}", 400); - // POST (add) an Athenz tenant with by- prefix - tester.assertResponse(request("/application/v4/tenant/by-tenant2", POST) - .userIdentity(USER_ID) - .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") - .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Athenz tenant name cannot have prefix 'by-'\"}", - 400); - // POST (add) an Athenz tenant with a reserved name tester.assertResponse(request("/application/v4/tenant/hosted-vespa", POST) .userIdentity(USER_ID) @@ -1089,12 +1115,16 @@ public class ApplicationApiTest extends ControllerContainerTest { 404); // GET non-existent application package of specific build - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package?build=42", GET).userIdentity(HOSTED_VESPA_OPERATOR), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET) + .properties(Map.of("build", "42")) + .userIdentity(HOSTED_VESPA_OPERATOR), "{\"error-code\":\"NOT_FOUND\",\"message\":\"No application package found for 'tenant1.application1' with build number 42\"}", 404); // GET non-existent application package of invalid build - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package?build=foobar", GET).userIdentity(HOSTED_VESPA_OPERATOR), + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET) + .properties(Map.of("build", "foobar")) + .userIdentity(HOSTED_VESPA_OPERATOR), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid build number: For input string: \\\"foobar\\\"\"}", 400); @@ -1334,26 +1364,13 @@ public class ApplicationApiTest extends ControllerContainerTest { createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, tenantAdmin); allowLaunchOfService(new com.yahoo.vespa.athenz.api.AthenzService(ATHENZ_TENANT_DOMAIN, "service")); - // Create tenant - // PUT (create) the authenticated user - tester.assertResponse(request("/application/v4/user?user=new_user&domain=by", PUT) - .userIdentity(userId), // Normalized to by-new-user by API - new File("create-user-response.json")); - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("domain1"), com.yahoo.config.provision.AthenzService.from("service")) .build(); - // POST (deploy) an application to a dev zone - MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); - tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-west-1/instance/default", POST) - .data(entity) - .userIdentity(userId), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"User user.new-user is not allowed to launch service domain1.service. Please reach out to the domain admin.\"}", - 400); - createTenantAndApplication(); - // POST (deploy) an application to dev through a deployment job + MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); + // POST (deploy) an application to dev through a deployment job, with user instance and a proper tenant tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/new-user/deploy/dev-us-east-1", POST) .data(entity) .userIdentity(userId), @@ -1365,12 +1382,6 @@ public class ApplicationApiTest extends ControllerContainerTest { .domains.get(ATHENZ_TENANT_DOMAIN) .admin(HostedAthenzIdentities.from(userId)); - // POST (deploy) an application to a dev zone - tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-east-1/instance/default", POST) - .data(entity) - .userIdentity(userId), - new File("deploy-result.json")); - // POST (deploy) an application to dev through a deployment job tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/new-user/deploy/dev-us-east-1", POST) .data(entity) @@ -1398,7 +1409,7 @@ public class ApplicationApiTest extends ControllerContainerTest { AthenzCredentials credentials = new AthenzCredentials( new AthenzPrincipal(new AthenzUser(developer.id())), sandboxDomain, OKTA_IT, OKTA_AT); tester.controller().tenants().create(tenantSpec, credentials); - tester.controller().applications().createApplication(TenantAndApplicationId.from("sandbox", "myapp"), Optional.of(credentials)); + tester.controller().applications().createApplication(TenantAndApplicationId.from("sandbox", "myapp"), credentials); // Create an application package referencing the service from the other domain ApplicationPackage applicationPackage = new ApplicationPackageBuilder() @@ -1437,19 +1448,21 @@ public class ApplicationApiTest extends ControllerContainerTest { } - - @Test public void applicationWithRoutingPolicy() { var app = deploymentTester.newDeploymentContext(createTenantAndApplication()); + var zone = ZoneId.from(Environment.prod, RegionName.from("us-west-1")); + deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone), + List.of(RoutingMethod.exclusive, RoutingMethod.shared)); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("domain"), AthenzService.from("service")) + .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION) .environment(Environment.prod) .instances("instance1") - .region("us-west-1") + .region(zone.region().value()) .build(); app.submit(applicationPackage).deploy(); - app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), true); - app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), false); + app.addInactiveRoutingPolicy(zone); // GET application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) @@ -1478,7 +1491,7 @@ public class ApplicationApiTest extends ControllerContainerTest { return streamer; } - private MultiPartStreamer createApplicationSubmissionData(ApplicationPackage applicationPackage, long projectId) { + static MultiPartStreamer createApplicationSubmissionData(ApplicationPackage applicationPackage, long projectId) { return new MultiPartStreamer().addJson(EnvironmentResource.SUBMIT_OPTIONS, "{\"repository\":\"repository1\",\"branch\":\"master\",\"commit\":\"commit1\"," + "\"projectId\":" + projectId + ",\"authorEmail\":\"a@b\"}") .addBytes(EnvironmentResource.APPLICATION_ZIP, applicationPackage.zippedContent()) @@ -1562,19 +1575,10 @@ public class ApplicationApiTest extends ControllerContainerTest { for (Instance instance : application.instances().values()) { for (Deployment deployment : instance.deployments().values()) { - Map<ClusterSpec.Id, ClusterInfo> clusterInfo = new HashMap<>(); - List<String> hostnames = new ArrayList<>(); - hostnames.add("host1"); - hostnames.add("host2"); - clusterInfo.put(ClusterSpec.Id.from("cluster1"), - new ClusterInfo("flavor1", 37, 2, 4, 50, - ClusterSpec.Type.content, hostnames)); DeploymentMetrics metrics = new DeploymentMetrics(1, 2, 3, 4, 5, Optional.of(Instant.ofEpochMilli(123123)), Map.of()); - lockedApplication = lockedApplication.with(instance.name(), - lockedInstance -> lockedInstance.withClusterInfo(deployment.zone(), clusterInfo) - .with(deployment.zone(), metrics) + lockedInstance -> lockedInstance.with(deployment.zone(), metrics) .recordActivityAt(Instant.parse("2018-06-01T10:15:30.00Z"), deployment.zone())); } deploymentTester.applications().store(lockedApplication); @@ -1610,7 +1614,7 @@ public class ApplicationApiTest extends ControllerContainerTest { private void assertGlobalRouting(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) { var changedAt = tester.controller().clock().instant(); - var westPolicies = tester.controller().applications().routingPolicies().get(deployment); + var westPolicies = tester.controller().routing().policies().get(deployment); assertEquals(1, westPolicies.size()); var westPolicy = westPolicies.values().iterator().next(); assertEquals(status, westPolicy.status().globalRouting().status()); @@ -1627,8 +1631,8 @@ public class ApplicationApiTest extends ControllerContainerTest { private OktaIdentityToken oktaIdentityToken; private OktaAccessToken oktaAccessToken; private String contentType = "application/json"; - private Map<String, List<String>> headers = new HashMap<>(); - private String recursive; + private final Map<String, List<String>> headers = new HashMap<>(); + private final Map<String, String> properties = new HashMap<>(); private RequestBuilder(String path, Request.Method method) { this.path = path; @@ -1636,7 +1640,7 @@ public class ApplicationApiTest extends ControllerContainerTest { } private RequestBuilder data(byte[] data) { this.data = data; return this; } - private RequestBuilder data(String data) { return data(data.getBytes(StandardCharsets.UTF_8)); } + private RequestBuilder data(String data) { return data(data.getBytes(UTF_8)); } private RequestBuilder data(MultiPartStreamer streamer) { return Exceptions.uncheck(() -> data(streamer.data().readAllBytes()).contentType(streamer.contentType())); } @@ -1646,7 +1650,8 @@ public class ApplicationApiTest extends ControllerContainerTest { private RequestBuilder oktaIdentityToken(OktaIdentityToken oktaIdentityToken) { this.oktaIdentityToken = oktaIdentityToken; return this; } private RequestBuilder oktaAccessToken(OktaAccessToken oktaAccessToken) { this.oktaAccessToken = oktaAccessToken; return this; } private RequestBuilder contentType(String contentType) { this.contentType = contentType; return this; } - private RequestBuilder recursive(String recursive) { this.recursive = recursive; return this; } + private RequestBuilder recursive(String recursive) {return properties(Map.of("recursive", recursive)); } + private RequestBuilder properties(Map<String, String> properties) { this.properties.putAll(properties); return this; } private RequestBuilder header(String name, String value) { this.headers.putIfAbsent(name, new ArrayList<>()); this.headers.get(name).add(value); @@ -1656,11 +1661,13 @@ public class ApplicationApiTest extends ControllerContainerTest { @Override public Request get() { Request request = new Request("http://localhost:8080" + path + - // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters - (recursive == null ? "" : "?recursive=" + recursive), + properties.entrySet().stream() + .map(entry -> encode(entry.getKey(), UTF_8) + "=" + encode(entry.getValue(), UTF_8)) + .collect(joining("&", "?", "")), data, method); request.getHeaders().addAll(headers); request.getHeaders().put("Content-Type", contentType); + // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters if (identity != null) { addIdentityToRequest(request, identity); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index db82adb4940..17d234b0c0c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -34,12 +34,10 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsCentral1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.FAILURE; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; import static org.junit.Assert.assertEquals; /** @@ -52,6 +50,9 @@ public class JobControllerApiHandlerHelperTest { public void testResponses() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .stagingTest() + .blockChange(true, true, "mon,tue", "7-13", "UTC") + .blockChange(false, true, "sun", "0-23", "CET") + .blockChange(true, false, "fri-sat", "8", "America/Los_Angeles") .region("us-central-1") .test("us-central-1") .parallel("us-west-1", "us-east-3") @@ -80,15 +81,10 @@ public class JobControllerApiHandlerHelperTest { tester.runner().run(); assertEquals(deploymentFailed, tester.jobs().last(app.instanceId(), productionUsEast3).get().status()); - ZoneId usWest1 = productionUsWest1.zone(tester.controller().system()); - tester.configServer().convergeServices(app.instanceId(), usWest1); - tester.configServer().convergeServices(app.testerId().id(), usWest1); - tester.setEndpoints(app.instanceId(), usWest1); - tester.setEndpoints(app.testerId().id(), usWest1); tester.runner().run(); - tester.cloud().set(FAILURE); + tester.clock().advance(Duration.ofHours(2).plusSeconds(1)); tester.runner().run(); - assertEquals(testFailure, tester.jobs().last(app.instanceId(), productionUsWest1).get().status()); + assertEquals(installationFailed, tester.jobs().last(app.instanceId(), productionUsWest1).get().status()); assertEquals(revision2, app.deployment(productionUsCentral1.zone(tester.controller().system())).applicationVersion()); assertEquals(revision1, app.deployment(productionUsEast3.zone(tester.controller().system())).applicationVersion()); assertEquals(revision2, app.deployment(productionUsWest1.zone(tester.controller().system())).applicationVersion()); @@ -140,7 +136,6 @@ public class JobControllerApiHandlerHelperTest { userApp.runJob(devAwsUsEast2a, applicationPackage); assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json"); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), userApp.instanceId(), URI.create("https://some.url:43/root/")), "overview-user-instance.json"); - assertResponse(JobControllerApiHandlerHelper.overviewResponse(tester.controller(), app.application().id(), URI.create("https://some.url:43/root/")), "deployment-overview-2.json"); } @@ -157,7 +152,6 @@ public class JobControllerApiHandlerHelperTest { tester.configServer().setLogStream("Nope, this won't be logged"); tester.configServer().convergeServices(app.instanceId(), zone); - tester.setEndpoints(app.instanceId(), zone); tester.runner().run(); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root")), "dev-overview.json"); @@ -183,7 +177,6 @@ public class JobControllerApiHandlerHelperTest { private void compare(HttpResponse response, String expected) throws JSONException, IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.render(baos); - System.err.println(baos.toString()); JSONObject actualJSON = new JSONObject(new String(baos.toByteArray())); JSONObject expectedJSON = new JSONObject(expected); assertEquals(expectedJSON.toString(), actualJSON.toString()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java index 9ef94cc9afd..57774c3f412 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.io.IOUtils; import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.ClusterView; import com.yahoo.vespa.serviceview.bindings.ServiceView; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json new file mode 100644 index 00000000000..2479f127f92 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json @@ -0,0 +1,13 @@ +[ + { + "tenant": "tenant1", + "application":"application1", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "instances": [ + { + "instance":"instance1", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" + } + ] + } +] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json new file mode 100644 index 00000000000..2ee72f150e5 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json @@ -0,0 +1,12 @@ +{ + "tenant": "tenant1", + "application": "application1", + "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/job/", + "instances": [], + "pemDeployKeys": [], + "metrics": { + "queryServiceQuality": 0.0, + "writeServiceQuality": 0.0 + }, + "activity": {} +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json index 7b84780d64e..5c073173544 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json @@ -14,7 +14,6 @@ "commit": "commit1" }, "projectId": 1000, - "compileVersion": "6.1.0", "majorVersion": 7, "instances": [ { @@ -40,75 +39,6 @@ "commit": "commit1" } }, - "deploymentJobs": [ - { - "type": "system-test", - "success": false, - "lastTriggered": { - "id": 1, - "version": "7.0.0", - "revision": { - "buildNumber": 1, - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "staging-test", - "success": false, - "lastTriggered": { - "id": 1, - "version": "7.0.0", - "revision": { - "buildNumber": 1, - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-west-1", - "success": false, - "lastTriggered": { - "id": 1, - "version": "7.0.0", - "revision": { - "buildNumber": 1, - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-east-3", - "success": false - } - ], "changeBlockers": [], "globalRotations": [ "https://instance1--application2--tenant2.global.vespa.oath.cloud:4443/" @@ -126,4 +56,3 @@ }, "activity": {} } - diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json index 3242fd343de..01fd88a599f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json @@ -14,7 +14,6 @@ "commit": "commit1" }, "projectId": 1000, - "compileVersion": "6.1.0", "instances": [ { "instance": "default", @@ -39,75 +38,6 @@ "commit": "commit1" } }, - "deploymentJobs": [ - { - "type": "system-test", - "success": false, - "lastTriggered": { - "id": 1, - "version": "7.0.0", - "revision": { - "buildNumber": 1, - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "staging-test", - "success": false, - "lastTriggered": { - "id": 1, - "version": "7.0.0", - "revision": { - "buildNumber": 1, - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-west-1", - "success": false, - "lastTriggered": { - "id": 1, - "version": "7.0.0", - "revision": { - "buildNumber": 1, - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-east-3", - "success": false - } - ], "changeBlockers": [], "globalRotations": [ "https://instance1--application2--tenant2.global.vespa.oath.cloud:4443/" @@ -123,4 +53,3 @@ }, "activity": {} } - diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json deleted file mode 100644 index d37e9120837..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "message": "Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.", - "run": 1 -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index fd1e8bc5f20..e45bd190d5f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -1,45 +1,150 @@ { "tenant": "tenant", "application": "application", + "projectId": 1001, "steps": [ { "type": "instance", "dependencies": [], "declared": true, - "instance": "default" + "instance": "default", + "readyAt": 0, + "deploying": { + "application": { + "build": 3, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } + }, + "latestVersions": { + "platform": { + "platform": "7.1.0", + "at": 0, + "upgrade": true, + "blockers": [ + { + "days": [ + "Mon", + "Tue" + ], + "hours": [ + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "zone": "UTC" + }, + { + "days": [ + "Sun" + ], + "hours": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ], + "zone": "CET" + } + ] + }, + "application": { + "application": { + "build": 3, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "at": 1000, + "upgrade": true, + "blockers": [ + { + "days": [ + "Mon", + "Tue" + ], + "hours": [ + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "zone": "UTC" + }, + { + "days": [ + "Fri", + "Sat" + ], + "hours": [ + 8 + ], + "zone": "America/Los_Angeles" + } + ] + } + } }, { "type": "test", "dependencies": [], "declared": false, "instance": "default", + "readyAt": 0, "jobName": "system-test", "url": "https://some.url:43/instance/default/job/system-test", "environment": "test", "region": "test.us-east-1", "toRun": [], - "readyAt": 0, "runs": [ { "id": 3, - "url": "https://some.url:43/instance/default/job/system-test/run/run 3 of system-test for tenant.application", - "start": 2000, - "end": 2000, + "url": "https://some.url:43/instance/default/job/system-test/run/3", + "start": 7203000, + "end": 7203000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -87,24 +192,24 @@ }, { "id": 2, - "url": "https://some.url:43/instance/default/job/system-test/run/run 2 of system-test for tenant.application", + "url": "https://some.url:43/instance/default/job/system-test/run/2", "start": 1000, "end": 1000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -152,17 +257,17 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/system-test/run/run 1 of system-test for tenant.application", + "url": "https://some.url:43/instance/default/job/system-test/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -215,6 +320,9 @@ "dependencies": [], "declared": true, "instance": "default", + "readyAt": 7953000, + "delayedUntil": 7953000, + "coolingDownUntil": 7953000, "jobName": "staging-test", "url": "https://some.url:43/instance/default/job/staging-test", "environment": "staging", @@ -224,45 +332,42 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } ], - "readyAt": 752000, - "delayedUntil": 752000, - "coolingDownUntil": 752000, "runs": [ { "id": 5, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 5 of staging-test for tenant.application", - "start": 102000, - "end": 102000, + "url": "https://some.url:43/instance/default/job/staging-test/run/5", + "start": 7303000, + "end": 7303000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -326,24 +431,24 @@ }, { "id": 4, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 4 of staging-test for tenant.application", - "start": 2000, - "end": 2000, + "url": "https://some.url:43/instance/default/job/staging-test/run/4", + "start": 7203000, + "end": 7203000, "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -407,24 +512,24 @@ }, { "id": 3, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 3 of staging-test for tenant.application", - "start": 2000, - "end": 2000, + "url": "https://some.url:43/instance/default/job/staging-test/run/3", + "start": 7203000, + "end": 7203000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -488,24 +593,24 @@ }, { "id": 2, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 2 of staging-test for tenant.application", + "url": "https://some.url:43/instance/default/job/staging-test/run/2", "start": 1000, "end": 1000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -569,17 +674,17 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/staging-test/run/run 1 of staging-test for tenant.application", + "url": "https://some.url:43/instance/default/job/staging-test/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -651,51 +756,43 @@ ], "declared": true, "instance": "default", + "readyAt": 7203000, "jobName": "production-us-central-1", "url": "https://some.url:43/instance/default/job/production-us-central-1", "environment": "prod", "region": "prod.us-central-1", "currentPlatform": "6.1.0", "currentApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "toRun": [], - "readyAt": 2000, "runs": [ { "id": 3, - "url": "https://some.url:43/instance/default/job/production-us-central-1/run/run 3 of production-us-central-1 for tenant.application", - "start": 2000, + "url": "https://some.url:43/instance/default/job/production-us-central-1/run/3", + "start": 7203000, "status": "running", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "succeeded" - }, - { - "name": "installTester", - "status": "unfinished" - }, - { "name": "deployReal", "status": "succeeded" }, @@ -704,18 +801,6 @@ "status": "unfinished" }, { - "name": "startTests", - "status": "unfinished" - }, - { - "name": "endTests", - "status": "unfinished" - }, - { - "name": "deactivateTester", - "status": "unfinished" - }, - { "name": "report", "status": "unfinished" } @@ -723,36 +808,28 @@ }, { "id": 2, - "url": "https://some.url:43/instance/default/job/production-us-central-1/run/run 2 of production-us-central-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-central-1/run/2", "start": 1000, "end": 1000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "succeeded" - }, - { - "name": "installTester", - "status": "succeeded" - }, - { "name": "deployReal", "status": "succeeded" }, @@ -761,18 +838,6 @@ "status": "succeeded" }, { - "name": "startTests", - "status": "succeeded" - }, - { - "name": "endTests", - "status": "succeeded" - }, - { - "name": "deactivateTester", - "status": "succeeded" - }, - { "name": "report", "status": "succeeded" } @@ -780,29 +845,21 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/production-us-central-1/run/run 1 of production-us-central-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-central-1/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "succeeded" - }, - { - "name": "installTester", - "status": "succeeded" - }, - { "name": "deployReal", "status": "succeeded" }, @@ -811,18 +868,6 @@ "status": "succeeded" }, { - "name": "startTests", - "status": "succeeded" - }, - { - "name": "endTests", - "status": "succeeded" - }, - { - "name": "deactivateTester", - "status": "succeeded" - }, - { "name": "report", "status": "succeeded" } @@ -846,17 +891,17 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -864,24 +909,24 @@ "runs": [ { "id": 2, - "url": "https://some.url:43/instance/default/job/test-us-central-1/run/run 2 of test-us-central-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/test-us-central-1/run/2", "start": 1000, "end": 1000, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -913,24 +958,24 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/test-us-central-1/run/run 1 of test-us-central-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/test-us-central-1/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -975,27 +1020,27 @@ "region": "prod.us-west-1", "currentPlatform": "6.1.0", "currentApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "toRun": [ { "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -1003,56 +1048,36 @@ "runs": [ { "id": 2, - "url": "https://some.url:43/instance/default/job/production-us-west-1/run/run 2 of production-us-west-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-west-1/run/2", "start": 1000, - "end": 1000, - "status": "testFailure", + "end": 7202000, + "status": "installationFailed", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "succeeded" - }, - { - "name": "installTester", - "status": "succeeded" - }, - { "name": "deployReal", "status": "succeeded" }, { "name": "installReal", - "status": "succeeded" - }, - { - "name": "startTests", - "status": "succeeded" - }, - { - "name": "endTests", "status": "failed" }, { - "name": "deactivateTester", - "status": "succeeded" - }, - { "name": "report", "status": "succeeded" } @@ -1060,29 +1085,21 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/production-us-west-1/run/run 1 of production-us-west-1 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-west-1/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "succeeded" - }, - { - "name": "installTester", - "status": "succeeded" - }, - { "name": "deployReal", "status": "succeeded" }, @@ -1091,18 +1108,6 @@ "status": "succeeded" }, { - "name": "startTests", - "status": "succeeded" - }, - { - "name": "endTests", - "status": "succeeded" - }, - { - "name": "deactivateTester", - "status": "succeeded" - }, - { "name": "report", "status": "succeeded" } @@ -1123,27 +1128,27 @@ "region": "prod.us-east-3", "currentPlatform": "6.1.0", "currentApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "toRun": [ { "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 3, - "commit": "commit1", + "build": 3, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -1151,56 +1156,36 @@ "runs": [ { "id": 2, - "url": "https://some.url:43/instance/default/job/production-us-east-3/run/run 2 of production-us-east-3 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-east-3/run/2", "start": 1000, "end": 1000, "status": "deploymentFailed", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 2, - "commit": "commit1", + "build": 2, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" }, "sourcePlatform": "6.1.0", "sourceApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "failed" - }, - { - "name": "installTester", - "status": "unfinished" - }, - { "name": "deployReal", - "status": "unfinished" + "status": "failed" }, { "name": "installReal", "status": "unfinished" }, { - "name": "startTests", - "status": "unfinished" - }, - { - "name": "endTests", - "status": "unfinished" - }, - { - "name": "deactivateTester", - "status": "succeeded" - }, - { "name": "report", "status": "succeeded" } @@ -1208,29 +1193,21 @@ }, { "id": 1, - "url": "https://some.url:43/instance/default/job/production-us-east-3/run/run 1 of production-us-east-3 for tenant.application", + "url": "https://some.url:43/instance/default/job/production-us-east-3/run/1", "start": 0, "end": 0, "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "succeeded" - }, - { - "name": "installTester", - "status": "succeeded" - }, - { "name": "deployReal", "status": "succeeded" }, @@ -1239,18 +1216,6 @@ "status": "succeeded" }, { - "name": "startTests", - "status": "succeeded" - }, - { - "name": "endTests", - "status": "succeeded" - }, - { - "name": "deactivateTester", - "status": "succeeded" - }, - { "name": "report", "status": "succeeded" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 33c83a30e38..36aa8a87689 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -1,37 +1,66 @@ { "tenant": "tenant1", "application": "application1", + "projectId": 123, "steps": [ { "type": "instance", "dependencies": [], "declared": true, - "instance": "instance1" + "instance": "instance1", + "readyAt": 0, + "deploying": { + "application": { + "build": 4, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } + }, + "latestVersions": { + "platform": { + "platform": "6.1.0", + "at": "(ignore)", + "upgrade": false, + "blockers": [] + }, + "application": { + "application": { + "build": 4, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "at": 1000, + "upgrade": false, + "blockers": [] + } + } }, { "type": "test", "dependencies": [], "declared": false, "instance": "instance1", + "readyAt": 0, "jobName": "system-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", "environment": "test", "region": "test.us-east-1", "toRun": [], - "readyAt": 0, "runs": [ { "id": 2, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/run 2 of system-test for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/2", "start": "(ignore)", "status": "running", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -79,17 +108,17 @@ }, { "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/run 1 of system-test for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1", "start": "(ignore)", "end": "(ignore)", "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -142,25 +171,25 @@ "dependencies": [], "declared": false, "instance": "instance1", + "readyAt": 0, "jobName": "staging-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test", "environment": "staging", "region": "staging.us-east-3", "toRun": [], - "readyAt": 0, "runs": [ { "id": 2, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/run 2 of staging-test for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/2", "start": "(ignore)", "status": "running", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -224,17 +253,17 @@ }, { "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/run 1 of staging-test for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/1", "start": "(ignore)", "end": "(ignore)", "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ @@ -314,10 +343,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -325,29 +354,21 @@ "runs": [ { "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/run/run 1 of production-us-central-1 for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/run/1", "start": "(ignore)", "end": "(ignore)", "status": "success", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "succeeded" - }, - { - "name": "installTester", - "status": "succeeded" - }, - { "name": "deployReal", "status": "succeeded" }, @@ -356,18 +377,6 @@ "status": "succeeded" }, { - "name": "startTests", - "status": "succeeded" - }, - { - "name": "endTests", - "status": "succeeded" - }, - { - "name": "deactivateTester", - "status": "succeeded" - }, - { "name": "report", "status": "succeeded" } @@ -391,10 +400,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -402,27 +411,19 @@ "runs": [ { "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/run/run 1 of production-us-west-1 for tenant1.application1.instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/run/1", "start": "(ignore)", "status": "aborted", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", - "sourceUrl": "repository1/tree/commit1" + "build": 1, + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "unfinished" - }, - { - "name": "installTester", - "status": "unfinished" - }, - { "name": "deployReal", "status": "unfinished" }, @@ -431,18 +432,6 @@ "status": "unfinished" }, { - "name": "startTests", - "status": "unfinished" - }, - { - "name": "endTests", - "status": "unfinished" - }, - { - "name": "deactivateTester", - "status": "unfinished" - }, - { "name": "report", "status": "unfinished" } @@ -466,39 +455,31 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } ], "runs": [ { - "id": 1, - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/run 1 of production-us-east-3 for tenant1.application1.instance1", + "id": 2, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/2", "start": "(ignore)", "status": "aborted", "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 1, - "commit": "commit1", + "build": 1, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } }, "steps": [ { - "name": "deployTester", - "status": "unfinished" - }, - { - "name": "installTester", - "status": "unfinished" - }, - { "name": "deployReal", "status": "unfinished" }, @@ -507,20 +488,33 @@ "status": "unfinished" }, { - "name": "startTests", + "name": "report", "status": "unfinished" - }, + } + ] + }, + { + "id": 1, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1", + "start": "(ignore)", + "end": "(ignore)", + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": {} + }, + "steps": [ { - "name": "endTests", - "status": "unfinished" + "name": "deployReal", + "status": "succeeded" }, { - "name": "deactivateTester", - "status": "unfinished" + "name": "installReal", + "status": "succeeded" }, { - "name": "report", - "status": "unfinished" + "name": "copyVespaLogs", + "status": "succeeded" } ] } @@ -533,19 +527,39 @@ 5 ], "declared": true, - "instance": "instance2" + "instance": "instance2", + "deploying": {}, + "latestVersions": { + "platform": { + "platform": "6.1.0", + "at": "(ignore)", + "upgrade": false, + "blockers": [] + }, + "application": { + "application": { + "build": 4, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "at": 1000, + "upgrade": false, + "blockers": [] + } + } }, { "type": "test", "dependencies": [], "declared": false, "instance": "instance2", + "readyAt": 0, "jobName": "system-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/system-test", "environment": "test", "region": "test.us-east-1", "toRun": [], - "readyAt": 0, "runs": [] }, { @@ -553,12 +567,12 @@ "dependencies": [], "declared": false, "instance": "instance2", + "readyAt": 0, "jobName": "staging-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/staging-test", "environment": "staging", "region": "staging.us-east-3", "toRun": [], - "readyAt": 0, "runs": [] }, { @@ -577,10 +591,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -603,10 +617,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } @@ -629,10 +643,10 @@ "versions": { "targetPlatform": "6.1.0", "targetApplication": { - "id": 4, - "commit": "commit1", + "build": 4, "compileVersion": "6.1.0", - "sourceUrl": "repository1/tree/commit1" + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json index 22ba83a4730..63e6e4b3937 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json @@ -8,12 +8,18 @@ { "cluster": "default", "tls": true, - "url": "https://instance1.application1.tenant1.us-west-1.vespa.oath.cloud/" + "url": "https://instance1.application1.tenant1.us-west-1.vespa.oath.cloud/", + "scope": "zone", + "routingMethod": "exclusive" + }, + { + "cluster": "default", + "tls": true, + "url": "https://instance1--application1--tenant1.us-west-1.vespa.oath.cloud:4443/", + "scope": "zone", + "routingMethod": "shared" } ], - "serviceUrls": [ - "https://instance1--application1--tenant1.us-west-1.prod.vespa:43" - ], "nodes": "http://localhost:8080/zone/v2/prod/us-west-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-west-1&application=tenant1.application1.instance1", "version": "(ignore)", @@ -36,12 +42,6 @@ }, "status": "complete", "activity": {}, - "cost": { - "tco": 0, - "waste": 0, - "utilization": 0.0, - "cluster": {} - }, "metrics": { "queriesPerSecond": 0.0, "writesPerSecond": 0.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index f9692a4afb7..928525a20d1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -4,9 +4,21 @@ "instance": "instance1", "environment": "prod", "region": "us-central-1", - "endpoints": [], - "serviceUrls": [ - "https://instance1--application1--tenant1.us-central-1.prod.vespa:43" + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1--application1--tenant1.us-central-1.vespa.oath.cloud:4443/", + "scope": "zone", + "routingMethod": "shared" + }, + { + "cluster": "", + "tls": true, + "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", + "scope": "global", + "routingMethod": "shared" + } ], "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", @@ -44,12 +56,6 @@ "lastQueriesPerSecond": 1.0, "lastWritesPerSecond": 2.0 }, - "cost": { - "tco": 0, - "waste": 0, - "utilization": 0.0, - "cluster": {} - }, "metrics": { "queriesPerSecond": 1.0, "writesPerSecond": 2.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json index fe90ddd772e..2053b5a80b1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json @@ -2,8 +2,8 @@ "1": { "id": 1, "status": "success", - "start": 102000, - "end": 102000, + "start": 7303000, + "end": 7303000, "wantedPlatform": "7.1", "wantedApplication": { "hash": "unknown" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json index bfaa62a602d..b37d0d41ae4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json @@ -29,5 +29,37 @@ ], "url": "https://some.url:43/root/dev-us-east-1" } - } + }, + "deployment": [ + { + "jobName": "dev-us-east-1", + "runs": [ + { + "id": 1, + "url": "https://some.url:43/root/run/1", + "start": 0, + "end": 0, + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": {} + }, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + } + ] + } + ] + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json index e1c2310ce7e..3a54ac70b0d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json @@ -28,6 +28,11 @@ { "at": 0, "type": "info", + "message": "######## Details for all nodes ########" + }, + { + "at": 0, + "type": "info", "message": "host-tenant:application:default-dev.us-east-1: unorchestrated" }, { @@ -49,7 +54,7 @@ } ] }, - "lastId": 7, + "lastId": 8, "steps": { "deployReal": { "status": "succeeded", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json index b63031fab4f..91b02e0193e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json @@ -6,7 +6,12 @@ { "at": 0, "type": "info", - "message": " |-- https://default--application--tenant.us-east-1.dev.vespa:43 (cluster 'default')" + "message": "- dev.us-east-1" + }, + { + "at": 0, + "type": "info", + "message": " |-- https://application--tenant.us-east-1.dev.vespa.oath.cloud:4443/ (cluster 'default')" }, { "at": 0, @@ -15,7 +20,7 @@ } ] }, - "lastId": 11, + "lastId": 12, "steps": { "deployReal": { "status": "succeeded", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index 204927f6954..83fa1983957 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -4,9 +4,14 @@ "instance": "instance1", "environment": "dev", "region": "us-east-1", - "endpoints": [], - "serviceUrls": [ - "https://instance1--application1--tenant1.us-east-1.dev.vespa:43" + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/", + "scope": "zone", + "routingMethod": "shared" + } ], "nodes": "http://localhost:8080/zone/v2/dev/us-east-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=dev®ion=us-east-1&application=tenant1.application1.instance1", @@ -20,12 +25,6 @@ "lastQueriesPerSecond": 1.0, "lastWritesPerSecond": 2.0 }, - "cost": { - "tco": 0, - "waste": 0, - "utilization": 0.0, - "cluster": {} - }, "metrics": { "queriesPerSecond": 1.0, "writesPerSecond": 2.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json index a21b1558aee..f7c512842fd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json @@ -1 +1,11 @@ -{"globalrotationoverride":["cluster1.application1.tenant1.us-west-1.prod",{"status":"in","reason":"","agent":"","timestamp":1497618757}]} +{ + "globalrotationoverride": [ + "instance1.application1.tenant1.us-west-1.prod", + { + "status": "in", + "reason": "", + "agent": "", + "timestamp": 1497618757 + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-list.json deleted file mode 100644 index 024ca11dbe3..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-list.json +++ /dev/null @@ -1,3 +0,0 @@ -[ - @include(instance-reference.json) -]
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json index 18f5127718f..33b1d95b5ca 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json @@ -11,178 +11,8 @@ "sourceUrl": "repository1/tree/commit1", "commit": "commit1", "projectId": 1000, - "deploymentJobs": [ - { - "type": "system-test", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "staging-test", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-west-1", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "1.0.1-commit1", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - } - ], "changeBlockers": [], - "compileVersion": "(ignore)", - "globalRotations": [ - "https://c0.instance1.application1.tenant1.global.vespa.oath.cloud/" - ], + "globalRotations": [], "instances": [ { "environment": "prod", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json deleted file mode 100644 index 6338306897c..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json +++ /dev/null @@ -1,315 +0,0 @@ -{ - "tenant": "tenant1", - "application": "application1", - "instance": "instance1", - "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1", - "projectId": 1000, - "deploymentJobs": [ - { - "type": "system-test", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "staging-test", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 3, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 3, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-west-1", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-east-3", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 2, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - } - ], - "changeBlockers": [], - "compileVersion": "(ignore)", - "globalRotations": [ - "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/" - ], - "rotationId": "rotation-id-1", - "instances": [ - { - "bcpStatus": { - "rotationStatus": "IN" - }, - "endpointStatus": [ - { - "endpointId": "default", - "rotationId": "rotation-id-1", - "clusterId": "foo", - "status": "IN", - "lastUpdated": "(ignore)" - } - ], - "applicationVersion": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "status": "complete", - "environment": "prod", - "region": "us-west-1", - "instance": "instance1", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1" - }, - { - "bcpStatus": { - "rotationStatus": "UNKNOWN" - }, - "endpointStatus": [ - { - "endpointId": "default", - "rotationId": "rotation-id-1", - "clusterId": "foo", - "status": "UNKNOWN", - "lastUpdated": "(ignore)" - } - ], - "applicationVersion": { - "hash": "1.0.1-commit1", - "build": 1, - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "status": "complete", - "environment": "prod", - "region": "us-east-3", - "instance": "instance1", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3" - } - ], - "pemDeployKeys": [], - "metrics": { - "queryServiceQuality": 0.5, - "writeServiceQuality": 0.7 - }, - "activity": { - "lastQueried": 1527848130000, - "lastWritten": 1527848130000, - "lastQueriesPerSecond": 1.0, - "lastWritesPerSecond": 2.0 - } -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json index fd8bc256ac5..1b2c0b4e237 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json @@ -13,8 +13,8 @@ "projectId": 123, "deploying": { "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -24,198 +24,6 @@ "commit": "commit1" } }, - "deploymentJobs": [ - { - "type": "system-test", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "staging-test", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-central-1", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-east-3", - "success": false, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-west-1", - "success": false - } - ], "changeBlockers": [ { "versions": true, @@ -241,7 +49,6 @@ ] } ], - "compileVersion": "6.0.0", "globalRotations": [ "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/" ], @@ -263,7 +70,7 @@ "rotationId": "rotation-id-1", "clusterId": "foo", "status": "IN", - "lastUpdated":"(ignore)" + "lastUpdated": "(ignore)" } ], "environment": "prod", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json index ee75d129241..19676decdb8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json @@ -13,8 +13,8 @@ "projectId": 123, "deploying": { "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", + "buildNumber": 1, + "hash": "1.0.1-commit1", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -24,198 +24,6 @@ "commit": "commit1" } }, - "deploymentJobs": [ - { - "type": "system-test", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "staging-test", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-central-1", - "success": true, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastCompleted": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - }, - "lastSuccess": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-east-3", - "success": false, - "lastTriggered": { - "id": 1, - "version": "(ignore)", - "revision": { - "buildNumber": "(ignore)", - "hash": "(ignore)", - "source": { - "gitRepository": "repository1", - "gitBranch": "master", - "gitCommit": "commit1" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1" - }, - "reason": "unknown reason", - "at": "(ignore)" - } - }, - { - "type": "production-us-west-1", - "success": false - } - ], "changeBlockers": [ { "versions": true, @@ -241,7 +49,6 @@ ] } ], - "compileVersion": "(ignore)", "globalRotations": [ "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/" ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json index 85a3245c308..1e43c6e2953 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json @@ -1,7 +1,8 @@ { - "devJobs": {}, - "deployments": [], "lastVersions": {}, "deploying": {}, - "jobs": {} + "deployments": [], + "jobs": {}, + "devJobs": {}, + "deployment": [] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index dc37c3b4bb4..b16ca4cc67c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -230,19 +230,13 @@ "commit": "commit1" }, "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", "deployReal": "succeeded", "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "deactivateTester": "succeeded", "report": "succeeded" }, "tasks": { "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" + "install": "succeeded" }, "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/run/1" } @@ -268,13 +262,8 @@ "commit": "commit1" }, "steps": { - "deployTester": "unfinished", - "installTester": "unfinished", "deployReal": "unfinished", "installReal": "unfinished", - "startTests": "unfinished", - "endTests": "unfinished", - "deactivateTester": "unfinished", "report": "unfinished" }, "tasks": {}, @@ -286,7 +275,7 @@ "us-east-3": { "runs": [ { - "id": 1, + "id": 2, "status": "aborted", "start": "(ignore)", "wantedPlatform": "6.1", @@ -302,16 +291,31 @@ "commit": "commit1" }, "steps": { - "deployTester": "unfinished", - "installTester": "unfinished", "deployReal": "unfinished", "installReal": "unfinished", - "startTests": "unfinished", - "endTests": "unfinished", - "deactivateTester": "unfinished", "report": "unfinished" }, "tasks": {}, + "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/2" + }, + { + "id": 1, + "status": "success", + "start": "(ignore)", + "end": "(ignore)", + "wantedPlatform": "6.1", + "wantedApplication": { + "hash": "unknown" + }, + "steps": { + "deployReal": "succeeded", + "installReal": "succeeded", + "copyVespaLogs": "succeeded" + }, + "tasks": { + "deploy": "succeeded", + "install": "succeeded" + }, "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1" } ], @@ -344,5 +348,37 @@ ], "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1" } - } + }, + "deployment": [ + { + "jobName": "dev-us-east-1", + "runs": [ + { + "id": 1, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/run/1", + "start": "(ignore)", + "end": "(ignore)", + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": {} + }, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + } + ] + } + ] + } + ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json index cf47e8671b0..d46b396b8cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json @@ -1,55 +1,86 @@ { + "lastVersions": { + "platform": { + "platform": "7.1", + "at": 0, + "completed": "0 of 0 complete" + }, + "application": { + "application": { + "hash": "1.0.3-commit1", + "build": 3, + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + }, + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + }, + "at": 1000, + "completed": "0 of 0 complete" + } + }, + "deploying": {}, + "deployments": [], + "jobs": {}, "devJobs": { "dev-aws-us-east-2a": { "runs": [ { + "id": 1, + "status": "success", + "start": 7303000, + "end": 7303000, "wantedPlatform": "7.1", - "log": "https://some.url:43/root/dev-aws-us-east-2a/run/1", "wantedApplication": { "hash": "unknown" }, - "start": 102000, - "end": 102000, - "id": 1, "steps": { "deployReal": "succeeded", "installReal": "succeeded", "copyVespaLogs": "succeeded" }, "tasks": { - "install": "succeeded", - "deploy": "succeeded" + "deploy": "succeeded", + "install": "succeeded" }, - "status": "success" + "log": "https://some.url:43/root/dev-aws-us-east-2a/run/1" } ], "url": "https://some.url:43/root/dev-aws-us-east-2a" } }, - "deployments": [], - "lastVersions": { - "application": { - "at": 1000, - "application": { - "build": 3, - "source": { - "gitRepository": "repository1", - "gitCommit": "commit1", - "gitBranch": "master" - }, - "sourceUrl": "repository1/tree/commit1", - "commit": "commit1", - "hash": "1.0.3-commit1" - }, - "completed": "0 of 0 complete" - }, - "platform": { - "at": 0, - "completed": "0 of 0 complete", - "platform": "7.1" + "deployment": [ + { + "jobName": "dev-aws-us-east-2a", + "runs": [ + { + "id": 1, + "url": "https://some.url:43/root//run/1", + "start": 7303000, + "end": 7303000, + "status": "success", + "versions": { + "targetPlatform": "7.1.0", + "targetApplication": {} + }, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + } + ] + } + ] } - }, - "deploying": {}, - "jobs": {} + ] } - diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json index ad00476154c..2c43aa45d06 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json @@ -37,7 +37,7 @@ "deployments": [ { "us-central-1": { - "at": 2000, + "at": 7203000, "platform": "6.1", "application": { "hash": "1.0.3-commit1", @@ -97,8 +97,8 @@ { "id": 3, "status": "success", - "start": 2000, - "end": 2000, + "start": 7203000, + "end": 7203000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -265,8 +265,8 @@ { "id": 5, "status": "installationFailed", - "start": 102000, - "end": 102000, + "start": 7303000, + "end": 7303000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -313,8 +313,8 @@ { "id": 4, "status": "installationFailed", - "start": 2000, - "end": 2000, + "start": 7203000, + "end": 7203000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -361,8 +361,8 @@ { "id": 3, "status": "success", - "start": 2000, - "end": 2000, + "start": 7203000, + "end": 7203000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -510,7 +510,7 @@ { "id": 3, "status": "running", - "start": 2000, + "start": 7203000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -536,13 +536,8 @@ "commit": "commit1" }, "steps": { - "deployTester": "succeeded", - "installTester": "unfinished", "deployReal": "succeeded", "installReal": "unfinished", - "startTests": "unfinished", - "endTests": "unfinished", - "deactivateTester": "unfinished", "report": "unfinished" }, "tasks": { @@ -581,19 +576,13 @@ "commit": "commit1" }, "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", "deployReal": "succeeded", "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "deactivateTester": "succeeded", "report": "succeeded" }, "tasks": { "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" + "install": "succeeded" }, "log": "https://some.url:43/root/production-us-central-1/run/2" }, @@ -615,19 +604,13 @@ "commit": "commit1" }, "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", "deployReal": "succeeded", "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "deactivateTester": "succeeded", "report": "succeeded" }, "tasks": { "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" + "install": "succeeded" }, "log": "https://some.url:43/root/production-us-central-1/run/1" } @@ -789,9 +772,9 @@ }, { "id": 2, - "status": "testFailure", + "status": "installationFailed", "start": 1000, - "end": 1000, + "end": 7202000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.2-commit1", @@ -817,19 +800,13 @@ "commit": "commit1" }, "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", "deployReal": "succeeded", - "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "failed", - "deactivateTester": "succeeded", + "installReal": "failed", "report": "succeeded" }, "tasks": { "deploy": "succeeded", - "install": "succeeded", - "test": "failed" + "install": "failed" }, "log": "https://some.url:43/root/production-us-west-1/run/2" }, @@ -851,19 +828,13 @@ "commit": "commit1" }, "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", "deployReal": "succeeded", "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "deactivateTester": "succeeded", "report": "succeeded" }, "tasks": { "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" + "install": "succeeded" }, "log": "https://some.url:43/root/production-us-west-1/run/1" } @@ -933,16 +904,13 @@ "commit": "commit1" }, "steps": { - "deployTester": "failed", - "installTester": "unfinished", - "deployReal": "unfinished", + "deployReal": "failed", "installReal": "unfinished", - "startTests": "unfinished", - "endTests": "unfinished", - "deactivateTester": "succeeded", "report": "succeeded" }, - "tasks": {}, + "tasks": { + "deploy": "failed" + }, "log": "https://some.url:43/root/production-us-east-3/run/2" }, { @@ -963,19 +931,13 @@ "commit": "commit1" }, "steps": { - "deployTester": "succeeded", - "installTester": "succeeded", "deployReal": "succeeded", "installReal": "succeeded", - "startTests": "succeeded", - "endTests": "succeeded", - "deactivateTester": "succeeded", "report": "succeeded" }, "tasks": { "deploy": "succeeded", - "install": "succeeded", - "test": "succeeded" + "install": "succeeded" }, "log": "https://some.url:43/root/production-us-east-3/run/1" } @@ -983,5 +945,6 @@ "url": "https://some.url:43/root/production-us-east-3" } }, - "devJobs": {} + "devJobs": {}, + "deployment": [] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 39824e22928..4ffe809297d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -7,9 +7,21 @@ "instance": "instance1", "environment": "prod", "region": "us-central-1", - "endpoints": [], - "serviceUrls": [ - "https://instance1--application1--tenant1.us-central-1.prod.vespa:43" + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://instance1--application1--tenant1.us-central-1.vespa.oath.cloud:4443/", + "scope": "zone", + "routingMethod": "shared" + }, + { + "cluster": "", + "tls": true, + "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/", + "scope": "global", + "routingMethod": "shared" + } ], "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", @@ -47,12 +59,6 @@ "lastQueriesPerSecond": 1.0, "lastWritesPerSecond": 2.0 }, - "cost": { - "tco": 0, - "waste": 0, - "utilization": 0.0, - "cluster": {} - }, "metrics": { "queriesPerSecond": 1.0, "writesPerSecond": 2.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json index 986245decca..d63a7ba7d56 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json @@ -1,9 +1,6 @@ { "resources":[ { - "url":"http://localhost:8080/application/v4/user/" - }, - { "url":"http://localhost:8080/application/v4/tenant/" } ] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json index 4238ce4b834..2ad35968732 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json @@ -94,8 +94,8 @@ "3": { "id": 3, "status": "success", - "start": 2000, - "end": 2000, + "start": 7203000, + "end": 7203000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -146,8 +146,8 @@ "4": { "id": 4, "status": "installationFailed", - "start": 2000, - "end": 2000, + "start": 7203000, + "end": 7203000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", @@ -194,8 +194,8 @@ "5": { "id": 5, "status": "installationFailed", - "start": 102000, - "end": 102000, + "start": 7303000, + "end": 7303000, "wantedPlatform": "6.1", "wantedApplication": { "hash": "1.0.3-commit1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json index 273887c26c4..ba51471d467 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -4,144 +4,149 @@ "log": { "deployTester": [ { - "at": 102000, + "at": 7303000, "type": "info", "message": "No services requiring restart." }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "Deployment successful." }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "foo" } ], "installTester": [ { - "at": 102000, + "at": 7303000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "--- platform 6.1" }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "--- platform 6.1" }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "--- platform 6.1" }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" } ], "deployInitialReal": [ { - "at": 102000, + "at": 7303000, "type": "info", "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..." }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "No services requiring restart." }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "Deployment successful." }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "foo" } ], "installInitialReal": [ { - "at": 102000, + "at": 7303000, + "type": "info", + "message": "######## Details for all nodes ########" + }, + { + "at": 7303000, "type": "info", "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "--- platform 6.1" }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" }, { - "at": 102000, + "at": 7303000, "type": "info", "message": "Deployment expired before installation was successful." } ], "deactivateReal": [ { - "at": 102000, + "at": 7303000, "type": "info", "message": "Deactivating deployment of tenant.application in staging.us-east-3 ..." } ], "deactivateTester": [ { - "at": 102000, + "at": 7303000, "type": "info", "message": "Deactivating tester of tenant.application in staging.us-east-3 ..." } ] }, - "lastId": 22, + "lastId": 23, "steps": { "deployTester": { "status": "succeeded", - "startMillis": 102000 + "startMillis": 7303000 }, "installTester": { "status": "unfinished", - "startMillis": 102000 + "startMillis": 7303000 }, "deployInitialReal": { "status": "succeeded", - "startMillis": 102000 + "startMillis": 7303000 }, "installInitialReal": { "status": "failed", - "startMillis": 102000, + "startMillis": 7303000, "convergence": { "nodes": 1, "down": 0, @@ -177,19 +182,19 @@ }, "copyVespaLogs": { "status": "succeeded", - "startMillis": 102000 + "startMillis": 7303000 }, "deactivateReal": { "status": "succeeded", - "startMillis": 102000 + "startMillis": 7303000 }, "deactivateTester": { "status": "succeeded", - "startMillis": 102000 + "startMillis": 7303000 }, "report": { "status": "succeeded", - "startMillis": 102000 + "startMillis": 7303000 } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json index dd3d16fc721..489d6a11b6a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json @@ -83,12 +83,41 @@ { "at": "(ignore)", "type": "info", - "message": "Endpoints not yet ready." + "message": "Tester container successfully installed!" + } + ], + "deployReal": [ + { + "at": "(ignore)", + "type": "info", + "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..." }, { "at": "(ignore)", "type": "info", - "message": "host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated" + "message": "No services requiring restart." + }, + { + "at": "(ignore)", + "type": "info", + "message": "Deployment successful." + }, + { + "at": "(ignore)", + "type": "info", + "message": "foo" + } + ], + "installReal": [ + { + "at": "(ignore)", + "type": "info", + "message": "######## Details for all nodes ########" + }, + { + "at": "(ignore)", + "type": "info", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" }, { "at": "(ignore)", @@ -98,47 +127,68 @@ { "at": "(ignore)", "type": "info", - "message": "Found endpoints:" + "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": "- test.us-east-1" + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" }, { "at": "(ignore)", "type": "info", - "message": " |-- https://instance1-t--application1--tenant1.us-east-1.test.vespa:43 (cluster 'default')" + "message": "--- platform 6.1" }, { "at": "(ignore)", "type": "info", - "message": "Tester container successfully installed!" - } - ], - "deployReal": [ + "message": "--- container on port 43 has config generation 1, wanted is 2" + }, { "at": "(ignore)", "type": "info", - "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..." + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" }, { "at": "(ignore)", "type": "info", - "message": "No services requiring restart." + "message": "--- platform 6.1" }, { "at": "(ignore)", "type": "info", - "message": "Deployment successful." + "message": "--- container on port 43 has config generation 1, wanted is 2" }, { "at": "(ignore)", "type": "info", - "message": "foo" - } - ], - "installReal": [ + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "info", + "message": "--- platform 6.1" + }, + { + "at": "(ignore)", + "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" + }, + { + "at": "(ignore)", + "type": "info", + "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated" + }, + { + "at": "(ignore)", + "type": "info", + "message": "--- platform 6.1" + }, + { + "at": "(ignore)", + "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" + }, { "at": "(ignore)", "type": "info", @@ -167,7 +217,7 @@ { "at": "(ignore)", "type": "info", - "message": " |-- https://instance1--application1--tenant1.us-east-1.test.vespa:43 (cluster 'default')" + "message": " |-- https://instance1--application1--tenant1.us-east-1.test.vespa.oath.cloud:4443/ (cluster 'default')" }, { "at": "(ignore)", @@ -194,7 +244,7 @@ { "at": "(ignore)", "type": "info", - "message": " |-- https://instance1--application1--tenant1.us-east-1.test.vespa:43 (cluster 'default')" + "message": " |-- https://instance1--application1--tenant1.us-east-1.test.vespa.oath.cloud:4443/ (cluster 'default')" }, { "at": "(ignore)", @@ -224,7 +274,7 @@ } ] }, - "lastId": 40, + "lastId": 50, "steps": { "deployTester": { "status": "succeeded", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json index 818dc72a9d9..0632ab7a67b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json @@ -5,18 +5,18 @@ "isCI": false, "endpoints": { "dev.us-east-1": [ - "https://us-east-1.dev.my-user" + "https://my-user--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/" ], "prod.us-central-1": [ - "https://us-central-1.prod.default" + "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/" ] }, "zoneEndpoints": { "dev.us-east-1": { - "default": "https://us-east-1.dev.my-user" + "default": "https://my-user--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/" }, "prod.us-central-1": { - "default": "https://us-central-1.prod.default" + "default": "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/" } }, "clusters": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json index c1ccbc7100f..c81ed767239 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json @@ -5,12 +5,12 @@ "isCI": false, "endpoints": { "prod.us-central-1": [ - "https://us-central-1.prod.default" + "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/" ] }, "zoneEndpoints": { "prod.us-central-1": { - "default": "https://us-central-1.prod.default" + "default": "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/" } }, "clusters": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json index 0fa4c541832..6c9315ca64b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json @@ -1,50 +1,27 @@ { + "active": false, + "status": "deploymentFailed", "log": { - "deployTester": [ + "deployReal": [ { "at": 1000, "type": "info", "message": "Failed to deploy application: ERROR!" } - ], - "deactivateTester": [ - { - "at": 1000, - "type": "info", - "message": "Deactivating tester of tenant.application in prod.us-east-3 ..." - } ] }, - "active": false, - "lastId": 2, + "lastId": 1, "steps": { - "startTests": { - "status": "unfinished" - }, - "deployTester": { - "startMillis": 1000, - "status": "failed" - }, - "report": { - "startMillis": 1000, - "status": "succeeded" - }, - "installTester": { - "status": "unfinished" - }, "deployReal": { - "status": "unfinished" + "status": "failed", + "startMillis": 1000 }, "installReal": { "status": "unfinished" }, - "deactivateTester": { - "startMillis": 1000, - "status": "succeeded" - }, - "endTests": { - "status": "unfinished" + "report": { + "status": "succeeded", + "startMillis": 1000 } - }, - "status": "deploymentFailed" + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user-which-exists.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user-which-exists.json deleted file mode 100644 index f2703677738..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user-which-exists.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "user": "myuser", - "tenants": @include(tenant-list-with-user.json), - "tenantExists": true -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json deleted file mode 100644 index 79b9a785801..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "user": "myuser", - "tenants": @include(tenant-list.json), - "tenantExists": false -}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java index 8aea26f21e3..c414a3680fc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; @@ -140,4 +139,4 @@ public class ConfigServerApiHandlerTest extends ControllerContainerTest { assertEquals(List.of(URI.create(target)), last.getTargets()); assertEquals(com.yahoo.jdisc.http.HttpRequest.Method.valueOf(method), last.getMethod()); } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index b21c588235e..8029d650b75 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -2,8 +2,11 @@ package com.yahoo.vespa.hosted.controller.restapi.controller; import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; @@ -15,6 +18,7 @@ import java.io.File; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; +import java.util.List; import java.util.Set; import static org.junit.Assert.assertFalse; @@ -156,4 +160,20 @@ public class ControllerApiTest extends ControllerContainerTest { tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/auditlog/"), new File("auditlog.json")); } + @Test + public void testMeteringApi() { + ApplicationId applicationId = ApplicationId.from("tenant", "app", "instance"); + Instant timestamp = Instant.ofEpochMilli(123456789); + ZoneId zoneId = ZoneId.defaultId(); + List<ResourceSnapshot> snapshots = List.of( + new ResourceSnapshot(applicationId, 12,48,1200, timestamp, zoneId), + new ResourceSnapshot(applicationId, 24, 96,2400, timestamp, zoneId) + ); + tester.controller().serviceRegistry().meteringService().consume(snapshots); + tester.assertResponse( + operatorRequest("http://localhost:8080/controller/v1/metering/tenant/tenantName/month/2020-02", "", Request.Method.GET), + new File("metering.json") + ); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 3371c5563c9..fbdf8caaed7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -10,9 +10,6 @@ "name": "CloudEventReporter" }, { - "name": "ClusterInfoMaintainer" - }, - { "name": "ContactInformationMaintainer" }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json new file mode 100644 index 00000000000..b64e8f26a63 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json @@ -0,0 +1,18 @@ +[ + { + "applicationId": "tenant.app.instance", + "timestamp": 123456789, + "zoneId": "prod.default", + "cpu": 12.0, + "memory": 48.0, + "disk": 1200.0 + }, + { + "applicationId": "tenant.app.instance", + "timestamp": 123456789, + "zoneId": "prod.default", + "cpu": 24.0, + "memory": 96.0, + "disk": 2400.0 + } +]
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java deleted file mode 100644 index f992a54a114..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.cost; - -import com.yahoo.application.container.handler.Request; -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; -import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; -import org.junit.Before; -import org.junit.Test; - -/** - * @author andreer - */ -public class CostApiTest extends ControllerContainerTest { - - private static final String responses = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/responses/"; - private static final AthenzIdentity operator = AthenzUser.fromUserId("operatorUser"); - private static final CloudName cloud1 = CloudName.from("yahoo"); - private static final CloudName cloud2 = CloudName.from("cloud2"); - private static final ZoneApi zone1 = ZoneApiMock.newBuilder().withId("prod.us-east-3").with(cloud1).build(); - private static final ZoneApi zone2 = ZoneApiMock.newBuilder().withId("prod.us-west-1").with(cloud1).build(); - private static final ZoneApi zone3 = ZoneApiMock.newBuilder().withId("prod.eu-west-1").with(cloud2).build(); - - private ContainerTester tester; - - @Before - public void before() { - tester = new ContainerTester(container, responses); - tester.serviceRegistry().zoneRegistry().setSystemName(SystemName.cd) - .setZones(zone1, zone2, zone3); - } - - @Test - public void test_api() { - assertResponse(new Request("http://localhost:8080/cost/v1/csv"), - "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n", 200); - } - - private void assertResponse(Request request, String body, int statusCode) { - addIdentityToRequest(request, operator); - tester.assertResponse(request, body, statusCode); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 6df2b00c9e5..c8036676fa9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -65,10 +65,13 @@ public class DeploymentApiTest extends ControllerContainerTest { deploymentTester.upgrader().maintain(); deploymentTester.triggerJobs(); productionApp.runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsWest1); - failingApp.runJob(JobType.systemTest).failDeployment(JobType.stagingTest); + failingApp.failDeployment(JobType.systemTest).failDeployment(JobType.stagingTest); deploymentTester.upgrader().maintain(); deploymentTester.triggerJobs(); + // Application fails application change + productionApp.submit(multiInstancePackage).failDeployment(JobType.systemTest); + tester.controller().updateVersionStatus(censorConfigServers(VersionStatus.compute(tester.controller()))); tester.assertResponse(authenticatedRequest("http://localhost:8080/deployment/v1/"), new File("root.json")); tester.assertResponse(authenticatedRequest("http://localhost:8080/api/deployment/v1/"), new File("root.json")); @@ -78,7 +81,7 @@ public class DeploymentApiTest extends ControllerContainerTest { List<VespaVersion> censored = new ArrayList<>(); for (VespaVersion version : versionStatus.versions()) { if (version.nodeVersions().size() > 0) { - version = new VespaVersion(version.statistics(), + version = new VespaVersion(version.versionNumber(), version.releaseCommit(), version.committedAt(), version.isControllerVersion(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index 2579eede1ae..dd953a3b6cf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -29,7 +29,87 @@ "productionSuccesses": 1 } ], - "deployingApplications": [] + "deployingApplications": [], + "applications": [ + { + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "upgrading": false, + "upgradePolicy": "default", + "jobs": [ + { + "name": "system-test", + "coolingDownUntil": "(ignore)" + }, + { + "name": "staging-test", + "coolingDownUntil": "(ignore)" + }, + { + "name": "production-us-west-1" + } + ], + "allRuns": { + "production-us-west-1": { + "success": { + "number": 1, + "start": "(ignore)", + "end": "(ignore)", + "status": "success" + } + } + }, + "upgradeRuns": { + "production-us-west-1": { + "success": { + "number": 1, + "start": "(ignore)", + "end": "(ignore)", + "status": "success" + } + } + } + }, + { + "tenant": "tenant2", + "application": "application2", + "instance": "i2", + "upgrading": false, + "upgradePolicy": "default", + "jobs": [ + { + "name": "system-test" + }, + { + "name": "staging-test" + }, + { + "name": "production-us-west-1" + } + ], + "allRuns": { + "production-us-west-1": { + "success": { + "number": 1, + "start": "(ignore)", + "end": "(ignore)", + "status": "success" + } + } + }, + "upgradeRuns": { + "production-us-west-1": { + "success": { + "number": 1, + "start": "(ignore)", + "end": "(ignore)", + "status": "success" + } + } + } + } + ] }, { "version": "5.1", @@ -53,6 +133,15 @@ "instance": "default", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", "upgradePolicy": "default", + "failing": "system-test", + "status": "error" + }, + { + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy": "default", "failing": "staging-test", "status": "error" } @@ -75,6 +164,14 @@ "instance": "default", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", "upgradePolicy": "default", + "running": "system-test" + }, + { + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy": "default", "running": "staging-test" }, { @@ -85,8 +182,217 @@ "upgradePolicy": "default", "running": "production-us-west-1" } + ], + "applications": [ + { + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "upgrading": true, + "upgradePolicy": "default", + "jobs": [ + { + "name": "system-test", + "coolingDownUntil": "(ignore)" + }, + { + "name": "staging-test", + "coolingDownUntil": "(ignore)" + }, + { + "name": "production-us-west-1" + } + ], + "allRuns": { + "system-test": { + "failing": { + "number": 2, + "start": "(ignore)", + "end": "(ignore)", + "status": "error" + }, + "running": { + "number": 3, + "start": "(ignore)", + "status": "running" + } + }, + "staging-test": { + "failing": { + "number": 2, + "start": "(ignore)", + "end": "(ignore)", + "status": "error" + }, + "running": { + "number": 3, + "start": "(ignore)", + "status": "running" + } + } + }, + "upgradeRuns": { + "system-test": { + "failing": { + "number": 2, + "start": "(ignore)", + "end": "(ignore)", + "status": "error" + }, + "running": { + "number": 3, + "start": "(ignore)", + "status": "running" + } + }, + "staging-test": { + "failing": { + "number": 2, + "start": "(ignore)", + "end": "(ignore)", + "status": "error" + }, + "running": { + "number": 3, + "start": "(ignore)", + "status": "running" + } + } + } + }, + { + "tenant": "tenant2", + "application": "application2", + "instance": "i1", + "upgrading": false, + "upgradePolicy": "default", + "jobs": [ + { + "name": "system-test", + "coolingDownUntil": "(ignore)" + }, + { + "name": "staging-test" + }, + { + "name": "production-us-west-1" + } + ], + "allRuns": { + "system-test": { + "failing": { + "number": 3, + "start": "(ignore)", + "end": "(ignore)", + "status": "error" + } + }, + "staging-test": { + "running": { + "number": 3, + "start": "(ignore)", + "status": "running" + } + }, + "production-us-west-1": { + "success": { + "number": 2, + "start": "(ignore)", + "end": "(ignore)", + "status": "success" + } + } + }, + "upgradeRuns": { + "system-test": {}, + "staging-test": {}, + "production-us-west-1": { + "success": { + "number": 2, + "start": "(ignore)", + "end": "(ignore)", + "status": "success" + } + } + } + }, + { + "tenant": "tenant2", + "application": "application2", + "instance": "i2", + "upgrading": true, + "upgradePolicy": "default", + "jobs": [ + { + "name": "system-test" + }, + { + "name": "staging-test" + }, + { + "name": "production-us-west-1" + } + ], + "allRuns": { + "production-us-west-1": { + "running": { + "number": 2, + "start": "(ignore)", + "status": "running" + } + }, + "system-test": { + "running": { + "number": 1, + "start": "(ignore)", + "status": "running" + } + }, + "staging-test": { + "running": { + "number": 1, + "start": "(ignore)", + "status": "running" + } + } + }, + "upgradeRuns": { + "production-us-west-1": { + "running": { + "number": 2, + "start": "(ignore)", + "status": "running" + } + }, + "system-test": {}, + "staging-test": {} + } + } ] } + ], + "jobs": [ + "system-test", + "staging-test", + "production-us-east-3", + "test-us-east-3", + "production-us-west-1", + "test-us-west-1", + "production-us-central-1", + "test-us-central-1", + "production-ap-northeast-1", + "test-ap-northeast-1", + "production-ap-northeast-2", + "test-ap-northeast-2", + "production-ap-southeast-1", + "test-ap-southeast-1", + "production-eu-west-1", + "test-eu-west-1", + "production-aws-us-east-1a", + "test-aws-us-east-1a", + "production-aws-us-west-2a", + "test-aws-us-west-2a", + "production-aws-us-east-1b", + "test-aws-us-east-1b" ] } - diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java index 82b97a5b144..5e50e80b7a7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.restapi.filter; import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzPrincipal; @@ -39,7 +38,6 @@ public class AthenzRoleFilterTest { private static final TenantName TENANT = TenantName.from("mytenant"); private static final TenantName TENANT2 = TenantName.from("othertenant"); private static final ApplicationName APPLICATION = ApplicationName.from("myapp"); - private static final InstanceName INSTANCE = InstanceName.from("john"); private static final URI NO_CONTEXT_PATH = URI.create("/application/v4/"); private static final URI TENANT_CONTEXT_PATH = URI.create("/application/v4/tenant/mytenant/"); private static final URI APPLICATION_CONTEXT_PATH = URI.create("/application/v4/tenant/mytenant/application/myapp/"); @@ -48,12 +46,11 @@ public class AthenzRoleFilterTest { private static final URI INSTANCE_CONTEXT_PATH = URI.create("/application/v4/tenant/mytenant/application/myapp/instance/john"); private static final URI INSTANCE2_CONTEXT_PATH = URI.create("/application/v4/tenant/mytenant/application/myapp/instance/jane"); - private ControllerTester tester; private AthenzRoleFilter filter; @Before public void setup() { - tester = new ControllerTester(); + ControllerTester tester = new ControllerTester(); filter = new AthenzRoleFilter(new AthenzClientFactoryMock(tester.athenzDb()), tester.controller()); @@ -70,16 +67,16 @@ public class AthenzRoleFilterTest { } @Test - public void testTranslations() { + public void testTranslations() throws Exception { // Hosted operators are always members of the hostedOperator role. - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, NO_CONTEXT_PATH)); - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, TENANT_CONTEXT_PATH)); - assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner()), + assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedSupporter()), filter.roles(HOSTED_OPERATOR, APPLICATION_CONTEXT_PATH)); // Tenant admins are members of the athenzTenantAdmin role within their tenant subtree. @@ -98,14 +95,14 @@ public class AthenzRoleFilterTest { assertEquals(Set.of(Role.athenzTenantAdmin(TENANT)), filter.roles(TENANT_ADMIN, APPLICATION2_CONTEXT_PATH)); - // Build services are members of the tenantPipeline role within their application subtree. + // Build services are members of the buildService role within their application subtree. assertEquals(Set.of(Role.everyone()), filter.roles(TENANT_PIPELINE, NO_CONTEXT_PATH)); assertEquals(Set.of(Role.everyone()), filter.roles(TENANT_PIPELINE, TENANT_CONTEXT_PATH)); - assertEquals(Set.of(Role.tenantPipeline(TENANT, APPLICATION)), + assertEquals(Set.of(Role.buildService(TENANT, APPLICATION)), filter.roles(TENANT_PIPELINE, APPLICATION_CONTEXT_PATH)); assertEquals(Set.of(Role.everyone()), @@ -115,7 +112,7 @@ public class AthenzRoleFilterTest { assertEquals(Set.of(Role.athenzTenantAdmin(TENANT)), filter.roles(TENANT_ADMIN_AND_PIPELINE, TENANT_CONTEXT_PATH)); - assertEquals(Set.of(Role.athenzTenantAdmin(TENANT), Role.tenantPipeline(TENANT, APPLICATION)), + assertEquals(Set.of(Role.athenzTenantAdmin(TENANT), Role.buildService(TENANT, APPLICATION)), filter.roles(TENANT_ADMIN_AND_PIPELINE, APPLICATION_CONTEXT_PATH)); // Users have nothing special under their instance diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java index a5520b42459..c95691fc120 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java @@ -43,6 +43,16 @@ public class ControllerAuthorizationFilterTest { } @Test + public void supporter() { + ControllerTester tester = new ControllerTester(); + SecurityContext securityContext = new SecurityContext(() -> "operator", Set.of(Role.hostedSupporter())); + ControllerAuthorizationFilter filter = createFilter(tester); + + assertIsForbidden(invokeFilter(filter, createRequest(Method.POST, "/zone/v2/path", securityContext))); + assertIsAllowed(invokeFilter(filter, createRequest(Method.GET, "/zone/v1/path", securityContext))); + } + + @Test public void unprivileged() { ControllerTester tester = new ControllerTester(); SecurityContext securityContext = new SecurityContext(() -> "user", Set.of(Role.everyone())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index 0a1e996696b..6e1480d45e7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -30,7 +30,6 @@ import java.security.PublicKey; import java.util.Set; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class SignatureFilterTest { @@ -95,21 +94,14 @@ public class SignatureFilterTest { verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody), new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"), Set.of(Role.reader(id.tenant()), - Role.developer(id.tenant())))); // TODO jonmv: Change to headless. - - // TODO jonmv: remove after Oct 2019. - // Signed request gets a build service role when a matching key is stored for the application and no X-Key header is provided. - verifySecurityContext(requestOf(signer.legacySigned(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody), - new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"), - Set.of(Role.reader(id.tenant()), - Role.developer(id.tenant())))); + Role.headless(id.tenant(), id.application())))); // Signed POST request with X-Key header gets a headless role. byte[] hiBytes = new byte[]{0x48, 0x69}; verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes), new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"), Set.of(Role.reader(id.tenant()), - Role.developer(id.tenant())))); // TODO jonmv: Change to headless. + Role.headless(id.tenant(), id.application())))); // Signed request gets a developer role when a matching developer key is stored for the tenant. tester.curator().writeTenant(new CloudTenant(appId.tenant(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json deleted file mode 100644 index dbaa6623fae..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json +++ /dev/null @@ -1,153 +0,0 @@ -{ - "versions": [ - { - "version": "0.0.0", - "targetVersion": false, - "cloud": "cloud1", - "nodes": [ - { - "hostname": "node-2-configserver-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-configserver-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-3-configserver-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-configserver-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-2-configserver-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-3-configserver-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-2-proxy-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-3-proxy-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-proxy-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-2-proxy-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-1-proxy-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-3-proxy-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-3-tenant-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-2-tenant-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-1-tenant-host-prod.us-east-3", - "environment": "prod", - "region": "us-east-3" - }, - { - "hostname": "node-3-tenant-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-2-tenant-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - }, - { - "hostname": "node-1-tenant-host-prod.us-west-1", - "environment": "prod", - "region": "us-west-1" - } - ] - }, - { - "version": "0.0.0", - "targetVersion": false, - "cloud": "cloud2", - "nodes": [ - { - "hostname": "node-1-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-configserver-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-1-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-proxy-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-1-tenant-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-3-tenant-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - }, - { - "hostname": "node-2-tenant-host-prod.eu-west-1", - "environment": "prod", - "region": "eu-west-1" - } - ] - } - ] -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index b7edcef97e4..fefd23eb67c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -2,10 +2,15 @@ package com.yahoo.vespa.hosted.controller.restapi.routing; import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.RoutingController; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import org.junit.Before; @@ -13,7 +18,6 @@ import org.junit.Test; import java.io.File; import java.util.List; -import java.util.Set; import static org.junit.Assert.assertNotEquals; @@ -34,19 +38,104 @@ public class RoutingApiTest extends ControllerContainerTest { } @Test - public void policy_based_routing() { - var context = deploymentTester.newDeploymentContext(); + public void discovery() { + // Deploy + var context = deploymentTester.newDeploymentContext("t1", "a1", "default"); + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + var applicationPackage = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context.submit(applicationPackage).deploy(); - // Deploy application + // GET root + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/", "", + Request.Method.GET), + new File("discovery/root.json")); + + // GET tenant + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1", "", + Request.Method.GET), + new File("discovery/tenant.json")); + + // GET application + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/", + "", + Request.Method.GET), + new File("discovery/application.json")); + + // GET instance + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/", + "", + Request.Method.GET), + new File("discovery/instance.json")); + + // GET environment + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/", "", + Request.Method.GET), + new File("discovery/environment.json")); + } + + @Test + public void recursion() { + var context1 = deploymentTester.newDeploymentContext("t1", "a1", "default"); var westZone = ZoneId.from("prod", "us-west-1"); var eastZone = ZoneId.from("prod", "us-east-3"); + var package1 = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context1.submit(package1).deploy(); + + var context2 = deploymentTester.newDeploymentContext("t1", "a2", "default"); + var package2 = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context2.submit(package2).deploy(); + + // GET tenant recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1?recursive=true", "", + Request.Method.GET), + new File("recursion/tenant.json")); + + // GET application recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1?recursive=true", "", + Request.Method.GET), + new File("recursion/application.json")); + + // GET instance recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default?recursive=true", "", + Request.Method.GET), + new File("recursion/application.json")); + + // GET environment recursively + tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment?recursive=true", "", + Request.Method.GET), + new File("recursion/environment.json")); + } + + @Test + public void exclusive_routing() { + var context = deploymentTester.newDeploymentContext(); + // Zones support direct routing + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + deploymentTester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(westZone), + ZoneApiMock.from(eastZone)); + // Deploy application var applicationPackage = new ApplicationPackageBuilder() + .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) + .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION) .region(westZone.region()) .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) .build(); context.submit(applicationPackage).deploy(); - context.addRoutingPolicy(westZone, true); - context.addRoutingPolicy(eastZone, true); // GET initial deployment status tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", @@ -56,7 +145,7 @@ public class RoutingApiTest extends ControllerContainerTest { // POST sets deployment out tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.POST), - "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'out'\"}"); + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("policy/deployment-status-out.json")); @@ -64,7 +153,7 @@ public class RoutingApiTest extends ControllerContainerTest { // DELETE sets deployment in tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.DELETE), - "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'in'\"}"); + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("policy/deployment-status-in.json")); @@ -77,7 +166,7 @@ public class RoutingApiTest extends ControllerContainerTest { // POST sets zone out tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1", "", Request.Method.POST), - "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'out'\"}"); + "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to OUT\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1", "", Request.Method.GET), new File("policy/zone-status-out.json")); @@ -85,16 +174,14 @@ public class RoutingApiTest extends ControllerContainerTest { // DELETE sets zone in tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1", "", Request.Method.DELETE), - "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'in'\"}"); + "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to IN\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1", "", Request.Method.GET), new File("policy/zone-status-in.json")); } @Test - public void rotation_based_routing() { - // No zones support direct routing - deploymentTester.controllerTester().zoneRegistry().setDirectlyRouted(Set.of()); + public void shared_routing() { // Deploy application var context = deploymentTester.newDeploymentContext(); var westZone = ZoneId.from("prod", "us-west-1"); @@ -102,7 +189,7 @@ public class RoutingApiTest extends ControllerContainerTest { var applicationPackage = new ApplicationPackageBuilder() .region(westZone.region()) .region(eastZone.region()) - .endpoint("default", "qrs", eastZone.region().value(), westZone.region().value()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) .build(); context.submit(applicationPackage).deploy(); @@ -116,7 +203,7 @@ public class RoutingApiTest extends ControllerContainerTest { // POST sets deployment out tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.POST), - "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'out'\"}"); + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("rotation/deployment-status-out.json")); @@ -124,7 +211,7 @@ public class RoutingApiTest extends ControllerContainerTest { // DELETE sets deployment in tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.DELETE), - "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'in'\"}"); + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("rotation/deployment-status-in.json")); @@ -137,7 +224,7 @@ public class RoutingApiTest extends ControllerContainerTest { // POST sets zone out tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1", "", Request.Method.POST), - "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'out'\"}"); + "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to OUT\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1", "", Request.Method.GET), new File("rotation/zone-status-out.json")); @@ -145,15 +232,33 @@ public class RoutingApiTest extends ControllerContainerTest { // DELETE sets zone in tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1", "", Request.Method.DELETE), - "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'in'\"}"); + "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to IN\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1", "", Request.Method.GET), new File("rotation/zone-status-in.json")); + } - // TODO(mpolden): Remove the following once a zone supports either of routing policy and rotation + // TODO(mpolden): Remove this once a zone supports either of routing policy and rotation + @Test + public void mixed_routing() { + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + + // One zone supports multiple routing methods + deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(westZone), + RoutingMethod.shared, + RoutingMethod.exclusive); + + // Deploy application + var context = deploymentTester.newDeploymentContext(); + var applicationPackage = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context.submit(applicationPackage).deploy(); // GET status with both policy and rotation assigned - context.addRoutingPolicy(westZone, true); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("multi-status-initial.json")); @@ -161,7 +266,7 @@ public class RoutingApiTest extends ControllerContainerTest { // POST sets deployment out tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.POST), - "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'out'\"}"); + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("multi-status-out.json")); @@ -169,7 +274,7 @@ public class RoutingApiTest extends ControllerContainerTest { // DELETE sets deployment in tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.DELETE), - "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'in'\"}"); + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("multi-status-in.json")); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json new file mode 100644 index 00000000000..deda734cbbf --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json @@ -0,0 +1,7 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json new file mode 100644 index 00000000000..1e06b279873 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json @@ -0,0 +1,43 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/environment/dev/region/aws-us-east-2a/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/dev/region/us-east-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/perf/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-northeast-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-northeast-2/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-southeast-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/aws-us-east-1a/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/eu-west-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-central-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/staging/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/test/region/us-east-1/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json new file mode 100644 index 00000000000..1a3ad823e14 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json @@ -0,0 +1,10 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-east-3/" + }, + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-west-1/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json new file mode 100644 index 00000000000..9b5630335aa --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json @@ -0,0 +1,10 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/" + }, + { + "url": "http://localhost:8080/routing/v1/status/environment/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json new file mode 100644 index 00000000000..acd05d35c8d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json @@ -0,0 +1,7 @@ +{ + "resources": [ + { + "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json index b3f3e90a9cd..1c23c6bb569 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "rotation", + "routingMethod": "shared", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", @@ -10,7 +10,7 @@ "changedAt": "(ignore)" }, { - "routingType": "policy", + "routingMethod": "exclusive", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json index a15a0cc8a99..eea78c1b963 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json @@ -1,16 +1,16 @@ { "deployments": [ { - "routingType": "rotation", + "routingMethod": "shared", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", "status": "in", - "agent": "operator", + "agent": "unknown", "changedAt": "(ignore)" }, { - "routingType": "policy", + "routingMethod": "exclusive", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json index 373d7076ffc..6cb90bdb673 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "rotation", + "routingMethod": "shared", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", @@ -10,7 +10,7 @@ "changedAt": "(ignore)" }, { - "routingType": "policy", + "routingMethod": "exclusive", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json index 74c8173d132..59519c33d06 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "policy", + "routingMethod": "exclusive", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json index 444b6a825ea..e95d9bcdc42 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "policy", + "routingMethod": "exclusive", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json index 3f5353505df..49b85775e63 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "policy", + "routingMethod": "exclusive", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json index 376b6c9c902..abf0a46ae3e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json @@ -1,5 +1,5 @@ { - "routingType": "policy", + "routingMethod": "exclusive", "environment": "prod", "region": "us-west-1", "status": "in", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json index 482fd920070..8328e1ffab1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json @@ -1,5 +1,5 @@ { - "routingType": "policy", + "routingMethod": "exclusive", "environment": "prod", "region": "us-west-1", "status": "in", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json index 5ab261067bf..d86ca2d56e6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json @@ -1,5 +1,5 @@ { - "routingType": "policy", + "routingMethod": "exclusive", "environment": "prod", "region": "us-west-1", "status": "out", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json new file mode 100644 index 00000000000..e0b0e5e9b7a --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json @@ -0,0 +1,22 @@ +{ + "deployments": [ + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json new file mode 100644 index 00000000000..f0dd0b7310d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json @@ -0,0 +1,108 @@ +{ + "zones": [ + { + "routingMethod": "shared", + "environment": "test", + "region": "us-east-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "staging", + "region": "us-east-3", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "dev", + "region": "us-east-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "dev", + "region": "aws-us-east-2a", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "perf", + "region": "us-east-3", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "aws-us-east-1a", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "ap-northeast-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "ap-northeast-2", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "ap-southeast-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "us-central-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + }, + { + "routingMethod": "shared", + "environment": "prod", + "region": "eu-west-1", + "status": "in", + "agent": "operator", + "changedAt": 0 + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json new file mode 100644 index 00000000000..1ee4e1b82ba --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json @@ -0,0 +1,40 @@ +{ + "deployments": [ + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a1:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a2:default", + "environment": "prod", + "region": "us-east-3", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + }, + { + "routingMethod": "shared", + "instance": "t1:a2:default", + "environment": "prod", + "region": "us-west-1", + "status": "in", + "agent": "unknown", + "changedAt": "(ignore)" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json index fd30a27fba5..5b15b72752c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "rotation", + "routingMethod": "shared", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json index 59ea77cf7ae..90b2317c1b3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "rotation", + "routingMethod": "shared", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json index 0240079f154..85e345c01d0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json @@ -1,7 +1,7 @@ { "deployments": [ { - "routingType": "rotation", + "routingMethod": "shared", "instance": "tenant:application:default", "environment": "prod", "region": "us-west-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json index a883fd3f342..eb06e9ee11d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json @@ -1,5 +1,5 @@ { - "routingType": "rotation", + "routingMethod": "shared", "environment": "prod", "region": "us-west-1", "status": "in", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json index a883fd3f342..eb06e9ee11d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json @@ -1,5 +1,5 @@ { - "routingType": "rotation", + "routingMethod": "shared", "environment": "prod", "region": "us-west-1", "status": "in", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json index 18803f56de3..440b80bc4d0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json @@ -1,5 +1,5 @@ { - "routingType": "rotation", + "routingMethod": "shared", "environment": "prod", "region": "us-west-1", "status": "out", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResultTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResultTest.java index 104bb91a8cb..b2a835b6b55 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResultTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResultTest.java @@ -27,7 +27,8 @@ public class SystemFlagsDeployResultTest { List.of( FlagDataChange.deleted(flagOne, controllerTarget)), List.of( - OperationError.deleteFailed("delete failed", controllerTarget, flagTwo))); + OperationError.deleteFailed("delete failed", controllerTarget, flagTwo)), + List.of()); WireSystemFlagsDeployResult wire = result.toWire(); assertThat(wire.changes).hasSize(1); @@ -49,11 +50,13 @@ public class SystemFlagsDeployResultTest { SystemFlagsDeployResult resultController = new SystemFlagsDeployResult( List.of(FlagDataChange.deleted(flagOne, controllerTarget)), - List.of(OperationError.deleteFailed("message", controllerTarget, flagTwo))); + List.of(OperationError.deleteFailed("message", controllerTarget, flagTwo)), + List.of()); SystemFlagsDeployResult resultProdUsWest1 = new SystemFlagsDeployResult( List.of(FlagDataChange.deleted(flagOne, prodUsWest1Target)), - List.of(OperationError.deleteFailed("message", prodUsWest1Target, flagTwo))); + List.of(OperationError.deleteFailed("message", prodUsWest1Target, flagTwo)), + List.of()); var results = List.of(resultController, resultProdUsWest1); SystemFlagsDeployResult mergedResult = merge(results); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java index 22ed079b501..475ac12f2fd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java @@ -58,7 +58,7 @@ public class SystemFlagsDeployerTest { .build(); SystemFlagsDeployer deployer = - new SystemFlagsDeployer(flagsClient, Set.of(controllerTarget, prodUsWest1Target, prodUsEast3Target)); + new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(controllerTarget, prodUsWest1Target, prodUsEast3Target)); SystemFlagsDeployResult result = deployer.deployFlags(archive, false); @@ -83,7 +83,7 @@ public class SystemFlagsDeployerTest { .addFile("main.json", defaultData) .build(); - SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, Set.of(controllerTarget)); + SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(controllerTarget)); SystemFlagsDeployResult result = deployer.deployFlags(archive, true); verify(flagsClient, times(1)).listFlagData(controllerTarget); @@ -107,20 +107,50 @@ public class SystemFlagsDeployerTest { .addFile("main.json", defaultData) .build(); - SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, Set.of(prodUsWest1Target, prodUsEast3Target)); + SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(prodUsWest1Target, prodUsEast3Target)); SystemFlagsDeployResult result = deployer.deployFlags(archive, false); - System.out.println(result); - assertThat(result.errors()).containsOnly( OperationError.listFailed(exception.getMessage(), prodUsWest1Target)); assertThat(result.flagChanges()).containsOnly( FlagDataChange.created(FLAG_ID, prodUsEast3Target, defaultData)); } + @Test + public void creates_error_entry_for_invalid_flag_archive() throws IOException { + FlagsClient flagsClient = mock(FlagsClient.class); + FlagData defaultData = flagData("flags/my-flag/main.json"); + SystemFlagsDataArchive archive = new SystemFlagsDataArchive.Builder() + .addFile("main.prod.unknown-region.json", defaultData) + .build(); + SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(controllerTarget)); + SystemFlagsDeployResult result = deployer.deployFlags(archive, false); + assertThat(result.flagChanges()) + .isEmpty(); + assertThat(result.errors()) + .containsOnly(OperationError.archiveValidationFailed("Unknown flag file: flags/my-flag/main.prod.unknown-region.json")); + } + + @Test + public void creates_warning_entry_for_existing_flag_data_for_undefined_flag() throws IOException { + FlagData prodUsEast3Data = flagData("flags/my-flag/main.prod.us-east-3.json"); + FlagsClient flagsClient = mock(FlagsClient.class); + when(flagsClient.listFlagData(prodUsEast3Target)) + .thenReturn(List.of(prodUsEast3Data)); + when(flagsClient.listDefinedFlags(prodUsEast3Target)) + .thenReturn(List.of()); + SystemFlagsDataArchive archive = new SystemFlagsDataArchive.Builder() + .addFile("main.prod.us-east-3.json", prodUsEast3Data) + .build(); + SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(prodUsEast3Target)); + SystemFlagsDeployResult result = deployer.deployFlags(archive, true); + assertThat(result.warnings()) + .containsOnly(SystemFlagsDeployResult.Warning.dataForUndefinedFlag(prodUsEast3Target, new FlagId("my-flag"))); + } + private static FlagData flagData(String filename) throws IOException { return FlagData.deserializeUtf8Json(Files.readAllBytes(Paths.get("src/test/resources/system-flags/" + filename))); } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index d1dd50cfb4c..51466e5b1e2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -71,11 +71,6 @@ public class UserApiTest extends ControllerContainerCloudTest { .data("{\"token\":\"hello\"}"), new File("tenant-without-applications.json")); - // PUT a tenant is not available to anyone. - tester.assertResponse(request("/application/v4/user/", PUT) - .roles(operator), - "{\"error-code\":\"FORBIDDEN\",\"message\":\"Not authenticated or not a user.\"}", 403); - // GET at user/v1 root fails as no access control is defined there. tester.assertResponse(request("/user/v1/"), accessDenied, 403); @@ -102,7 +97,7 @@ public class UserApiTest extends ControllerContainerCloudTest { tester.assertResponse(request("/user/v1/tenant/my-tenant/application/my-app", POST) .roles(Set.of(Role.administrator(TenantName.from("my-tenant")))) .data("{\"user\":\"headless@app\",\"roleName\":\"headless\"}"), - "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"NullPointerException\"}", 500); + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"role 'headless' of 'my-app' owned by 'my-tenant' not found\"}", 400); // POST an application is allowed for a tenant developer. tester.assertResponse(request("/application/v4/tenant/my-tenant/application/my-app", POST) @@ -193,17 +188,20 @@ public class UserApiTest extends ControllerContainerCloudTest { .data("{\"user\":\"administrator@tenant\",\"roleName\":\"administrator\"}"), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can't remove the last administrator of a tenant.\"}", 400); - // DELETE the tenant is available to the tenant owner. + // DELETE the tenant is not allowed tester.assertResponse(request("/application/v4/tenant/my-tenant", DELETE) - .roles(Set.of(Role.tenantOwner(id.tenant()))), - new File("tenant-without-applications.json")); + .roles(Set.of(Role.developer(id.tenant()))), + "{\n" + + " \"code\" : 403,\n" + + " \"message\" : \"Access denied\"\n" + + "}", 403); } @Test public void userMetadataTest() { ContainerTester tester = new ContainerTester(container, responseFiles); ControllerTester controller = new ControllerTester(tester); - Set<Role> operator = Set.of(Role.hostedOperator()); + Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter()); User user = new User("dev@domail", "Joe Developer", "dev", null); tester.assertResponse(request("/api/user/v1/user") diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json index 079e2c9c388..56108dce94f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json @@ -11,57 +11,21 @@ "administrator", "developer", "reader" - ], - "applications": { - "app1": { - "instances": [ - "default" - ] - }, - "app2": { - "instances": [ - "default", - "dev" - ] - } - } + ] }, "tenant1": { "roles": [ "administrator", "developer", "reader" - ], - "applications": { - "app1": { - "instances": [ - "default" - ] - }, - "app2": { - "instances": [ - "default", - "myinstance" - ] - }, - "app3": { - "instances": [] - } - } + ] }, "tenant2": { "roles": [ "administrator", "developer", "reader" - ], - "applications": { - "app2": { - "instances": [ - "test" - ] - } - } + ] } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json index f0ea10ed888..ea76aa977ce 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json @@ -10,53 +10,17 @@ "roles": [ "developer", "reader" - ], - "applications": { - "app1": { - "instances": [ - "default" - ] - }, - "app2": { - "instances": [ - "default", - "dev" - ] - } - } + ] }, "tenant1": { "roles": [ "administrator" - ], - "applications": { - "app1": { - "instances": [ - "default" - ] - }, - "app2": { - "instances": [ - "default", - "myinstance" - ] - }, - "app3": { - "instances": [] - } - } + ] }, "tenant2": { "roles": [ "developer" - ], - "applications": { - "app2": { - "instances": [ - "test" - ] - } - } + ] } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json index 17489bb15d8..400fe8d4d9b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json @@ -7,6 +7,7 @@ }, "tenants": {}, "operator": [ - "hostedOperator" + "hostedOperator", + "hostedSupporter" ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java index eaeba420bc9..d5031267b27 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; import org.junit.Before; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java index f00363989e6..a7adac7f89d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java @@ -8,7 +8,6 @@ import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java index 674f084a8b7..136ed508a33 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java @@ -1,21 +1,24 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.rotation; +import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; -import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import java.net.URI; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -48,16 +51,9 @@ public class RotationRepositoryTest { .region("us-west-1") .build(); - private DeploymentTester tester; - private RotationRepository repository; - private DeploymentContext application; - - @Before - public void before() { - tester = new DeploymentTester(new ControllerTester(rotationsConfig)); - repository = tester.applications().rotationRepository(); - application = tester.newDeploymentContext("tenant1", "app1", "default"); - } + private final DeploymentTester tester = new DeploymentTester(new ControllerTester(rotationsConfig)); + private final RotationRepository repository = tester.controller().routing().rotations(); + private final DeploymentContext application = tester.newDeploymentContext("tenant1", "app1", "default"); @Test public void assigns_and_reuses_rotation() { @@ -67,23 +63,37 @@ public class RotationRepositoryTest { assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"), - application.instance().endpointsIn(SystemName.main).main().get().url()); + tester.controller().routing().endpointsOf(application.instanceId()).primary().get().url()); try (RotationLock lock = repository.lock()) { List<AssignedRotation> rotations = repository.getOrAssignRotations(application.application().deploymentSpec(), application.instance(), lock); assertSingleRotation(expected, rotations, repository); + assertEquals(Set.of(RegionName.from("us-west-1"), RegionName.from("us-east-3")), + application.instance().rotations().get(0).regions()); } // Submitting once more assigns same rotation - application.submit(applicationPackage); + application.submit(applicationPackage).deploy(); assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); + + // Adding region updates rotation + var applicationPackage = new ApplicationPackageBuilder() + .globalServiceId("foo") + .region("us-east-3") + .region("us-west-1") + .region("us-central-1") + .build(); + application.submit(applicationPackage).deploy(); + assertEquals(Set.of(RegionName.from("us-west-1"), RegionName.from("us-east-3"), + RegionName.from("us-central-1")), + application.instance().rotations().get(0).regions()); } @Test public void strips_whitespace_in_rotation_fqdn() { - tester = new DeploymentTester(new ControllerTester(rotationsConfigWhitespaces)); - RotationRepository repository = tester.controller().applications().rotationRepository(); + var tester = new DeploymentTester(new ControllerTester(rotationsConfigWhitespaces)); + RotationRepository repository = tester.controller().routing().rotations(); var application2 = tester.newDeploymentContext("tenant1", "app2", "default"); application2.submit(applicationPackage); @@ -106,7 +116,7 @@ public class RotationRepositoryTest { // We're now out of rotations thrown.expect(IllegalStateException.class); - thrown.expectMessage("no rotations available"); + thrown.expectMessage("out of rotations"); var application3 = tester.newDeploymentContext("tenant3", "app3", "default"); application3.submit(applicationPackage); } @@ -136,14 +146,19 @@ public class RotationRepositoryTest { public void prefixes_system_when_not_main() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .globalServiceId("foo") - .region("us-east-3") - .region("us-west-1") + .region("cd-us-central-1") + .region("cd-us-west-1") .build(); + var zones = List.of(ZoneApiMock.fromId("prod.cd-us-central-1"), ZoneApiMock.fromId("prod.cd-us-west-1")); + tester.controllerTester().zoneRegistry() + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.shared) + .setSystemName(SystemName.cd); var application2 = tester.newDeploymentContext("tenant2", "app2", "default"); application2.submit(applicationPackage); assertEquals(List.of(new RotationId("foo-1")), rotationIds(application2.instance().rotations())); assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/", - application2.instance().endpointsIn(SystemName.cd).main().get().url().toString()); + tester.controller().routing().endpointsOf(application2.instanceId()).primary().get().url().toString()); } @Test @@ -159,9 +174,9 @@ public class RotationRepositoryTest { assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - instance1.instance().endpointsIn(SystemName.main).main().get().url()); + tester.controller().routing().endpointsOf(instance1.instanceId()).primary().get().url()); assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - instance2.instance().endpointsIn(SystemName.main).main().get().url()); + tester.controller().routing().endpointsOf(instance2.instanceId()).primary().get().url()); } @Test @@ -179,9 +194,9 @@ public class RotationRepositoryTest { assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - instance1.instance().endpointsIn(SystemName.main).main().get().url()); + tester.controller().routing().endpointsOf(instance1.instanceId()).primary().get().url()); assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - instance2.instance().endpointsIn(SystemName.main).main().get().url()); + tester.controller().routing().endpointsOf(instance2.instanceId()).primary().get().url()); } private void assertSingleRotation(Rotation expected, List<AssignedRotation> assignedRotations, RotationRepository repository) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index ec169633433..e44a1364185 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -1,43 +1,53 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing; +import com.google.common.collect.Sets; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.RoutingController; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.Test; -import java.net.URI; import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; /** @@ -50,10 +60,9 @@ public class RoutingPoliciesTest { private final ZoneId zone2 = ZoneId.from("prod", "us-central-1"); private final ZoneId zone3 = ZoneId.from("prod", "us-east-3"); - private final ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .build(); + private final ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) + .region(zone2.region()) + .build(); @Test public void global_routing_policies() { @@ -62,7 +71,7 @@ public class RoutingPoliciesTest { var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); int clustersPerZone = 2; int numberOfDeployments = 2; - var applicationPackage = new ApplicationPackageBuilder() + var applicationPackage = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .endpoint("r0", "c0") @@ -72,7 +81,7 @@ public class RoutingPoliciesTest { tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); // Creates alias records - context1.submit(applicationPackage).deploy(); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2); @@ -81,7 +90,7 @@ public class RoutingPoliciesTest { tester.policiesOf(context1.instance().id()).size()); // Applications gains a new deployment - ApplicationPackage applicationPackage2 = new ApplicationPackageBuilder() + ApplicationPackage applicationPackage2 = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .region(zone3.region()) @@ -91,7 +100,7 @@ public class RoutingPoliciesTest { .build(); numberOfDeployments++; tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone3); - context1.submit(applicationPackage2).deploy(); + context1.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Endpoints are updated to contain cluster in new deployment tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2, zone3); @@ -101,22 +110,22 @@ public class RoutingPoliciesTest { // Another application is deployed with a single cluster and global endpoint var endpoint4 = "r0.app2.tenant1.global.vespa.oath.cloud"; tester.provisionLoadBalancers(1, context2.instanceId(), zone1, zone2); - var applicationPackage3 = new ApplicationPackageBuilder() + var applicationPackage3 = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .endpoint("r0", "c0") .build(); - context2.submit(applicationPackage3).deploy(); + context2.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context2.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); // All endpoints for app1 are removed - ApplicationPackage applicationPackage4 = new ApplicationPackageBuilder() + ApplicationPackage applicationPackage4 = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .region(zone3.region()) .allow(ValidationId.globalEndpointChange) .build(); - context1.submit(applicationPackage4).deploy(); + context1.submit(applicationPackage4).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0); tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0); tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 0); @@ -136,7 +145,7 @@ public class RoutingPoliciesTest { // Deploy application int clustersPerZone = 2; tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); - context1.submit(applicationPackage).deploy(); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Deployment creates records and policies for all clusters in all zones Set<String> expectedRecords = Set.of( @@ -149,13 +158,13 @@ public class RoutingPoliciesTest { assertEquals(4, tester.policiesOf(context1.instanceId()).size()); // Next deploy does nothing - context1.submit(applicationPackage).deploy(); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(expectedRecords, tester.recordNames()); assertEquals(4, tester.policiesOf(context1.instanceId()).size()); // Add 1 cluster in each zone and deploy tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), zone1, zone2); - context1.submit(applicationPackage).deploy(); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", "c1.app1.tenant1.us-west-1.vespa.oath.cloud", @@ -169,7 +178,7 @@ public class RoutingPoliciesTest { // Deploy another application tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), zone1, zone2); - context2.submit(applicationPackage).deploy(); + context2.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", "c1.app1.tenant1.us-west-1.vespa.oath.cloud", @@ -182,12 +191,12 @@ public class RoutingPoliciesTest { "c0.app2.tenant1.us-west-1.vespa.oath.cloud", "c1.app2.tenant1.us-west-1.vespa.oath.cloud" ); - assertEquals(expectedRecords, tester.recordNames()); + assertEquals(expectedRecords.stream().sorted().collect(Collectors.toList()), tester.recordNames().stream().sorted().collect(Collectors.toList())); assertEquals(4, tester.policiesOf(context2.instanceId()).size()); // Deploy removes cluster from app1 tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); - context1.submit(applicationPackage).deploy(); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", "c1.app1.tenant1.us-west-1.vespa.oath.cloud", @@ -224,11 +233,11 @@ public class RoutingPoliciesTest { var context = tester.newDeploymentContext("tenant1", "app1", "default"); tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); - var applicationPackage = new ApplicationPackageBuilder() + var applicationPackage = applicationPackageBuilder() .region(zone1.region().value()) .endpoint("r0", "c0") .build(); - context.submit(applicationPackage).deploy(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); var endpoint = "r0.app1.tenant1.global.vespa.oath.cloud"; assertEquals(endpoint + " points to c0 in all regions", @@ -240,22 +249,6 @@ public class RoutingPoliciesTest { } @Test - public void cluster_endpoints_resolve_from_policies() { - var tester = new RoutingPoliciesTester(); - var context = tester.newDeploymentContext("tenant1", "app1", "default"); - tester.provisionLoadBalancers(3, context.instanceId(), zone1); - context.submit(applicationPackage).deploy(); - tester.controllerTester().serviceRegistry().routingGeneratorMock().putEndpoints(context.deploymentIdIn(zone1), Collections.emptyList()); - assertEquals(Map.of(ClusterSpec.Id.from("c0"), - URI.create("https://c0.app1.tenant1.us-west-1.vespa.oath.cloud/"), - ClusterSpec.Id.from("c1"), - URI.create("https://c1.app1.tenant1.us-west-1.vespa.oath.cloud/"), - ClusterSpec.Id.from("c2"), - URI.create("https://c2.app1.tenant1.us-west-1.vespa.oath.cloud/")), - tester.controllerTester().controller().applications().clusterEndpoints(context.deploymentIdIn(zone1))); - } - - @Test public void manual_deployment_creates_routing_policy() { // Empty application package is valid in manually deployed environments var tester = new RoutingPoliciesTester(); @@ -265,8 +258,7 @@ public class RoutingPoliciesTest { var zoneApi = ZoneApiMock.from(zone.environment(), zone.region()); tester.controllerTester().serviceRegistry().zoneRegistry() .setZones(zoneApi) - .setDirectlyRouted(zoneApi); - tester.provisionLoadBalancers(1, context.instanceId(), zone); + .exclusiveRoutingIn(zoneApi); // Deploy to dev tester.controllerTester().controller().applications().deploy(context.instanceId(), zone, Optional.of(emptyApplicationPackage), DeployOptions.none()); @@ -275,7 +267,7 @@ public class RoutingPoliciesTest { // Routing policy is created and DNS is updated assertEquals(1, tester.policiesOf(context.instanceId()).size()); - assertEquals(Set.of("c0.app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames()); + assertEquals(Set.of("app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames()); } @Test @@ -288,19 +280,19 @@ public class RoutingPoliciesTest { var zoneApi = ZoneApiMock.from(zone.environment(), zone.region()); tester.controllerTester().serviceRegistry().zoneRegistry() .setZones(zoneApi) - .setDirectlyRouted(zoneApi); - + .exclusiveRoutingIn(zoneApi); + var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud"); + assertEquals(prodRecords, tester.recordNames()); // Deploy to dev under different instance var devInstance = context.application().id().instance("user"); - tester.provisionLoadBalancers(1, devInstance, zone); tester.controllerTester().controller().applications().deploy(devInstance, zone, Optional.of(applicationPackage), DeployOptions.none()); assertEquals("DeploymentSpec is persisted", applicationPackage.deploymentSpec(), context.application().deploymentSpec()); context.flushDnsUpdates(); // Routing policy is created and DNS is updated assertEquals(1, tester.policiesOf(devInstance).size()); - assertEquals(Set.of("c0.user.app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames()); + assertEquals(Sets.union(prodRecords, Set.of("user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")), tester.recordNames()); } @Test @@ -310,12 +302,12 @@ public class RoutingPoliciesTest { // Initial load balancer is provisioned tester.provisionLoadBalancers(1, context.instanceId(), zone1); - var applicationPackage = new ApplicationPackageBuilder() + var applicationPackage = applicationPackageBuilder() .region(zone1.region()) .build(); // Application is deployed - context.submit(applicationPackage).deploy(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); var expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud" ); @@ -334,10 +326,10 @@ public class RoutingPoliciesTest { newHostname, LoadBalancer.State.active, Optional.of("dns-zone-1")); - tester.controllerTester().configServer().addLoadBalancers(zone1, List.of(loadBalancer)); + tester.controllerTester().configServer().putLoadBalancers(zone1, List.of(loadBalancer)); // Application redeployment preserves DNS record - context.submit(applicationPackage).deploy(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(expectedRecords, tester.recordNames()); assertEquals(1, tester.policiesOf(context.instanceId()).size()); assertEquals("CNAME points to current load blancer", newHostname.value() + ".", @@ -351,13 +343,13 @@ public class RoutingPoliciesTest { // Provision load balancers and deploy application tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); - var applicationPackage = new ApplicationPackageBuilder() + var applicationPackage = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) .build(); - context.submit(applicationPackage).deploy(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Global DNS record is created tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); @@ -386,7 +378,7 @@ public class RoutingPoliciesTest { assertEquals(Instant.EPOCH, policy2.status().globalRouting().changedAt()); // Next deployment does not affect status - context.submit(applicationPackage).deploy(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); @@ -405,24 +397,24 @@ public class RoutingPoliciesTest { assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt()); // Deployment is set out through a new deployment.xml - var applicationPackage2 = new ApplicationPackageBuilder() + var applicationPackage2 = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region(), false) .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) .build(); - context.submit(applicationPackage2).deploy(); + context.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1); // ... back in - var applicationPackage3 = new ApplicationPackageBuilder() + var applicationPackage3 = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) .build(); - context.submit(applicationPackage3).deploy(); + context.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); } @@ -435,14 +427,14 @@ public class RoutingPoliciesTest { var contexts = List.of(context1, context2); // Deploy applications - var applicationPackage = new ApplicationPackageBuilder() + var applicationPackage = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .endpoint("default", "c0", zone1.region().value(), zone2.region().value()) .build(); for (var context : contexts) { tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); - context.submit(applicationPackage).deploy(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); } @@ -482,6 +474,112 @@ public class RoutingPoliciesTest { tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); } + + @Test + public void non_production_deployment_is_not_registered_in_global_endpoint() { + var tester = new RoutingPoliciesTester(SystemName.Public); + + // Configure the system to use the same region for test, staging and prod + var sharedRegion = RegionName.from("aws-us-east-1c"); + var prodZone = ZoneId.from(Environment.prod, sharedRegion); + var stagingZone = ZoneId.from(Environment.staging, sharedRegion); + var testZone = ZoneId.from(Environment.test, sharedRegion); + var zones = List.of(ZoneApiMock.from(prodZone), + ZoneApiMock.from(stagingZone), + ZoneApiMock.from(testZone)); + tester.controllerTester().zoneRegistry() + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.exclusive); + tester.controllerTester().configServer().bootstrap(List.of(prodZone, stagingZone, testZone), + SystemApplication.all()); + + var context = tester.tester.newDeploymentContext(); + var endpointId = EndpointId.of("r0"); + var applicationPackage = applicationPackageBuilder() + .trustDefaultCertificate() + .region(sharedRegion) + .endpoint(endpointId.id(), "default") + .build(); + + // Application starts deployment + context = context.submit(applicationPackage); + for (var testJob : List.of(JobType.systemTest, JobType.stagingTest)) { + context = context.runJob(testJob); + // Since runJob implicitly tears down the deployment and immediately deletes DNS records associated with the + // deployment, we consume only one DNS update at a time here + do { + context = context.flushDnsUpdates(1); + tester.assertTargets(context.instanceId(), endpointId, 0); + } while (!tester.recordNames().isEmpty()); + } + + // Deployment completes + context.completeRollout(); + tester.assertTargets(context.instanceId(), endpointId, 0, prodZone); + } + + @Test + public void changing_global_routing_status_never_removes_all_members() { + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + + // Provision load balancers and deploy application + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + var applicationPackage = applicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + // Global DNS record is created, pointing to all configured zones + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + + // Global routing status is overridden for one deployment + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + + // Setting other deployment out implicitly sets all deployments in + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.out, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + + // One inactive deployment is put back in. Global DNS record now points to the only active deployment + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); + + // Setting zone (containing active deployment) out puts all deployments in + tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.out); + context.flushDnsUpdates(); + assertEquals(GlobalRouting.Status.out, tester.routingPolicies().get(zone1).globalRouting().status()); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + + // Setting zone back in removes the currently inactive deployment + tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.in); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); + + // Inactive deployment is set in + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.in, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + for (var policy : tester.routingPolicies().get(context.instanceId()).values()) { + assertSame(GlobalRouting.Status.in, policy.status().globalRouting().status()); + } + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + } + + /** Returns an application package builder that satisfies requirements for a directly routed endpoint */ + private static ApplicationPackageBuilder applicationPackageBuilder() { + return new ApplicationPackageBuilder() + .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) + .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION); + } private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, int count) { List<LoadBalancer> loadBalancers = new ArrayList<>(); @@ -503,11 +601,15 @@ public class RoutingPoliciesTest { private final DeploymentTester tester; public RoutingPoliciesTester() { - this(new DeploymentTester()); + this(SystemName.main); + } + + public RoutingPoliciesTester(SystemName system) { + this(new DeploymentTester(new ControllerTester(new ServiceRegistryMock(system)))); } public RoutingPolicies routingPolicies() { - return tester.controllerTester().controller().applications().routingPolicies(); + return tester.controllerTester().controller().routing().policies(); } public DeploymentContext newDeploymentContext(String tenant, String application, String instance) { @@ -520,12 +622,14 @@ public class RoutingPoliciesTest { public RoutingPoliciesTester(DeploymentTester tester) { this.tester = tester; + // Make all zones directly routed + tester.controllerTester().zoneRegistry().exclusiveRoutingIn(tester.controllerTester().zoneRegistry().zones().all().zones()); } private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) { for (ZoneId zone : zones) { tester.configServer().removeLoadBalancers(application, zone); - tester.configServer().addLoadBalancers(zone, createLoadBalancers(zone, application, clustersPerZone)); + tester.configServer().putLoadBalancers(zone, createLoadBalancers(zone, application, clustersPerZone)); } } @@ -555,12 +659,12 @@ public class RoutingPoliciesTest { } private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) { - var prefix = ""; - if (!endpointId.equals(EndpointId.defaultId())) { - prefix = endpointId.id() + "."; - } - var endpoint = prefix + application.application().value() + "." + application.tenant().value() + - ".global.vespa.oath.cloud"; + var endpoint = tester.controller().routing().endpointsOf(application) + .named(endpointId) + .targets(List.of(zone)) + .primary() + .map(Endpoint::dnsName) + .orElse("<none>"); var zoneTargets = Arrays.stream(zone) .map(z -> "lb-" + loadBalancerId + "--" + application.serializedForm() + "--" + z.value() + "/dns-zone-1/" + z.value()) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index c983d3610e8..f60d11693d8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -1,7 +1,6 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; -import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.provision.ApplicationId; @@ -11,10 +10,13 @@ import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; @@ -23,6 +25,7 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -30,6 +33,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static java.util.stream.Collectors.toSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -146,36 +150,68 @@ public class VersionStatusTest { tester.triggerJobs(); // - app1 is in production on version1, but then fails in system test on version2 - context1.submit(applicationPackage) - .timeOutConvergence(systemTest); + context1.timeOutConvergence(systemTest); // - app2 is partially in production on version1 and partially on version2 - context2.submit(applicationPackage) - .runJob(systemTest) + context2.runJob(systemTest) .runJob(stagingTest) .runJob(productionUsWest1) .failDeployment(productionUsEast3); // - app3 is in production on version1, but then fails in staging test on version2 - context3.submit(applicationPackage) - .timeOutUpgrade(stagingTest); + context3.timeOutUpgrade(stagingTest); + tester.triggerJobs(); tester.controllerTester().computeVersionStatus(); List<VespaVersion> versions = tester.controller().versionStatus().versions(); assertEquals("The two versions above exist", 2, versions.size()); VespaVersion v1 = versions.get(0); assertEquals(version1, v1.versionNumber()); - assertEquals("No applications are failing on version1.", ImmutableSet.of(), v1.statistics().failing()); - assertEquals("All applications have at least one active production deployment on version 1.", ImmutableSet.of(context1.instanceId(), context2.instanceId(), context3.instanceId()), v1.statistics().production()); - assertEquals("No applications have active deployment jobs on version1.", ImmutableSet.of(), v1.statistics().deploying()); + var statistics = DeploymentStatistics.compute(List.of(version1, version2), tester.deploymentStatuses()); + var statistics1 = statistics.get(0); + assertJobsRun("No runs are failing on version1.", + Map.of(context1.instanceId(), List.of(), + context2.instanceId(), List.of(), + context3.instanceId(), List.of()), + statistics1.failingUpgrades()); + assertJobsRun("All applications have at least one active production deployment on version 1.", + Map.of(context1.instanceId(), List.of(productionUsWest1, productionUsEast3), + context2.instanceId(), List.of(productionUsEast3), + context3.instanceId(), List.of(productionUsWest1, productionUsEast3)), + statistics1.productionSuccesses()); + assertEquals("No applications have active deployment jobs on version1.", + List.of(), + statistics1.runningUpgrade()); VespaVersion v2 = versions.get(1); assertEquals(version2, v2.versionNumber()); - assertEquals("All applications have failed on version2 in at least one zone.", ImmutableSet.of(context1.instanceId(), context2.instanceId(), context3.instanceId()), v2.statistics().failing()); - assertEquals("Only app2 has successfully deployed to production on version2.", ImmutableSet.of(context2.instanceId()), v2.statistics().production()); - // Should test the below, but can't easily be done with current test framework. This test passes in DeploymentApiTest. - // assertEquals("All applications are being retried on version2.", ImmutableSet.of(app1.id(), app2.id(), app3.id()), v2.statistics().deploying()); + var statistics2 = statistics.get(1); + assertJobsRun("All applications have failed on version2 in at least one zone.", + Map.of(context1.instanceId(), List.of(systemTest), + context2.instanceId(), List.of(productionUsEast3), + context3.instanceId(), List.of(stagingTest)), + statistics2.failingUpgrades()); + assertJobsRun("Only app2 has successfully deployed to production on version2.", + Map.of(context1.instanceId(), List.of(), + context2.instanceId(), List.of(productionUsWest1), + context3.instanceId(), List.of()), + statistics2.productionSuccesses()); + assertJobsRun("All applications are being retried on version2.", + Map.of(context1.instanceId(), List.of(systemTest, stagingTest), + context2.instanceId(), List.of(productionUsEast3), + context3.instanceId(), List.of(systemTest, stagingTest)), + statistics2.runningUpgrade()); } - + + private static void assertJobsRun(String assertion, Map<ApplicationId, List<JobType>> jobs, List<Run> runs) { + assertEquals(assertion, + jobs.entrySet().stream() + .flatMap(entry -> entry.getValue().stream().map(type -> new JobId(entry.getKey(), type))) + .collect(toSet()), + runs.stream() + .map(run -> run.id().job()) + .collect(toSet())); + } + @Test public void testVersionConfidence() { DeploymentTester tester = new DeploymentTester().atMondayMorning(); @@ -340,10 +376,7 @@ public class VersionStatusTest { // Test version order List<VespaVersion> versions = tester.controller().versionStatus().versions(); - assertEquals(3, versions.size()); - assertEquals("6.2", versions.get(0).versionNumber().toString()); - assertEquals("6.4", versions.get(1).versionNumber().toString()); - assertEquals("6.5", versions.get(2).versionNumber().toString()); + assertEquals(List.of("6.2", "6.4", "6.5"), versions.stream().map(version -> version.versionNumber().toString()).collect(Collectors.toList())); // Check release status is correct (static data in MockMavenRepository). assertTrue(versions.get(0).isReleased()); @@ -379,7 +412,7 @@ public class VersionStatusTest { // Stale override was removed assertFalse("Stale override removed", tester.controller().curator().readConfidenceOverrides() - .keySet().contains(version0)); + .containsKey(version0)); } @Test @@ -516,7 +549,7 @@ public class VersionStatusTest { .failDeployment(productionUsWest1); tester.controllerTester().computeVersionStatus(); for (var version : List.of(version0, version1)) { - assertOnVersion(version, context.instanceId(), tester.controllerTester()); + assertOnVersion(version, context.instanceId(), tester); } // System is upgraded and application starts upgrading to next version @@ -531,14 +564,14 @@ public class VersionStatusTest { .failDeployment(productionUsWest1); tester.controllerTester().computeVersionStatus(); for (var version : List.of(version0, version1, version2)) { - assertOnVersion(version, context.instanceId(), tester.controllerTester()); + assertOnVersion(version, context.instanceId(), tester); } // Upgrade succeeds context.deployPlatform(version2); tester.controllerTester().computeVersionStatus(); assertEquals(1, tester.controller().versionStatus().versions().size()); - assertOnVersion(version2, context.instanceId(), tester.controllerTester()); + assertOnVersion(version2, context.instanceId(), tester); // System is upgraded and application starts upgrading to next version var version3 = Version.fromString("7.4"); @@ -552,17 +585,17 @@ public class VersionStatusTest { tester.controllerTester().computeVersionStatus(); assertEquals(2, tester.controller().versionStatus().versions().size()); for (var version : List.of(version2, version3)) { - assertOnVersion(version, context.instanceId(), tester.controllerTester()); + assertOnVersion(version, context.instanceId(), tester); } } - private void assertOnVersion(Version version, ApplicationId instance, ControllerTester tester) { + private void assertOnVersion(Version version, ApplicationId instance, DeploymentTester tester) { var vespaVersion = tester.controller().versionStatus().version(version); assertNotNull("Statistics for version " + version + " exist", vespaVersion); - var statistics = vespaVersion.statistics(); - assertTrue("Application is on version " + version, statistics.production().contains(instance) || - statistics.failing().contains(instance) || - statistics.deploying().contains(instance)); + var statistics = DeploymentStatistics.compute(List.of(version), tester.deploymentStatuses()).get(0); + assertTrue("Application is on version " + version, + Stream.of(statistics.productionSuccesses(), statistics.failingUpgrades(), statistics.runningUpgrade()) + .anyMatch(runs -> runs.stream().anyMatch(run -> run.id().application().equals(instance)))); } private static void writeControllerVersion(HostName hostname, Version version, CuratorDb db) { @@ -571,7 +604,7 @@ public class VersionStatusTest { private Confidence confidence(Controller controller, Version version) { return controller.versionStatus().versions().stream() - .filter(v -> v.statistics().version().equals(version)) + .filter(v -> v.versionNumber().equals(version)) .findFirst() .map(VespaVersion::confidence) .orElseThrow(() -> new IllegalArgumentException("Expected to find version: " + version)); |