diff options
author | Jon Bratseth <bratseth@verizonmedia.com> | 2019-10-07 09:45:48 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@verizonmedia.com> | 2019-10-07 09:45:48 +0200 |
commit | 8729925b15b81bd3a5d0a0835c631843cd791178 (patch) | |
tree | c7e6dc8f3c2fb429644004f45429c8db5bf9e6ad /controller-server/src/test | |
parent | 3188f79fdad37e3ea30f84f8c3be67b0c645386d (diff) | |
parent | 260e989c42beb61608f4e8ebbffbe54a59ef4602 (diff) |
Merge with master
Diffstat (limited to 'controller-server/src/test')
33 files changed, 454 insertions, 319 deletions
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index fab1ed2ab20..e3682a78b7d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -507,7 +507,7 @@ public class ControllerTest { tester.deployAndNotify(tester.defaultInstance(app1.id()).id(), Optional.of(applicationPackage), true, systemTest); tester.applications().deactivate(app1.id().defaultInstance(), ZoneId.from(Environment.test, RegionName.from("us-east-1"))); tester.applications().deactivate(app1.id().defaultInstance(), ZoneId.from(Environment.staging, RegionName.from("us-east-3"))); - tester.applications().deleteApplication(app1.id().tenant(), app1.id().application(), tester.controllerTester().credentialsFor(app1.id())); + tester.applications().deleteApplication(app1.id(), tester.controllerTester().credentialsFor(app1.id())); try (RotationLock lock = tester.applications().rotationRepository().lock()) { assertTrue("Rotation is unassigned", tester.applications().rotationRepository().availableRotations(lock) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index cefdc3bed61..2c88d122e8f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -233,11 +233,12 @@ public final class ControllerTester { } public Application createApplication(TenantName tenant, String applicationName, String instanceName, long projectId) { - ApplicationId applicationId = ApplicationId.from(tenant.value(), applicationName, instanceName); - controller().applications().createApplication(applicationId, credentialsFor(TenantAndApplicationId.from(applicationId))); - controller().applications().lockApplicationOrThrow(TenantAndApplicationId.from(applicationId), application -> + TenantAndApplicationId applicationId = TenantAndApplicationId.from(tenant.value(), applicationName); + controller().applications().createApplication(applicationId, credentialsFor(applicationId)); + controller().applications().lockApplicationOrThrow(applicationId, application -> controller().applications().store(application.withProjectId(OptionalLong.of(projectId)))); - Application application = controller().applications().requireApplication(TenantAndApplicationId.from(applicationId)); + controller().applications().createInstance(applicationId.instance(instanceName)); + Application application = controller().applications().requireApplication(applicationId); assertTrue(application.projectId().isPresent()); return application; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 5dc6fb183a2..61b393efbff 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -148,8 +148,13 @@ public class DeploymentTester { /** Upgrade system applications in all zones to given version */ public void upgradeSystemApplications(Version version) { + upgradeSystemApplications(version, SystemApplication.all()); + } + + /** Upgrade given system applications in all zones to version */ + public void upgradeSystemApplications(Version version, List<SystemApplication> systemApplications) { for (ZoneApi zone : tester.zoneRegistry().zones().all().zones()) { - for (SystemApplication application : SystemApplication.all()) { + for (SystemApplication application : systemApplications) { tester.configServer().setVersion(application.id(), zone.getId(), version); tester.configServer().convergeServices(application.id(), zone.getId()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 4a7ee8bcb63..6da77a967f1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; import com.google.inject.Inject; @@ -110,7 +110,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer List<Node> nodes = IntStream.rangeClosed(1, 3) .mapToObj(i -> new Node( HostName.from("node-" + i + "-" + application.id().application() - .value()), + .value() + + "-" + zone.value()), Node.State.active, application.nodeType(), Optional.of(application.id()), initialVersion, @@ -150,9 +151,16 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer /** Set version for an application in a given zone */ public void setVersion(ApplicationId application, ZoneId zone, Version version) { + setVersion(application, zone, version, -1); + } + + /** Set version for nodeCount number of nodes in application in a given zone */ + public void setVersion(ApplicationId application, ZoneId zone, Version version, int nodeCount) { + int n = 0; for (Node node : nodeRepository().list(zone, application)) { nodeRepository().putByHostname(zone, new Node(node.hostname(), node.state(), node.type(), node.owner(), version, version)); + if (++n == nodeCount) break; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index ff245e2e488..c6bd4bde410 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -218,7 +218,7 @@ public class JobRunnerTest { // Thread is still trying to deploy tester -- delete application, and see all data is garbage collected. assertEquals(Collections.singletonList(runId), jobs.active().stream().map(run -> run.id()).collect(Collectors.toList())); - tester.controllerTester().controller().applications().deleteApplication(id.tenant(), id.application(), tester.controllerTester().credentialsFor(TenantAndApplicationId.from(id))); + tester.controllerTester().controller().applications().deleteApplication(TenantAndApplicationId.from(id), tester.controllerTester().credentialsFor(TenantAndApplicationId.from(id))); assertEquals(Collections.emptyList(), jobs.active()); assertEquals(runId, jobs.last(id, systemTest).get().id()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 4fc952b0b15..9cb40d60677 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -1,9 +1,10 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -11,14 +12,17 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import org.junit.Test; import java.time.Duration; +import java.util.List; import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; @@ -213,6 +217,51 @@ public class MetricsReporterTest { assertEquals("Queue consumed", 0, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); } + @Test + public void test_nodes_failing_system_upgrade() { + var tester = new DeploymentTester(); + var reporter = createReporter(tester.controller()); + var zone1 = ZoneApiMock.fromId("prod.eu-west-1"); + tester.controllerTester().zoneRegistry().setUpgradePolicy(UpgradePolicy.create().upgrade(zone1)); + var systemUpgrader = new SystemUpgrader(tester.controller(), Duration.ofDays(1), + new JobControl(tester.controllerTester().curator())); + tester.configServer().bootstrap(List.of(zone1.getId()), SystemApplication.configServer); + + // System on initial version + var version0 = Version.fromString("7.0"); + tester.upgradeSystem(version0); + reporter.maintain(); + assertEquals(0, getNodesFailingUpgrade()); + + for (var version : List.of(Version.fromString("7.1"), Version.fromString("7.2"))) { + // System starts upgrading to next version + tester.upgradeController(version); + reporter.maintain(); + assertEquals(0, getNodesFailingUpgrade()); + systemUpgrader.maintain(); + + // 30 minutes pass and nothing happens + tester.clock().advance(Duration.ofMinutes(30)); + tester.computeVersionStatus(); + reporter.maintain(); + assertEquals(0, getNodesFailingUpgrade()); + + // 1/3 nodes upgrade within timeout + tester.configServer().setVersion(SystemApplication.configServer.id(), zone1.getId(), version, 1); + tester.clock().advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); + tester.computeVersionStatus(); + reporter.maintain(); + assertEquals(2, getNodesFailingUpgrade()); + + // 3/3 nodes upgrade + tester.configServer().setVersion(SystemApplication.configServer.id(), zone1.getId(), version); + tester.computeVersionStatus(); + reporter.maintain(); + assertEquals(0, getNodesFailingUpgrade()); + assertEquals(version, tester.controller().systemVersion()); + } + } + private Duration getAverageDeploymentDuration(ApplicationId id) { return Duration.ofSeconds(getMetric(MetricsReporter.DEPLOYMENT_AVERAGE_DURATION, id).longValue()); } @@ -225,6 +274,10 @@ public class MetricsReporterTest { return getMetric(MetricsReporter.DEPLOYMENT_WARNINGS, id).intValue(); } + private int getNodesFailingUpgrade() { + return metrics.getMetric(MetricsReporter.NODES_FAILING_SYSTEM_UPGRADE).intValue(); + } + private Number getMetric(String name, ApplicationId id) { return metrics.getMetric((dimensions) -> id.tenant().value().equals(dimensions.get("tenant")) && appDimension(id).equals(dimensions.get("app")), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 9677df6fd18..72b26aca588 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -1,10 +1,9 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -59,6 +58,7 @@ public class SystemUpgraderTest { systemUpgrader.maintain(); assertCurrentVersion(SystemApplication.configServer, version1, zone1, zone2, zone3, zone4); assertCurrentVersion(SystemApplication.proxy, version1, zone1, zone2, zone3, zone4); + assertSystemVersion(version1); // Controller upgrades Version version2 = Version.fromString("6.6"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 3ba1181f762..08963b9fec7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -1,13 +1,13 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.common.collect.ImmutableBiMap; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.security.KeyUtils; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; @@ -16,7 +16,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; @@ -37,6 +36,7 @@ import org.junit.Test; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.security.PublicKey; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -48,7 +48,6 @@ import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import java.util.stream.Collectors; import static com.yahoo.config.provision.SystemName.main; import static java.util.Optional.empty; @@ -64,6 +63,15 @@ public class ApplicationSerializerTest { private static final Path testData = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/"); private static final ZoneId zone1 = ZoneId.from("prod", "us-west-1"); private static final ZoneId zone2 = ZoneId.from("prod", "us-east-3"); + private static final PublicKey publicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" + + "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + + "-----END PUBLIC KEY-----\n"); + private static final PublicKey otherPublicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" + + "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" + + "-----END PUBLIC KEY-----\n"); + @Test public void testSerialization() { @@ -134,7 +142,7 @@ public class ApplicationSerializerTest { Optional.of(User.from("by-username")), OptionalInt.of(7), new ApplicationMetrics(0.5, 0.9), - Set.of("-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n\n-----END PUBLIC KEY-----", "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----"), + Set.of(publicKey, otherPublicKey), projectId, true, instances); @@ -178,17 +186,14 @@ public class ApplicationSerializerTest { assertEquals(original.owner(), serialized.owner()); assertEquals(original.majorVersion(), serialized.majorVersion()); assertEquals(original.change(), serialized.change()); - assertEquals(original.pemDeployKeys(), serialized.pemDeployKeys()); + assertEquals(original.deployKeys(), serialized.deployKeys()); assertEquals(original.require(id1.instance()).rotations(), serialized.require(id1.instance()).rotations()); assertEquals(original.require(id1.instance()).rotationStatus(), serialized.require(id1.instance()).rotationStatus()); // Test cluster utilization assertEquals(0, serialized.require(id1.instance()).deployments().get(zone1).clusterUtils().size()); - assertEquals(3, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().size()); - assertEquals(0.4, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id2")).getCpu(), 0.01); - assertEquals(0.2, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id1")).getCpu(), 0.01); - assertEquals(0.2, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id1")).getMemory(), 0.01); + assertEquals(0, serialized.require(id1.instance()).deployments().get(zone2).clusterUtils().size()); // Test cluster info assertEquals(3, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().size()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index 51df0e4b08b..ff1c952c2a5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.persistence;// Copyright 2018 Yahoo Ho import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.TenantName; +import com.yahoo.security.KeyUtils; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; @@ -15,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import org.junit.Test; import java.net.URI; +import java.security.PublicKey; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -29,6 +31,14 @@ import static org.junit.Assert.assertTrue; public class TenantSerializerTest { private static final TenantSerializer serializer = new TenantSerializer(); + private static final PublicKey publicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" + + "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + + "-----END PUBLIC KEY-----\n"); + private static final PublicKey otherPublicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" + + "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" + + "-----END PUBLIC KEY-----\n"); @Test public void athenz_tenant() { @@ -78,12 +88,12 @@ public class TenantSerializerTest { public void cloud_tenant() { CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"), new BillingInfo("old cat lady", "vespa"), - ImmutableBiMap.of("-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n\n-----END PUBLIC KEY-----", new SimplePrincipal("joe"), - "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----", new SimplePrincipal("jane"))); + ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"), + otherPublicKey, new SimplePrincipal("jane"))); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.name(), serialized.name()); assertEquals(tenant.billingInfo(), serialized.billingInfo()); - assertEquals(tenant.pemDeveloperKeys(), serialized.pemDeveloperKeys()); + assertEquals(tenant.developerKeys(), serialized.developerKeys()); } private Contact contact() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index a1e22b4fc64..5d65cf0381e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -1,20 +1,23 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; +import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Test; +import java.nio.file.Files; +import java.nio.file.Paths; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; import static java.time.temporal.ChronoUnit.MILLIS; import static org.junit.Assert.assertEquals; @@ -36,9 +39,11 @@ public class VersionStatusSerializerTest { ApplicationId.from("tenant2", "success2", "default")) ); vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, false, - true, asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + true, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"), + Instant.ofEpochMilli(123), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); vespaVersions.add(new VespaVersion(statistics, "cafe", Instant.now(), true, true, - false, asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + false, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"), + Instant.ofEpochMilli(456), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); VersionStatus status = new VersionStatus(vespaVersions); VersionStatusSerializer serializer = new VersionStatusSerializer(); VersionStatus deserialized = serializer.fromSlime(serializer.toSlime(status)); @@ -53,14 +58,48 @@ public class VersionStatusSerializerTest { assertEquals(a.isSystemVersion(), b.isSystemVersion()); assertEquals(a.isReleased(), b.isReleased()); assertEquals(a.statistics(), b.statistics()); - assertEquals(a.systemApplicationHostnames(), b.systemApplicationHostnames()); + assertEquals(a.nodeVersions(), b.nodeVersions()); assertEquals(a.confidence(), b.confidence()); } } - private static List<HostName> asHostnames(String... hostname) { - return Arrays.stream(hostname).map(HostName::from).collect(Collectors.toList()); + @Test + public void testLegacySerialization() throws Exception { + var data = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json")); + var serializer = new VersionStatusSerializer(); + var deserializedStatus = serializer.fromSlime(SlimeUtils.jsonToSlime(data)); + + var statistics = new DeploymentStatistics( + Version.fromString("7.0"), + List.of(), + List.of(), + List.of() + ); + var vespaVersion = new VespaVersion(statistics, "badc0ffee", + Instant.ofEpochMilli(123), true, + true, true, + nodeVersions(Version.emptyVersion, Version.emptyVersion, + Instant.EPOCH, "cfg1", "cfg2", "cfg3"), + VespaVersion.Confidence.normal); + + VespaVersion deserialized = deserializedStatus.versions().get(0); + assertEquals(vespaVersion.releaseCommit(), deserialized.releaseCommit()); + assertEquals(vespaVersion.committedAt().truncatedTo(MILLIS), deserialized.committedAt()); + assertEquals(vespaVersion.isControllerVersion(), deserialized.isControllerVersion()); + assertEquals(vespaVersion.isSystemVersion(), deserialized.isSystemVersion()); + assertEquals(vespaVersion.isReleased(), deserialized.isReleased()); + assertEquals(vespaVersion.statistics(), deserialized.statistics()); + assertEquals(vespaVersion.nodeVersions(), deserialized.nodeVersions()); + assertEquals(vespaVersion.confidence(), deserialized.confidence()); + } + + private static NodeVersions nodeVersions(Version version, Version wantedVersion, Instant changedAt, String... hostnames) { + var nodeVersions = new ArrayList<NodeVersion>(); + for (var hostname : hostnames) { + nodeVersions.add(new NodeVersion(HostName.from(hostname), version, wantedVersion, changedAt)); + } + return NodeVersions.EMPTY.with(nodeVersions); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json index 8ab277a3795..1c660726d61 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json @@ -17,11 +17,11 @@ "queryQuality": 100, "writeQuality": 99.99894341115082, "pemDeployKeys": [ - "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----" + "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----" ], "pemDeveloperKeys": [ { - "key": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----", + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----", "user": "joe@dev" } ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json new file mode 100644 index 00000000000..96ca22e1c1a --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json @@ -0,0 +1,23 @@ +{ + "versions": [ + { + "releaseCommit": "badc0ffee", + "releasedAt": 123, + "isCurrentControllerVersion": true, + "isCurrentSystemVersion": true, + "isReleased": true, + "deploymentStatistics": { + "version": "7.0", + "failing": [], + "production": [], + "deploying": [] + }, + "confidence": "normal", + "configServerHostnames": [ + "cfg1", + "cfg2", + "cfg3" + ] + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 2d8c937097a..80e52f373d7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -79,8 +79,10 @@ public class ContainerControllerTester { Optional.of(new PropertyId("1234"))); controller().tenants().create(tenantSpec, credentials); - ApplicationId app = ApplicationId.from(tenant, application, instance); - return controller().applications().createApplication(app, Optional.of(credentials)); + TenantAndApplicationId id = TenantAndApplicationId.from(tenant, application); + controller().applications().createApplication(id, Optional.of(credentials)); + controller().applications().createInstance(id.instance(instance)); + return controller().applications().requireApplication(id); } public void deploy(ApplicationId id, ApplicationPackage applicationPackage, ZoneId zone) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 3ff21bb2261..307496ace5a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -29,6 +29,9 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -49,11 +52,8 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildJob; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; @@ -111,6 +111,11 @@ import static org.junit.Assert.assertTrue; public class ApplicationApiTest extends ControllerContainerTest { private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/"; + private static final String pemPublicKey = "-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" + + "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + + "-----END PUBLIC KEY-----\n"; + private static final String quotedPemPublicKey = pemPublicKey.replaceAll("\\n", "\\\\n"); private static final ApplicationPackage applicationPackageDefault = new ApplicationPackageBuilder() .instances("default") @@ -320,7 +325,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .region("us-west-1") .build(); - tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", POST) + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/default", POST) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), new File("application-reference-2.json")); @@ -360,14 +365,14 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST a pem deploy key tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/key", POST) .userIdentity(USER_ID) - .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----\"}"), - "{\"message\":\"Added deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)_\\n-----END PUBLIC KEY-----\"}"); + .data("{\"key\":\"" + pemPublicKey + "\"}"), + "{\"keys\":[\"-----BEGIN PUBLIC KEY-----\\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\\n-----END PUBLIC KEY-----\\n\"]}"); // PATCH in a pem deploy key at deprecated path tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/default", PATCH) .userIdentity(USER_ID) - .data("{\"pemDeployKey\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----\"}"), - "{\"message\":\"Added deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)_\\n-----END PUBLIC KEY-----\"}"); + .data("{\"pemDeployKey\":\"" + pemPublicKey + "\"}"), + "{\"message\":\"Added deploy key " + quotedPemPublicKey + "\"}"); // GET an application with a major version override tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", GET) @@ -383,8 +388,8 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE the pem deploy key tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/key", DELETE) .userIdentity(USER_ID) - .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)_\\n-----END PUBLIC KEY-----\"}"), - "{\"message\":\"Removed deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)_\\n-----END PUBLIC KEY-----\"}"); + .data("{\"key\":\"" + pemPublicKey + "\"}"), + "{\"keys\":[]}"); tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", GET) .userIdentity(USER_ID), @@ -556,6 +561,11 @@ public class ApplicationApiTest extends ControllerContainerTest { .oktaAccessToken(OKTA_AT), new File("delete-with-active-deployments.json"), 400); + // GET test-config for local tests against a prod deployment + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/test-config", GET) + .userIdentity(USER_ID), + new File("test-config.json")); + // DELETE (deactivate) a deployment - dev tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1", DELETE) .userIdentity(USER_ID), @@ -1061,7 +1071,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .oktaAccessToken(OKTA_AT) .userIdentity(USER_ID), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create 'tenant1.application1.instance1': Application already exists\"}", + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create 'tenant1.application1.instance1': Instance already exists\"}", 400); ConfigServerMock configServer = serviceRegistry().configServerMock(); @@ -1111,7 +1121,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) .oktaAccessToken(OKTA_AT) .userIdentity(USER_ID), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete application 'tenant1.application1.instance1': Application not found\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete instance 'tenant1.application1.instance1': Instance not found\"}", 404); // DELETE tenant diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json index 331aabd32d0..9d76654fbc0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json @@ -80,8 +80,8 @@ "majorVersion": 7, "globalRotations": [], "instances": [], - "pemDeployKey": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----", - "pemDeployKeys": ["-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----"], + "pemDeployKey": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n", + "pemDeployKeys": ["-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n"], "metrics": { "queryServiceQuality": 0.0, "writeServiceQuality": 0.0 diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index 25948e998f1..d62e39e42e7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -28,40 +28,10 @@ "lastWritesPerSecond": 2.0 }, "cost": { - "tco": 74, + "tco": 0, "waste": 0, - "utilization": 2.9999999999999996, - "cluster": { - "cluster1": { - "count": 2, - "resource": "cpu", - "utilization": 2.9999999999999996, - "tco": 74, - "waste": 0, - "flavor": "flavor1", - "flavorCost":37.0, - "flavorCpu":2.0, - "flavorMem":4.0, - "flavorDisk":50.0, - "type": "content", - "util": { - "cpu": 2.9999999999999996, - "mem": 0.4285714285714286, - "disk": 0.5714285714285715, - "diskBusy": 1.0 - }, - "usage": { - "cpu": 0.6, - "mem": 0.3, - "disk": 0.4, - "diskBusy": 0.3 - }, - "hostnames": [ - "host1", - "host2" - ] - } - } + "utilization": 0.0, + "cluster": {} }, "metrics": { "queriesPerSecond": 1.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index 1a2025e4de2..c56a269b9d4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -25,40 +25,10 @@ "lastWritesPerSecond": 2.0 }, "cost": { - "tco": 74, + "tco": 0, "waste": 0, - "utilization": 2.9999999999999996, - "cluster": { - "cluster1": { - "count": 2, - "resource": "cpu", - "utilization": 2.9999999999999996, - "tco": 74, - "waste": 0, - "flavor": "flavor1", - "flavorCost": 37.0, - "flavorCpu": 2.0, - "flavorMem": 4.0, - "flavorDisk": 50.0, - "type": "content", - "util": { - "cpu": 2.9999999999999996, - "mem": 0.4285714285714286, - "disk": 0.5714285714285715, - "diskBusy": 1.0 - }, - "usage": { - "cpu": 0.6, - "mem": 0.3, - "disk": 0.4, - "diskBusy": 0.3 - }, - "hostnames": [ - "host1", - "host2" - ] - } - } + "utilization": 0.0, + "cluster": {} }, "metrics": { "queriesPerSecond": 1.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index bb68904bee6..140be562fe9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -37,40 +37,10 @@ "lastWritesPerSecond": 2.0 }, "cost": { - "tco": 74, + "tco": 0, "waste": 0, - "utilization": 2.9999999999999996, - "cluster": { - "cluster1": { - "count": 2, - "resource": "cpu", - "utilization": 2.9999999999999996, - "tco": 74, - "waste": 0, - "flavor": "flavor1", - "flavorCost": 37.0, - "flavorCpu": 2.0, - "flavorMem": 4.0, - "flavorDisk": 50.0, - "type": "content", - "util": { - "cpu": 2.9999999999999996, - "mem": 0.4285714285714286, - "disk": 0.5714285714285715, - "diskBusy": 1.0 - }, - "usage": { - "cpu": 0.6, - "mem": 0.3, - "disk": 0.4, - "diskBusy": 0.3 - }, - "hostnames": [ - "host1", - "host2" - ] - } - } + "utilization": 0.0, + "cluster": {} }, "metrics": { "queriesPerSecond": 1.0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json new file mode 100644 index 00000000000..2338543b019 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json @@ -0,0 +1,20 @@ +{ + "application": "tenant1:application1:instance1", + "zone": "prod.us-central-1", + "system": "main", + "endpoints": { + "prod.us-central-1": [ + "http://old-endpoint.vespa.yahooapis.com:4080" + ] + }, + "zoneEndpoints": { + "prod.us-central-1": { + "default": "http://old-endpoint.vespa.yahooapis.com:4080" + } + }, + "clusters": { + "prod.us-central-1": [ + "music" + ] + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 084b235943e..0a4d046e318 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -1,26 +1,26 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.deployment; -import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; +import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Test; import java.io.File; +import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; /** * @author bratseth @@ -69,16 +69,15 @@ public class DeploymentApiTest extends ControllerContainerTest { private VersionStatus censorConfigServers(VersionStatus versionStatus, Controller controller) { List<VespaVersion> censored = new ArrayList<>(); for (VespaVersion version : versionStatus.versions()) { - if (!version.systemApplicationHostnames().isEmpty()) { + if (version.nodeVersions().size() > 0) { version = new VespaVersion(version.statistics(), version.releaseCommit(), version.committedAt(), version.isControllerVersion(), version.isSystemVersion(), version.isReleased(), - ImmutableSet.of("config1.test", "config2.test").stream() - .map(HostName::from) - .collect(Collectors.toSet()), + NodeVersions.EMPTY.with(List.of(new NodeVersion(HostName.from("config1.test"), version.versionNumber(), version.versionNumber(), Instant.EPOCH), + new NodeVersion(HostName.from("config2.test"), version.versionNumber(), version.versionNumber(), Instant.EPOCH))), VespaVersion.confidenceFrom(version.statistics(), controller) ); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index 3b7d55f8cef..0a1e996696b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -3,15 +3,21 @@ package com.yahoo.vespa.hosted.controller.restapi.filter; import ai.vespa.hosted.api.Method; import ai.vespa.hosted.api.RequestSigner; +import com.google.common.collect.ImmutableBiMap; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.ApplicationId; import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.security.KeyUtils; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; +import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.restapi.ApplicationRequestToDiscFilterRequestWrapper; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import org.junit.Before; import org.junit.Test; @@ -19,6 +25,8 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.net.URI; import java.net.http.HttpRequest; +import java.security.PrivateKey; +import java.security.PublicKey; import java.util.Set; import static org.junit.Assert.assertEquals; @@ -27,21 +35,21 @@ import static org.junit.Assert.assertTrue; public class SignatureFilterTest { - private static final String publicKey = "-----BEGIN PUBLIC KEY-----\n" + - "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" + - "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + - "-----END PUBLIC KEY-----\n"; + private static final PublicKey publicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" + + "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + + "-----END PUBLIC KEY-----\n"); - private static final String otherPublicKey = "-----BEGIN PUBLIC KEY-----\n" + - "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" + - "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" + - "-----END PUBLIC KEY-----\n"; + private static final PublicKey otherPublicKey = KeyUtils.fromPemEncodedPublicKey("-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" + + "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" + + "-----END PUBLIC KEY-----\n"); - private static final String privateKey = "-----BEGIN EC PRIVATE KEY-----\n" + - "MHcCAQEEIJUmbIX8YFLHtpRgkwqDDE3igU9RG6JD9cYHWAZii9j7oAoGCCqGSM49\n" + - "AwEHoUQDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9z/4jKSTHwbYR8wdsOSrJGVEU\n" + - "PbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + - "-----END EC PRIVATE KEY-----\n"; + private static final PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey("-----BEGIN EC PRIVATE KEY-----\n" + + "MHcCAQEEIJUmbIX8YFLHtpRgkwqDDE3igU9RG6JD9cYHWAZii9j7oAoGCCqGSM49\n" + + "AwEHoUQDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9z/4jKSTHwbYR8wdsOSrJGVEU\n" + + "PbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + + "-----END EC PRIVATE KEY-----\n"); private static final TenantAndApplicationId appId = TenantAndApplicationId.from("my-tenant", "my-app"); private static final ApplicationId id = appId.defaultInstance(); @@ -58,10 +66,10 @@ public class SignatureFilterTest { filter = new SignatureFilter(tester.controller()); signer = new RequestSigner(privateKey, id.serializedForm(), tester.clock()); - tester.createApplication(tester.createTenant(id.tenant().value(), "unused", 496L), - id.application().value(), - id.instance().value(), - 28L); + tester.curator().writeTenant(new CloudTenant(appId.tenant(), + new BillingInfo("id", "code"), + ImmutableBiMap.of())); + tester.curator().writeApplication(new Application(appId, tester.clock().instant())); } @Test @@ -69,42 +77,57 @@ public class SignatureFilterTest { // Unsigned request gets no role. HttpRequest.Builder request = HttpRequest.newBuilder(URI.create("https://host:123/path/./..//..%2F?query=empty&%3F=%26")); byte[] emptyBody = new byte[0]; - DiscFilterRequest unsigned = requestOf(request.method("GET", HttpRequest.BodyPublishers.ofByteArray(emptyBody)).build(), emptyBody); - filter.filter(unsigned); - assertNull(unsigned.getAttribute(SecurityContext.ATTRIBUTE_NAME)); + verifySecurityContext(requestOf(request.copy().method("GET", HttpRequest.BodyPublishers.ofByteArray(emptyBody)).build(), emptyBody), + null); // Signed request gets no role when no key is stored for the application. - DiscFilterRequest signed = requestOf(signer.signed(request, Method.GET, InputStream::nullInputStream), emptyBody); - filter.filter(signed); - assertNull(signed.getAttribute(SecurityContext.ATTRIBUTE_NAME)); - - // Signed request gets no role when a non-matching key is stored for the application. - applications.lockApplicationOrThrow(appId, application -> applications.store(application.withPemDeployKey(otherPublicKey))); - filter.filter(signed); - assertNull(signed.getAttribute(SecurityContext.ATTRIBUTE_NAME)); - - // Signed request gets a build service role when a matching key is stored for the application. - applications.lockApplicationOrThrow(appId, application -> applications.store(application.withPemDeployKey(publicKey))); - assertTrue(filter.filter(signed).isEmpty()); - SecurityContext securityContext = (SecurityContext) signed.getAttribute(SecurityContext.ATTRIBUTE_NAME); - assertEquals("buildService@my-tenant.my-app", securityContext.principal().getName()); - assertEquals(Set.of(Role.buildService(id.tenant(), id.application()), - Role.applicationDeveloper(id.tenant(), id.application())), - securityContext.roles()); - - // Signed POST request also gets a build service role. + verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody), + null); + + // Signed request gets no role when only non-matching keys are stored for the application. + applications.lockApplicationOrThrow(appId, application -> applications.store(application.withDeployKey(otherPublicKey))); + // Signed request gets no role when no key is stored for the application. + verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody), + null); + + // Signed request gets a headless role when a matching key is stored for the application. + applications.lockApplicationOrThrow(appId, application -> applications.store(application.withDeployKey(publicKey))); + verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody), + new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"), + Set.of(Role.reader(id.tenant()), + Role.developer(id.tenant())))); // TODO jonmv: Change to headless. + + // TODO jonmv: remove after Oct 2019. + // Signed request gets a build service role when a matching key is stored for the application and no X-Key header is provided. + verifySecurityContext(requestOf(signer.legacySigned(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody), + new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"), + Set.of(Role.reader(id.tenant()), + Role.developer(id.tenant())))); + + // Signed POST request with X-Key header gets a headless role. byte[] hiBytes = new byte[]{0x48, 0x69}; - signed = requestOf(signer.signed(request, Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes); - filter.filter(signed); - securityContext = (SecurityContext) signed.getAttribute(SecurityContext.ATTRIBUTE_NAME); - assertEquals("buildService@my-tenant.my-app", securityContext.principal().getName()); - assertEquals(Set.of(Role.buildService(id.tenant(), id.application()), - Role.applicationDeveloper(id.tenant(), id.application())), - securityContext.roles()); + verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes), + new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"), + Set.of(Role.reader(id.tenant()), + Role.developer(id.tenant())))); // TODO jonmv: Change to headless. + + // Signed request gets a developer role when a matching developer key is stored for the tenant. + tester.curator().writeTenant(new CloudTenant(appId.tenant(), + new BillingInfo("id", "code"), + ImmutableBiMap.of(publicKey, () -> "user"))); + verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes), + new SecurityContext(new SimplePrincipal("user"), + Set.of(Role.reader(id.tenant()), + Role.developer(id.tenant())))); // Unsigned requests still get no roles. - filter.filter(unsigned); - assertNull(unsigned.getAttribute(SecurityContext.ATTRIBUTE_NAME)); + verifySecurityContext(requestOf(request.copy().method("GET", HttpRequest.BodyPublishers.ofByteArray(emptyBody)).build(), emptyBody), + null); + } + + private void verifySecurityContext(DiscFilterRequest request, SecurityContext securityContext) { + assertTrue(filter.filter(request).isEmpty()); + assertEquals(securityContext, request.getAttribute(SecurityContext.ATTRIBUTE_NAME)); } private static DiscFilterRequest requestOf(HttpRequest request, byte[] body) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json index 17f90259fa8..01af1bd70dd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json @@ -6,92 +6,92 @@ "cloud": "cloud1", "nodes": [ { - "hostname": "node-1-configserver-host", + "hostname": "node-2-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-1-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-1-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-2-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-2-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-1-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-2-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-1-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-3-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-3-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-2-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-1-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-3-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-2-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-1-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" } @@ -103,47 +103,47 @@ "cloud": "cloud2", "nodes": [ { - "hostname": "node-1-configserver-host", + "hostname": "node-1-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-2-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-1-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-2-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-1-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-3-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-2-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json index 86bc272fcd1..dbaa6623fae 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json @@ -6,92 +6,92 @@ "cloud": "cloud1", "nodes": [ { - "hostname": "node-1-configserver-host", + "hostname": "node-2-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-1-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-1-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-2-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-2-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-1-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-2-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-1-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-3-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-3-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-2-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-1-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-3-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-2-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-1-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" } @@ -103,47 +103,47 @@ "cloud": "cloud2", "nodes": [ { - "hostname": "node-1-configserver-host", + "hostname": "node-1-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-2-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-1-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-2-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-1-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-3-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-2-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json index e8007fbf6c5..2b907c1156c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json @@ -6,47 +6,47 @@ "cloud": "cloud1", "nodes": [ { - "hostname": "node-1-configserver-host", + "hostname": "node-1-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-2-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-2-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-1-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-3-proxy-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-3-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-2-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-1-tenant-host-prod.us-west-1", "environment": "prod", "region": "us-west-1" } @@ -58,47 +58,47 @@ "cloud": "cloud1", "nodes": [ { - "hostname": "node-1-configserver-host", + "hostname": "node-2-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-1-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-2-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-1-proxy-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-3-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-2-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-1-tenant-host-prod.us-east-3", "environment": "prod", "region": "us-east-3" } @@ -110,47 +110,47 @@ "cloud": "cloud2", "nodes": [ { - "hostname": "node-1-configserver-host", + "hostname": "node-1-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-configserver-host", + "hostname": "node-2-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-configserver-host", + "hostname": "node-3-configserver-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-1-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-2-proxy-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-tenant-host", + "hostname": "node-1-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-1-tenant-host", + "hostname": "node-3-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-3-tenant-host", + "hostname": "node-2-tenant-host-prod.eu-west-1", "environment": "prod", "region": "eu-west-1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index b17dda7f810..b1f5f33b960 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -12,7 +12,6 @@ import java.io.File; import java.util.Set; import static com.yahoo.application.container.handler.Request.Method.DELETE; -import static com.yahoo.application.container.handler.Request.Method.PATCH; import static com.yahoo.application.container.handler.Request.Method.POST; import static com.yahoo.application.container.handler.Request.Method.PUT; import static org.junit.Assert.assertEquals; @@ -23,6 +22,17 @@ import static org.junit.Assert.assertEquals; public class UserApiTest extends ControllerContainerCloudTest { private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/"; + private static final String pemPublicKey = "-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\n" + + "z/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n" + + "-----END PUBLIC KEY-----\n"; + private static final String otherPemPublicKey = "-----BEGIN PUBLIC KEY-----\n" + + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\n" + + "pDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n" + + "-----END PUBLIC KEY-----\n"; + private static final String quotedPemPublicKey = pemPublicKey.replaceAll("\\n", "\\\\n"); + private static final String otherQuotedPemPublicKey = otherPemPublicKey.replaceAll("\\n", "\\\\n"); + @Test public void testUserManagement() { @@ -132,30 +142,30 @@ public class UserApiTest extends ControllerContainerCloudTest { // POST a pem deploy key tester.assertResponse(request("/application/v4/tenant/my-tenant/application/my-app/key", POST) .roles(Set.of(Role.tenantOperator(id.tenant()))) - .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----\"}"), - "{\"message\":\"Added deploy key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)_\\n-----END PUBLIC KEY-----\"}"); + .data("{\"key\":\"" + pemPublicKey + "\"}"), + new File("first-deploy-key.json")); // POST a pem developer key tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST) .user("joe@dev") .roles(Set.of(Role.tenantOperator(id.tenant()))) - .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----\"}"), - "{\"message\":\"Set developer key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)_\\n-----END PUBLIC KEY----- for joe@dev\"}"); + .data("{\"key\":\"" + pemPublicKey + "\"}"), + new File("first-developer-key.json")); // POST the same pem developer key for a different user is forbidden tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST) .user("operator@tenant") .roles(Set.of(Role.tenantOperator(id.tenant()))) - .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----\"}"), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Multiple entries with same key: -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)_\\n-----END PUBLIC KEY-----=operator@tenant and -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)_\\n-----END PUBLIC KEY-----=joe@dev\"}", + .data("{\"key\":\"" + pemPublicKey + "\"}"), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Key "+ quotedPemPublicKey + " is already owned by joe@dev\"}", 400); // PATCH in a different pem developer key tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST) .user("operator@tenant") .roles(Set.of(Role.tenantOperator(id.tenant()))) - .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n-----END PUBLIC KEY-----\"}"), - "{\"message\":\"Set developer key -----BEGIN PUBLIC KEY-----\\nƪ(`▿▿▿▿´ƪ)\\n-----END PUBLIC KEY----- for operator@tenant\"}"); + .data("{\"key\":\"" + otherPemPublicKey + "\"}"), + new File("both-developer-keys.json")); // GET tenant information with keys tester.assertResponse(request("/application/v4/tenant/my-tenant/") @@ -165,8 +175,8 @@ public class UserApiTest extends ControllerContainerCloudTest { // DELETE a pem developer key tester.assertResponse(request("/application/v4/tenant/my-tenant/key", DELETE) .roles(Set.of(Role.tenantOperator(id.tenant()))) - .data("{\"key\":\"-----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)_\\n-----END PUBLIC KEY-----\"}"), - "{\"message\":\"Removed developer key -----BEGIN PUBLIC KEY-----\\n∠( ᐛ 」∠)_\\n-----END PUBLIC KEY----- for joe@dev\"}"); + .data("{\"key\":\"" + pemPublicKey + "\"}"), + new File("second-developer-key.json")); // DELETE an application role is allowed for an application admin. tester.assertResponse(request("/user/v1/tenant/my-tenant/application/my-app", DELETE) @@ -180,8 +190,7 @@ public class UserApiTest extends ControllerContainerCloudTest { "{\"message\":\"Deleted application my-tenant.my-app\"}"); // DELETE a tenant role is available to tenant admins. - // DELETE the tenantOperator role clears any developer key. - // TODO jonmv: Change to developer, when this role exists. + // DELETE the developer role clears any developer key. tester.assertResponse(request("/user/v1/tenant/my-tenant", DELETE) .roles(Set.of(Role.tenantAdmin(id.tenant()))) .data("{\"user\":\"operator@tenant\",\"roleName\":\"tenantOperator\"}"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json index 6cf4dc76173..31bdb07b26b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json @@ -1,6 +1,5 @@ { "tenant": "my-tenant", "application": "my-app", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/my-tenant/application/my-app/instance/default" + "url": "http://localhost:8080/application/v4/tenant/my-tenant/application/my-app" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/both-developer-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/both-developer-keys.json new file mode 100644 index 00000000000..2ff1c29fe29 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/both-developer-keys.json @@ -0,0 +1,12 @@ +{ + "keys": [ + { + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n", + "user": "joe@dev" + }, + { + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n", + "user": "operator@tenant" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-deploy-key.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-deploy-key.json new file mode 100644 index 00000000000..1c86877b77d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-deploy-key.json @@ -0,0 +1,5 @@ +{ + "keys": [ + "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n" + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json new file mode 100644 index 00000000000..b7d48f283f3 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/first-developer-key.json @@ -0,0 +1,9 @@ +{ + "keys": [ + { + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n", + "user": "joe@dev" + } + ] +} + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/second-developer-key.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/second-developer-key.json new file mode 100644 index 00000000000..f7d90f31116 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/second-developer-key.json @@ -0,0 +1,8 @@ +{ + "keys": [ + { + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n", + "user": "operator@tenant" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json index 5aaa900c3f0..b7970a48963 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json @@ -1,27 +1,14 @@ { "tenant": "my-tenant", "type": "CLOUD", - "pemDeployKeys": [ - { - "key": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----", - "application": "my-app" - } - ], "pemDeveloperKeys": [ { - "key": "-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)_\n-----END PUBLIC KEY-----", + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\n-----END PUBLIC KEY-----\n", "user": "joe@dev" }, { - "key": "-----BEGIN PUBLIC KEY-----\nƪ(`▿▿▿▿´ƪ)\n-----END PUBLIC KEY-----", + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\n-----END PUBLIC KEY-----\n", "user": "operator@tenant" }], - "applications": [ - { - "tenant":"my-tenant", - "application":"my-app", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/my-tenant/application/my-app/instance/default" - } - ] + "applications": [] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json index a89a0f5360c..39b6cccbab0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json @@ -1,7 +1,6 @@ { "tenant": "my-tenant", "type": "CLOUD", - "pemDeployKeys": [], "pemDeveloperKeys": [], "applications": [] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index ba8309de286..83223b0e041 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -8,7 +8,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; |