From a078c9d8d365682639477c6d499a48fe12b56ecd Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Thu, 14 Nov 2019 12:25:37 +0100 Subject: Created maintainer for resource tagging --- .../api/integration/ServiceRegistry.java | 3 ++ .../api/integration/aws/MockResourceTagger.java | 28 ++++++++++++ .../api/integration/aws/ResourceTagger.java | 28 ++++++++++++ .../api/integration/configserver/Node.java | 12 +++-- .../integration/configserver/NodeRepository.java | 5 +- .../maintenance/ControllerMaintenance.java | 3 ++ .../maintenance/ResourceTagMaintainer.java | 53 ++++++++++++++++++++++ .../controller/integration/ConfigServerMock.java | 7 ++- .../controller/integration/NodeRepositoryMock.java | 10 +++- .../integration/ServiceRegistryMock.java | 8 ++++ .../maintenance/ClusterInfoMaintainerTest.java | 2 + .../controller/maintenance/OsUpgraderTest.java | 4 +- .../maintenance/ResourceTagMaintainerTest.java | 53 ++++++++++++++++++++++ .../controller/maintenance/SystemUpgraderTest.java | 4 +- .../restapi/controller/responses/maintenance.json | 3 ++ .../hosted/controller/restapi/os/OsApiTest.java | 2 +- .../controller/versions/VersionStatusTest.java | 4 +- 17 files changed, 214 insertions(+), 15 deletions(-) create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index 4eb4f669225..6ca5cae0455 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.vespa.hosted.controller.api.integration.aws.AwsEventFetcher; +import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; @@ -77,4 +78,6 @@ public interface ServiceRegistry { ZoneRegistry zoneRegistry(); + ResourceTagger resourceTagger(); + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java new file mode 100644 index 00000000000..6992e813b10 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java @@ -0,0 +1,28 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.aws; + +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.config.provision.zone.ZoneId; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author olaa + */ +public class MockResourceTagger implements ResourceTagger { + + Map> values = new HashMap<>(); + + @Override + public int tagResources(ZoneApi zone, Map tenantOfHosts) { + values.put(zone.getId(), tenantOfHosts); + return 0; + } + + public Map> getValues() { + return values; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java new file mode 100644 index 00000000000..46561ccf401 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java @@ -0,0 +1,28 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.aws; + +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneApi; + +import java.util.Map; + +/** + * @author olaa + */ +public interface ResourceTagger { + + /** + * Returns number of tagged resources + */ + int tagResources(ZoneApi zone, Map tenantOfHosts); + + static ResourceTagger empty() { + return new ResourceTagger() { + @Override + public int tagResources(ZoneApi zone, Map tenantOfHosts) { + return 0; + } + }; + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index 4ffe221b555..e8ae7c181d8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -19,6 +19,7 @@ import java.util.Optional; public class Node { private final HostName hostname; + private final Optional parentHostname; private final State state; private final NodeType type; private final Optional owner; @@ -41,11 +42,12 @@ public class Node { private final String clusterId; private final ClusterType clusterType; - public Node(HostName hostname, State state, NodeType type, Optional owner, + public Node(HostName hostname, Optional parentHostname, State state, NodeType type, Optional owner, Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, ServiceState serviceState, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, double vcpu, double memoryGb, double diskGb, double bandwidthGbps, boolean fastDisk, int cost, String canonicalFlavor, String clusterId, ClusterType clusterType) { this.hostname = hostname; + this.parentHostname = parentHostname; this.state = state; this.type = type; this.owner = owner; @@ -70,9 +72,9 @@ public class Node { } @TestOnly - public Node(HostName hostname, State state, NodeType type, Optional owner, + public Node(HostName hostname, Optional parentHostname, State state, NodeType type, Optional owner, Version currentVersion, Version wantedVersion) { - this(hostname, state, type, owner, currentVersion, wantedVersion, + this(hostname, parentHostname, state, type, owner, currentVersion, wantedVersion, Version.emptyVersion, Version.emptyVersion, ServiceState.unorchestrated, 0, 0, 0, 0, 2, 8, 50, 1, true, 0, "d-2-8-50", "cluster", ClusterType.container); } @@ -81,6 +83,10 @@ public class Node { return hostname; } + public Optional parentHostname() { + return parentHostname; + } + public State state() { return state; } public NodeType type() { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index b36eba7d063..2dcbf4ca930 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; @@ -75,7 +76,9 @@ public interface NodeRepository { var application = Optional.ofNullable(node.getOwner()) .map(owner -> ApplicationId.from(owner.getTenant(), owner.getApplication(), owner.getInstance())); - return new Node(com.yahoo.config.provision.HostName.from(node.getHostname()), + var parentHostname = Optional.ofNullable(node.getParentHostname()).map(HostName::from); + return new Node(HostName.from(node.getHostname()), + parentHostname, fromJacksonState(node.getState()), fromJacksonType(node.getType()), application, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 1743cad32e4..47959abfab4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -46,6 +46,7 @@ public class ControllerMaintenance extends AbstractComponent { private final BillingMaintainer billingMaintainer; private final CloudEventReporter cloudEventReporter; private final RotationStatusUpdater rotationStatusUpdater; + private final ResourceTagMaintainer resourceTagMaintainer; @Inject @SuppressWarnings("unused") // instantiated by Dependency Injection @@ -77,6 +78,7 @@ public class ControllerMaintenance extends AbstractComponent { billingMaintainer = new BillingMaintainer(controller, Duration.ofDays(3), jobControl); cloudEventReporter = new CloudEventReporter(controller, Duration.ofDays(1), jobControl); rotationStatusUpdater = new RotationStatusUpdater(controller, maintenanceInterval, jobControl); + resourceTagMaintainer = new ResourceTagMaintainer(controller, Duration.ofMinutes(30), jobControl, controller.serviceRegistry().resourceTagger()); } public Upgrader upgrader() { return upgrader; } @@ -107,6 +109,7 @@ public class ControllerMaintenance extends AbstractComponent { billingMaintainer.deconstruct(); cloudEventReporter.deconstruct(); rotationStatusUpdater.deconstruct(); + resourceTagMaintainer.deconstruct(); } /** Create one OS upgrader per cloud found in the zone registry of controller */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java new file mode 100644 index 00000000000..32c8d64bf29 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java @@ -0,0 +1,53 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; + +import java.time.Duration; +import java.util.Map; +import java.util.logging.Level; +import java.util.stream.Collectors; + +/** + * @author olaa + */ +public class ResourceTagMaintainer extends Maintainer { + + private final ResourceTagger resourceTagger; + + public ResourceTagMaintainer(Controller controller, Duration interval, JobControl jobControl, ResourceTagger resourceTagger) { + super(controller, interval, jobControl); + this.resourceTagger = resourceTagger; + } + + @Override + public void maintain() { + controller().zoneRegistry().zones() + .ofCloud(CloudName.from("aws")) + .reachable() + .zones().forEach(zone -> { + Map tenantOfHosts = getTenantOfParentHosts(zone.getId()); + int taggedResources = resourceTagger.tagResources(zone, tenantOfHosts); + if (taggedResources > 0) + log.log(Level.INFO, "Tagged " + taggedResources + " resources in " + zone.getId()); + }); + + + } + + private Map getTenantOfParentHosts(ZoneId zoneId) { + return controller().serviceRegistry().configServer().nodeRepository() + .list(zoneId) + .stream() + .filter(node -> node.parentHostname().isPresent() && node.owner().isPresent()) + .collect(Collectors.toMap( + node -> node.parentHostname().get(), + node -> node.owner().get().tenant() + )); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 1025901dc15..d466681255a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -85,6 +85,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer /** Assigns a reserved tenant node to the given deployment, with initial versions. */ public void provision(ZoneId zone, ApplicationId application) { nodeRepository().putByHostname(zone, new Node(hostFor(application, zone), + Optional.empty(), Node.State.reserved, NodeType.tenant, Optional.of(application), @@ -113,6 +114,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer HostName.from("node-" + i + "-" + application.id().application() .value() + "-" + zone.value()), + Optional.empty(), Node.State.active, application.nodeType(), Optional.of(application.id()), initialVersion, @@ -175,14 +177,14 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer for (Node node : nodeRepository().list(zone, application)) { Node newNode; if (osVersion) { - newNode = new Node(node.hostname(), node.state(), node.type(), node.owner(), node.currentVersion(), + newNode = new Node(node.hostname(), Optional.empty(), node.state(), node.type(), node.owner(), node.currentVersion(), node.wantedVersion(), version, version, node.serviceState(), node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), node.wantedRebootGeneration(), node.vcpu(), node.memoryGb(), node.diskGb(), node.bandwidthGbps(), node.fastDisk(), node.cost(), node.canonicalFlavor(), node.clusterId(), node.clusterType()); } else { - newNode = new Node(node.hostname(), node.state(), node.type(), node.owner(), version, + newNode = new Node(node.hostname(), Optional.empty(), node.state(), node.type(), node.owner(), version, version, node.currentOsVersion(), node.wantedOsVersion(), node.serviceState(), node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), node.wantedRebootGeneration(), node.vcpu(), node.memoryGb(), node.diskGb(), @@ -306,6 +308,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer List nodes = nodeRepository.list(deployment.zoneId(), deployment.applicationId()); for (Node node : nodes) { nodeRepository.putByHostname(deployment.zoneId(), new Node(node.hostname(), + Optional.empty(), Node.State.active, node.type(), node.owner(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 7cec6a00f8e..d3b33581214 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -67,6 +67,7 @@ public class NodeRepositoryMock implements NodeRepository { public void addFixedNodes(ZoneId zone) { var nodeA = new Node(HostName.from("hostA"), + Optional.of(HostName.from("parentHostA")), Node.State.active, NodeType.tenant, Optional.of(ApplicationId.from("tenant1", "app1", "default")), @@ -89,6 +90,7 @@ public class NodeRepositoryMock implements NodeRepository { "clusterA", Node.ClusterType.container); var nodeB = new Node(HostName.from("hostB"), + Optional.of(HostName.from("parentHostB")), Node.State.active, NodeType.tenant, Optional.of(ApplicationId.from("tenant2", "app2", "default")), @@ -160,7 +162,7 @@ public class NodeRepositoryMock implements NodeRepository { nodeRepository.getOrDefault(zone, Collections.emptyMap()).values() .stream() .filter(node -> node.type() == type) - .map(node -> new Node(node.hostname(), node.state(), node.type(), node.owner(), + .map(node -> new Node(node.hostname(), node.parentHostname(), node.state(), node.type(), node.owner(), node.currentVersion(), version)) .forEach(node -> putByHostname(zone, node)); } @@ -191,7 +193,7 @@ public class NodeRepositoryMock implements NodeRepository { public void doUpgrade(DeploymentId deployment, Optional hostName, Version version) { modifyNodes(deployment, hostName, node -> { assert node.wantedVersion().equals(version); - return new Node(node.hostname(), node.state(), node.type(), node.owner(), version, version); + return new Node(node.hostname(), node.parentHostname(), node.state(), node.type(), node.owner(), version, version); }); } @@ -205,6 +207,7 @@ public class NodeRepositoryMock implements NodeRepository { public void requestRestart(DeploymentId deployment, Optional hostname) { modifyNodes(deployment, hostname, node -> new Node(node.hostname(), + node.parentHostname(), node.state(), node.type(), node.owner(), @@ -230,6 +233,7 @@ public class NodeRepositoryMock implements NodeRepository { public void doRestart(DeploymentId deployment, Optional hostname) { modifyNodes(deployment, hostname, node -> new Node(node.hostname(), + node.parentHostname(), node.state(), node.type(), node.owner(), @@ -255,6 +259,7 @@ public class NodeRepositoryMock implements NodeRepository { public void requestReboot(DeploymentId deployment, Optional hostname) { modifyNodes(deployment, hostname, node -> new Node(node.hostname(), + node.parentHostname(), node.state(), node.type(), node.owner(), @@ -280,6 +285,7 @@ public class NodeRepositoryMock implements NodeRepository { public void doReboot(DeploymentId deployment, Optional hostname) { modifyNodes(deployment, hostname, node -> new Node(node.hostname(), + node.parentHostname(), node.state(), node.type(), node.owner(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index eced161cebc..bd82807342e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -8,6 +8,8 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher; +import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; +import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; @@ -57,6 +59,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final ApplicationStoreMock applicationStoreMock = new ApplicationStoreMock(); private final MockRunDataStore mockRunDataStore = new MockRunDataStore(); private final MockTenantCost mockTenantCost = new MockTenantCost(); + private final MockResourceTagger mockResourceTagger = new MockResourceTagger(); public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); @@ -180,6 +183,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return zoneRegistryMock; } + @Override + public ResourceTagger resourceTagger() { + return mockResourceTagger; + } + public ConfigServerMock configServerMock() { return configServerMock; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java index 9739a24af01..bab0bb0615e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java @@ -54,6 +54,7 @@ public class ClusterInfoMaintainerTest { private void addNodes(ZoneId zone) { var nodeA = new Node(HostName.from("hostA"), + Optional.empty(), Node.State.active, NodeType.tenant, Optional.of(ApplicationId.from("tenant1", "app1", "default")), @@ -76,6 +77,7 @@ public class ClusterInfoMaintainerTest { "clusterA", Node.ClusterType.container); var nodeB = new Node(HostName.from("hostB"), + Optional.empty(), Node.State.active, NodeType.tenant, Optional.of(ApplicationId.from("tenant1", "app1", "default")), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 8a5dc7594f9..f39bff416dd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -154,7 +154,7 @@ public class OsUpgraderTest { throw new IllegalArgumentException("No nodes allocated to " + application.id()); } Node node = nodes.get(0); - nodeRepository().putByHostname(zone, new Node(node.hostname(), Node.State.failed, node.type(), node.owner(), + nodeRepository().putByHostname(zone, new Node(node.hostname(), node.parentHostname(), Node.State.failed, node.type(), node.owner(), node.currentVersion(), node.wantedVersion())); } @@ -164,7 +164,7 @@ public class OsUpgraderTest { for (ZoneId zone : zones) { for (Node node : nodesRequiredToUpgrade(zone, application)) { nodeRepository().putByHostname(zone, new Node( - node.hostname(), node.state(), node.type(), node.owner(), node.currentVersion(), + node.hostname(), node.parentHostname(), node.state(), node.type(), node.owner(), node.currentVersion(), node.wantedVersion(), version, version, node.serviceState(), node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), node.wantedRebootGeneration(), node.vcpu(), node.memoryGb(), node.diskGb(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java new file mode 100644 index 00000000000..2421a477568 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java @@ -0,0 +1,53 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import org.junit.Test; + +import java.time.Duration; +import java.util.Map; + +import static org.junit.Assert.*; + +/** + * @author olaa + */ +public class ResourceTagMaintainerTest { + + ControllerTester tester = new ControllerTester(); + + @Test + public void maintain() { + setUpZones(); + MockResourceTagger mockResourceTagger = new MockResourceTagger(); + ResourceTagMaintainer resourceTagMaintainer = new ResourceTagMaintainer(tester.controller(), + Duration.ofMinutes(5), + new JobControl(tester.curator()), + mockResourceTagger); + resourceTagMaintainer.maintain(); + assertEquals(2, mockResourceTagger.getValues().size()); + Map tenanForHost = mockResourceTagger.getValues().get(ZoneId.from("prod.region-2")); + assertEquals(TenantName.from("tenant1"), tenanForHost.get(HostName.from("parentHostA"))); + assertEquals(TenantName.from("tenant2"), tenanForHost.get(HostName.from("parentHostB"))); + + + } + + private void setUpZones() { + ZoneApiMock nonAwsZone = ZoneApiMock.newBuilder().withId("test.region-1").build(); + ZoneApiMock awsZone1 = ZoneApiMock.newBuilder().withId("prod.region-2").withCloud("aws").build(); + ZoneApiMock awsZone2 = ZoneApiMock.newBuilder().withId("test.region-3").withCloud("aws").build(); + tester.zoneRegistry().setZones( + nonAwsZone, + awsZone1, + awsZone2); + tester.configServer().nodeRepository().addFixedNodes(nonAwsZone.getId()); + tester.configServer().nodeRepository().addFixedNodes(awsZone1.getId()); + tester.configServer().nodeRepository().addFixedNodes(awsZone2.getId()); + } + +} \ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index f3c18499f16..d83b0d2eafe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -301,7 +301,7 @@ public class SystemUpgraderTest { for (Node node : listNodes(zone, application)) { nodeRepository().putByHostname( zone.getId(), - new Node(node.hostname(), node.state(), node.type(), node.owner(), node.wantedVersion(), node.wantedVersion())); + new Node(node.hostname(), node.parentHostname(), node.state(), node.type(), node.owner(), node.wantedVersion(), node.wantedVersion())); } assertCurrentVersion(application, version, zone); @@ -326,7 +326,7 @@ public class SystemUpgraderTest { Node node = nodes.get(0); nodeRepository().putByHostname( zone.getId(), - new Node(node.hostname(), Node.State.failed, node.type(), node.owner(), node.currentVersion(), node.wantedVersion())); + new Node(node.hostname(), node.parentHostname(), Node.State.failed, node.type(), node.owner(), node.currentVersion(), node.wantedVersion())); } private void assertSystemVersion(Version version) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 28d2980181f..3371c5563c9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -51,6 +51,9 @@ { "name": "ResourceMeterMaintainer" }, + { + "name": "ResourceTagMaintainer" + }, { "name": "RotationStatusUpdater" }, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index 89f11eae5ae..3b5b3ba7f9d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -145,7 +145,7 @@ public class OsApiTest extends ControllerContainerTest { for (Node node : nodeRepository().list(zone, application.id())) { var version = targetVersion.orElse(node.wantedOsVersion()); nodeRepository().putByHostname(zone, new Node( - node.hostname(), node.state(), node.type(), node.owner(), node.currentVersion(), + node.hostname(), node.parentHostname(), node.state(), node.type(), node.owner(), node.currentVersion(), node.wantedVersion(), version, version, node.serviceState(), node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), node.wantedRebootGeneration(), node.vcpu(), node.memoryGb(), node.diskGb(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 06260da833f..5456c280d95 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -66,7 +66,7 @@ public class VersionStatusTest { // Upgrade some config servers for (ZoneApi zone : tester.zoneRegistry().zones().all().zones()) { for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) { - Node upgradedNode = new Node(node.hostname(), node.state(), node.type(), node.owner(), version1, node.wantedVersion()); + Node upgradedNode = new Node(node.hostname(), node.parentHostname(), node.state(), node.type(), node.owner(), version1, node.wantedVersion()); tester.configServer().nodeRepository().putByHostname(zone.getId(), upgradedNode); break; } @@ -110,7 +110,7 @@ public class VersionStatusTest { Version ancientVersion = Version.fromString("5.1"); for (ZoneApi zone : tester.controller().zoneRegistry().zones().all().zones()) { for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) { - Node downgradedNode = new Node(node.hostname(), node.state(), node.type(), node.owner(), ancientVersion, node.wantedVersion()); + Node downgradedNode = new Node(node.hostname(), node.parentHostname(), node.state(), node.type(), node.owner(), ancientVersion, node.wantedVersion()); tester.configServer().nodeRepository().putByHostname(zone.getId(), downgradedNode); break; } -- cgit v1.2.3 From da978ae5a7ae163f33dd604831dded50cb1c6ee3 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Fri, 15 Nov 2019 16:06:40 +0100 Subject: Map hostname to applicationId --- .../controller/api/integration/aws/MockResourceTagger.java | 8 ++++---- .../hosted/controller/api/integration/aws/ResourceTagger.java | 6 +++--- .../hosted/controller/maintenance/ResourceTagMaintainer.java | 10 +++++----- .../controller/maintenance/ResourceTagMaintainerTest.java | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java index 6992e813b10..0fd114095d7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/MockResourceTagger.java @@ -1,8 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.aws; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; @@ -14,15 +14,15 @@ import java.util.Map; */ public class MockResourceTagger implements ResourceTagger { - Map> values = new HashMap<>(); + Map> values = new HashMap<>(); @Override - public int tagResources(ZoneApi zone, Map tenantOfHosts) { + public int tagResources(ZoneApi zone, Map tenantOfHosts) { values.put(zone.getId(), tenantOfHosts); return 0; } - public Map> getValues() { + public Map> getValues() { return values; } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java index 46561ccf401..9d0e6481d38 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java @@ -1,8 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.aws; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneApi; import java.util.Map; @@ -15,12 +15,12 @@ public interface ResourceTagger { /** * Returns number of tagged resources */ - int tagResources(ZoneApi zone, Map tenantOfHosts); + int tagResources(ZoneApi zone, Map tenantOfHosts); static ResourceTagger empty() { return new ResourceTagger() { @Override - public int tagResources(ZoneApi zone, Map tenantOfHosts) { + public int tagResources(ZoneApi zone, Map tenantOfHosts) { return 0; } }; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java index 32c8d64bf29..44e21b3aed9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java @@ -1,9 +1,9 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; @@ -31,8 +31,8 @@ public class ResourceTagMaintainer extends Maintainer { .ofCloud(CloudName.from("aws")) .reachable() .zones().forEach(zone -> { - Map tenantOfHosts = getTenantOfParentHosts(zone.getId()); - int taggedResources = resourceTagger.tagResources(zone, tenantOfHosts); + Map applicationOfHosts = getTenantOfParentHosts(zone.getId()); + int taggedResources = resourceTagger.tagResources(zone, applicationOfHosts); if (taggedResources > 0) log.log(Level.INFO, "Tagged " + taggedResources + " resources in " + zone.getId()); }); @@ -40,14 +40,14 @@ public class ResourceTagMaintainer extends Maintainer { } - private Map getTenantOfParentHosts(ZoneId zoneId) { + private Map getTenantOfParentHosts(ZoneId zoneId) { return controller().serviceRegistry().configServer().nodeRepository() .list(zoneId) .stream() .filter(node -> node.parentHostname().isPresent() && node.owner().isPresent()) .collect(Collectors.toMap( node -> node.parentHostname().get(), - node -> node.owner().get().tenant() + node -> node.owner().get() )); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java index 2421a477568..e5e124d6647 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java @@ -1,7 +1,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; @@ -30,9 +30,9 @@ public class ResourceTagMaintainerTest { mockResourceTagger); resourceTagMaintainer.maintain(); assertEquals(2, mockResourceTagger.getValues().size()); - Map tenanForHost = mockResourceTagger.getValues().get(ZoneId.from("prod.region-2")); - assertEquals(TenantName.from("tenant1"), tenanForHost.get(HostName.from("parentHostA"))); - assertEquals(TenantName.from("tenant2"), tenanForHost.get(HostName.from("parentHostB"))); + Map applicationForHost = mockResourceTagger.getValues().get(ZoneId.from("prod.region-2")); + assertEquals(ApplicationId.from("tenant1", "app1", "default"), applicationForHost.get(HostName.from("parentHostA"))); + assertEquals(ApplicationId.from("tenant2", "app2", "default"), applicationForHost.get(HostName.from("parentHostB"))); } -- cgit v1.2.3