From 894c4fdd551f11f415c08c09af64c3228a57d74a Mon Sep 17 00:00:00 2001 From: Morten Tokle Date: Wed, 16 Jan 2019 13:37:32 +0100 Subject: Maintainer updating name service for load balancers --- .../vespa/hosted/controller/LockedApplication.java | 12 ++- .../hosted/controller/application/Deployment.java | 24 +++-- .../maintenance/ControllerMaintenance.java | 2 + .../maintenance/LoadBalancerMaintainer.java | 120 +++++++++++++++++++++ .../persistence/ApplicationSerializer.java | 18 +++- .../controller/integration/ConfigServerMock.java | 14 ++- .../maintenance/LoadBalancerMaintainerTest.java | 114 ++++++++++++++++++++ .../persistence/ApplicationSerializerTest.java | 10 +- .../restapi/controller/responses/maintenance.json | 3 + 9 files changed, 302 insertions(+), 15 deletions(-) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 6ac62284661..30921715f75 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -10,11 +10,11 @@ import com.yahoo.config.provision.HostName; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -26,6 +26,7 @@ import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.time.Instant; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -68,8 +69,7 @@ public class LockedApplication { application.deployments(), application.deploymentJobs(), application.change(), application.outstandingChange(), application.ownershipIssueId(), application.owner(), application.majorVersion(), application.metrics(), - application.rotation(), - application.rotationStatus()); + application.rotation(), application.rotationStatus()); } private LockedApplication(Lock lock, ApplicationId id, Instant createdAt, @@ -147,7 +147,7 @@ public class LockedApplication { previousDeployment.clusterUtils(), previousDeployment.clusterInfo(), previousDeployment.metrics(), - previousDeployment.activity()); + previousDeployment.activity(), Collections.emptyMap()); return with(newDeployment); } @@ -248,6 +248,10 @@ public class LockedApplication { outstandingChange, ownershipIssueId, owner, majorVersion, metrics, rotation, rotationStatus); } + public LockedApplication withDeploymentLoadBalancers(ZoneId zoneId, Map loadBalancers) { + return with(deployments.get(zoneId).withLoadBalancers(loadBalancers)); + } + /** Don't expose non-leaf sub-objects. */ private LockedApplication with(Deployment deployment) { Map deployments = new LinkedHashMap<>(this.deployments); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index ed06ff5bddc..7b9cc6be6ca 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; import com.yahoo.config.provision.ClusterSpec.Id; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; @@ -28,16 +29,17 @@ public class Deployment { private final Map clusterInfo; private final DeploymentMetrics metrics; private final DeploymentActivity activity; + private final Map loadBalancers; public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime) { this(zone, applicationVersion, version, deployTime, Collections.emptyMap(), Collections.emptyMap(), - DeploymentMetrics.none, DeploymentActivity.none); + DeploymentMetrics.none, DeploymentActivity.none, Collections.emptyMap()); } public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime, Map clusterUtilization, Map clusterInfo, DeploymentMetrics metrics, - DeploymentActivity activity) { + DeploymentActivity activity, Map loadBalancers) { this.zone = Objects.requireNonNull(zone, "zone cannot be null"); this.applicationVersion = Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null"); this.version = Objects.requireNonNull(version, "version cannot be null"); @@ -46,6 +48,7 @@ public class Deployment { this.clusterInfo = Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null"); this.metrics = Objects.requireNonNull(metrics, "deploymentMetrics cannot be null"); this.activity = Objects.requireNonNull(activity, "activity cannot be null"); + this.loadBalancers = Objects.requireNonNull(loadBalancers, "loadBalancers cannot be null"); } /** Returns the zone this was deployed to */ @@ -78,24 +81,33 @@ public class Deployment { return clusterUtilization; } + public Map loadBalancers() { + return loadBalancers; + } + public Deployment recordActivityAt(Instant instant) { return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics, - activity.recordAt(instant, metrics)); + activity.recordAt(instant, metrics), loadBalancers); } public Deployment withClusterUtils(Map clusterUtilization) { return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics, - activity); + activity, loadBalancers); } public Deployment withClusterInfo(Map newClusterInfo) { return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, newClusterInfo, metrics, - activity); + activity, loadBalancers); } public Deployment withMetrics(DeploymentMetrics metrics) { return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics, - activity); + activity, loadBalancers); + } + + public Deployment withLoadBalancers(Map loadBalancers) { + return new Deployment(zone, applicationVersion, version, deployTime, clusterUtilization, clusterInfo, metrics, + activity, loadBalancers); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 08e5be2ea1f..a2015710070 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -52,6 +52,7 @@ public class ControllerMaintenance extends AbstractComponent { private final JobRunner jobRunner; private final ContactInformationMaintainer contactInformationMaintainer; private final CostReportMaintainer costReportMaintainer; + private final LoadBalancerMaintainer loadbalancerMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator, @@ -81,6 +82,7 @@ public class ControllerMaintenance extends AbstractComponent { osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl); contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, contactRetriever); costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig); + loadbalancerMaintainer = new LoadBalancerMaintainer(controller, Duration.ofMinutes(5), jobControl, nameService); } public Upgrader upgrader() { return upgrader; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java new file mode 100644 index 00000000000..f54cbc875f2 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java @@ -0,0 +1,120 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.google.common.base.Strings; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.LockedApplication; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Deployment; + +import java.time.Duration; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +public class LoadBalancerMaintainer extends Maintainer { + + private static final Logger log = Logger.getLogger(LoadBalancerMaintainer.class.getName()); + private static final String IGNORE_ENDPOINT_VALUE = "default"; + + private final NameService nameService; + + public LoadBalancerMaintainer(Controller controller, + Duration interval, + JobControl jobControl, + NameService nameService) { + super(controller, interval, jobControl); + this.nameService = nameService; + } + + @Override + protected void maintain() { + // update application object with load balancer information + controller().applications().asList().forEach(this::updateApplicationLoadBalancers); + + // Create or update cnames + List applications = controller().applications().asList(); + applications.forEach(this::registerLoadBalancerEndpoint); + + // Delete removed application rotations + // TODO + } + + private void updateApplicationLoadBalancers(Application application) { + // Get a list of all load balancers for this applications (for all zones and clusters) + Map> applicationZoneLoadBalancers = new HashMap<>(); + for (ZoneId zoneId : application.deployments().keySet()) { + try { + applicationZoneLoadBalancers.put(zoneId, controller().applications().configServer().getLoadBalancers(new DeploymentId(application.id(), zoneId))); + } catch (Exception e) { + log.log(LogLevel.INFO, + String.format("Got exception fetching load balancers for application: %s, in zone: %s", + application.id().toShortString(), zoneId.value()), + e); + } + } + + // store the load balancers on the deployments + controller().applications().lockIfPresent(application.id(), lockedApplication -> storeApplicationWithLoadBalancers(lockedApplication, applicationZoneLoadBalancers)); + } + + private void storeApplicationWithLoadBalancers(LockedApplication lockedApplication, Map> loadBalancers) { + for (Map.Entry> entry : loadBalancers.entrySet()) { + Map loadbalancerClusterMap = entry.getValue().stream() + .collect(Collectors.toMap(lb -> ClusterSpec.Id.from(lb.id()), lb -> HostName.from(lb.hostname()))); + lockedApplication = lockedApplication.withDeploymentLoadBalancers(entry.getKey(), loadbalancerClusterMap); + + } + controller().applications().store(lockedApplication); + } + + private void registerLoadBalancerEndpoint(Application application) { + for (Map.Entry deploymentEntry : application.deployments().entrySet()) { + ZoneId zone = deploymentEntry.getKey(); + Deployment deployment = deploymentEntry.getValue(); + for (Map.Entry loadBalancers : deployment.loadBalancers().entrySet()) { + try { + RecordName recordName = RecordName.from(getEndpointName(loadBalancers.getKey(), application.id(), zone)); + RecordData recordData = RecordData.fqdn(loadBalancers.getValue().value()); + nameService.createCname(recordName, recordData); + } catch (Exception e) { + // Catching any exception, will be retried on next run + log.log(LogLevel.INFO, + String.format("Got exception updating name service for application: %s, cluster: %s in zone: %s", + application.id().toShortString(), loadBalancers.getKey().value(), zone.value()), + e); + } + } + } + } + + static String getEndpointName(ClusterSpec.Id clusterId, ApplicationId applicationId, ZoneId zoneId) { + List endpointTerms = Arrays.asList(ignorePartIfDefault(clusterId.value()), + ignorePartIfDefault(applicationId.instance().value()), + applicationId.application().value(), + applicationId.tenant().value(), + zoneId.value(), + "vespa.oath.cloud" + ); + return endpointTerms.stream() + .filter(s -> !Strings.isNullOrEmpty((s))) + .collect(Collectors.joining(".")); + } + + private static String ignorePartIfDefault(String s) { + return IGNORE_ENDPOINT_VALUE.equalsIgnoreCase(s) ? "" : s; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index ffb900f0fac..dfea0b23fe4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -88,6 +88,7 @@ public class ApplicationSerializer { private final String lastWrittenField = "lastWritten"; private final String lastQueriesPerSecondField = "lastQueriesPerSecond"; private final String lastWritesPerSecondField = "lastWritesPerSecond"; + private final String loadBalancers = "loadBalancers"; // DeploymentJobs fields private final String projectIdField = "projectId"; @@ -179,6 +180,7 @@ public class ApplicationSerializer { deployment.activity().lastWritten().ifPresent(instant -> object.setLong(lastWrittenField, instant.toEpochMilli())); deployment.activity().lastQueriesPerSecond().ifPresent(value -> object.setDouble(lastQueriesPerSecondField, value)); deployment.activity().lastWritesPerSecond().ifPresent(value -> object.setDouble(lastWritesPerSecondField, value)); + loadBalancersToSlime(deployment.loadBalancers(), object); } private void deploymentMetricsToSlime(DeploymentMetrics metrics, Cursor object) { @@ -301,6 +303,13 @@ public class ApplicationSerializer { }); } + private void loadBalancersToSlime(Map loadBalancerMap, Cursor parentObject) { + Cursor root = parentObject.setObject(loadBalancers); + for (Map.Entry entry : loadBalancerMap.entrySet()) { + root.setString(entry.getKey().value(), entry.getValue().value()); + } + } + // ------------------ Deserialization public Application fromSlime(Slime slime) { @@ -343,7 +352,8 @@ public class ApplicationSerializer { DeploymentActivity.create(optionalInstant(deploymentObject.field(lastQueriedField)), optionalInstant(deploymentObject.field(lastWrittenField)), optionalDouble(deploymentObject.field(lastQueriesPerSecondField)), - optionalDouble(deploymentObject.field(lastWritesPerSecondField)))); + optionalDouble(deploymentObject.field(lastWritesPerSecondField))), + loadBalancerMapFromSlime(deploymentObject.field(loadBalancers))); } private DeploymentMetrics deploymentMetricsFromSlime(Inspector object) { @@ -499,6 +509,12 @@ public class ApplicationSerializer { return field.valid() ? optionalString(field).map(RotationId::new) : Optional.empty(); } + private Map loadBalancerMapFromSlime(Inspector object) { + Map map = new HashMap<>(); + object.traverse((String name, Inspector value) -> map.put(new ClusterSpec.Id(name), HostName.from(value.asString()))); + return map; + } + private OptionalLong optionalLong(Inspector field) { return field.valid() ? OptionalLong.of(field.asLong()) : OptionalLong.empty(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index f011baef39a..5beb95493c3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -7,7 +7,6 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; -import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; @@ -16,6 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -28,8 +28,6 @@ import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.ClusterView; import com.yahoo.vespa.serviceview.bindings.ServiceView; -import java.io.IOException; -import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -55,6 +53,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final Map serviceStatus = new HashMap<>(); private final Version initialVersion = new Version(6, 1, 0); private final Set suspendedApplications = new HashSet<>(); + private final Map> loadBalancers = new HashMap<>(); private Version lastPrepareVersion = null; private RuntimeException prepareException = null; @@ -172,6 +171,15 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return Optional.ofNullable(serviceStatus.get(deployment)); } + @Override + public List getLoadBalancers(DeploymentId deployment) { + return loadBalancers.getOrDefault(deployment, Collections.emptyList()); + } + + public void addLoadBalancers(ZoneId zoneId, ApplicationId applicationId, List loadBalancers) { + this.loadBalancers.put(new DeploymentId(applicationId, zoneId), loadBalancers); + } + @Override public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set rotationCnames, Set rotationNames, byte[] content) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java new file mode 100644 index 00000000000..68a29c77957 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java @@ -0,0 +1,114 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class LoadBalancerMaintainerTest { + + @Test + public void maintains_loadbalancer_records_correctly () { + DeploymentTester tester = new DeploymentTester(); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + + LoadBalancerMaintainer loadbalancerMaintainer = new LoadBalancerMaintainer(tester.controller(), Duration.ofHours(12), + new JobControl(new MockCuratorDb()), + tester.controllerTester().nameService()); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-west-1") + .region("us-central-1") + .build(); + + int numberOfClusters = 2; + + // Deploy application + tester.deployCompletely(application, applicationPackage); + tester.controller().applications().get(application.id()).orElseThrow(()->new RuntimeException("No deployments")).deployments().keySet() + .forEach(zone -> tester.configServer() + .addLoadBalancers(zone, application.id(), getLoadBalancers(zone, application.id(), numberOfClusters))); + + + loadbalancerMaintainer.maintain(); + Map records = tester.controllerTester().nameService().records(); + long recordCount = records.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); + records.entrySet().stream().forEach(entry -> System.out.println("entry = " + entry)); + assertEquals(4,recordCount); + + // no update + loadbalancerMaintainer.maintain(); + Map records2 = tester.controllerTester().nameService().records(); + long recordCount2 = records2.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); + assertEquals(recordCount, recordCount2); + assertEquals(records, records2); + + + // add cluster + tester.controller().applications().get(application.id()).orElseThrow(()->new RuntimeException("No deployments")).deployments().keySet() + .forEach(zone -> tester.configServer() + .addLoadBalancers(zone, application.id(), getLoadBalancers(zone, application.id(), numberOfClusters + 1))); + + loadbalancerMaintainer.maintain(); + Map records3 = tester.controllerTester().nameService().records(); + long recordCount3 = records3.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); + assertEquals(6,recordCount3); + } + + + + @Test + public void test_endpoint_names() { + ZoneId zoneId = ZoneId.from("prod", "us-north-1"); + ApplicationId withInstanceName = ApplicationId.from("tenant", "application", "instance"); + testLoadBalancerName("instance.application.tenant.prod.us-north-1.vespa.oath.cloud", "default", withInstanceName, zoneId); + testLoadBalancerName("cluster.instance.application.tenant.prod.us-north-1.vespa.oath.cloud", "cluster", withInstanceName, zoneId); + + ApplicationId withDefaultInstance = ApplicationId.from("tenant", "application", "default"); + testLoadBalancerName("application.tenant.prod.us-north-1.vespa.oath.cloud", "default", withDefaultInstance, zoneId); + testLoadBalancerName("cluster.application.tenant.prod.us-north-1.vespa.oath.cloud", "cluster", withDefaultInstance, zoneId); + } + + private void testLoadBalancerName(String expected, String clusterName, ApplicationId applicationId, ZoneId zoneId) { + assertEquals(expected, + LoadBalancerMaintainer.getEndpointName(ClusterSpec.Id.from(clusterName), applicationId, zoneId)); + } + + private List getLoadBalancers(ZoneId zone, ApplicationId applicationId, int loadBalancerCount) { + List loadBalancers = new ArrayList<>(); + for (int i = 0; i < loadBalancerCount; i++) { + loadBalancers.add( + new LoadBalancer("LB-"+ i + "-Z-"+zone.value(), + new TenantId(applicationId.tenant().value()), + new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(applicationId.application().value()), + new InstanceId(applicationId.instance().value()), + "cluster-"+i, + "loadbalancer-"+i+"-zone-"+zone.value() + )); + } + return loadBalancers; + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index d4cd7c0fe85..e46119dc0b5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; +import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; @@ -78,7 +79,7 @@ public class ApplicationSerializerTest { createClusterUtils(3, 0.2), createClusterInfo(3, 4), new DeploymentMetrics(2, 3, 4, 5, 6, Optional.of(Instant.now().truncatedTo(ChronoUnit.MILLIS))), DeploymentActivity.create(Optional.of(activityAt), Optional.of(activityAt), - OptionalDouble.of(200), OptionalDouble.of(10)))); + OptionalDouble.of(200), OptionalDouble.of(10)), createLoadBalancers("default", "foo.bar"))); OptionalLong projectId = OptionalLong.of(123L); List statusList = new ArrayList<>(); @@ -198,6 +199,9 @@ public class ApplicationSerializerTest { Application original6 = writable(original).withOutstandingChange(Change.of(ApplicationVersion.from(new SourceRevision("a", "b", "c"), 42))).get(); Application serialized6 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original6)); assertEquals(original6.outstandingChange(), serialized6.outstandingChange()); + + assertEquals(1, serialized.deployments().get(zone2).loadBalancers().size()); + assertEquals(original.deployments().get(zone2).loadBalancers(), serialized.deployments().get(zone2).loadBalancers()); } } @@ -231,6 +235,10 @@ public class ApplicationSerializerTest { return result; } + private Map createLoadBalancers(String clusterId, String hostName) { + return ImmutableMap.of(ClusterSpec.Id.from(clusterId), HostName.from(hostName)); + } + @Test public void testCompleteApplicationDeserialization() throws Exception { byte[] applicationJson = Files.readAllBytes(testData.resolve("complete-application.json")); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index dcee9717ecc..75e98d2d5e9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -33,6 +33,9 @@ { "name": "JobRunner" }, + { + "name": "LoadBalancerMaintainer" + }, { "name": "MetricsReporter" }, -- cgit v1.2.3 From 8a3ac6e851a97ec44a84dfc5942b3f1d74afe092 Mon Sep 17 00:00:00 2001 From: Morten Tokle Date: Thu, 17 Jan 2019 09:48:24 +0100 Subject: Review fixes --- .../api/integration/configserver/LoadBalancer.java | 17 +++++++---- .../api/integration/dns/MemoryNameService.java | 3 +- .../vespa/hosted/controller/LockedApplication.java | 4 +-- .../hosted/controller/application/Deployment.java | 7 +++-- .../maintenance/ControllerMaintenance.java | 1 + .../maintenance/LoadBalancerMaintainer.java | 35 +++++++++++++++------- .../persistence/ApplicationSerializer.java | 6 ++-- .../maintenance/LoadBalancerMaintainerTest.java | 7 +++-- 8 files changed, 52 insertions(+), 28 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java index a0cb76860a7..e8300262a89 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java @@ -1,19 +1,26 @@ // Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +/** + * A load balancer + * + * @author mortent + */ public class LoadBalancer { private final String id; private final TenantId tenant; private final ApplicationId application; private final InstanceId instance; - private final String cluster; - private final String hostname; + private final ClusterSpec.Id cluster; + private final HostName hostname; - public LoadBalancer(String id, TenantId tenant, ApplicationId application, InstanceId instance, String cluster, String hostname) { + public LoadBalancer(String id, TenantId tenant, ApplicationId application, InstanceId instance, ClusterSpec.Id cluster, HostName hostname) { this.id = id; this.tenant = tenant; this.application = application; @@ -38,11 +45,11 @@ public class LoadBalancer { return instance; } - public String cluster() { + public ClusterSpec.Id cluster() { return cluster; } - public String hostname() { + public HostName hostname() { return hostname; } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java index aca7562df6f..06599aa0b82 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java @@ -7,6 +7,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; import java.util.stream.Collectors; /** @@ -24,7 +25,7 @@ public class MemoryNameService implements NameService { @Override public RecordId createCname(RecordName alias, RecordData canonicalName) { - RecordId id = new RecordId(alias.asString()); + RecordId id = new RecordId(UUID.randomUUID().toString()); records.put(id, new Record(id, Record.Type.CNAME, alias, canonicalName)); return id; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 30921715f75..5d0b71a809b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -26,7 +26,6 @@ import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.time.Instant; -import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -147,7 +146,8 @@ public class LockedApplication { previousDeployment.clusterUtils(), previousDeployment.clusterInfo(), previousDeployment.metrics(), - previousDeployment.activity(), Collections.emptyMap()); + previousDeployment.activity(), + previousDeployment.loadBalancers()); return with(newDeployment); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 7b9cc6be6ca..3fd2932c3e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; +import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ClusterSpec.Id; import com.yahoo.config.provision.HostName; @@ -44,11 +45,11 @@ public class Deployment { this.applicationVersion = Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null"); this.version = Objects.requireNonNull(version, "version cannot be null"); this.deployTime = Objects.requireNonNull(deployTime, "deployTime cannot be null"); - this.clusterUtilization = Objects.requireNonNull(clusterUtilization, "clusterUtilization cannot be null"); - this.clusterInfo = Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null"); + this.clusterUtilization = ImmutableMap.copyOf(Objects.requireNonNull(clusterUtilization, "clusterUtilization cannot be null")); + this.clusterInfo = ImmutableMap.copyOf(Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null")); this.metrics = Objects.requireNonNull(metrics, "deploymentMetrics cannot be null"); this.activity = Objects.requireNonNull(activity, "activity cannot be null"); - this.loadBalancers = Objects.requireNonNull(loadBalancers, "loadBalancers cannot be null"); + this.loadBalancers = ImmutableMap.copyOf(Objects.requireNonNull(loadBalancers, "loadBalancers cannot be null")); } /** Returns the zone this was deployed to */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index a2015710070..8b9c8fd4fb2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -110,6 +110,7 @@ public class ControllerMaintenance extends AbstractComponent { jobRunner.deconstruct(); contactInformationMaintainer.deconstruct(); costReportMaintainer.deconstruct(); + loadbalancerMaintainer.deconstruct(); } /** Create one OS upgrader per cloud found in the zone registry of controller */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java index f54cbc875f2..48e22b6baf2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.LockedApplication; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; @@ -22,9 +23,16 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.logging.Logger; import java.util.stream.Collectors; +/** + * Maintains loadbalancer endpoints. + * Reads load balancer information for each application in all zones and updates name service. + * + * @author mortent + */ public class LoadBalancerMaintainer extends Maintainer { private static final Logger log = Logger.getLogger(LoadBalancerMaintainer.class.getName()); @@ -55,26 +63,26 @@ public class LoadBalancerMaintainer extends Maintainer { private void updateApplicationLoadBalancers(Application application) { // Get a list of all load balancers for this applications (for all zones and clusters) - Map> applicationZoneLoadBalancers = new HashMap<>(); + Map> zoneLoadBalancers = new HashMap<>(); for (ZoneId zoneId : application.deployments().keySet()) { try { - applicationZoneLoadBalancers.put(zoneId, controller().applications().configServer().getLoadBalancers(new DeploymentId(application.id(), zoneId))); + zoneLoadBalancers.put(zoneId, controller().applications().configServer().getLoadBalancers(new DeploymentId(application.id(), zoneId))); } catch (Exception e) { - log.log(LogLevel.INFO, - String.format("Got exception fetching load balancers for application: %s, in zone: %s", - application.id().toShortString(), zoneId.value()), + log.log(LogLevel.WARNING, + String.format("Got exception fetching load balancers for application: %s, in zone: %s. Retrying in %s", + application.id().toShortString(), zoneId.value(), maintenanceInterval()), e); } } // store the load balancers on the deployments - controller().applications().lockIfPresent(application.id(), lockedApplication -> storeApplicationWithLoadBalancers(lockedApplication, applicationZoneLoadBalancers)); + controller().applications().lockIfPresent(application.id(), lockedApplication -> storeApplicationWithLoadBalancers(lockedApplication, zoneLoadBalancers)); } private void storeApplicationWithLoadBalancers(LockedApplication lockedApplication, Map> loadBalancers) { for (Map.Entry> entry : loadBalancers.entrySet()) { Map loadbalancerClusterMap = entry.getValue().stream() - .collect(Collectors.toMap(lb -> ClusterSpec.Id.from(lb.id()), lb -> HostName.from(lb.hostname()))); + .collect(Collectors.toMap(LoadBalancer::cluster, LoadBalancer::hostname)); lockedApplication = lockedApplication.withDeploymentLoadBalancers(entry.getKey(), loadbalancerClusterMap); } @@ -89,12 +97,17 @@ public class LoadBalancerMaintainer extends Maintainer { try { RecordName recordName = RecordName.from(getEndpointName(loadBalancers.getKey(), application.id(), zone)); RecordData recordData = RecordData.fqdn(loadBalancers.getValue().value()); - nameService.createCname(recordName, recordData); + Optional existingRecord = nameService.findRecord(Record.Type.CNAME, recordName); + if(existingRecord.isPresent()) { + nameService.updateRecord(existingRecord.get().id(), recordData); + } else { + nameService.createCname(recordName, recordData); + } } catch (Exception e) { // Catching any exception, will be retried on next run - log.log(LogLevel.INFO, - String.format("Got exception updating name service for application: %s, cluster: %s in zone: %s", - application.id().toShortString(), loadBalancers.getKey().value(), zone.value()), + log.log(LogLevel.WARNING, + String.format("Got exception updating name service for application: %s, cluster: %s in zone: %s. Retrying in %s", + application.id().toShortString(), loadBalancers.getKey().value(), zone.value(), maintenanceInterval()), e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index dfea0b23fe4..415cb76cf4d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -510,9 +510,9 @@ public class ApplicationSerializer { } private Map loadBalancerMapFromSlime(Inspector object) { - Map map = new HashMap<>(); - object.traverse((String name, Inspector value) -> map.put(new ClusterSpec.Id(name), HostName.from(value.asString()))); - return map; + Map loadBalancers = new HashMap<>(); + object.traverse((String name, Inspector value) -> loadBalancers.put(new ClusterSpec.Id(name), HostName.from(value.asString()))); + return loadBalancers; } private OptionalLong optionalLong(Inspector field) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java index 68a29c77957..380281a273b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; @@ -25,7 +26,7 @@ import java.util.Map; import static org.junit.Assert.assertEquals; /** - * @author mpolden + * @author mortent */ public class LoadBalancerMaintainerTest { @@ -105,8 +106,8 @@ public class LoadBalancerMaintainerTest { new TenantId(applicationId.tenant().value()), new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(applicationId.application().value()), new InstanceId(applicationId.instance().value()), - "cluster-"+i, - "loadbalancer-"+i+"-zone-"+zone.value() + ClusterSpec.Id.from("cluster-"+i), + HostName.from("loadbalancer-" + i + "-zone-" + zone.value()) )); } return loadBalancers; -- cgit v1.2.3 From 01c8dd3582c3c86a7fab4699617ef9d1eec13681 Mon Sep 17 00:00:00 2001 From: Morten Tokle Date: Thu, 24 Jan 2019 15:17:51 +0100 Subject: Support for removing load balancer names --- .../api/integration/configserver/LoadBalancer.java | 2 +- .../controller/loadbalancer/LoadBalancerName.java | 52 +++++++++++++ .../maintenance/ControllerMaintenance.java | 2 +- .../maintenance/LoadBalancerMaintainer.java | 89 ++++++++++++++++++++-- .../hosted/controller/persistence/CuratorDb.java | 19 +++++ .../persistence/LoadBalancerNameSerializer.java | 56 ++++++++++++++ .../controller/integration/ConfigServerMock.java | 4 + .../maintenance/LoadBalancerMaintainerTest.java | 76 +++++++++++++++--- .../LoadBalancerNameSerializerTest.java | 33 ++++++++ 9 files changed, 312 insertions(+), 21 deletions(-) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/loadbalancer/LoadBalancerName.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializer.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializerTest.java (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java index e8300262a89..114ae4dbfe7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java @@ -1,4 +1,4 @@ -// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; import com.yahoo.config.provision.ClusterSpec; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/loadbalancer/LoadBalancerName.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/loadbalancer/LoadBalancerName.java new file mode 100644 index 00000000000..b2963a62eba --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/loadbalancer/LoadBalancerName.java @@ -0,0 +1,52 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.loadbalancer; + +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; + +import java.util.Objects; + +/** + * Represents a pair of RecordId and RecordName + * + * @author mortent + */ +public class LoadBalancerName { + private final RecordId recordId; + private final RecordName recordName; + + public LoadBalancerName(RecordId recordId, RecordName recordName) { + this.recordId = recordId; + this.recordName = recordName; + } + + public RecordId recordId() { + return recordId; + } + + public RecordName recordName() { + return recordName; + } + + @Override + public String toString() { + return "LoadBalancerName{" + + "recordId=" + recordId + + ", recordName=" + recordName + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LoadBalancerName that = (LoadBalancerName) o; + return recordId.equals(that.recordId) && + recordName.equals(that.recordName); + } + + @Override + public int hashCode() { + return Objects.hash(recordId, recordName); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 8b9c8fd4fb2..132b4a28514 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -82,7 +82,7 @@ public class ControllerMaintenance extends AbstractComponent { osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl); contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, contactRetriever); costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig); - loadbalancerMaintainer = new LoadBalancerMaintainer(controller, Duration.ofMinutes(5), jobControl, nameService); + loadbalancerMaintainer = new LoadBalancerMaintainer(controller, Duration.ofMinutes(5), jobControl, nameService, curator); } public Upgrader upgrader() { return upgrader; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java index 48e22b6baf2..71b2482a650 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java @@ -1,4 +1,4 @@ -// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.base.Strings; @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedApplication; @@ -14,18 +15,25 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalanc import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Maintains loadbalancer endpoints. @@ -39,13 +47,16 @@ public class LoadBalancerMaintainer extends Maintainer { private static final String IGNORE_ENDPOINT_VALUE = "default"; private final NameService nameService; + private final CuratorDb curatorDb; public LoadBalancerMaintainer(Controller controller, Duration interval, JobControl jobControl, - NameService nameService) { + NameService nameService, + CuratorDb curatorDb) { super(controller, interval, jobControl); this.nameService = nameService; + this.curatorDb = curatorDb; } @Override @@ -55,10 +66,69 @@ public class LoadBalancerMaintainer extends Maintainer { // Create or update cnames List applications = controller().applications().asList(); - applications.forEach(this::registerLoadBalancerEndpoint); + Map> applicationEndpointMap = applications.stream() + .collect(Collectors.toMap(Application::id, this::registerLoadBalancerEndpoint)); - // Delete removed application rotations - // TODO + try (Lock lock = curatorDb.lockLoadBalancerNames()) { + updatePersistedLoadBalancerNames(applicationEndpointMap); + removeObsoleteLoadBalancerNames(); + } + } + + + private void removeObsoleteLoadBalancerNames() { + Map> persistedLoadBalancerNames = new HashMap<>(curatorDb.readLoadBalancerNames()); + Map> result = new HashMap<>(); + + Map> wantedLoadBalancers = new HashMap<>(); + + for (Application application : controller().applications().asList()) { + for (Map.Entry entry : application.deployments().entrySet()) { + Map loadBalancers = entry.getValue().loadBalancers(); + List loadBalancerNames = loadBalancers.keySet().stream() + .map(cluster -> getEndpointName(cluster, application.id(), entry.getKey())) + .collect(Collectors.toList()); + wantedLoadBalancers.merge(application.id(), loadBalancerNames, (v1, v2) -> + Stream.concat(v1.stream(), v2.stream()).collect(Collectors.toList())); + } + } + + for (Map.Entry> loadbalancerEntry : persistedLoadBalancerNames.entrySet()) { + List wanted = wantedLoadBalancers.getOrDefault(loadbalancerEntry.getKey(), Collections.emptyList()); + List current = loadbalancerEntry.getValue(); + List toBeRemoved = current.stream() + .filter(lbname -> !wanted.contains(lbname.recordName().asString())) + .collect(Collectors.toList()); + removeLoadBalancerNames(toBeRemoved); + List resultingLoadBalancers = current.stream().filter(lb -> !toBeRemoved.contains(lb)).collect(Collectors.toList()); + result.put(loadbalancerEntry.getKey(), resultingLoadBalancers); + } + + curatorDb.writeLoadBalancerNames(result); + } + + private void updatePersistedLoadBalancerNames(Map> applicationEndpointMap) { + // Read current list of maintained load balancer endpoints + Map> existingApplicationEndpointList = curatorDb.readLoadBalancerNames(); + + // Update ZK with new load balancer endpoints + Map> allCreated = new HashMap<>(applicationEndpointMap); + existingApplicationEndpointList.forEach((k, v) -> allCreated.merge(k, v, (v1, v2) -> + // Merge the two lists, removing duplicates + List.copyOf(new LinkedHashSet( + Stream.concat(v1.stream(), v2.stream()).collect(Collectors.toList())) + ))); + + curatorDb.writeLoadBalancerNames(allCreated); + + } + + private void removeLoadBalancerNames(List loadBalancerNames) { + if(loadBalancerNames.isEmpty()) return; + log.log(LogLevel.INFO, String.format("Removing %d Load Balancer names", loadBalancerNames.size())); + for (LoadBalancerName loadBalancerName : loadBalancerNames) { + nameService.removeRecord(loadBalancerName.recordId()); + } } private void updateApplicationLoadBalancers(Application application) { @@ -89,7 +159,8 @@ public class LoadBalancerMaintainer extends Maintainer { controller().applications().store(lockedApplication); } - private void registerLoadBalancerEndpoint(Application application) { + private List registerLoadBalancerEndpoint(Application application) { + List hostNamesRegistered = new ArrayList<>(); for (Map.Entry deploymentEntry : application.deployments().entrySet()) { ZoneId zone = deploymentEntry.getKey(); Deployment deployment = deploymentEntry.getValue(); @@ -98,11 +169,14 @@ public class LoadBalancerMaintainer extends Maintainer { RecordName recordName = RecordName.from(getEndpointName(loadBalancers.getKey(), application.id(), zone)); RecordData recordData = RecordData.fqdn(loadBalancers.getValue().value()); Optional existingRecord = nameService.findRecord(Record.Type.CNAME, recordName); + final RecordId recordId; if(existingRecord.isPresent()) { + recordId = existingRecord.get().id(); nameService.updateRecord(existingRecord.get().id(), recordData); } else { - nameService.createCname(recordName, recordData); + recordId = nameService.createCname(recordName, recordData); } + hostNamesRegistered.add(new LoadBalancerName(recordId, recordName)); } catch (Exception e) { // Catching any exception, will be retried on next run log.log(LogLevel.WARNING, @@ -112,6 +186,7 @@ public class LoadBalancerMaintainer extends Maintainer { } } } + return hostNamesRegistered; } static String getEndpointName(ClusterSpec.Id clusterId, ApplicationId applicationId, ZoneId zoneId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 74e60a40b4c..5fac0896246 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; +import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.UserTenant; @@ -81,6 +82,7 @@ public class CuratorDb { private final RunSerializer runSerializer = new RunSerializer(); private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer(); private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer); + private final LoadBalancerNameSerializer loadBalancerNameSerializer = new LoadBalancerNameSerializer(); private final Curator curator; private final Duration tryLockTimeout; @@ -174,6 +176,9 @@ public class CuratorDb { return lock(lockRoot.append("osVersionStatus"), defaultLockTimeout); } + public Lock lockLoadBalancerNames() { + return lock(lockRoot.append("loadBalancerNames"), defaultLockTimeout); + } // -------------- Helpers ------------------------------------------ /** Try locking with a low timeout, meaning it is OK to fail lock acquisition. @@ -455,6 +460,16 @@ public class CuratorDb { curator.set(openStackServerPoolPath(), data); } + // -------------- Load balancer names ------------------------------------- + + public void writeLoadBalancerNames(Map> loadBalancerNames) { + curator.set(loadBalancerNamePath(), asJson(loadBalancerNameSerializer.toSlime(loadBalancerNames))); + } + + public Map> readLoadBalancerNames() { + return readSlime(loadBalancerNamePath()).map(loadBalancerNameSerializer::fromSlime).orElseGet(Collections::emptyMap); + } + // -------------- Paths --------------------------------------------------- private Path lockPath(TenantName tenant) { @@ -530,6 +545,10 @@ public class CuratorDb { return root.append("versionStatus"); } + private static Path loadBalancerNamePath() { + return root.append("loadBalancerNames"); + } + private static Path provisionStatePath() { return root.append("provisioning").append("states"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializer.java new file mode 100644 index 00000000000..a76e1189294 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializer.java @@ -0,0 +1,56 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Serializer and deserializer for LoadBalancerName + * + * @author mortent + */ +public class LoadBalancerNameSerializer { + + private static final String loadBalancerNamesField = "loadBalancerNames"; + + public Slime toSlime(Map> loadBalancerNames) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor applicationEndpoints = root.setObject(loadBalancerNamesField); + + for (Map.Entry> entry : loadBalancerNames.entrySet()) { + ApplicationId applicationId = entry.getKey(); + Cursor cursor = applicationEndpoints.setArray(applicationId.serializedForm()); + entry.getValue().forEach( lb -> cursor.addObject().setString(lb.recordId().asString(), lb.recordName().asString())); + } + return slime; + } + + public Map> fromSlime(Slime slime) { + + Inspector object = slime.get().field(loadBalancerNamesField); + Map> loadBalancerNames = new HashMap<>(); + object.traverse((ObjectTraverser) (appId, inspector) -> + loadBalancerNames.put(ApplicationId.fromSerializedForm(appId), loadBalancerNamesFromSlime(inspector))); + + return loadBalancerNames; + } + + private List loadBalancerNamesFromSlime(Inspector root) { + List names = new ArrayList<>(); + root.traverse((ArrayTraverser) (i, inspector) -> inspector.traverse((ObjectTraverser)(x,y)-> names.add(new LoadBalancerName(new RecordId(x), RecordName.from(y.asString()))))); + return names; + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 5beb95493c3..45e87859576 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -180,6 +180,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer this.loadBalancers.put(new DeploymentId(applicationId, zoneId), loadBalancers); } + public void removeLoadBalancers(DeploymentId deployment) { + this.loadBalancers.remove(deployment); + } + @Override public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set rotationCnames, Set rotationNames, byte[] content) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java index 380281a273b..05c3de79023 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java @@ -1,4 +1,4 @@ -// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; @@ -15,11 +16,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import org.junit.Test; import java.time.Duration; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -37,7 +40,8 @@ public class LoadBalancerMaintainerTest { LoadBalancerMaintainer loadbalancerMaintainer = new LoadBalancerMaintainer(tester.controller(), Duration.ofHours(12), new JobControl(new MockCuratorDb()), - tester.controllerTester().nameService()); + tester.controllerTester().nameService(), + tester.controllerTester().curator()); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) @@ -45,21 +49,22 @@ public class LoadBalancerMaintainerTest { .region("us-central-1") .build(); - int numberOfClusters = 2; + int numberOfClustersPerZone = 2; // Deploy application tester.deployCompletely(application, applicationPackage); - tester.controller().applications().get(application.id()).orElseThrow(()->new RuntimeException("No deployments")).deployments().keySet() - .forEach(zone -> tester.configServer() - .addLoadBalancers(zone, application.id(), getLoadBalancers(zone, application.id(), numberOfClusters))); - + setupClustersWithLoadBalancers(tester, application, numberOfClustersPerZone); loadbalancerMaintainer.maintain(); Map records = tester.controllerTester().nameService().records(); long recordCount = records.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); - records.entrySet().stream().forEach(entry -> System.out.println("entry = " + entry)); assertEquals(4,recordCount); + Map> loadBalancerNames = tester.controller().curator().readLoadBalancerNames(); + List names = loadBalancerNames.get(application.id()); + assertEquals(4, names.size()); + + // no update loadbalancerMaintainer.maintain(); Map records2 = tester.controllerTester().nameService().records(); @@ -68,17 +73,64 @@ public class LoadBalancerMaintainerTest { assertEquals(records, records2); - // add cluster - tester.controller().applications().get(application.id()).orElseThrow(()->new RuntimeException("No deployments")).deployments().keySet() - .forEach(zone -> tester.configServer() - .addLoadBalancers(zone, application.id(), getLoadBalancers(zone, application.id(), numberOfClusters + 1))); + // add 1 cluster per zone + setupClustersWithLoadBalancers(tester, application, numberOfClustersPerZone + 1); loadbalancerMaintainer.maintain(); Map records3 = tester.controllerTester().nameService().records(); long recordCount3 = records3.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); assertEquals(6,recordCount3); + + Map> loadBalancerNames3 = tester.controller().curator().readLoadBalancerNames(); + List names3 = loadBalancerNames3.get(application.id()); + assertEquals(6, names3.size()); + + + // Add application + Application application2 = tester.createApplication("app2", "tenant1", 1, 1L); + tester.deployCompletely(application2, applicationPackage); + setupClustersWithLoadBalancers(tester, application2, numberOfClustersPerZone); + + loadbalancerMaintainer.maintain(); + Map records4 = tester.controllerTester().nameService().records(); + long recordCount4 = records4.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); + assertEquals(10,recordCount4); + + Map> loadBalancerNames4 = tester.controller().curator().readLoadBalancerNames(); + List names4 = loadBalancerNames4.get(application2.id()); + assertEquals(4, names4.size()); + + + // Remove cluster in app1 + setupClustersWithLoadBalancers(tester, application, numberOfClustersPerZone); + + loadbalancerMaintainer.maintain(); + Map records5 = tester.controllerTester().nameService().records(); + long recordCount5 = records5.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); + assertEquals(8,recordCount5); + + // Remove application app2 + tester.controller().applications().get(application2.id()) + .map(app -> app.deployments().keySet()) + .orElse(Collections.emptySet()) + .forEach(zone -> tester.controller().applications().deactivate(application2.id(), zone)); + + loadbalancerMaintainer.maintain(); + Map records6 = tester.controllerTester().nameService().records(); + long recordCount6 = records6.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); + assertEquals(4,recordCount6); + } + private void setupClustersWithLoadBalancers(DeploymentTester tester, Application application, int numberOfClustersPerZone) { + tester.controller().applications().get(application.id()).orElseThrow(()->new RuntimeException("No deployments")).deployments().keySet() + .forEach(zone -> tester.configServer() + .removeLoadBalancers(new DeploymentId(application.id(), zone))); + tester.controller().applications().get(application.id()).orElseThrow(()->new RuntimeException("No deployments")).deployments().keySet() + .forEach(zone -> tester.configServer() + .addLoadBalancers(zone, application.id(), getLoadBalancers(zone, application.id(), numberOfClustersPerZone))); + + } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializerTest.java new file mode 100644 index 00000000000..a66fe127cd5 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializerTest.java @@ -0,0 +1,33 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; + +/** + * @author mortent + */ +public class LoadBalancerNameSerializerTest { + + @Test + public void test_serialization() throws IOException { + LoadBalancerNameSerializer serializer = new LoadBalancerNameSerializer(); + ImmutableMap> lbnames = ImmutableMap.of( + ApplicationId.from("foo", "bar", "default"), Lists.newArrayList(new LoadBalancerName(new RecordId("123.4123.:124123"), RecordName.from("foo.bar")))); + Map> deserialized = serializer.fromSlime(serializer.toSlime(lbnames)); + + assertThat(deserialized, equalTo(lbnames)); + } +} \ No newline at end of file -- cgit v1.2.3 From 8d90a1fd5a0e6b08ab03ab9497e2508a8e15fcae Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 25 Jan 2019 14:45:21 +0100 Subject: Simplify DNS record cleanup --- .../api/integration/configserver/LoadBalancer.java | 18 +- .../vespa/hosted/controller/LockedApplication.java | 10 +- .../controller/application/LoadBalancerAlias.java | 93 +++++++++ .../controller/loadbalancer/LoadBalancerName.java | 52 ------ .../maintenance/ControllerMaintenance.java | 6 +- .../maintenance/LoadBalancerAliasMaintainer.java | 147 +++++++++++++++ .../maintenance/LoadBalancerMaintainer.java | 208 --------------------- .../hosted/controller/persistence/CuratorDb.java | 31 +-- .../persistence/LoadBalancerAliasSerializer.java | 52 ++++++ .../persistence/LoadBalancerNameSerializer.java | 56 ------ .../application/LoadBalancerAliasTest.java | 33 ++++ .../LoadBalancerAliasMaintainerTest.java | 146 +++++++++++++++ .../maintenance/LoadBalancerMaintainerTest.java | 167 ----------------- .../LoadBalancerAliasSerializerTest.java | 35 ++++ .../LoadBalancerNameSerializerTest.java | 33 ---- .../restapi/controller/responses/maintenance.json | 2 +- 16 files changed, 549 insertions(+), 540 deletions(-) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAlias.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/loadbalancer/LoadBalancerName.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerAliasMaintainer.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerAliasSerializer.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializer.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAliasTest.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerAliasMaintainerTest.java delete mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerAliasSerializerTest.java delete mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializerTest.java (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java index 114ae4dbfe7..b77b63cab8a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/LoadBalancer.java @@ -7,12 +7,15 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import java.util.Objects; + /** - * A load balancer + * Represents an exclusive load balancer, assigned to an application's cluster. * * @author mortent */ public class LoadBalancer { + private final String id; private final TenantId tenant; private final ApplicationId application; @@ -21,12 +24,12 @@ public class LoadBalancer { private final HostName hostname; public LoadBalancer(String id, TenantId tenant, ApplicationId application, InstanceId instance, ClusterSpec.Id cluster, HostName hostname) { - this.id = id; - this.tenant = tenant; - this.application = application; - this.instance = instance; - this.cluster = cluster; - this.hostname = hostname; + this.id = Objects.requireNonNull(id, "id must be non-null"); + this.tenant = Objects.requireNonNull(tenant, "tenant must be non-null"); + this.application = Objects.requireNonNull(application, "application must be non-null"); + this.instance = Objects.requireNonNull(instance, "instance must be non-null"); + this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); + this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); } public String id() { @@ -52,4 +55,5 @@ public class LoadBalancer { public HostName hostname() { return hostname; } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 5d0b71a809b..130519335ce 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; @@ -27,10 +28,12 @@ import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.time.Instant; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; +import java.util.stream.Collectors; /** * An application that has been locked for modification. Provides methods for modifying an application's fields. @@ -248,8 +251,11 @@ public class LockedApplication { outstandingChange, ownershipIssueId, owner, majorVersion, metrics, rotation, rotationStatus); } - public LockedApplication withDeploymentLoadBalancers(ZoneId zoneId, Map loadBalancers) { - return with(deployments.get(zoneId).withLoadBalancers(loadBalancers)); + public LockedApplication withLoadBalancersIn(ZoneId zoneId, List loadBalancers) { + Map loadBalancersByCluster = loadBalancers.stream() + .collect(Collectors.toUnmodifiableMap(LoadBalancer::cluster, + LoadBalancer::hostname)); + return with(deployments.get(zoneId).withLoadBalancers(loadBalancersByCluster)); } /** Don't expose non-leaf sub-objects. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAlias.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAlias.java new file mode 100644 index 00000000000..4e1c79248a9 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAlias.java @@ -0,0 +1,93 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.google.common.base.Strings; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Represents a DNS alias for a load balancer. + * + * @author mortent + */ +public class LoadBalancerAlias { + + private static final String ignoredEndpointPart = "default"; + private final ApplicationId owner; + private final String id; + private final HostName alias; + private final HostName canonicalName; + + public LoadBalancerAlias(ApplicationId owner, String id, HostName alias, HostName canonicalName) { + this.owner = Objects.requireNonNull(owner, "owner must be non-null"); + this.id = Objects.requireNonNull(id, "id must be non-null"); + this.alias = Objects.requireNonNull(alias, "alias must be non-null"); + this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); + } + + /** The application owning this */ + public ApplicationId owner() { + return owner; + } + + /** The ID of the DNS record represented by this */ + public String id() { + return id; + } + + /** This alias (lhs of the CNAME record) */ + public HostName alias() { + return alias; + } + + /** The canonical name of this (rhs of the CNAME record) */ + public HostName canonicalName() { + return canonicalName; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LoadBalancerAlias that = (LoadBalancerAlias) o; + return Objects.equals(owner, that.owner) && + Objects.equals(id, that.id) && + Objects.equals(alias, that.alias) && + Objects.equals(canonicalName, that.canonicalName); + } + + @Override + public int hashCode() { + return Objects.hash(owner, id, alias, canonicalName); + } + + @Override + public String toString() { + return String.format("%s: %s -> %s, owned by %s", id, alias, canonicalName, owner.toShortString()); + } + + public static String createAlias(ClusterSpec.Id clusterId, ApplicationId applicationId, ZoneId zoneId) { + List parts = Arrays.asList(ignorePartIfDefault(clusterId.value()), + ignorePartIfDefault(applicationId.instance().value()), + applicationId.application().value(), + applicationId.tenant().value(), + zoneId.value(), + "vespa.oath.cloud" + ); + return parts.stream() + .filter(s -> !Strings.isNullOrEmpty((s))) + .collect(Collectors.joining("--")); + } + + private static String ignorePartIfDefault(String s) { + return ignoredEndpointPart.equalsIgnoreCase(s) ? "" : s; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/loadbalancer/LoadBalancerName.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/loadbalancer/LoadBalancerName.java deleted file mode 100644 index b2963a62eba..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/loadbalancer/LoadBalancerName.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.loadbalancer; - -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; - -import java.util.Objects; - -/** - * Represents a pair of RecordId and RecordName - * - * @author mortent - */ -public class LoadBalancerName { - private final RecordId recordId; - private final RecordName recordName; - - public LoadBalancerName(RecordId recordId, RecordName recordName) { - this.recordId = recordId; - this.recordName = recordName; - } - - public RecordId recordId() { - return recordId; - } - - public RecordName recordName() { - return recordName; - } - - @Override - public String toString() { - return "LoadBalancerName{" + - "recordId=" + recordId + - ", recordName=" + recordName + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - LoadBalancerName that = (LoadBalancerName) o; - return recordId.equals(that.recordId) && - recordName.equals(that.recordName); - } - - @Override - public int hashCode() { - return Objects.hash(recordId, recordName); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 132b4a28514..b38f55826d6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -52,7 +52,7 @@ public class ControllerMaintenance extends AbstractComponent { private final JobRunner jobRunner; private final ContactInformationMaintainer contactInformationMaintainer; private final CostReportMaintainer costReportMaintainer; - private final LoadBalancerMaintainer loadbalancerMaintainer; + private final LoadBalancerAliasMaintainer loadBalancerAliasMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator, @@ -82,7 +82,7 @@ public class ControllerMaintenance extends AbstractComponent { osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl); contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, contactRetriever); costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig); - loadbalancerMaintainer = new LoadBalancerMaintainer(controller, Duration.ofMinutes(5), jobControl, nameService, curator); + loadBalancerAliasMaintainer = new LoadBalancerAliasMaintainer(controller, Duration.ofMinutes(5), jobControl, nameService, curator); } public Upgrader upgrader() { return upgrader; } @@ -110,7 +110,7 @@ public class ControllerMaintenance extends AbstractComponent { jobRunner.deconstruct(); contactInformationMaintainer.deconstruct(); costReportMaintainer.deconstruct(); - loadbalancerMaintainer.deconstruct(); + loadBalancerAliasMaintainer.deconstruct(); } /** Create one OS upgrader per cloud found in the zone registry of controller */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerAliasMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerAliasMaintainer.java new file mode 100644 index 00000000000..6c5000e3e3d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerAliasMaintainer.java @@ -0,0 +1,147 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.LoadBalancerAlias; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Maintains DNS aliases for all load balancers in this system. + * + * @author mortent + */ +public class LoadBalancerAliasMaintainer extends Maintainer { + + private static final Logger log = Logger.getLogger(LoadBalancerAliasMaintainer.class.getName()); + + private final NameService nameService; + private final CuratorDb db; + private final ApplicationController applications; + + public LoadBalancerAliasMaintainer(Controller controller, + Duration interval, + JobControl jobControl, + NameService nameService, + CuratorDb db) { + super(controller, interval, jobControl); + this.nameService = nameService; + this.db = db; + this.applications = controller.applications(); + } + + @Override + protected void maintain() { + updateDnsRecords(); + removeObsoleteDnsRecords(); + } + + /** Create DNS records for all exclusive load balancers */ + private void updateDnsRecords() { + for (Application application : applications.asList()) { + for (ZoneId zone : application.deployments().keySet()) { + List loadBalancers = findLoadBalancersIn(zone, application.id()); + if (loadBalancers.isEmpty()) continue; + + applications.lockIfPresent(application.id(), (locked) -> { + applications.store(locked.withLoadBalancersIn(zone, loadBalancers)); + }); + + try (Lock lock = db.lockLoadBalancerAliases()) { + Set aliases = new LinkedHashSet<>(db.readLoadBalancerAliases(application.id())); + for (LoadBalancer loadBalancer : loadBalancers) { + try { + aliases.add(registerDnsAlias(application.id(), zone, loadBalancer)); + } catch (Exception e) { + log.log(LogLevel.WARNING, "Failed to create or update DNS record for load balancer " + + loadBalancer.hostname() + ". Retrying in " + maintenanceInterval(), + e); + } + } + db.writeLoadBalancerAliases(application.id(), aliases); + } + } + } + } + + /** Register DNS alias for given load balancer */ + private LoadBalancerAlias registerDnsAlias(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer) { + HostName alias = HostName.from(LoadBalancerAlias.createAlias(loadBalancer.cluster(), application, zone)); + RecordName name = RecordName.from(alias.value()); + RecordData data = RecordData.fqdn(loadBalancer.hostname().value()); + Optional existingRecord = nameService.findRecord(Record.Type.CNAME, name); + RecordId id; + if(existingRecord.isPresent()) { + id = existingRecord.get().id(); + nameService.updateRecord(existingRecord.get().id(), data); + } else { + id = nameService.createCname(name, data); + } + return new LoadBalancerAlias(application, id.asString(), alias, loadBalancer.hostname()); + } + + /** Find all load balancers assigned to application in given zone */ + private List findLoadBalancersIn(ZoneId zone, ApplicationId application) { + try { + return controller().applications().configServer().getLoadBalancers(new DeploymentId(application, zone)); + } catch (Exception e) { + log.log(LogLevel.WARNING, + String.format("Got exception fetching load balancers for application: %s, in zone: %s. Retrying in %s", + application.toShortString(), zone.value(), maintenanceInterval()), e); + } + return Collections.emptyList(); + } + + /** Remove all DNS records that point to non-existing load balancers */ + private void removeObsoleteDnsRecords() { + try (Lock lock = db.lockLoadBalancerAliases()) { + List removalCandidates = new ArrayList<>(db.readLoadBalancerAliases()); + Set activeLoadBalancers = controller().applications().asList().stream() + .map(Application::deployments) + .map(Map::values) + .flatMap(Collection::stream) + .map(Deployment::loadBalancers) + .map(Map::values) + .flatMap(Collection::stream) + .collect(Collectors.toUnmodifiableSet()); + + // Remove any active load balancers + removalCandidates.removeIf(lb -> activeLoadBalancers.contains(lb.canonicalName())); + for (LoadBalancerAlias alias : removalCandidates) { + try { + nameService.removeRecord(new RecordId(alias.id())); + } catch (Exception e) { + log.log(LogLevel.WARNING, "Failed to remove DNS record with ID '" + alias.id() + + "'. Retrying in " + maintenanceInterval()); + } + } + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java deleted file mode 100644 index 71b2482a650..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainer.java +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.google.common.base.Strings; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.LockedApplication; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Maintains loadbalancer endpoints. - * Reads load balancer information for each application in all zones and updates name service. - * - * @author mortent - */ -public class LoadBalancerMaintainer extends Maintainer { - - private static final Logger log = Logger.getLogger(LoadBalancerMaintainer.class.getName()); - private static final String IGNORE_ENDPOINT_VALUE = "default"; - - private final NameService nameService; - private final CuratorDb curatorDb; - - public LoadBalancerMaintainer(Controller controller, - Duration interval, - JobControl jobControl, - NameService nameService, - CuratorDb curatorDb) { - super(controller, interval, jobControl); - this.nameService = nameService; - this.curatorDb = curatorDb; - } - - @Override - protected void maintain() { - // update application object with load balancer information - controller().applications().asList().forEach(this::updateApplicationLoadBalancers); - - // Create or update cnames - List applications = controller().applications().asList(); - Map> applicationEndpointMap = applications.stream() - .collect(Collectors.toMap(Application::id, this::registerLoadBalancerEndpoint)); - - try (Lock lock = curatorDb.lockLoadBalancerNames()) { - updatePersistedLoadBalancerNames(applicationEndpointMap); - removeObsoleteLoadBalancerNames(); - } - } - - - private void removeObsoleteLoadBalancerNames() { - Map> persistedLoadBalancerNames = new HashMap<>(curatorDb.readLoadBalancerNames()); - Map> result = new HashMap<>(); - - Map> wantedLoadBalancers = new HashMap<>(); - - for (Application application : controller().applications().asList()) { - for (Map.Entry entry : application.deployments().entrySet()) { - Map loadBalancers = entry.getValue().loadBalancers(); - List loadBalancerNames = loadBalancers.keySet().stream() - .map(cluster -> getEndpointName(cluster, application.id(), entry.getKey())) - .collect(Collectors.toList()); - wantedLoadBalancers.merge(application.id(), loadBalancerNames, (v1, v2) -> - Stream.concat(v1.stream(), v2.stream()).collect(Collectors.toList())); - } - } - - for (Map.Entry> loadbalancerEntry : persistedLoadBalancerNames.entrySet()) { - List wanted = wantedLoadBalancers.getOrDefault(loadbalancerEntry.getKey(), Collections.emptyList()); - List current = loadbalancerEntry.getValue(); - List toBeRemoved = current.stream() - .filter(lbname -> !wanted.contains(lbname.recordName().asString())) - .collect(Collectors.toList()); - removeLoadBalancerNames(toBeRemoved); - List resultingLoadBalancers = current.stream().filter(lb -> !toBeRemoved.contains(lb)).collect(Collectors.toList()); - result.put(loadbalancerEntry.getKey(), resultingLoadBalancers); - } - - curatorDb.writeLoadBalancerNames(result); - } - - private void updatePersistedLoadBalancerNames(Map> applicationEndpointMap) { - // Read current list of maintained load balancer endpoints - Map> existingApplicationEndpointList = curatorDb.readLoadBalancerNames(); - - // Update ZK with new load balancer endpoints - Map> allCreated = new HashMap<>(applicationEndpointMap); - existingApplicationEndpointList.forEach((k, v) -> allCreated.merge(k, v, (v1, v2) -> - // Merge the two lists, removing duplicates - List.copyOf(new LinkedHashSet( - Stream.concat(v1.stream(), v2.stream()).collect(Collectors.toList())) - ))); - - curatorDb.writeLoadBalancerNames(allCreated); - - } - - private void removeLoadBalancerNames(List loadBalancerNames) { - if(loadBalancerNames.isEmpty()) return; - log.log(LogLevel.INFO, String.format("Removing %d Load Balancer names", loadBalancerNames.size())); - for (LoadBalancerName loadBalancerName : loadBalancerNames) { - nameService.removeRecord(loadBalancerName.recordId()); - } - } - - private void updateApplicationLoadBalancers(Application application) { - // Get a list of all load balancers for this applications (for all zones and clusters) - Map> zoneLoadBalancers = new HashMap<>(); - for (ZoneId zoneId : application.deployments().keySet()) { - try { - zoneLoadBalancers.put(zoneId, controller().applications().configServer().getLoadBalancers(new DeploymentId(application.id(), zoneId))); - } catch (Exception e) { - log.log(LogLevel.WARNING, - String.format("Got exception fetching load balancers for application: %s, in zone: %s. Retrying in %s", - application.id().toShortString(), zoneId.value(), maintenanceInterval()), - e); - } - } - - // store the load balancers on the deployments - controller().applications().lockIfPresent(application.id(), lockedApplication -> storeApplicationWithLoadBalancers(lockedApplication, zoneLoadBalancers)); - } - - private void storeApplicationWithLoadBalancers(LockedApplication lockedApplication, Map> loadBalancers) { - for (Map.Entry> entry : loadBalancers.entrySet()) { - Map loadbalancerClusterMap = entry.getValue().stream() - .collect(Collectors.toMap(LoadBalancer::cluster, LoadBalancer::hostname)); - lockedApplication = lockedApplication.withDeploymentLoadBalancers(entry.getKey(), loadbalancerClusterMap); - - } - controller().applications().store(lockedApplication); - } - - private List registerLoadBalancerEndpoint(Application application) { - List hostNamesRegistered = new ArrayList<>(); - for (Map.Entry deploymentEntry : application.deployments().entrySet()) { - ZoneId zone = deploymentEntry.getKey(); - Deployment deployment = deploymentEntry.getValue(); - for (Map.Entry loadBalancers : deployment.loadBalancers().entrySet()) { - try { - RecordName recordName = RecordName.from(getEndpointName(loadBalancers.getKey(), application.id(), zone)); - RecordData recordData = RecordData.fqdn(loadBalancers.getValue().value()); - Optional existingRecord = nameService.findRecord(Record.Type.CNAME, recordName); - final RecordId recordId; - if(existingRecord.isPresent()) { - recordId = existingRecord.get().id(); - nameService.updateRecord(existingRecord.get().id(), recordData); - } else { - recordId = nameService.createCname(recordName, recordData); - } - hostNamesRegistered.add(new LoadBalancerName(recordId, recordName)); - } catch (Exception e) { - // Catching any exception, will be retried on next run - log.log(LogLevel.WARNING, - String.format("Got exception updating name service for application: %s, cluster: %s in zone: %s. Retrying in %s", - application.id().toShortString(), loadBalancers.getKey().value(), zone.value(), maintenanceInterval()), - e); - } - } - } - return hostNamesRegistered; - } - - static String getEndpointName(ClusterSpec.Id clusterId, ApplicationId applicationId, ZoneId zoneId) { - List endpointTerms = Arrays.asList(ignorePartIfDefault(clusterId.value()), - ignorePartIfDefault(applicationId.instance().value()), - applicationId.application().value(), - applicationId.tenant().value(), - zoneId.value(), - "vespa.oath.cloud" - ); - return endpointTerms.stream() - .filter(s -> !Strings.isNullOrEmpty((s))) - .collect(Collectors.joining(".")); - } - - private static String ignorePartIfDefault(String s) { - return IGNORE_ENDPOINT_VALUE.equalsIgnoreCase(s) ? "" : s; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 5fac0896246..9c498233809 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -18,7 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; -import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName; +import com.yahoo.vespa.hosted.controller.application.LoadBalancerAlias; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.UserTenant; @@ -72,6 +72,7 @@ public class CuratorDb { private static final Path applicationRoot = root.append("applications"); private static final Path jobRoot = root.append("jobs"); private static final Path controllerRoot = root.append("controllers"); + private static final Path loadBalancerAliasesRoot = root.append("loadBalancerAliases"); private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); private final VersionStatusSerializer versionStatusSerializer = new VersionStatusSerializer(); @@ -82,7 +83,7 @@ public class CuratorDb { private final RunSerializer runSerializer = new RunSerializer(); private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer(); private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer); - private final LoadBalancerNameSerializer loadBalancerNameSerializer = new LoadBalancerNameSerializer(); + private final LoadBalancerAliasSerializer loadBalancerAliasSerializer = new LoadBalancerAliasSerializer(); private final Curator curator; private final Duration tryLockTimeout; @@ -176,8 +177,8 @@ public class CuratorDb { return lock(lockRoot.append("osVersionStatus"), defaultLockTimeout); } - public Lock lockLoadBalancerNames() { - return lock(lockRoot.append("loadBalancerNames"), defaultLockTimeout); + public Lock lockLoadBalancerAliases() { + return lock(lockRoot.append("loadBalancerAliases"), defaultLockTimeout); } // -------------- Helpers ------------------------------------------ @@ -460,14 +461,22 @@ public class CuratorDb { curator.set(openStackServerPoolPath(), data); } - // -------------- Load balancer names ------------------------------------- + // -------------- Load balancer aliases------------------------------------ - public void writeLoadBalancerNames(Map> loadBalancerNames) { - curator.set(loadBalancerNamePath(), asJson(loadBalancerNameSerializer.toSlime(loadBalancerNames))); + public void writeLoadBalancerAliases(ApplicationId application, Set aliases) { + curator.set(loadBalancerAliasPath(application), asJson(loadBalancerAliasSerializer.toSlime(aliases))); } - public Map> readLoadBalancerNames() { - return readSlime(loadBalancerNamePath()).map(loadBalancerNameSerializer::fromSlime).orElseGet(Collections::emptyMap); + public Set readLoadBalancerAliases() { + return curator.getChildren(loadBalancerAliasesRoot).stream() + .map(ApplicationId::fromSerializedForm) + .flatMap(application -> readLoadBalancerAliases(application).stream()) + .collect(Collectors.toUnmodifiableSet()); + } + + public Set readLoadBalancerAliases(ApplicationId application) { + return readSlime(loadBalancerAliasPath(application)).map(slime -> loadBalancerAliasSerializer.fromSlime(application, slime)) + .orElseGet(Collections::emptySet); } // -------------- Paths --------------------------------------------------- @@ -545,8 +554,8 @@ public class CuratorDb { return root.append("versionStatus"); } - private static Path loadBalancerNamePath() { - return root.append("loadBalancerNames"); + private static Path loadBalancerAliasPath(ApplicationId application) { + return loadBalancerAliasesRoot.append(application.serializedForm()); } private static Path provisionStatePath() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerAliasSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerAliasSerializer.java new file mode 100644 index 00000000000..0e7682eaf96 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerAliasSerializer.java @@ -0,0 +1,52 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.application.LoadBalancerAlias; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Serializer and deserializer for a {@link LoadBalancerAlias}. + * + * @author mortent + */ +public class LoadBalancerAliasSerializer { + + private static final String aliasesField = "aliases"; + private static final String idField = "id"; + private static final String aliasField = "alias"; + private static final String canonicalNameField = "canonicalName"; + + public Slime toSlime(Set aliases) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor aliasArray = root.setArray(aliasesField); + aliases.forEach(alias -> { + Cursor nameObject = aliasArray.addObject(); + nameObject.setString(idField, alias.id()); + nameObject.setString(aliasField, alias.alias().value()); + nameObject.setString(canonicalNameField, alias.canonicalName().value()); + }); + return slime; + } + + public Set fromSlime(ApplicationId owner, Slime slime) { + Set names = new LinkedHashSet<>(); + slime.get().field(aliasesField).traverse((ArrayTraverser) (i, inspect) -> { + names.add(new LoadBalancerAlias(owner, + inspect.field(idField).asString(), + HostName.from(inspect.field(aliasField).asString()), + HostName.from(inspect.field(canonicalNameField).asString()))); + }); + + return Collections.unmodifiableSet(names); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializer.java deleted file mode 100644 index a76e1189294..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializer.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Serializer and deserializer for LoadBalancerName - * - * @author mortent - */ -public class LoadBalancerNameSerializer { - - private static final String loadBalancerNamesField = "loadBalancerNames"; - - public Slime toSlime(Map> loadBalancerNames) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor applicationEndpoints = root.setObject(loadBalancerNamesField); - - for (Map.Entry> entry : loadBalancerNames.entrySet()) { - ApplicationId applicationId = entry.getKey(); - Cursor cursor = applicationEndpoints.setArray(applicationId.serializedForm()); - entry.getValue().forEach( lb -> cursor.addObject().setString(lb.recordId().asString(), lb.recordName().asString())); - } - return slime; - } - - public Map> fromSlime(Slime slime) { - - Inspector object = slime.get().field(loadBalancerNamesField); - Map> loadBalancerNames = new HashMap<>(); - object.traverse((ObjectTraverser) (appId, inspector) -> - loadBalancerNames.put(ApplicationId.fromSerializedForm(appId), loadBalancerNamesFromSlime(inspector))); - - return loadBalancerNames; - } - - private List loadBalancerNamesFromSlime(Inspector root) { - List names = new ArrayList<>(); - root.traverse((ArrayTraverser) (i, inspector) -> inspector.traverse((ObjectTraverser)(x,y)-> names.add(new LoadBalancerName(new RecordId(x), RecordName.from(y.asString()))))); - return names; - } -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAliasTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAliasTest.java new file mode 100644 index 00000000000..965a639a68c --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAliasTest.java @@ -0,0 +1,33 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import org.junit.Test; + +import static com.yahoo.vespa.hosted.controller.application.LoadBalancerAlias.createAlias; +import static org.junit.Assert.*; + +/** + * @author mpolden + */ +public class LoadBalancerAliasTest { + + @Test + public void test_endpoint_names() { + ZoneId zoneId = ZoneId.from("prod", "us-north-1"); + ApplicationId withInstanceName = ApplicationId.from("tenant", "application", "instance"); + testAlias("instance--application--tenant--prod.us-north-1--vespa.oath.cloud", "default", withInstanceName, zoneId); + testAlias("cluster--instance--application--tenant--prod.us-north-1--vespa.oath.cloud", "cluster", withInstanceName, zoneId); + + ApplicationId withDefaultInstance = ApplicationId.from("tenant", "application", "default"); + testAlias("application--tenant--prod.us-north-1--vespa.oath.cloud", "default", withDefaultInstance, zoneId); + testAlias("cluster--application--tenant--prod.us-north-1--vespa.oath.cloud", "cluster", withDefaultInstance, zoneId); + } + + private void testAlias(String expected, String clusterName, ApplicationId applicationId, ZoneId zoneId) { + assertEquals(expected, createAlias(ClusterSpec.Id.from(clusterName), applicationId, zoneId)); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerAliasMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerAliasMaintainerTest.java new file mode 100644 index 00000000000..fa3fe505f3c --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerAliasMaintainerTest.java @@ -0,0 +1,146 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; +import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.application.LoadBalancerAlias; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +/** + * @author mortent + */ +public class LoadBalancerAliasMaintainerTest { + + @Test + public void maintains_load_balancer_records_correctly() { + DeploymentTester tester = new DeploymentTester(); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + LoadBalancerAliasMaintainer maintainer = new LoadBalancerAliasMaintainer(tester.controller(), Duration.ofHours(12), + new JobControl(new MockCuratorDb()), + tester.controllerTester().nameService(), + tester.controllerTester().curator()); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-west-1") + .region("us-central-1") + .build(); + + int numberOfClustersPerZone = 2; + + // Deploy application + tester.deployCompletely(application, applicationPackage); + setupClustersWithLoadBalancers(tester, application, numberOfClustersPerZone); + + maintainer.maintain(); + Map records = tester.controllerTester().nameService().records(); + long recordCount = records.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); + assertEquals(4, recordCount); + + Set loadBalancerAliases = tester.controller().curator().readLoadBalancerAliases(application.id()); + assertEquals(4, loadBalancerAliases.size()); + + + // no update + maintainer.maintain(); + Map records2 = tester.controllerTester().nameService().records(); + long recordCount2 = records2.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); + assertEquals(recordCount, recordCount2); + assertEquals(records, records2); + + + // add 1 cluster per zone + setupClustersWithLoadBalancers(tester, application, numberOfClustersPerZone + 1); + + maintainer.maintain(); + Map records3 = tester.controllerTester().nameService().records(); + long recordCount3 = records3.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); + assertEquals(6,recordCount3); + + Set aliases3 = tester.controller().curator().readLoadBalancerAliases(application.id()); + assertEquals(6, aliases3.size()); + + + // Add application + Application application2 = tester.createApplication("app2", "tenant1", 1, 1L); + tester.deployCompletely(application2, applicationPackage); + setupClustersWithLoadBalancers(tester, application2, numberOfClustersPerZone); + + maintainer.maintain(); + Map records4 = tester.controllerTester().nameService().records(); + long recordCount4 = records4.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); + assertEquals(10,recordCount4); + + Set aliases4 = tester.controller().curator().readLoadBalancerAliases(application2.id()); + assertEquals(4, aliases4.size()); + + + // Remove cluster in app1 + setupClustersWithLoadBalancers(tester, application, numberOfClustersPerZone); + + maintainer.maintain(); + Map records5 = tester.controllerTester().nameService().records(); + long recordCount5 = records5.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); + assertEquals(8, recordCount5); + + // Remove application app2 + tester.controller().applications().get(application2.id()) + .map(app -> app.deployments().keySet()) + .orElse(Collections.emptySet()) + .forEach(zone -> tester.controller().applications().deactivate(application2.id(), zone)); + + maintainer.maintain(); + Map records6 = tester.controllerTester().nameService().records(); + long recordCount6 = records6.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); + assertEquals(4, recordCount6); + } + + private void setupClustersWithLoadBalancers(DeploymentTester tester, Application application, int numberOfClustersPerZone) { + tester.controller().applications().get(application.id()).orElseThrow(()->new RuntimeException("No deployments")).deployments().keySet() + .forEach(zone -> tester.configServer() + .removeLoadBalancers(new DeploymentId(application.id(), zone))); + tester.controller().applications().get(application.id()).orElseThrow(()->new RuntimeException("No deployments")).deployments().keySet() + .forEach(zone -> tester.configServer() + .addLoadBalancers(zone, application.id(), makeLoadBalancers(zone, application.id(), numberOfClustersPerZone))); + + } + + private List makeLoadBalancers(ZoneId zone, ApplicationId applicationId, int count) { + List loadBalancers = new ArrayList<>(); + for (int i = 0; i < count; i++) { + loadBalancers.add( + new LoadBalancer("LB-" + i + "-Z-" + zone.value(), + new TenantId(applicationId.tenant().value()), + new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(applicationId.application().value()), + new InstanceId(applicationId.instance().value()), + ClusterSpec.Id.from("cluster-" + i), + HostName.from("loadbalancer-" + i + "-" + applicationId.serializedForm() + "-zone-" + zone.value()) + )); + } + return loadBalancers; + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java deleted file mode 100644 index 05c3de79023..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/LoadBalancerMaintainerTest.java +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; -import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName; -import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import org.junit.Test; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * @author mortent - */ -public class LoadBalancerMaintainerTest { - - @Test - public void maintains_loadbalancer_records_correctly () { - DeploymentTester tester = new DeploymentTester(); - Application application = tester.createApplication("app1", "tenant1", 1, 1L); - - LoadBalancerMaintainer loadbalancerMaintainer = new LoadBalancerMaintainer(tester.controller(), Duration.ofHours(12), - new JobControl(new MockCuratorDb()), - tester.controllerTester().nameService(), - tester.controllerTester().curator()); - - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .environment(Environment.prod) - .region("us-west-1") - .region("us-central-1") - .build(); - - int numberOfClustersPerZone = 2; - - // Deploy application - tester.deployCompletely(application, applicationPackage); - setupClustersWithLoadBalancers(tester, application, numberOfClustersPerZone); - - loadbalancerMaintainer.maintain(); - Map records = tester.controllerTester().nameService().records(); - long recordCount = records.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); - assertEquals(4,recordCount); - - Map> loadBalancerNames = tester.controller().curator().readLoadBalancerNames(); - List names = loadBalancerNames.get(application.id()); - assertEquals(4, names.size()); - - - // no update - loadbalancerMaintainer.maintain(); - Map records2 = tester.controllerTester().nameService().records(); - long recordCount2 = records2.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); - assertEquals(recordCount, recordCount2); - assertEquals(records, records2); - - - // add 1 cluster per zone - setupClustersWithLoadBalancers(tester, application, numberOfClustersPerZone + 1); - - loadbalancerMaintainer.maintain(); - Map records3 = tester.controllerTester().nameService().records(); - long recordCount3 = records3.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); - assertEquals(6,recordCount3); - - Map> loadBalancerNames3 = tester.controller().curator().readLoadBalancerNames(); - List names3 = loadBalancerNames3.get(application.id()); - assertEquals(6, names3.size()); - - - // Add application - Application application2 = tester.createApplication("app2", "tenant1", 1, 1L); - tester.deployCompletely(application2, applicationPackage); - setupClustersWithLoadBalancers(tester, application2, numberOfClustersPerZone); - - loadbalancerMaintainer.maintain(); - Map records4 = tester.controllerTester().nameService().records(); - long recordCount4 = records4.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); - assertEquals(10,recordCount4); - - Map> loadBalancerNames4 = tester.controller().curator().readLoadBalancerNames(); - List names4 = loadBalancerNames4.get(application2.id()); - assertEquals(4, names4.size()); - - - // Remove cluster in app1 - setupClustersWithLoadBalancers(tester, application, numberOfClustersPerZone); - - loadbalancerMaintainer.maintain(); - Map records5 = tester.controllerTester().nameService().records(); - long recordCount5 = records5.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); - assertEquals(8,recordCount5); - - // Remove application app2 - tester.controller().applications().get(application2.id()) - .map(app -> app.deployments().keySet()) - .orElse(Collections.emptySet()) - .forEach(zone -> tester.controller().applications().deactivate(application2.id(), zone)); - - loadbalancerMaintainer.maintain(); - Map records6 = tester.controllerTester().nameService().records(); - long recordCount6 = records6.entrySet().stream().filter(entry -> entry.getValue().data().asString().contains("loadbalancer")).count(); - assertEquals(4,recordCount6); - - } - - private void setupClustersWithLoadBalancers(DeploymentTester tester, Application application, int numberOfClustersPerZone) { - tester.controller().applications().get(application.id()).orElseThrow(()->new RuntimeException("No deployments")).deployments().keySet() - .forEach(zone -> tester.configServer() - .removeLoadBalancers(new DeploymentId(application.id(), zone))); - tester.controller().applications().get(application.id()).orElseThrow(()->new RuntimeException("No deployments")).deployments().keySet() - .forEach(zone -> tester.configServer() - .addLoadBalancers(zone, application.id(), getLoadBalancers(zone, application.id(), numberOfClustersPerZone))); - - } - - - @Test - public void test_endpoint_names() { - ZoneId zoneId = ZoneId.from("prod", "us-north-1"); - ApplicationId withInstanceName = ApplicationId.from("tenant", "application", "instance"); - testLoadBalancerName("instance.application.tenant.prod.us-north-1.vespa.oath.cloud", "default", withInstanceName, zoneId); - testLoadBalancerName("cluster.instance.application.tenant.prod.us-north-1.vespa.oath.cloud", "cluster", withInstanceName, zoneId); - - ApplicationId withDefaultInstance = ApplicationId.from("tenant", "application", "default"); - testLoadBalancerName("application.tenant.prod.us-north-1.vespa.oath.cloud", "default", withDefaultInstance, zoneId); - testLoadBalancerName("cluster.application.tenant.prod.us-north-1.vespa.oath.cloud", "cluster", withDefaultInstance, zoneId); - } - - private void testLoadBalancerName(String expected, String clusterName, ApplicationId applicationId, ZoneId zoneId) { - assertEquals(expected, - LoadBalancerMaintainer.getEndpointName(ClusterSpec.Id.from(clusterName), applicationId, zoneId)); - } - - private List getLoadBalancers(ZoneId zone, ApplicationId applicationId, int loadBalancerCount) { - List loadBalancers = new ArrayList<>(); - for (int i = 0; i < loadBalancerCount; i++) { - loadBalancers.add( - new LoadBalancer("LB-"+ i + "-Z-"+zone.value(), - new TenantId(applicationId.tenant().value()), - new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(applicationId.application().value()), - new InstanceId(applicationId.instance().value()), - ClusterSpec.Id.from("cluster-"+i), - HostName.from("loadbalancer-" + i + "-zone-" + zone.value()) - )); - } - return loadBalancers; - } -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerAliasSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerAliasSerializerTest.java new file mode 100644 index 00000000000..6ef15be775e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerAliasSerializerTest.java @@ -0,0 +1,35 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.hosted.controller.application.LoadBalancerAlias; +import org.junit.Test; + +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +/** + * @author mortent + */ +public class LoadBalancerAliasSerializerTest { + + @Test + public void test_serialization() { + LoadBalancerAliasSerializer serializer = new LoadBalancerAliasSerializer(); + ApplicationId owner = ApplicationId.defaultId(); + Set names = ImmutableSet.of(new LoadBalancerAlias(owner, + "record-id-1", + HostName.from("my-pretty-alias"), + HostName.from("long-and-ugly-name")), + new LoadBalancerAlias(owner, + "record-id-2", + HostName.from("my-pretty-alias-2"), + HostName.from("long-and-ugly-name-2"))); + Set serialized = serializer.fromSlime(owner, serializer.toSlime(names)); + assertEquals(names, serialized); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializerTest.java deleted file mode 100644 index a66fe127cd5..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/LoadBalancerNameSerializerTest.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.persistence; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.loadbalancer.LoadBalancerName; -import org.junit.Test; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertThat; - -/** - * @author mortent - */ -public class LoadBalancerNameSerializerTest { - - @Test - public void test_serialization() throws IOException { - LoadBalancerNameSerializer serializer = new LoadBalancerNameSerializer(); - ImmutableMap> lbnames = ImmutableMap.of( - ApplicationId.from("foo", "bar", "default"), Lists.newArrayList(new LoadBalancerName(new RecordId("123.4123.:124123"), RecordName.from("foo.bar")))); - Map> deserialized = serializer.fromSlime(serializer.toSlime(lbnames)); - - assertThat(deserialized, equalTo(lbnames)); - } -} \ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 75e98d2d5e9..169c86fabbe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -34,7 +34,7 @@ "name": "JobRunner" }, { - "name": "LoadBalancerMaintainer" + "name": "LoadBalancerAliasMaintainer" }, { "name": "MetricsReporter" -- cgit v1.2.3