From ada377ef8e8cf29102a2ddb696332dca37f71d57 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 18 Nov 2021 15:52:01 +0100 Subject: Move rotation to routing package --- .../yahoo/vespa/hosted/controller/Instance.java | 2 +- .../vespa/hosted/controller/RoutingController.java | 4 +- .../controller/application/AssignedRotation.java | 2 +- .../controller/maintenance/MetricsReporter.java | 2 +- .../persistence/ApplicationSerializer.java | 6 +- .../restapi/application/ApplicationApiHandler.java | 6 +- .../vespa/hosted/controller/rotation/Rotation.java | 50 ----- .../hosted/controller/rotation/RotationId.java | 42 ---- .../hosted/controller/rotation/RotationLock.java | 25 --- .../controller/rotation/RotationRepository.java | 191 ------------------ .../hosted/controller/rotation/RotationState.java | 20 -- .../hosted/controller/rotation/RotationStatus.java | 103 ---------- .../controller/routing/rotation/Rotation.java | 50 +++++ .../controller/routing/rotation/RotationId.java | 42 ++++ .../controller/routing/rotation/RotationLock.java | 25 +++ .../routing/rotation/RotationRepository.java | 191 ++++++++++++++++++ .../controller/routing/rotation/RotationState.java | 20 ++ .../routing/rotation/RotationStatus.java | 103 ++++++++++ .../vespa/hosted/controller/ControllerTest.java | 4 +- .../persistence/ApplicationSerializerTest.java | 6 +- .../rotation/RotationRepositoryTest.java | 218 --------------------- .../routing/rotation/RotationRepositoryTest.java | 218 +++++++++++++++++++++ 22 files changed, 665 insertions(+), 665 deletions(-) delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java delete mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index ea2bcfcac4b..6e31c93dbdd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -14,7 +14,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; -import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import java.time.Instant; import java.util.Collection; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 2f5b92ca4c1..c832b6672d0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -31,8 +31,8 @@ import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.rotation.RotationLock; -import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationRepository; import com.yahoo.vespa.hosted.controller.routing.RoutingId; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java index 1596456b7cc..ab9304e75f3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; import java.util.Collection; import java.util.Objects; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 47df7a9da92..2939d10f99e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; import com.yahoo.vespa.hosted.controller.deployment.JobList; -import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index e8a7f7729fb..4b060846090 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -31,9 +31,9 @@ import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.rotation.RotationState; -import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import java.security.PublicKey; import java.time.Instant; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 5cd5a70e4a4..5d4b45fa82b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -97,9 +97,9 @@ import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.SupportAccessSerializer; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.rotation.RotationState; -import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java deleted file mode 100644 index ca5d2d5915f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; - -import com.yahoo.text.Text; -import java.util.Objects; - -/** - * Represents a global routing rotation. - * - * @author mpolden - */ -public class Rotation { - - private final RotationId id; - private final String name; - - public Rotation(RotationId id, String name) { - this.id = Objects.requireNonNull(id); - this.name = Objects.requireNonNull(name); - } - - /** The ID of the allocated rotation. This value is generated by global routing system */ - public RotationId id() { - return id; - } - - /** The global rotation FQDN */ - public String name() { - return name; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof Rotation)) return false; - final Rotation rotation = (Rotation) o; - return id().equals(rotation.id()) && name().equals(rotation.name()); - } - - @Override - public int hashCode() { - return Objects.hash(id(), name()); - } - - @Override - public String toString() { - return Text.format("rotation %s -> %s", id().asString(), name()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java deleted file mode 100644 index 2b75777fbbd..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; - -import java.util.Objects; - -/** - * ID of a global rotation. - * - * @author mpolden - */ -public class RotationId { - - private final String id; - - public RotationId(String id) { - this.id = id; - } - - /** Rotation ID, e.g. rotation-42.vespa.global.routing */ - public String asString() { - return id; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - RotationId that = (RotationId) o; - return Objects.equals(id, that.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public String toString() { - return "rotation ID " + id; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java deleted file mode 100644 index fe9280b1193..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; - -import com.yahoo.vespa.curator.Lock; - -import java.util.Objects; - -/** - * A lock for the rotation repository. This is a type-safe wrapper for a curator lock. - * - * @author mpolden - */ -public class RotationLock implements AutoCloseable { - - private final Lock lock; - - RotationLock(Lock lock) { - this.lock = Objects.requireNonNull(lock, "lock cannot be null"); - } - - @Override - public void close() { - lock.close(); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java deleted file mode 100644 index 5b24f39717b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; - -import com.yahoo.config.application.api.DeploymentInstanceSpec; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.Endpoint; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.ApplicationController; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.application.AssignedRotation; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static java.util.stream.Collectors.collectingAndThen; - -/** - * The rotation repository offers global rotations to Vespa applications. - * - * The list of rotations comes from RotationsConfig, which is set in the controller's services.xml. - * - * @author Oyvind Gronnesby - * @author mpolden - */ -public class RotationRepository { - - private static final Logger log = Logger.getLogger(RotationRepository.class.getName()); - - private final Map allRotations; - private final ApplicationController applications; - private final CuratorDb curator; - - public RotationRepository(RotationsConfig rotationsConfig, ApplicationController applications, CuratorDb curator) { - this.allRotations = from(rotationsConfig); - this.applications = applications; - this.curator = curator; - } - - /** Acquire a exclusive lock for this */ - public RotationLock lock() { - return new RotationLock(curator.lockRotations()); - } - - /** Get rotation by given rotationId */ - public Optional getRotation(RotationId rotationId) { - return Optional.of(allRotations.get(rotationId)); - } - - /** - * Returns a single rotation for the given application. This is only used when a rotation is assigned through the - * use of a global service ID. - * - * If a rotation is already assigned to the application, that rotation will be returned. - * If no rotation is assigned, return an available rotation. The caller is responsible for assigning the rotation. - * - * @param instanceSpec the instance deployment spec - * @param instance the instance requesting a rotation - * @param lock lock which must be acquired by the caller - */ - private AssignedRotation assignRotationTo(String globalServiceId, DeploymentInstanceSpec instanceSpec, - Instance instance, RotationLock lock) { - RotationId rotation; - if (instance.rotations().isEmpty()) { - rotation = findAvailableRotation(instance.id(), lock).id(); - } else { - rotation = instance.rotations().get(0).rotationId(); - } - var productionRegions = instanceSpec.zones().stream() - .filter(zone -> zone.environment().isProduction()) - .flatMap(zone -> zone.region().stream()) - .collect(Collectors.toSet()); - if (productionRegions.size() < 2) { - throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined " + - "in instance '" + instance.name() + "'"); - } - return new AssignedRotation(new ClusterSpec.Id(globalServiceId), - EndpointId.defaultId(), - rotation, - productionRegions); - } - - /** - * Returns rotation assignments for all endpoints in application. - * - * If rotations are already assigned, these will be returned. - * If rotations are not assigned, a new assignment will be created taking new rotations from the repository. - * This method supports both global-service-id as well as the new endpoints tag. - * - * @param deploymentSpec The deployment spec of the application - * @param instance The application requesting rotations - * @param lock Lock which by acquired by the caller - * @return List of rotation assignments - either new or existing - */ - public List getOrAssignRotations(DeploymentSpec deploymentSpec, Instance instance, RotationLock lock) { - // Skip assignment if no rotations are configured in this system - if (allRotations.isEmpty()) { - return List.of(); - } - - // Only allow one kind of configuration syntax - var instanceSpec = deploymentSpec.requireInstance(instance.name()); - if ( instanceSpec.globalServiceId().isPresent() - && ! instanceSpec.endpoints().isEmpty()) { - throw new IllegalArgumentException("Cannot provision rotations with both global-service-id and 'endpoints'"); - } - - // Support legacy global-service-id - if (instanceSpec.globalServiceId().isPresent()) { - return List.of(assignRotationTo(instanceSpec.globalServiceId().get(), instanceSpec, instance, lock)); - } - - return assignRotationsTo(instanceSpec.endpoints(), instance, lock); - } - - private List assignRotationsTo(List endpoints, Instance instance, RotationLock lock) { - if (endpoints.isEmpty()) return List.of(); // No endpoints declared, nothing to assign. - var availableRotations = new ArrayList<>(availableRotations(lock).values()); - var assignedRotationsByEndpointId = instance.rotations().stream() - .collect(Collectors.toMap(AssignedRotation::endpointId, - Function.identity())); - var assignments = new ArrayList(); - for (var endpoint : endpoints) { - var endpointId = EndpointId.of(endpoint.endpointId()); - var assignedRotation = assignedRotationsByEndpointId.get(endpointId); - RotationId rotationId; - if (assignedRotation == null) { // No rotation is assigned to this endpoint - rotationId = requireNonEmpty(availableRotations).remove(0).id(); - } else { // Rotation already assigned to this endpoint, reuse it - rotationId = assignedRotation.rotationId(); - } - assignments.add(new AssignedRotation(ClusterSpec.Id.from(endpoint.containerId()), endpointId, rotationId, Set.copyOf(endpoint.regions()))); - } - return Collections.unmodifiableList(assignments); - } - - /** - * Returns all unassigned rotations - * @param lock Lock which must be acquired by the caller - */ - public Map availableRotations(@SuppressWarnings("unused") RotationLock lock) { - List assignedRotations = applications.asList().stream() - .flatMap(application -> application.instances().values().stream()) - .flatMap(instance -> instance.rotations().stream()) - .map(AssignedRotation::rotationId) - .collect(Collectors.toList()); - Map unassignedRotations = new LinkedHashMap<>(this.allRotations); - assignedRotations.forEach(unassignedRotations::remove); - return Collections.unmodifiableMap(unassignedRotations); - } - - private Rotation findAvailableRotation(ApplicationId id, RotationLock lock) { - Map availableRotations = availableRotations(lock); - // Return first available rotation - RotationId rotation = requireNonEmpty(availableRotations.keySet()).iterator().next(); - log.info(Text.format("Offering %s to application %s", rotation, id)); - return allRotations.get(rotation); - } - - /** Returns a immutable map of rotation ID to rotation sorted by rotation ID */ - private static Map from(RotationsConfig rotationConfig) { - return rotationConfig.rotations().entrySet().stream() - .map(entry -> new Rotation(new RotationId(entry.getKey()), entry.getValue().trim())) - .sorted(Comparator.comparing(rotation -> rotation.id().asString())) - .collect(collectingAndThen(Collectors.toMap(Rotation::id, - rotation -> rotation, - (k, v) -> v, - LinkedHashMap::new), - Collections::unmodifiableMap)); - } - - private static > T requireNonEmpty(T rotations) { - if (rotations.isEmpty()) throw new IllegalStateException("Hosted Vespa ran out of rotations, unable to assign rotation"); - return rotations; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java deleted file mode 100644 index 032f01433b3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; - -/** - * The possible states of a global rotation. - * - * @author mpolden - */ -public enum RotationState { - - /** Rotation has status 'in' and is receiving traffic */ - in, - - /** Rotation has status 'out' and is *NOT* receiving traffic */ - out, - - /** Rotation status is currently unknown, or no global rotation has been assigned */ - unknown - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java deleted file mode 100644 index 1ddbd640e53..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; - -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.Deployment; - -import java.time.Instant; -import java.util.Map; -import java.util.Objects; - -/** - * The status of all rotations assigned to an application. - * - * @author mpolden - */ -public class RotationStatus { - - public static final RotationStatus EMPTY = new RotationStatus(Map.of()); - - private final Map status; - - private RotationStatus(Map status) { - this.status = Map.copyOf(Objects.requireNonNull(status)); - } - - public Map asMap() { - return status; - } - - /** Get targets of given rotation, if any */ - public Targets of(RotationId rotation) { - return status.getOrDefault(rotation, Targets.NONE); - } - - /** Get status of deployment in given rotation, if any */ - public RotationState of(RotationId rotation, Deployment deployment) { - return of(rotation).asMap().entrySet().stream() - .filter(kv -> kv.getKey().equals(deployment.zone())) - .map(Map.Entry::getValue) - .findFirst() - .orElse(RotationState.unknown); - } - - @Override - public String toString() { - return "rotation status " + status; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - RotationStatus that = (RotationStatus) o; - return status.equals(that.status); - } - - @Override - public int hashCode() { - return Objects.hash(status); - } - - public static RotationStatus from(Map targets) { - return targets.isEmpty() ? EMPTY : new RotationStatus(targets); - } - - /** Targets of a rotation */ - public static class Targets { - - public static final Targets NONE = new Targets(Map.of(), Instant.EPOCH); - - private final Map targets; - private final Instant lastUpdated; - - public Targets(Map targets, Instant lastUpdated) { - this.targets = Map.copyOf(Objects.requireNonNull(targets, "states must be non-null")); - this.lastUpdated = Objects.requireNonNull(lastUpdated, "lastUpdated must be non-null"); - } - - public Map asMap() { - return targets; - } - - public Instant lastUpdated() { - return lastUpdated; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Targets targets1 = (Targets) o; - return targets.equals(targets1.targets) && - lastUpdated.equals(targets1.lastUpdated); - } - - @Override - public int hashCode() { - return Objects.hash(targets, lastUpdated); - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java new file mode 100644 index 00000000000..0cf7101cac0 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.rotation; + +import com.yahoo.text.Text; +import java.util.Objects; + +/** + * Represents a global routing rotation. + * + * @author mpolden + */ +public class Rotation { + + private final RotationId id; + private final String name; + + public Rotation(RotationId id, String name) { + this.id = Objects.requireNonNull(id); + this.name = Objects.requireNonNull(name); + } + + /** The ID of the allocated rotation. This value is generated by global routing system */ + public RotationId id() { + return id; + } + + /** The global rotation FQDN */ + public String name() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Rotation)) return false; + final Rotation rotation = (Rotation) o; + return id().equals(rotation.id()) && name().equals(rotation.name()); + } + + @Override + public int hashCode() { + return Objects.hash(id(), name()); + } + + @Override + public String toString() { + return Text.format("rotation %s -> %s", id().asString(), name()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java new file mode 100644 index 00000000000..4d97962a40a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.rotation; + +import java.util.Objects; + +/** + * ID of a global rotation. + * + * @author mpolden + */ +public class RotationId { + + private final String id; + + public RotationId(String id) { + this.id = id; + } + + /** Rotation ID, e.g. rotation-42.vespa.global.routing */ + public String asString() { + return id; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RotationId that = (RotationId) o; + return Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public String toString() { + return "rotation ID " + id; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java new file mode 100644 index 00000000000..36a43f80e9a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.rotation; + +import com.yahoo.vespa.curator.Lock; + +import java.util.Objects; + +/** + * A lock for the rotation repository. This is a type-safe wrapper for a curator lock. + * + * @author mpolden + */ +public class RotationLock implements AutoCloseable { + + private final Lock lock; + + RotationLock(Lock lock) { + this.lock = Objects.requireNonNull(lock, "lock cannot be null"); + } + + @Override + public void close() { + lock.close(); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java new file mode 100644 index 00000000000..961fdc6dd9c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java @@ -0,0 +1,191 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.rotation; + +import com.yahoo.config.application.api.DeploymentInstanceSpec; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.Endpoint; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.text.Text; +import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.collectingAndThen; + +/** + * The rotation repository offers global rotations to Vespa applications. + * + * The list of rotations comes from RotationsConfig, which is set in the controller's services.xml. + * + * @author Oyvind Gronnesby + * @author mpolden + */ +public class RotationRepository { + + private static final Logger log = Logger.getLogger(RotationRepository.class.getName()); + + private final Map allRotations; + private final ApplicationController applications; + private final CuratorDb curator; + + public RotationRepository(RotationsConfig rotationsConfig, ApplicationController applications, CuratorDb curator) { + this.allRotations = from(rotationsConfig); + this.applications = applications; + this.curator = curator; + } + + /** Acquire a exclusive lock for this */ + public RotationLock lock() { + return new RotationLock(curator.lockRotations()); + } + + /** Get rotation by given rotationId */ + public Optional getRotation(RotationId rotationId) { + return Optional.of(allRotations.get(rotationId)); + } + + /** + * Returns a single rotation for the given application. This is only used when a rotation is assigned through the + * use of a global service ID. + * + * If a rotation is already assigned to the application, that rotation will be returned. + * If no rotation is assigned, return an available rotation. The caller is responsible for assigning the rotation. + * + * @param instanceSpec the instance deployment spec + * @param instance the instance requesting a rotation + * @param lock lock which must be acquired by the caller + */ + private AssignedRotation assignRotationTo(String globalServiceId, DeploymentInstanceSpec instanceSpec, + Instance instance, RotationLock lock) { + RotationId rotation; + if (instance.rotations().isEmpty()) { + rotation = findAvailableRotation(instance.id(), lock).id(); + } else { + rotation = instance.rotations().get(0).rotationId(); + } + var productionRegions = instanceSpec.zones().stream() + .filter(zone -> zone.environment().isProduction()) + .flatMap(zone -> zone.region().stream()) + .collect(Collectors.toSet()); + if (productionRegions.size() < 2) { + throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined " + + "in instance '" + instance.name() + "'"); + } + return new AssignedRotation(new ClusterSpec.Id(globalServiceId), + EndpointId.defaultId(), + rotation, + productionRegions); + } + + /** + * Returns rotation assignments for all endpoints in application. + * + * If rotations are already assigned, these will be returned. + * If rotations are not assigned, a new assignment will be created taking new rotations from the repository. + * This method supports both global-service-id as well as the new endpoints tag. + * + * @param deploymentSpec The deployment spec of the application + * @param instance The application requesting rotations + * @param lock Lock which by acquired by the caller + * @return List of rotation assignments - either new or existing + */ + public List getOrAssignRotations(DeploymentSpec deploymentSpec, Instance instance, RotationLock lock) { + // Skip assignment if no rotations are configured in this system + if (allRotations.isEmpty()) { + return List.of(); + } + + // Only allow one kind of configuration syntax + var instanceSpec = deploymentSpec.requireInstance(instance.name()); + if ( instanceSpec.globalServiceId().isPresent() + && ! instanceSpec.endpoints().isEmpty()) { + throw new IllegalArgumentException("Cannot provision rotations with both global-service-id and 'endpoints'"); + } + + // Support legacy global-service-id + if (instanceSpec.globalServiceId().isPresent()) { + return List.of(assignRotationTo(instanceSpec.globalServiceId().get(), instanceSpec, instance, lock)); + } + + return assignRotationsTo(instanceSpec.endpoints(), instance, lock); + } + + private List assignRotationsTo(List endpoints, Instance instance, RotationLock lock) { + if (endpoints.isEmpty()) return List.of(); // No endpoints declared, nothing to assign. + var availableRotations = new ArrayList<>(availableRotations(lock).values()); + var assignedRotationsByEndpointId = instance.rotations().stream() + .collect(Collectors.toMap(AssignedRotation::endpointId, + Function.identity())); + var assignments = new ArrayList(); + for (var endpoint : endpoints) { + var endpointId = EndpointId.of(endpoint.endpointId()); + var assignedRotation = assignedRotationsByEndpointId.get(endpointId); + RotationId rotationId; + if (assignedRotation == null) { // No rotation is assigned to this endpoint + rotationId = requireNonEmpty(availableRotations).remove(0).id(); + } else { // Rotation already assigned to this endpoint, reuse it + rotationId = assignedRotation.rotationId(); + } + assignments.add(new AssignedRotation(ClusterSpec.Id.from(endpoint.containerId()), endpointId, rotationId, Set.copyOf(endpoint.regions()))); + } + return Collections.unmodifiableList(assignments); + } + + /** + * Returns all unassigned rotations + * @param lock Lock which must be acquired by the caller + */ + public Map availableRotations(@SuppressWarnings("unused") RotationLock lock) { + List assignedRotations = applications.asList().stream() + .flatMap(application -> application.instances().values().stream()) + .flatMap(instance -> instance.rotations().stream()) + .map(AssignedRotation::rotationId) + .collect(Collectors.toList()); + Map unassignedRotations = new LinkedHashMap<>(this.allRotations); + assignedRotations.forEach(unassignedRotations::remove); + return Collections.unmodifiableMap(unassignedRotations); + } + + private Rotation findAvailableRotation(ApplicationId id, RotationLock lock) { + Map availableRotations = availableRotations(lock); + // Return first available rotation + RotationId rotation = requireNonEmpty(availableRotations.keySet()).iterator().next(); + log.info(Text.format("Offering %s to application %s", rotation, id)); + return allRotations.get(rotation); + } + + /** Returns a immutable map of rotation ID to rotation sorted by rotation ID */ + private static Map from(RotationsConfig rotationConfig) { + return rotationConfig.rotations().entrySet().stream() + .map(entry -> new Rotation(new RotationId(entry.getKey()), entry.getValue().trim())) + .sorted(Comparator.comparing(rotation -> rotation.id().asString())) + .collect(collectingAndThen(Collectors.toMap(Rotation::id, + rotation -> rotation, + (k, v) -> v, + LinkedHashMap::new), + Collections::unmodifiableMap)); + } + + private static > T requireNonEmpty(T rotations) { + if (rotations.isEmpty()) throw new IllegalStateException("Hosted Vespa ran out of rotations, unable to assign rotation"); + return rotations; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java new file mode 100644 index 00000000000..19e816a0b51 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.rotation; + +/** + * The possible states of a global rotation. + * + * @author mpolden + */ +public enum RotationState { + + /** Rotation has status 'in' and is receiving traffic */ + in, + + /** Rotation has status 'out' and is *NOT* receiving traffic */ + out, + + /** Rotation status is currently unknown, or no global rotation has been assigned */ + unknown + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java new file mode 100644 index 00000000000..6d95ad9a230 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java @@ -0,0 +1,103 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.rotation; + +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Deployment; + +import java.time.Instant; +import java.util.Map; +import java.util.Objects; + +/** + * The status of all rotations assigned to an application. + * + * @author mpolden + */ +public class RotationStatus { + + public static final RotationStatus EMPTY = new RotationStatus(Map.of()); + + private final Map status; + + private RotationStatus(Map status) { + this.status = Map.copyOf(Objects.requireNonNull(status)); + } + + public Map asMap() { + return status; + } + + /** Get targets of given rotation, if any */ + public Targets of(RotationId rotation) { + return status.getOrDefault(rotation, Targets.NONE); + } + + /** Get status of deployment in given rotation, if any */ + public RotationState of(RotationId rotation, Deployment deployment) { + return of(rotation).asMap().entrySet().stream() + .filter(kv -> kv.getKey().equals(deployment.zone())) + .map(Map.Entry::getValue) + .findFirst() + .orElse(RotationState.unknown); + } + + @Override + public String toString() { + return "rotation status " + status; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RotationStatus that = (RotationStatus) o; + return status.equals(that.status); + } + + @Override + public int hashCode() { + return Objects.hash(status); + } + + public static RotationStatus from(Map targets) { + return targets.isEmpty() ? EMPTY : new RotationStatus(targets); + } + + /** Targets of a rotation */ + public static class Targets { + + public static final Targets NONE = new Targets(Map.of(), Instant.EPOCH); + + private final Map targets; + private final Instant lastUpdated; + + public Targets(Map targets, Instant lastUpdated) { + this.targets = Map.copyOf(Objects.requireNonNull(targets, "states must be non-null")); + this.lastUpdated = Objects.requireNonNull(lastUpdated, "lastUpdated must be non-null"); + } + + public Map asMap() { + return targets; + } + + public Instant lastUpdated() { + return lastUpdated; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Targets targets1 = (Targets) o; + return targets.equals(targets1.targets) && + lastUpdated.equals(targets1.lastUpdated); + } + + @Override + public int hashCode() { + return Objects.hash(targets, lastUpdated); + } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index be180f27af6..132e6caa3ca 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -38,8 +38,8 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.Test; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index f1421b5affd..b33f8f6f7e7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -23,9 +23,9 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.rotation.RotationState; -import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import org.junit.Test; import java.nio.file.Files; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java deleted file mode 100644 index e7c2eacbd02..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; - -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.AssignedRotation; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; -import org.junit.Test; - -import java.net.URI; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author Oyvind Gronnesby - * @author mpolden - */ -public class RotationRepositoryTest { - - private static final RotationsConfig rotationsConfig = new RotationsConfig( - new RotationsConfig.Builder() - .rotations("foo-1", "foo-1.com") - .rotations("foo-2", "foo-2.com") - ); - - private static final RotationsConfig rotationsConfigWhitespaces = new RotationsConfig( - new RotationsConfig.Builder() - .rotations("foo-1", "\n \t foo-1.com \n") - .rotations("foo-2", "foo-2.com") - ); - - private static final ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .globalServiceId("foo") - .region("us-east-3") - .region("us-west-1") - .build(); - - private final DeploymentTester tester = new DeploymentTester(new ControllerTester(rotationsConfig, SystemName.main)); - private final RotationRepository repository = tester.controller().routing().rotations(); - private final DeploymentContext application = tester.newDeploymentContext("tenant1", "app1", "default"); - - @Test - public void assigns_and_reuses_rotation() { - // Deploying assigns a rotation - application.submit(applicationPackage).deploy(); - Rotation expected = new Rotation(new RotationId("foo-1"), "foo-1.com"); - - assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); - assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(application.instanceId()).primary().get().url()); - try (RotationLock lock = repository.lock()) { - List rotations = repository.getOrAssignRotations(application.application().deploymentSpec(), - application.instance(), - lock); - assertSingleRotation(expected, rotations, repository); - assertEquals(Set.of(RegionName.from("us-west-1"), RegionName.from("us-east-3")), - application.instance().rotations().get(0).regions()); - } - - // Submitting once more assigns same rotation - application.submit(applicationPackage).deploy(); - assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); - - // Adding region updates rotation - var applicationPackage = new ApplicationPackageBuilder() - .globalServiceId("foo") - .region("us-east-3") - .region("us-west-1") - .region("us-central-1") - .build(); - application.submit(applicationPackage).deploy(); - assertEquals(Set.of(RegionName.from("us-west-1"), RegionName.from("us-east-3"), - RegionName.from("us-central-1")), - application.instance().rotations().get(0).regions()); - } - - @Test - public void strips_whitespace_in_rotation_fqdn() { - var tester = new DeploymentTester(new ControllerTester(rotationsConfigWhitespaces, SystemName.main)); - RotationRepository repository = tester.controller().routing().rotations(); - var application2 = tester.newDeploymentContext("tenant1", "app2", "default"); - - application2.submit(applicationPackage); - - try (RotationLock lock = repository.lock()) { - List rotations = repository.getOrAssignRotations(application2.application().deploymentSpec(), application2.instance(), lock); - Rotation assignedRotation = new Rotation(new RotationId("foo-1"), "foo-1.com"); - assertSingleRotation(assignedRotation, rotations, repository); - } - } - - @Test - public void out_of_rotations() { - // Assigns 1 rotation - application.submit(applicationPackage).deploy(); - - // Assigns 1 more - var application2 = tester.newDeploymentContext("tenant2", "app2", "default"); - application2.submit(applicationPackage).deploy(); - - // We're now out of rotations and next deployment fails - var application3 = tester.newDeploymentContext("tenant3", "app3", "default"); - application3.submit(applicationPackage) - .runJobExpectingFailure(JobType.systemTest, Optional.of("out of rotations")); - } - - @Test - public void too_few_zones() { - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .globalServiceId("foo") - .region("us-east-3") - .build(); - application.submit(applicationPackage).runJobExpectingFailure(JobType.systemTest, Optional.of("less than 2 prod zones are defined")); - } - - @Test - public void no_rotation_assigned_for_application_without_service_id() { - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .region("us-east-3") - .region("us-west-1") - .build(); - application.submit(applicationPackage); - assertTrue(application.instance().rotations().isEmpty()); - } - - @Test - public void prefixes_system_when_not_main() { - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .globalServiceId("foo") - .region("cd-us-east-1") - .region("cd-us-west-1") - .build(); - var zones = List.of( - ZoneApiMock.fromId("test.cd-us-west-1"), - ZoneApiMock.fromId("staging.cd-us-west-1"), - ZoneApiMock.fromId("prod.cd-us-east-1"), - ZoneApiMock.fromId("prod.cd-us-west-1")); - tester.controllerTester().zoneRegistry() - .setZones(zones) - .setRoutingMethod(zones, RoutingMethod.shared) - .setSystemName(SystemName.cd); - tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.notController()); - var application2 = tester.newDeploymentContext("tenant2", "app2", "default"); - application2.submit(applicationPackage).deploy(); - assertEquals(List.of(new RotationId("foo-1")), rotationIds(application2.instance().rotations())); - assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/", - tester.controller().routing().readDeclaredEndpointsOf(application2.instanceId()).primary().get().url().toString()); - } - - @Test - public void multiple_instances_with_similar_global_service_id() { - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .instances("instance1,instance2") - .region("us-central-1") - .parallel("us-west-1", "us-east-3") - .globalServiceId("global") - .build(); - var instance1 = tester.newDeploymentContext("tenant1", "application1", "instance1") - .submit(applicationPackage) - .deploy(); - var instance2 = tester.newDeploymentContext("tenant1", "application1", "instance2"); - assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); - assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); - assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).primary().get().url()); - assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).primary().get().url()); - } - - @Test - public void multiple_instances_with_similar_endpoints() { - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .instances("instance1,instance2") - .region("us-central-1") - .parallel("us-west-1", "us-east-3") - .endpoint("default", "foo", "us-central-1", "us-west-1") - .build(); - var instance1 = tester.newDeploymentContext("tenant1", "application1", "instance1") - .submit(applicationPackage) - .deploy(); - var instance2 = tester.newDeploymentContext("tenant1", "application1", "instance2"); - - assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); - assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); - - assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).primary().get().url()); - assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).primary().get().url()); - } - - private void assertSingleRotation(Rotation expected, List assignedRotations, RotationRepository repository) { - assertEquals(1, assignedRotations.size()); - var rotationId = assignedRotations.get(0).rotationId(); - var rotation = repository.getRotation(rotationId); - assertTrue(rotationId + " exists", rotation.isPresent()); - assertEquals(expected, rotation.get()); - } - - private static List rotationIds(List assignedRotations) { - return assignedRotations.stream().map(AssignedRotation::rotationId).collect(Collectors.toUnmodifiableList()); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java new file mode 100644 index 00000000000..9a3ac8b547d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java @@ -0,0 +1,218 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.rotation; + +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import org.junit.Test; + +import java.net.URI; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author Oyvind Gronnesby + * @author mpolden + */ +public class RotationRepositoryTest { + + private static final RotationsConfig rotationsConfig = new RotationsConfig( + new RotationsConfig.Builder() + .rotations("foo-1", "foo-1.com") + .rotations("foo-2", "foo-2.com") + ); + + private static final RotationsConfig rotationsConfigWhitespaces = new RotationsConfig( + new RotationsConfig.Builder() + .rotations("foo-1", "\n \t foo-1.com \n") + .rotations("foo-2", "foo-2.com") + ); + + private static final ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .globalServiceId("foo") + .region("us-east-3") + .region("us-west-1") + .build(); + + private final DeploymentTester tester = new DeploymentTester(new ControllerTester(rotationsConfig, SystemName.main)); + private final RotationRepository repository = tester.controller().routing().rotations(); + private final DeploymentContext application = tester.newDeploymentContext("tenant1", "app1", "default"); + + @Test + public void assigns_and_reuses_rotation() { + // Deploying assigns a rotation + application.submit(applicationPackage).deploy(); + Rotation expected = new Rotation(new RotationId("foo-1"), "foo-1.com"); + + assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); + assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"), + tester.controller().routing().readDeclaredEndpointsOf(application.instanceId()).primary().get().url()); + try (RotationLock lock = repository.lock()) { + List rotations = repository.getOrAssignRotations(application.application().deploymentSpec(), + application.instance(), + lock); + assertSingleRotation(expected, rotations, repository); + assertEquals(Set.of(RegionName.from("us-west-1"), RegionName.from("us-east-3")), + application.instance().rotations().get(0).regions()); + } + + // Submitting once more assigns same rotation + application.submit(applicationPackage).deploy(); + assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); + + // Adding region updates rotation + var applicationPackage = new ApplicationPackageBuilder() + .globalServiceId("foo") + .region("us-east-3") + .region("us-west-1") + .region("us-central-1") + .build(); + application.submit(applicationPackage).deploy(); + assertEquals(Set.of(RegionName.from("us-west-1"), RegionName.from("us-east-3"), + RegionName.from("us-central-1")), + application.instance().rotations().get(0).regions()); + } + + @Test + public void strips_whitespace_in_rotation_fqdn() { + var tester = new DeploymentTester(new ControllerTester(rotationsConfigWhitespaces, SystemName.main)); + RotationRepository repository = tester.controller().routing().rotations(); + var application2 = tester.newDeploymentContext("tenant1", "app2", "default"); + + application2.submit(applicationPackage); + + try (RotationLock lock = repository.lock()) { + List rotations = repository.getOrAssignRotations(application2.application().deploymentSpec(), application2.instance(), lock); + Rotation assignedRotation = new Rotation(new RotationId("foo-1"), "foo-1.com"); + assertSingleRotation(assignedRotation, rotations, repository); + } + } + + @Test + public void out_of_rotations() { + // Assigns 1 rotation + application.submit(applicationPackage).deploy(); + + // Assigns 1 more + var application2 = tester.newDeploymentContext("tenant2", "app2", "default"); + application2.submit(applicationPackage).deploy(); + + // We're now out of rotations and next deployment fails + var application3 = tester.newDeploymentContext("tenant3", "app3", "default"); + application3.submit(applicationPackage) + .runJobExpectingFailure(JobType.systemTest, Optional.of("out of rotations")); + } + + @Test + public void too_few_zones() { + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .globalServiceId("foo") + .region("us-east-3") + .build(); + application.submit(applicationPackage).runJobExpectingFailure(JobType.systemTest, Optional.of("less than 2 prod zones are defined")); + } + + @Test + public void no_rotation_assigned_for_application_without_service_id() { + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .region("us-east-3") + .region("us-west-1") + .build(); + application.submit(applicationPackage); + assertTrue(application.instance().rotations().isEmpty()); + } + + @Test + public void prefixes_system_when_not_main() { + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .globalServiceId("foo") + .region("cd-us-east-1") + .region("cd-us-west-1") + .build(); + var zones = List.of( + ZoneApiMock.fromId("test.cd-us-west-1"), + ZoneApiMock.fromId("staging.cd-us-west-1"), + ZoneApiMock.fromId("prod.cd-us-east-1"), + ZoneApiMock.fromId("prod.cd-us-west-1")); + tester.controllerTester().zoneRegistry() + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.shared) + .setSystemName(SystemName.cd); + tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.notController()); + var application2 = tester.newDeploymentContext("tenant2", "app2", "default"); + application2.submit(applicationPackage).deploy(); + assertEquals(List.of(new RotationId("foo-1")), rotationIds(application2.instance().rotations())); + assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/", + tester.controller().routing().readDeclaredEndpointsOf(application2.instanceId()).primary().get().url().toString()); + } + + @Test + public void multiple_instances_with_similar_global_service_id() { + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("instance1,instance2") + .region("us-central-1") + .parallel("us-west-1", "us-east-3") + .globalServiceId("global") + .build(); + var instance1 = tester.newDeploymentContext("tenant1", "application1", "instance1") + .submit(applicationPackage) + .deploy(); + var instance2 = tester.newDeploymentContext("tenant1", "application1", "instance2"); + assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); + assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); + assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), + tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).primary().get().url()); + assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), + tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).primary().get().url()); + } + + @Test + public void multiple_instances_with_similar_endpoints() { + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("instance1,instance2") + .region("us-central-1") + .parallel("us-west-1", "us-east-3") + .endpoint("default", "foo", "us-central-1", "us-west-1") + .build(); + var instance1 = tester.newDeploymentContext("tenant1", "application1", "instance1") + .submit(applicationPackage) + .deploy(); + var instance2 = tester.newDeploymentContext("tenant1", "application1", "instance2"); + + assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); + assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); + + assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), + tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).primary().get().url()); + assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), + tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).primary().get().url()); + } + + private void assertSingleRotation(Rotation expected, List assignedRotations, RotationRepository repository) { + assertEquals(1, assignedRotations.size()); + var rotationId = assignedRotations.get(0).rotationId(); + var rotation = repository.getRotation(rotationId); + assertTrue(rotationId + " exists", rotation.isPresent()); + assertEquals(expected, rotation.get()); + } + + private static List rotationIds(List assignedRotations) { + return assignedRotations.stream().map(AssignedRotation::rotationId).collect(Collectors.toUnmodifiableList()); + } + +} -- cgit v1.2.3