Ensure deactivate removes all generated records

6 files changed, 49 insertions, 15 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index bf3a3e1e489..2f7b9f92316 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -932,7 +932,7 @@ public class ApplicationController {
         DeploymentId id = new DeploymentId(instanceId, zone);
         interface CleanCloseable extends AutoCloseable { void close(); }
         try (CleanCloseable postDeactivation = () -> {
-            application.ifPresent(app -> controller.routing().of(id).activate(app.get().deploymentSpec(), EndpointList.EMPTY));
+            application.ifPresent(app -> controller.routing().of(id).deactivate(app.get().deploymentSpec()));
             if (id.zoneId().environment().isManuallyDeployed())
                 applicationStore.putMetaTombstone(id, clock.instant());
             if ( ! id.zoneId().environment().isTest())
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GeneratedEndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GeneratedEndpointList.java
index 13d6f465c30..376ac578363 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GeneratedEndpointList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GeneratedEndpointList.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.routing;
 
 import com.yahoo.collections.AbstractFilteringList;
 import com.yahoo.config.provision.zone.AuthMethod;
+import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
 
 import java.util.Collection;
@@ -21,6 +22,11 @@ public class GeneratedEndpointList extends AbstractFilteringList<GeneratedEndpoi
         super(items, negate, GeneratedEndpointList::new);
     }
 
+    /** Returns the subset of endpoints which are generated for given endpoint ID */
+    public GeneratedEndpointList declared(EndpointId endpoint) {
+        return matching(e -> e.endpoint().isPresent() && e.endpoint().get().equals(endpoint));
+    }
+
     /** Returns the subset of endpoints which are generated for endpoints declared in {@link com.yahoo.config.application.api.DeploymentSpec} */
     public GeneratedEndpointList declared() {
         return matching(GeneratedEndpoint::declared);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
index c702d76218d..8f60bdb3a51 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
@@ -514,7 +514,7 @@ public class RoutingPolicies {
             Set<Endpoint> endpoints = new LinkedHashSet<>();
             policyByCluster.forEach((cluster, clusterPolicies) -> {
                 List<DeploymentId> deployments = clusterPolicies.stream().map(p -> p.id().deployment()).toList();
-                GeneratedEndpointList generated = declaredGeneratedEndpoints(clusterPolicies);
+                GeneratedEndpointList generated = declaredGeneratedEndpoints(id.endpointId(), clusterPolicies);
                 endpoints.addAll(controller.routing().declaredEndpointsOf(id, cluster, deployments, generated)
                                            .not().requiresRotation()
                                            .named(id.endpointId(), Endpoint.Scope.global).asList());
@@ -552,7 +552,7 @@ public class RoutingPolicies {
                 Map<DeploymentId, Integer> deployments = clusterPolicies.stream()
                                                                  .map(p -> p.id().deployment())
                                                                  .collect(Collectors.toMap(Function.identity(), (ignored) -> 1));
-                GeneratedEndpointList generated = declaredGeneratedEndpoints(clusterPolicies);
+                GeneratedEndpointList generated = declaredGeneratedEndpoints(id.endpointId(), clusterPolicies);
                 endpoints.addAll(controller.routing().declaredEndpointsOf(application, id.endpointId(), cluster,
                                                                           deployments, generated).asList());
             });
@@ -586,9 +586,9 @@ public class RoutingPolicies {
         return routingIdsFrom(allocation, true);
     }
 
-    private static GeneratedEndpointList declaredGeneratedEndpoints(List<RoutingPolicy> clusterPolicies) {
+    private static GeneratedEndpointList declaredGeneratedEndpoints(EndpointId endpoint, List<RoutingPolicy> clusterPolicies) {
         return GeneratedEndpointList.copyOf(clusterPolicies.stream()
-                                                           .flatMap(p -> p.generatedEndpoints().declared().asList().stream())
+                                                           .flatMap(p -> p.generatedEndpoints().declared(endpoint).asList().stream())
                                                            .distinct()
                                                            .toList());
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
index fe1e3d3d880..df0226176a2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
@@ -54,6 +54,11 @@ public abstract class DeploymentRoutingContext implements RoutingContext {
         routing.policies().refresh(deployment, deploymentSpec, generatedEndpoints);
     }
 
+    /** Deactivate routing configuration for the deployment in this context, using given deployment spec */
+    public final void deactivate(DeploymentSpec deploymentSpec) {
+        routing.policies().refresh(deployment, deploymentSpec, EndpointList.EMPTY);
+    }
+
     /** Routing method of this context */
     public final RoutingMethod routingMethod() {
         return method;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index 967a821c506..9f6cacb557b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -6,9 +6,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
-import com.yahoo.config.provision.ClusterSpec.Id;
 import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.security.KeyAlgorithm;
 import com.yahoo.security.KeyUtils;
@@ -29,16 +27,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevisi
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner.Timeouts;
 import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
 import com.yahoo.vespa.hosted.controller.maintenance.JobRunner;
 import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher;
-import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
-import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
-import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
 
 import javax.security.auth.x500.X500Principal;
 import java.math.BigInteger;
@@ -47,7 +41,6 @@ import java.security.cert.X509Certificate;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.HashMap;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -262,9 +255,8 @@ public class DeploymentContext {
         try {
             dispatcher.run();
             return this;
-        }
-        finally {
-            dispatcher.shutdown();
+        } finally {
+            dispatcher.awaitShutdown();
         }
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
index 45ee1cca4cc..7147c7ea709 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
@@ -1083,6 +1083,37 @@ public class RoutingPoliciesTest {
         context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         assertEquals(expectedRecords, tester.recordNames());
         assertEquals(containerEndpointsInProd, tester.containerEndpoints(Environment.prod));
+
+        // One endpoint is removed
+        applicationPackage = applicationPackageBuilder().region(zone1.region())
+                                                        .region(zone2.region())
+                                                        .container("c0", AuthMethod.mtls)
+                                                        .container("c1", AuthMethod.mtls, AuthMethod.token)
+                                                        .applicationEndpoint("bar", "c0", Map.of(zone1.region().value(), Map.of(InstanceName.defaultName(), 1)))
+                                                        .allow(ValidationId.globalEndpointChange)
+                                                        .build();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+        assertEquals(List.of(
+                "b36bf591.cafed00d.z.vespa-app.cloud",
+                "bar.app1.tenant1.a.vespa-app.cloud",
+                "bc50b636.cafed00d.z.vespa-app.cloud",
+                "c0.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud",
+                "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud",
+                "c1.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud",
+                "c1.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud",
+                "c33db5ed.cafed00d.z.vespa-app.cloud",
+                "d71005bf.cafed00d.z.vespa-app.cloud",
+                "dd0971b4.cafed00d.z.vespa-app.cloud",
+                "eb48ad53.cafed00d.z.vespa-app.cloud",
+                "f4a4d111.cafed00d.a.vespa-app.cloud"
+        ), tester.recordNames());
+
+        // Removing application removes all records
+        context.submit(ApplicationPackageBuilder.fromDeploymentXml("<deployment version='1.0'/>",
+                                                                   ValidationId.deploymentRemoval,
+                                                                   ValidationId.globalEndpointChange));
+        context.flushDnsUpdates();
+        assertEquals(List.of(), tester.recordNames());
     }
 
     private void addCertificateToPool(String id, UnassignedCertificate.State state, RoutingPoliciesTester tester) {