Create ArchiveUriUpdater

6 files changed, 175 insertions, 0 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java
new file mode 100644
index 00000000000..fe219f12af4
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java
@@ -0,0 +1,65 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.ApplicationController;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
+
+import java.net.URI;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.Map;
+
+/**
+ * Updates archive URIs for tenants in all zones.
+ *
+ * @author freva
+ */
+public class ArchiveUriUpdater extends ControllerMaintainer {
+
+    private final ApplicationController applications;
+    private final NodeRepository nodeRepository;
+    private final ArchiveService archiveService;
+
+    public ArchiveUriUpdater(Controller controller, Duration duration) {
+        super(controller, duration, ArchiveUriUpdater.class.getSimpleName(), SystemName.all());
+        this.applications = controller.applications();
+        this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
+        this.archiveService = controller.serviceRegistry().archiveService();
+    }
+
+    @Override
+    protected boolean maintain() {
+        Map<ZoneId, Set<TenantName>> tenantsByZone = new HashMap<>();
+        for (var application : applications.asList()) {
+            for (var instance : application.instances().values()) {
+                for (var deployment : instance.deployments().values()) {
+                    tenantsByZone.computeIfAbsent(deployment.zone(), zone -> new HashSet<>())
+                            .add(instance.id().tenant());
+                }
+            }
+        }
+
+        tenantsByZone.forEach((zone, tenants) -> {
+            Map<TenantName, URI> zoneArchiveUris = nodeRepository.getArchiveUris(zone);
+            for (TenantName tenant : tenants) {
+                archiveService.archiveUriFor(zone, tenant)
+                        .filter(uri -> !uri.equals(zoneArchiveUris.get(tenant)))
+                        .ifPresent(uri -> nodeRepository.setArchiveUri(zone, tenant, uri));
+            }
+
+            zoneArchiveUris.keySet().stream()
+                    .filter(tenant -> ! tenants.contains(tenant))
+                    .forEach(tenant -> nodeRepository.removeArchiveUri(zone, tenant));
+        });
+
+        return true;
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index f7ab4d30088..9f9a0f6d56f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -65,6 +65,7 @@ public class ControllerMaintenance extends AbstractComponent {
         maintainers.add(new ReindexingTriggerer(controller, intervals.reindexingTriggerer));
         maintainers.add(new EndpointCertificateMaintainer(controller, intervals.endpointCertificateMaintainer));
         maintainers.add(new TrafficShareUpdater(controller, intervals.trafficFractionUpdater));
+        maintainers.add(new ArchiveUriUpdater(controller, intervals.archiveUriUpdater));
     }
 
     public Upgrader upgrader() { return upgrader; }
@@ -115,6 +116,7 @@ public class ControllerMaintenance extends AbstractComponent {
         private final Duration reindexingTriggerer;
         private final Duration endpointCertificateMaintainer;
         private final Duration trafficFractionUpdater;
+        private final Duration archiveUriUpdater;
 
         public Intervals(SystemName system) {
             this.system = Objects.requireNonNull(system);
@@ -142,6 +144,7 @@ public class ControllerMaintenance extends AbstractComponent {
             this.reindexingTriggerer = duration(1, HOURS);
             this.endpointCertificateMaintainer = duration(12, HOURS);
             this.trafficFractionUpdater = duration(5, MINUTES);
+            this.archiveUriUpdater = duration(5, MINUTES);
         }
 
         private Duration duration(long amount, TemporalUnit unit) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
index 96240f2b6c7..0c558ef3ea8 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
@@ -8,6 +8,7 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Application;
@@ -18,6 +19,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState;
 
+import java.net.URI;
 import java.time.Duration;
 import java.util.Collection;
 import java.util.Collections;
@@ -42,6 +44,7 @@ public class NodeRepositoryMock implements NodeRepository {
     private final Map<ZoneId, TargetVersions> targetVersions = new HashMap<>();
     private final Map<Integer, Duration> osUpgradeBudgets = new HashMap<>();
     private final Map<DeploymentId, Pair<Double, Double>> trafficFractions = new HashMap<>();
+    private final Map<ZoneId, Map<TenantName, URI>> archiveUris = new HashMap<>();
 
     private boolean allowPatching = false;
 
@@ -192,6 +195,21 @@ public class NodeRepositoryMock implements NodeRepository {
     }
 
     @Override
+    public Map<TenantName, URI> getArchiveUris(ZoneId zone) {
+        return Map.copyOf(archiveUris.getOrDefault(zone, Map.of()));
+    }
+
+    @Override
+    public void setArchiveUri(ZoneId zone, TenantName tenantName, URI archiveUri) {
+        archiveUris.computeIfAbsent(zone, z -> new HashMap<>()).put(tenantName, archiveUri);
+    }
+
+    @Override
+    public void removeArchiveUri(ZoneId zone, TenantName tenantName) {
+        Optional.ofNullable(archiveUris.get(zone)).ifPresent(map -> map.remove(tenantName));
+    }
+
+    @Override
     public void upgrade(ZoneId zone, NodeType type, Version version) {
         this.targetVersions.compute(zone, (ignored, targetVersions) -> {
             if (targetVersions == null) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
index 69960ed393b..136b37b02e1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
@@ -7,6 +7,8 @@ import com.yahoo.component.AbstractComponent;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry;
+import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService;
+import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService;
 import com.yahoo.vespa.hosted.controller.api.integration.aws.RoleService;
 import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher;
 import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger;
@@ -67,6 +69,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     private final BillingController billingController = new MockBillingController();
     private final ContainerRegistryMock containerRegistry = new ContainerRegistryMock();
     private final NoopTenantSecretService tenantSecretService = new NoopTenantSecretService();
+    private final ArchiveService archiveService = new MockArchiveService();
 
     public ServiceRegistryMock(SystemName system) {
         this.zoneRegistryMock = new ZoneRegistryMock(system);
@@ -213,6 +216,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
         return tenantSecretService;
     }
 
+    @Override
+    public ArchiveService archiveService() {
+        return archiveService;
+    }
+
     public ConfigServerMock configServerMock() {
         return configServerMock;
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java
new file mode 100644
index 00000000000..90a58c47677
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java
@@ -0,0 +1,78 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import org.junit.Test;
+
+import java.net.URI;
+import java.time.Duration;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author freva
+ */
+public class ArchiveUriUpdaterTest {
+
+    private final DeploymentTester tester = new DeploymentTester();
+
+    @Test
+    public void archive_uri_test() {
+        var updater = new ArchiveUriUpdater(tester.controller(), Duration.ofDays(1));
+
+        var tenant1 = TenantName.from("tenant1");
+        var tenant2 = TenantName.from("tenant2");
+        var application = tester.newDeploymentContext(tenant1.value(), "app1", "instance1");
+        ZoneId zone = ZoneId.from("prod", "ap-northeast-1");
+
+        // Initially we should not set any archive URIs as the archive service does not return any
+        updater.maintain();
+        assertArchiveUris(Map.of(), zone);
+
+        // Archive service now has URI for tenant1, but tenant1 is not deployed in zone
+        setArchiveUriInService(Map.of(tenant1, "uri-1"), zone);
+        updater.maintain();
+        assertArchiveUris(Map.of(), zone);
+
+        deploy(application, zone);
+        updater.maintain();
+        assertArchiveUris(Map.of(tenant1, "uri-1"), zone);
+
+        // URI for tenant1 should be updated and removed for tenant2
+        setArchiveUriInNodeRepo(Map.of(tenant1, "wrong-uri", tenant2, "uri-2"), zone);
+        updater.maintain();
+        assertArchiveUris(Map.of(tenant1, "uri-1"), zone);
+    }
+
+    private void assertArchiveUris(Map<TenantName, String> expectedUris, ZoneId zone) {
+        Map<TenantName, String> actualUris = tester.controller().serviceRegistry().configServer().nodeRepository()
+                .getArchiveUris(zone).entrySet().stream()
+                .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()));
+        assertEquals(expectedUris, actualUris);
+    }
+
+    private void setArchiveUriInService(Map<TenantName, String> archiveUris, ZoneId zone) {
+        MockArchiveService archiveService = (MockArchiveService) tester.controller().serviceRegistry().archiveService();
+        archiveUris.forEach((tenant, uri) -> archiveService.setArchiveUri(zone, tenant, URI.create(uri)));
+    }
+
+    private void setArchiveUriInNodeRepo(Map<TenantName, String> archiveUris, ZoneId zone) {
+        NodeRepository nodeRepository = tester.controller().serviceRegistry().configServer().nodeRepository();
+        archiveUris.forEach((tenant, uri) -> nodeRepository.setArchiveUri(zone, tenant, URI.create(uri)));
+    }
+
+    private void deploy(DeploymentContext application, ZoneId zone) {
+        application.runJob(JobType.from(SystemName.main, zone).orElseThrow(), new ApplicationPackage(new byte[0]), Version.fromString("7.1"));
+    }
+}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index 245e8cbebfe..935a90a1e43 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -7,6 +7,9 @@
       "name": "ApplicationOwnershipConfirmer"
     },
     {
+      "name": "ArchiveUriUpdater"
+    },
+    {
       "name": "CloudEventReporter"
     },
     {