Merge pull request #22478 from vespa-engine/freva/gcp-artifact-retention

Prepare for adding GCP ArtifactExpirer

8 files changed, 145 insertions, 129 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java
new file mode 100644
index 00000000000..81fbb85e3d7
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java
@@ -0,0 +1,87 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.artifact.Artifact;
+import com.yahoo.vespa.hosted.controller.api.integration.artifact.ArtifactRegistry;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+/**
+ * Periodically expire unused artifacts, e.g. container images and RPMs.
+ *
+ * @author mpolden
+ */
+public class ArtifactExpirer extends ControllerMaintainer {
+
+    private static final Logger log = Logger.getLogger(ArtifactExpirer.class.getName());
+
+    private static final Duration MIN_AGE = Duration.ofDays(14);
+
+    public ArtifactExpirer(Controller controller, Duration interval) {
+        super(controller, interval, null, expiringSystems());
+    }
+
+    @Override
+    protected double maintain() {
+        VersionStatus versionStatus = controller().readVersionStatus();
+        return controller().clouds().stream()
+                .flatMapToDouble(cloud ->
+                    controller().serviceRegistry().artifactRegistry(cloud).stream()
+                            .mapToDouble(artifactRegistry -> maintain(versionStatus, cloud, artifactRegistry)))
+                .average()
+                .orElse(1);
+    }
+
+    private double maintain(VersionStatus versionStatus, CloudName cloudName, ArtifactRegistry artifactRegistry) {
+        try {
+            Instant now = controller().clock().instant();
+            List<Artifact> artifactsToExpire = artifactRegistry.list().stream()
+                    .filter(artifact -> isExpired(artifact, now, versionStatus))
+                    .collect(Collectors.toList());
+            if (!artifactsToExpire.isEmpty()) {
+                log.log(Level.INFO, "Expiring " + artifactsToExpire.size() + " artifacts: " + artifactsToExpire);
+                artifactRegistry.deleteAll(artifactsToExpire);
+            }
+            return 1;
+        } catch (RuntimeException e) {
+            log.log(Level.WARNING, "Failed to expire artifacts in " + cloudName + ". Will retry in " + interval(), e);
+            return 0;
+        }
+    }
+
+    /** Returns whether given artifact is expired */
+    private boolean isExpired(Artifact artifact, Instant now, VersionStatus versionStatus) {
+        List<VespaVersion> versions = versionStatus.versions();
+        if (versions.isEmpty()) return false;
+
+        if (versionStatus.isActive(artifact.version())) return false;
+        if (artifact.createdAt().isAfter(now.minus(MIN_AGE))) return false;
+
+        Version maxVersion = versions.stream().map(VespaVersion::versionNumber).max(Comparator.naturalOrder()).get();
+        if (artifact.version().isAfter(maxVersion)) return false; // A future version
+
+        return true;
+    }
+
+    /** Returns systems where artifacts can be expired */
+    private static Set<SystemName> expiringSystems() {
+        // Run only in public and main. Public systems have distinct container registries, while main and CD have
+        // shared registries.
+        return EnumSet.of(SystemName.Public, SystemName.PublicCd, SystemName.main);
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java
deleted file mode 100644
index c87dd262fa3..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java
+++ /dev/null
@@ -1,71 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.maintenance;
-
-import com.yahoo.component.Version;
-import com.yahoo.config.provision.SystemName;
-import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.api.integration.container.ContainerImage;
-import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
-import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.util.Comparator;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-
-/**
- * Periodically expire unused container images.
- *
- * @author mpolden
- */
-public class ContainerImageExpirer extends ControllerMaintainer {
-
-    private static final Logger log = Logger.getLogger(ContainerImageExpirer.class.getName());
-
-    private static final Duration MIN_AGE = Duration.ofDays(14);
-
-    public ContainerImageExpirer(Controller controller, Duration interval) {
-        super(controller, interval, null, expiringSystems());
-    }
-
-    @Override
-    protected double maintain() {
-        Instant now = controller().clock().instant();
-        VersionStatus versionStatus = controller().readVersionStatus();
-        List<ContainerImage> imagesToExpire = controller().serviceRegistry().containerRegistry().list().stream()
-                                                          .filter(image -> isExpired(image, now, versionStatus))
-                                                          .collect(Collectors.toList());
-        if (!imagesToExpire.isEmpty()) {
-            log.log(Level.INFO, "Expiring " + imagesToExpire.size() + " container images: " + imagesToExpire);
-            controller().serviceRegistry().containerRegistry().deleteAll(imagesToExpire);
-        }
-        return 1.0;
-    }
-
-    /** Returns whether given image is expired */
-    private boolean isExpired(ContainerImage image, Instant now, VersionStatus versionStatus) {
-        List<VespaVersion> versions = versionStatus.versions();
-        if (versions.isEmpty()) return false;
-
-        if (versionStatus.isActive(image.version())) return false;
-        if (image.createdAt().isAfter(now.minus(MIN_AGE))) return false;
-
-        Version maxVersion = versions.stream().map(VespaVersion::versionNumber).max(Comparator.naturalOrder()).get();
-        if (image.version().isAfter(maxVersion)) return false; // A future version
-
-        return true;
-    }
-
-    /** Returns systems where images can be expired */
-    private static Set<SystemName> expiringSystems() {
-        // Run only in public and main. Public systems have distinct container registries, while main and CD have
-        // shared registries.
-        return EnumSet.of(SystemName.Public, SystemName.PublicCd, SystemName.main);
-    }
-
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index e4aa80b7b93..193d171e334 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -63,7 +63,7 @@ public class ControllerMaintenance extends AbstractComponent {
         maintainers.add(new ResourceTagMaintainer(controller, intervals.resourceTagMaintainer, controller.serviceRegistry().resourceTagger()));
         maintainers.add(new SystemRoutingPolicyMaintainer(controller, intervals.systemRoutingPolicyMaintainer));
         maintainers.add(new ApplicationMetaDataGarbageCollector(controller, intervals.applicationMetaDataGarbageCollector));
-        maintainers.add(new ContainerImageExpirer(controller, intervals.containerImageExpirer));
+        maintainers.add(new ArtifactExpirer(controller, intervals.containerImageExpirer));
         maintainers.add(new HostInfoUpdater(controller, intervals.hostInfoUpdater));
         maintainers.add(new ReindexingTriggerer(controller, intervals.reindexingTriggerer));
         maintainers.add(new EndpointCertificateMaintainer(controller, intervals.endpointCertificateMaintainer));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRegistryMock.java
new file mode 100644
index 00000000000..ef0445e06c6
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRegistryMock.java
@@ -0,0 +1,39 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.integration;
+
+import com.yahoo.vespa.hosted.controller.api.integration.artifact.Artifact;
+import com.yahoo.vespa.hosted.controller.api.integration.artifact.ArtifactRegistry;
+
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @author mpolden
+ */
+public class ArtifactRegistryMock implements ArtifactRegistry {
+
+    private static final Comparator<Artifact> comparator = Comparator.comparing(Artifact::registry)
+                                                                     .thenComparing(Artifact::repository)
+                                                                     .thenComparing(Artifact::version);
+
+    private final Map<String, Artifact> images = new HashMap<>();
+
+    public ArtifactRegistryMock add(Artifact image) {
+        images.put(image.id(), image);
+        return this;
+    }
+
+    @Override
+    public void deleteAll(List<Artifact> artifacts) {
+        artifacts.forEach(image -> this.images.remove(image.id()));
+    }
+
+    @Override
+    public List<Artifact> list() {
+        return images.values().stream().sorted(comparator).collect(Collectors.toUnmodifiableList());
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ContainerRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ContainerRegistryMock.java
deleted file mode 100644
index 4791017548d..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ContainerRegistryMock.java
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.integration;
-
-import com.yahoo.vespa.hosted.controller.api.integration.container.ContainerImage;
-import com.yahoo.vespa.hosted.controller.api.integration.container.ContainerRegistry;
-
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/**
- * @author mpolden
- */
-public class ContainerRegistryMock implements ContainerRegistry {
-
-    private static final Comparator<ContainerImage> comparator = Comparator.comparing(ContainerImage::registry)
-                                                                           .thenComparing(ContainerImage::repository)
-                                                                           .thenComparing(ContainerImage::version);
-
-    private final Map<String, ContainerImage> images = new HashMap<>();
-
-    public ContainerRegistryMock add(ContainerImage image) {
-        images.put(image.id(), image);
-        return this;
-    }
-
-    @Override
-    public void deleteAll(List<ContainerImage> images) {
-        images.forEach(image -> this.images.remove(image.id()));
-    }
-
-    @Override
-    public List<ContainerImage> list() {
-        return images.values().stream().sorted(comparator).collect(Collectors.toUnmodifiableList());
-    }
-
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
index 7b42dbff488..a91d81bb0c5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
@@ -5,6 +5,7 @@ import com.yahoo.component.annotation.Inject;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.component.Version;
+import com.yahoo.config.provision.CloudName;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.test.ManualClock;
@@ -50,6 +51,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainerMock
 import com.yahoo.vespa.hosted.controller.api.integration.vcmr.MockChangeRequestClient;
 
 import java.time.Instant;
+import java.util.Optional;
 
 /**
  * A mock implementation of a {@link ServiceRegistry} for testing purposes.
@@ -82,7 +84,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     private final MockResourceTagger mockResourceTagger = new MockResourceTagger();
     private final RoleService roleService = new MockRoleService();
     private final BillingController billingController = new MockBillingController(clock);
-    private final ContainerRegistryMock containerRegistry = new ContainerRegistryMock();
+    private final ArtifactRegistryMock containerRegistry = new ArtifactRegistryMock();
     private final NoopTenantSecretService tenantSecretService = new NoopTenantSecretService();
     private final ArchiveService archiveService = new MockArchiveService();
     private final MockChangeRequestClient changeRequestClient = new MockChangeRequestClient();
@@ -236,8 +238,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     }
 
     @Override
-    public ContainerRegistryMock containerRegistry() {
-        return containerRegistry;
+    public Optional<ArtifactRegistryMock> artifactRegistry(CloudName cloudName) {
+        return Optional.of(containerRegistry);
     }
 
     @Override
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirerTest.java
index baebf7239ee..7703266c9ba 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirerTest.java
@@ -2,35 +2,33 @@
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
-import com.yahoo.vespa.hosted.controller.api.integration.container.ContainerImage;
-import com.yahoo.vespa.hosted.controller.api.integration.container.ContainerImage.Architecture;
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.vespa.hosted.controller.api.integration.artifact.Artifact;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
-import com.yahoo.vespa.hosted.controller.integration.ContainerRegistryMock;
+import com.yahoo.vespa.hosted.controller.integration.ArtifactRegistryMock;
 import org.junit.Test;
 
 import java.time.Duration;
 import java.time.Instant;
 import java.util.List;
-import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
 
 /**
  * @author mpolden
  */
-public class ContainerImageExpirerTest {
+public class ArtifactExpirerTest {
 
     @Test
     public void maintain() {
         DeploymentTester tester = new DeploymentTester();
-        ContainerImageExpirer expirer = new ContainerImageExpirer(tester.controller(), Duration.ofDays(1));
-        ContainerRegistryMock registry = tester.controllerTester().serviceRegistry().containerRegistry();
+        ArtifactExpirer expirer = new ArtifactExpirer(tester.controller(), Duration.ofDays(1));
+        ArtifactRegistryMock registry = tester.controllerTester().serviceRegistry().artifactRegistry(CloudName.defaultName()).orElseThrow();
 
         Instant instant = tester.clock().instant();
-        ContainerImage image0 = new ContainerImage("image0", "registry.example.com", "vespa/vespa", instant, Version.fromString("7.1"), Optional.empty());
-        ContainerImage image1 = new ContainerImage("image1", "registry.example.com", "vespa/vespa", instant, Version.fromString("7.2"), Optional.of(Architecture.amd64));
-        ContainerImage image2 = new ContainerImage("image2", "registry.example.com", "vespa/vespa", instant, Version.fromString("7.4"), Optional.of(Architecture.amd64));
-
+        Artifact image0 = new Artifact("image0", "registry.example.com", "vespa/vespa", "7.1", instant, Version.fromString("7.1"));
+        Artifact image1 = new Artifact("image1", "registry.example.com", "vespa/vespa", "7.2-amd64", instant, Version.fromString("7.2"));
+        Artifact image2 = new Artifact("image2", "registry.example.com", "vespa/vespa", "7.4-amd64", instant, Version.fromString("7.4"));
         registry.add(image0)
                 .add(image1)
                 .add(image2);
@@ -53,7 +51,7 @@ public class ContainerImageExpirerTest {
         assertEquals(List.of(image1, image2), registry.list());
 
         // A new version becomes active. The active and future version are kept
-        ContainerImage image3 = new ContainerImage("image3", "registry.example.com", "vespa/vespa", tester.clock().instant(), Version.fromString("7.3"), Optional.of(Architecture.amd64));
+        Artifact image3 = new Artifact("image3", "registry.example.com", "vespa/vespa", "7.3-arm64", tester.clock().instant(), Version.fromString("7.3"));
         registry.add(image3);
         tester.controllerTester().upgradeSystem(image3.version());
         expirer.maintain();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index 79e11fa1140..8b2e5578ae0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -13,6 +13,9 @@
       "name": "ArchiveUriUpdater"
     },
     {
+      "name": "ArtifactExpirer"
+    },
+    {
       "name": "BillingDatabaseMaintainer"
     },
     {
@@ -28,9 +31,6 @@
       "name": "ContactInformationMaintainer"
     },
     {
-      "name": "ContainerImageExpirer"
-    },
-    {
       "name": "CostReportMaintainer"
     },
     {