Replace image atomically

4 files changed, 57 insertions, 24 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index d8ea19b7677..845d2763501 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -127,7 +127,8 @@ public class NodeRepository extends AbstractComponent {
              Clock.systemUTC(),
              zone,
              new DnsNameResolver(),
-             DockerImage.fromString(config.containerImage()),
+             DockerImage.fromString(config.containerImage())
+                        .withReplacedBy(DockerImage.fromString(config.containerImageReplacement())),
              flagSource,
              config.useCuratorClientCache(),
              zone.environment().isProduction() && !zone.getCloud().dynamicProvisioning() ? 1 : 0,
@@ -144,7 +145,7 @@ public class NodeRepository extends AbstractComponent {
                           Clock clock,
                           Zone zone,
                           NameResolver nameResolver,
-                          DockerImage dockerImage,
+                          DockerImage containerImage,
                           FlagSource flagSource,
                           boolean useCuratorClientCache,
                           int spareCount,
@@ -164,7 +165,7 @@ public class NodeRepository extends AbstractComponent {
         this.osVersions = new OsVersions(this);
         this.infrastructureVersions = new InfrastructureVersions(db);
         this.firmwareChecks = new FirmwareChecks(db, clock);
-        this.containerImages = new ContainerImages(db, dockerImage);
+        this.containerImages = new ContainerImages(db, containerImage, flagSource);
         this.jobControl = new JobControl(new JobControlFlags(db, flagSource));
         this.applications = new Applications(db);
         this.spareCount = spareCount;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java
index 92518239258..45156c57481 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java
@@ -6,6 +6,9 @@ import com.google.common.base.Suppliers;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
 
 import java.time.Duration;
@@ -23,12 +26,12 @@ import java.util.logging.Logger;
  */
 public class ContainerImages {
 
-    private static final Duration defaultCacheTtl = Duration.ofMinutes(1);
+    private static final Duration cacheTtl = Duration.ofMinutes(1);
     private static final Logger log = Logger.getLogger(ContainerImages.class.getName());
 
     private final CuratorDatabaseClient db;
     private final DockerImage defaultImage;
-    private final Duration cacheTtl;
+    private final BooleanFlag replaceImage;
 
     /**
      * The container image is read on every request to /nodes/v2/node/[fqdn]. Cache current images to avoid
@@ -37,14 +40,10 @@ public class ContainerImages {
      */
     private volatile Supplier<Map<NodeType, DockerImage>> images;
 
-    public ContainerImages(CuratorDatabaseClient db, DockerImage defaultImage) {
-        this(db, defaultImage, defaultCacheTtl);
-    }
-
-    ContainerImages(CuratorDatabaseClient db, DockerImage defaultImage, Duration cacheTtl) {
+    public ContainerImages(CuratorDatabaseClient db, DockerImage defaultImage, FlagSource flagSource) {
         this.db = db;
         this.defaultImage = defaultImage;
-        this.cacheTtl = cacheTtl;
+        this.replaceImage = Flags.REGIONAL_CONTAINER_REGISTRY.bindTo(flagSource);
         createCache();
     }
 
@@ -58,15 +57,14 @@ public class ContainerImages {
         return images.get();
     }
 
-    /** Returns the current docker image for given node type, or the type for corresponding child nodes
-     * if it is a Docker host, or default */
+    /** Returns the container image to use for given node type */
     public DockerImage imageFor(NodeType type) {
         NodeType typeToUseForLookup = type.isHost() ? type.childNodeType() : type;
         DockerImage image = getImages().get(typeToUseForLookup);
         if (image == null) {
-            return defaultImage;
+            image = defaultImage;
         }
-        return image.withRegistry(defaultImage.registry()); // Always use the registry configured for this zone.
+        return rewriteRegistry(image);
     }
 
     /** Set the docker image for nodes of given type */
@@ -84,4 +82,13 @@ public class ContainerImages {
         }
     }
 
+    /** Rewrite the registry part of given image, using this zone's default image */
+    private DockerImage rewriteRegistry(DockerImage image) {
+        DockerImage zoneImage = defaultImage;
+        if (zoneImage.replacedBy().isPresent() && replaceImage.value()) {
+            zoneImage = zoneImage.replacedBy().get();
+        }
+        return image.withRegistry(zoneImage.registry());
+    }
+
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImagesTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImagesTest.java
index 320feed5d66..94b670f8e5e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImagesTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImagesTest.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import org.junit.Test;
 
@@ -48,4 +49,26 @@ public class ContainerImagesTest {
         }
     }
 
+    @Test
+    public void image_replacement() {
+        var flagSource = new InMemoryFlagSource();
+        var defaultImage = DockerImage.fromString("foo.example.com/vespa/vespa")
+                                      .withReplacedBy(DockerImage.fromString("bar.example.com/vespa/vespa"));
+        var tester = new ProvisioningTester.Builder().defaultImage(defaultImage).flagSource(flagSource).build();
+        var hosts = tester.makeReadyNodes(2, "default", NodeType.host);
+        tester.activateTenantHosts();
+
+        // Default image is used initially
+        for (var host : hosts) {
+            assertEquals(defaultImage, tester.nodeRepository().containerImages().imageFor(host.type()));
+        }
+
+        // Enabling flag switches to replacement
+        flagSource.withBooleanFlag(Flags.REGIONAL_CONTAINER_REGISTRY.id(), true);
+        for (var host : hosts) {
+            assertEquals(defaultImage.replacedBy().get().asString(),
+                         tester.nodeRepository().containerImages().imageFor(host.type()).asString());
+        }
+    }
+
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index dd8b04ebf69..994f35afe60 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -89,11 +89,12 @@ public class ProvisioningTester {
     private int nextHost = 0;
     private int nextIP = 0;
 
-    public ProvisioningTester(Curator curator,
+    private ProvisioningTester(Curator curator,
                               NodeFlavors nodeFlavors,
                               HostResourcesCalculator resourcesCalculator,
                               Zone zone,
                               NameResolver nameResolver,
+                              DockerImage containerImage,
                               Orchestrator orchestrator,
                               HostProvisioner hostProvisioner,
                               LoadBalancerServiceMock loadBalancerService,
@@ -109,7 +110,7 @@ public class ProvisioningTester {
                                                  clock,
                                                  zone,
                                                  nameResolver,
-                                                 DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
+                                                 containerImage,
                                                  flagSource,
                                                  true,
                                                  spareCount,
@@ -596,9 +597,9 @@ public class ProvisioningTester {
         private NameResolver nameResolver;
         private Orchestrator orchestrator;
         private HostProvisioner hostProvisioner;
-        private LoadBalancerServiceMock loadBalancerService;
         private FlagSource flagSource;
         private int spareCount = 0;
+        private DockerImage defaultImage = DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa");
 
         public Builder curator(Curator curator) {
             this.curator = curator;
@@ -636,6 +637,11 @@ public class ProvisioningTester {
             return this;
         }
 
+        public Builder defaultImage(DockerImage defaultImage) {
+            this.defaultImage = defaultImage;
+            return this;
+        }
+
         public Builder orchestrator(Orchestrator orchestrator) {
             this.orchestrator = orchestrator;
             return this;
@@ -646,11 +652,6 @@ public class ProvisioningTester {
             return this;
         }
 
-        public Builder loadBalancerService(LoadBalancerServiceMock loadBalancerService) {
-            this.loadBalancerService = loadBalancerService;
-            return this;
-        }
-
         public Builder flagSource(FlagSource flagSource) {
             this.flagSource = flagSource;
             return this;
@@ -678,9 +679,10 @@ public class ProvisioningTester {
                                           resourcesCalculator,
                                           Optional.ofNullable(zone).orElseGet(Zone::defaultZone),
                                           Optional.ofNullable(nameResolver).orElseGet(() -> new MockNameResolver().mockAnyLookup()),
+                                          defaultImage,
                                           orchestrator,
                                           hostProvisioner,
-                                          Optional.ofNullable(loadBalancerService).orElseGet(LoadBalancerServiceMock::new),
+                                          new LoadBalancerServiceMock(),
                                           Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new),
                                           spareCount);
         }