Stop reading container images from ZK

7 files changed, 35 insertions, 91 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 67de210127d..7f0c52a5aca 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -80,6 +80,7 @@ public class NodeRepository extends AbstractComponent {
              zone,
              new DnsNameResolver(),
              DockerImage.fromString(config.containerImage()),
+             Optional.of(config.tenantContainerImage()).filter(s -> !s.isEmpty()).map(DockerImage::fromString),
              flagSource,
              metricsDb,
              config.useCuratorClientCache(),
@@ -98,6 +99,7 @@ public class NodeRepository extends AbstractComponent {
                           Zone zone,
                           NameResolver nameResolver,
                           DockerImage containerImage,
+                          Optional<DockerImage> tenantContainerImage,
                           FlagSource flagSource,
                           MetricsDb metricsDb,
                           boolean useCuratorClientCache,
@@ -118,7 +120,7 @@ public class NodeRepository extends AbstractComponent {
         this.osVersions = new OsVersions(this);
         this.infrastructureVersions = new InfrastructureVersions(db);
         this.firmwareChecks = new FirmwareChecks(db, clock);
-        this.containerImages = new ContainerImages(db, containerImage);
+        this.containerImages = new ContainerImages(containerImage, tenantContainerImage);
         this.archiveUris = new ArchiveUris(db);
         this.jobControl = new JobControl(new JobControlFlags(db, flagSource));
         this.applications = new Applications(db);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index 45aaedd9550..0a05d9d5bf4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -6,7 +6,6 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationLockException;
 import com.yahoo.config.provision.ApplicationTransaction;
-import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeType;
@@ -67,7 +66,7 @@ public class CuratorDatabaseClient {
     private static final Path inactiveJobsPath = root.append("inactiveJobs");
     private static final Path infrastructureVersionsPath = root.append("infrastructureVersions");
     private static final Path osVersionsPath = root.append("osVersions");
-    private static final Path containerImagesPath = root.append("dockerImages");
+    private static final Path containerImagesPath = root.append("dockerImages"); // TODO(mpolden): Delete this path from ZK after 2021-11-01
     private static final Path firmwareCheckPath = root.append("firmwareCheck");
     private static final Path archiveUrisPath = root.append("archiveUris");
 
@@ -98,7 +97,6 @@ public class CuratorDatabaseClient {
         db.create(inactiveJobsPath);
         db.create(infrastructureVersionsPath);
         db.create(osVersionsPath);
-        db.create(containerImagesPath);
         db.create(firmwareCheckPath);
         db.create(archiveUrisPath);
         db.create(loadBalancersPath);
@@ -410,24 +408,6 @@ public class CuratorDatabaseClient {
         return db.lock(lockPath.append("osVersionsLock"), defaultLockTimeout);
     }
 
-    // Container images -----------------------------------------------------------
-
-    public Map<NodeType, DockerImage> readContainerImages() {
-        return read(containerImagesPath, NodeTypeContainerImagesSerializer::fromJson).orElseGet(TreeMap::new);
-    }
-
-    public void writeContainerImages(Map<NodeType, DockerImage> images) {
-        NestedTransaction transaction = new NestedTransaction();
-        CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction);
-        curatorTransaction.add(CuratorOperations.setData(containerImagesPath.getAbsolute(),
-                                                         NodeTypeContainerImagesSerializer.toJson(images)));
-        transaction.commit();
-    }
-
-    public Lock lockContainerImages() {
-        return db.lock(lockPath.append("dockerImagesLock"), defaultLockTimeout);
-    }
-
     // Firmware checks -----------------------------------------------------------
 
     /** Stores the instant after which a firmware check is required, or clears any outstanding ones if empty is given. */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java
index cd9303d7f6e..a3227061e2c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ContainerImages.java
@@ -3,73 +3,49 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.NodeType;
-import com.yahoo.lang.CachedSupplier;
-import com.yahoo.vespa.curator.Lock;
-import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
+import com.yahoo.vespa.hosted.provision.Node;
 
-import java.time.Duration;
-import java.util.Collections;
-import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
-import java.util.logging.Logger;
 
 /**
- * Multi-thread safe class to get and set container images for given node types. Images are stored in ZooKeeper so that
- * nodes receive the same image from all config servers.
+ * This class decides the container image to use for a given node. Two sources are considered, in the following order:
+ *
+ * 1. Requested image (from node allocation, this is set by either a feature flag or through services.xml)
+ * 2. Default image, specified in the node repository config file
+ *
+ * Independent of source, the registry part of the image is rewritten to match the one set in the node repository config
+ * file.
  *
  * @author freva
+ * @author mpolden
  */
 public class ContainerImages {
 
-    private static final Duration cacheTtl = Duration.ofMinutes(1);
-    private static final Logger log = Logger.getLogger(ContainerImages.class.getName());
-
-    private final CuratorDatabaseClient db;
     private final DockerImage defaultImage;
+    private final Optional<DockerImage> tenantImage;
 
-    /**
-     * The container image is read on every request to /nodes/v2/node/[fqdn]. Cache current images to avoid
-     * unnecessary ZK reads. When images change, some nodes may need to wait for TTL until they see the new image,
-     * this is fine.
-     */
-    private final CachedSupplier<Map<NodeType, DockerImage>> images;
-
-    public ContainerImages(CuratorDatabaseClient db, DockerImage defaultImage) {
-        this.db = db;
-        this.defaultImage = defaultImage;
-        this.images = new CachedSupplier<>(() -> Collections.unmodifiableMap(db.readContainerImages()), cacheTtl);
-    }
-
-    /** Returns the current images for each node type */
-    public Map<NodeType, DockerImage> getImages() {
-        return images.get();
+    public ContainerImages(DockerImage defaultImage, Optional<DockerImage> tenantContainerImage) {
+        this.defaultImage = Objects.requireNonNull(defaultImage);
+        this.tenantImage = Objects.requireNonNull(tenantContainerImage);
     }
 
-    /** Returns the container image to use for given node type */
-    public DockerImage imageFor(NodeType type) {
-        NodeType typeToUseForLookup = type.isHost() ? type.childNodeType() : type;
-        DockerImage image = getImages().get(typeToUseForLookup);
-        if (image == null) {
+    /** Returns the container image to use for given node */
+    public DockerImage get(Node node) {
+        Optional<DockerImage> requestedImage = node.allocation()
+                                                   .flatMap(allocation -> allocation.membership().cluster().dockerImageRepo());
+        NodeType nodeType = node.type().isHost() ? node.type().childNodeType() : node.type();
+        final DockerImage image;
+        if (requestedImage.isPresent()) {
+            image = requestedImage.get();
+        } else if (nodeType == NodeType.tenant) {
+            image = tenantImage.orElse(defaultImage);
+        } else {
             image = defaultImage;
         }
         return rewriteRegistry(image);
     }
 
-    /** Set the docker image for nodes of given type */
-    public void setImage(NodeType nodeType, Optional<DockerImage> image) {
-        if (nodeType.isHost()) {
-            throw new IllegalArgumentException("Setting container image for " + nodeType + " nodes is unsupported");
-        }
-        try (Lock lock = db.lockContainerImages()) {
-            Map<NodeType, DockerImage> images = db.readContainerImages();
-            image.ifPresentOrElse(img -> images.put(nodeType, img),
-                                  () -> images.remove(nodeType));
-            db.writeContainerImages(images);
-            this.images.invalidate(); // Throw away current cache
-            log.info("Set container image for " + nodeType + " nodes to " + image.map(DockerImage::asString).orElse(null));
-        }
-    }
-
     /** Rewrite the registry part of given image, using this zone's default image */
     private DockerImage rewriteRegistry(DockerImage image) {
         return image.withRegistry(defaultImage.registry());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java
index df6e24aa8a3..02b426ed6fc 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java
@@ -146,8 +146,7 @@ class NodesResponse extends SlimeJsonResponse {
             toSlime(allocation.membership(), object.setObject("membership"));
             object.setLong("restartGeneration", allocation.restartGeneration().wanted());
             object.setLong("currentRestartGeneration", allocation.restartGeneration().current());
-            object.setString("wantedDockerImage", allocation.membership().cluster().dockerImage()
-                    .orElseGet(() -> nodeRepository.containerImages().imageFor(node.type()).withTag(allocation.membership().cluster().vespaVersion()).asString()));
+            object.setString("wantedDockerImage", nodeRepository.containerImages().get(node).withTag(allocation.membership().cluster().vespaVersion()).asString());
             object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString());
             NodeResourcesSerializer.toSlime(allocation.requestedResources(), object.setObject("requestedResources"));
             allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts")));
@@ -214,7 +213,7 @@ class NodesResponse extends SlimeJsonResponse {
                    .or(() -> Optional.of(node)
                                      .filter(n -> n.flavor().getType() != Flavor.Type.DOCKER_CONTAINER)
                                      .flatMap(n -> n.status().vespaVersion()
-                                                    .map(version -> nodeRepository.containerImages().imageFor(n.type()).withTag(version))));
+                                                    .map(version -> nodeRepository.containerImages().get(n).withTag(version))));
     }
 
     private void ipAddressesToSlime(Set<String> ipAddresses, Cursor array) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java
index fa89e2e58f0..1163890addf 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.restapi;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.NodeFlavors;
@@ -358,12 +357,11 @@ public class NodesV2ApiHandler extends LoggingRequestHandler {
 
     private MessageResponse setTargetVersions(String nodeTypeS, Inspector inspector) {
         NodeType nodeType = NodeType.valueOf(nodeTypeS.toLowerCase());
-        List<String> messageParts = new ArrayList<>(4);
+        List<String> messageParts = new ArrayList<>();
 
         boolean force = inspector.field("force").asBool();
         Inspector versionField = inspector.field("version");
         Inspector osVersionField = inspector.field("osVersion");
-        Inspector containerImageField = inspector.field("dockerImage");
         Inspector upgradeBudgetField = inspector.field("upgradeBudget");
 
         if (versionField.valid()) {
@@ -396,16 +394,8 @@ public class NodesV2ApiHandler extends LoggingRequestHandler {
             }
         }
 
-        if (containerImageField.valid()) {
-            Optional<DockerImage> dockerImage = Optional.of(containerImageField.asString())
-                    .filter(s -> !s.isEmpty())
-                    .map(DockerImage::fromString);
-            nodeRepository.containerImages().setImage(nodeType, dockerImage);
-            messageParts.add("container image to " + dockerImage.map(DockerImage::asString).orElse(null));
-        }
-
         if (messageParts.isEmpty()) {
-            throw new IllegalArgumentException("At least one of 'version', 'osVersion' or 'dockerImage' must be set");
+            throw new IllegalArgumentException("At least one of 'version' or 'osVersion' must be set");
         }
 
         return new MessageResponse("Set " + String.join(", ", messageParts) +
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java
index cc5d7745d2f..e03edecff5e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java
@@ -21,13 +21,11 @@ public class UpgradeResponse extends HttpResponse {
 
     private final InfrastructureVersions infrastructureVersions;
     private final OsVersions osVersions;
-    private final ContainerImages containerImages;
 
     public UpgradeResponse(InfrastructureVersions infrastructureVersions, OsVersions osVersions, ContainerImages containerImages) {
         super(200);
         this.infrastructureVersions = infrastructureVersions;
         this.osVersions = osVersions;
-        this.containerImages = containerImages;
     }
 
     @Override
@@ -41,9 +39,7 @@ public class UpgradeResponse extends HttpResponse {
         Cursor osVersionsObject = root.setObject("osVersions");
         osVersions.readChange().targets().forEach((nodeType, target) -> osVersionsObject.setString(nodeType.name(), target.version().toFullString()));
 
-
-        Cursor dockerImagesObject = root.setObject("dockerImages");
-        containerImages.getImages().forEach((nodeType, image) -> dockerImagesObject.setString(nodeType.name(), image.asString()));
+        root.setObject("dockerImages"); // Unused, but present to avoid breaking API
 
         new JsonFormat(true).encode(stream, slime);
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index 00937fe22ce..1a2d5294aa5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -67,6 +67,7 @@ public class MockNodeRepository extends NodeRepository {
               Zone.defaultZone(),
               new MockNameResolver().mockAnyLookup(),
               DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
+              Optional.empty(),
               new InMemoryFlagSource(),
               new MemoryMetricsDb(Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z"))),
               true,