Ask other config servers to start downloading files

To have redundancy, ask the other config servers to start downloading
a file, so we have redundancy in case the config server running the code
goes down after deployment has finished

11 files changed, 57 insertions, 19 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java
index 3f1c0046a85..77d7d557520 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java
@@ -24,8 +24,19 @@ public interface FileDistribution {
      * @param hostName       host which should be notified about file references to download
      * @param fileReferences set of file references to start downloading
      */
+    // TODO: Remove when 6.170 is the last version in use
     void startDownload(String hostName, Set<FileReference> fileReferences);
 
+    /**
+     * Notifies client which file references to download. Used to start downloading early (while
+     * preparing application package).
+     *
+     * @param hostName       host which should be notified about file references to download
+     * @param port           port which should be used when notifying
+     * @param fileReferences set of file references to start downloading
+     */
+    void startDownload(String hostName, int port, Set<FileReference> fileReferences);
+
     void reloadDeployFileDistributor();
 
     void removeDeploymentsThatHaveDifferentApplicationId(Collection<String> targetHostnames);
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
index 2a3ff1ca905..da034619d94 100644
--- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
+++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
@@ -64,7 +64,7 @@ public class MockRoot extends AbstractConfigProducerRoot {
         super(rootConfigId);
         hostSystem = new HostSystem(this, "hostsystem", deployState.getProvisioner());
         this.deployState = deployState;
-        fileDistributor = new FileDistributor(deployState.getFileRegistry());
+        fileDistributor = new FileDistributor(deployState.getFileRegistry(), null);
     }
 
     public FileDistributionConfigProducer getFileDistributionConfigProducer() {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
index eda8b564ffd..47a644a39d4 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
@@ -164,7 +164,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
     
     /** Creates a mutable model with no services instantiated */
     public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException {
-        return new VespaModel(new NullConfigModelRegistry(), deployState, false, new FileDistributor(deployState.getFileRegistry()));
+        return new VespaModel(new NullConfigModelRegistry(), deployState, false, new FileDistributor(deployState.getFileRegistry(), null));
     }
 
     private void validateWrapExceptions() {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
index 24e56472688..b9903ccd163 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
@@ -84,7 +84,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu
         FileDistributionOptions fileDistributionOptions = FileDistributionOptions.defaultOptions();
         fileDistributionOptions.disableFiledistributor(disableFiledistributor);
         fileDistributionOptions = new DomFileDistributionOptionsBuilder(fileDistributionOptions).build(XML.getChild(adminElement, "filedistribution"));
-        return new FileDistributionConfigProducer.Builder(fileDistributionOptions).build(parent, fileRegistry);
+        return new FileDistributionConfigProducer.Builder(fileDistributionOptions).build(parent, fileRegistry, configServerSpecs);
     }
 
     private Element getChildWithFallback(Element parent, String childName, String alternativeChildName) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java
index c863b180b21..85b937fee19 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java
@@ -2,11 +2,13 @@
 package com.yahoo.vespa.model.filedistribution;
 
 import com.yahoo.config.application.api.FileRegistry;
+import com.yahoo.config.model.api.ConfigServerSpec;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.vespa.model.Host;
 import com.yahoo.vespa.model.admin.FileDistributionOptions;
 
 import java.util.IdentityHashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -46,8 +48,8 @@ public class FileDistributionConfigProducer extends AbstractConfigProducer {
             this.options = fileDistributionOptions;
         }
 
-        public FileDistributionConfigProducer build(AbstractConfigProducer ancestor, FileRegistry fileRegistry) {
-            FileDistributor fileDistributor = new FileDistributor(fileRegistry);
+        public FileDistributionConfigProducer build(AbstractConfigProducer ancestor, FileRegistry fileRegistry, List<ConfigServerSpec> configServerSpec) {
+            FileDistributor fileDistributor = new FileDistributor(fileRegistry, configServerSpec);
             return new FileDistributionConfigProducer(ancestor, fileDistributor, options);
         }
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
index f3aad88a446..7bd0b9c2831 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
@@ -2,8 +2,10 @@
 package com.yahoo.vespa.model.filedistribution;
 
 import com.yahoo.config.FileReference;
+import com.yahoo.config.model.api.ConfigServerSpec;
 import com.yahoo.config.model.api.FileDistribution;
 import com.yahoo.config.application.api.FileRegistry;
+import com.yahoo.vespa.model.ConfigProxy;
 import com.yahoo.vespa.model.Host;
 
 import java.util.*;
@@ -17,6 +19,7 @@ import java.util.stream.Collectors;
 public class FileDistributor {
 
     private final FileRegistry fileRegistry;
+    private final List<ConfigServerSpec> configServerSpecs;
 
     /** A map from files to the hosts to which that file should be distributed */
     private final Map<FileReference, Set<Host>> filesToHosts = new LinkedHashMap<>();
@@ -54,8 +57,9 @@ public class FileDistributor {
         return hosts;
     }
 
-    public FileDistributor(FileRegistry fileRegistry) {
+    public FileDistributor(FileRegistry fileRegistry, List<ConfigServerSpec> configServerSpecs) {
         this.fileRegistry = fileRegistry;
+        this.configServerSpecs = configServerSpecs;
     }
 
     /** Returns the files which has been marked for distribution to the given host */
@@ -96,9 +100,15 @@ public class FileDistributor {
         for (Host host : getTargetHosts()) {
             if ( ! host.getHostname().equals(fileSourceHost)) {
                 dbHandler.sendDeployedFiles(host.getHostname(), filesToSendToHost(host));
-                dbHandler.startDownload(host.getHostname(), filesToSendToHost(host));
+                dbHandler.startDownload(host.getHostname(), ConfigProxy.BASEPORT, filesToSendToHost(host));
             }
         }
+        // Ask other config server to download, for redundancy
+        if (configServerSpecs != null)
+            configServerSpecs.stream()
+                    .filter(configServerSpec -> !configServerSpec.getHostName().equals(fileSourceHost))
+                    .forEach(spec -> dbHandler.startDownload(spec.getHostName(), spec.getConfigServerPort(), allFilesToSend()));
+
         dbHandler.sendDeployedFiles(fileSourceHost, allFilesToSend());
         dbHandler.removeDeploymentsThatHaveDifferentApplicationId(getTargetHostnames());
     }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
index e557e3674b5..27301ab60b2 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
@@ -30,7 +30,7 @@ import java.util.Optional;
  */
 public class ContentClusterUtils {
 
-    public static MockRoot createMockRoot(String[] hosts) throws Exception {
+    public static MockRoot createMockRoot(String[] hosts) {
         return createMockRoot(hosts, SearchDefinitionBuilder.createSearchDefinitions("test"));
     }
 
@@ -46,7 +46,7 @@ public class ContentClusterUtils {
         return new MockRoot("", deployStateBuilder.build());
     }
 
-    public static MockRoot createMockRoot(String[] hosts, List<String> searchDefinitions) throws Exception {
+    public static MockRoot createMockRoot(String[] hosts, List<String> searchDefinitions) {
         return createMockRoot(new InMemoryProvisioner(true, hosts), searchDefinitions);
     }
 
@@ -58,10 +58,11 @@ public class ContentClusterUtils {
         return createMockRoot(new SingleNodeProvisioner(), searchDefinitions, deployStateBuilder);
     }
 
-    public static ContentCluster createCluster(String clusterXml, MockRoot root) throws Exception {
+    public static ContentCluster createCluster(String clusterXml, MockRoot root) {
         Document doc = XML.getDocument(clusterXml);
         Admin admin = new Admin(root, new DefaultMonitoring("vespa", 60), new Metrics(), Collections.emptyMap(), false,
-                                new FileDistributionConfigProducer.Builder(FileDistributionOptions.defaultOptions()).build(root, new MockFileRegistry()));
+                                new FileDistributionConfigProducer.Builder(FileDistributionOptions.defaultOptions())
+                                        .build(root, new MockFileRegistry(), null));
         ConfigModelContext context = ConfigModelContext.create(null, root.getDeployState(), null, root, null);
         
         return new ContentCluster.Builder(admin).build(Collections.emptyList(), context, doc.getDocumentElement());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
index 9b4853d4224..26d08bc1677 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
@@ -20,7 +20,7 @@ public class FileDistributorTestCase {
     public void fileDistributor() {
         MockHosts hosts = new MockHosts();
 
-        FileDistributor fileDistributor = new FileDistributor(new MockFileRegistry());
+        FileDistributor fileDistributor = new FileDistributor(new MockFileRegistry(), null);
 
         FileReference ref1 = fileDistributor.sendFileToHosts("components/path1", Arrays.asList(hosts.host1, hosts.host2));
         FileReference ref2 = fileDistributor.sendFileToHosts("path2", Arrays.asList(hosts.host3));
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyDistribution.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyDistribution.java
index 819f9a9d5d6..155479a7fd1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyDistribution.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyDistribution.java
@@ -8,8 +8,8 @@ import com.yahoo.jrt.Spec;
 import com.yahoo.jrt.StringArray;
 import com.yahoo.jrt.Supervisor;
 import com.yahoo.jrt.Target;
-import com.yahoo.jrt.Transport;
 import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.model.ConfigProxy;
 
 import java.util.Collection;
 import java.util.Set;
@@ -36,10 +36,14 @@ public class CombinedLegacyDistribution implements FileDistribution {
         legacy.sendDeployedFiles(hostName, fileReferences);
     }
 
-    @Override
     public void startDownload(String hostName, Set<FileReference> fileReferences) {
+        startDownload(hostName, ConfigProxy.BASEPORT, fileReferences);
+    }
+
+    @Override
+    public void startDownload(String hostName, int port, Set<FileReference> fileReferences) {
         if (disableFileDistributor)
-            startDownloadingFileReferences(hostName, fileReferences);
+            startDownloadingFileReferences(hostName, port, fileReferences);
     }
 
     @Override
@@ -54,8 +58,8 @@ public class CombinedLegacyDistribution implements FileDistribution {
 
     // Notifies config proxy which file references it should start downloading. It's OK if the call does not succeed,
     // as downloading will then start synchronously when a service requests a file reference instead
-    private void startDownloadingFileReferences(String hostName, Set<FileReference> fileReferences) {
-        Target target = supervisor.connect(new Spec(hostName, 19090));
+    private void startDownloadingFileReferences(String hostName, int port, Set<FileReference> fileReferences) {
+        Target target = supervisor.connect(new Spec(hostName, port));
         double timeout = 0.1;
         Request request = new Request("filedistribution.setFileReferencesToDownload");
         request.parameters().add(new StringArray(fileReferences.stream().map(FileReference::value).toArray(String[]::new)));
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBHandler.java
index 9b3f4c39a45..7dea85c59c3 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBHandler.java
@@ -29,12 +29,16 @@ public class FileDBHandler implements FileDistribution {
         manager.setDeployedFiles(hostName, referencesAsString);
     }
 
-    @Override
     public void startDownload(String hostName, Set<FileReference> fileReferences) {
         throw new UnsupportedOperationException("Not valid for this Filedistribution implementation");
     }
 
     @Override
+    public void startDownload(String hostName, int port, Set<FileReference> fileReferences) {
+        throw new UnsupportedOperationException("Not valid for this Filedistribution implementation");
+    }
+
+    @Override
     public void removeDeploymentsThatHaveDifferentApplicationId(Collection<String> targetHostnames) {
         manager.removeDeploymentsThatHaveDifferentApplicationId(targetHostnames);
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java
index caf64cca4d0..9ba8daf87e4 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java
@@ -20,8 +20,14 @@ public class MockFileDBHandler implements FileDistribution {
         sendDeployedFilesCalled++;
     }
 
+    public void startDownload(String hostName, Set<FileReference> fileReferences) {
+        throw new UnsupportedOperationException("Not valid for this Filedistribution implementation");
+    }
+
     @Override
-    public void startDownload(String hostName, Set<FileReference> fileReferences) { /* not implemented */ }
+    public void startDownload(String hostName, int port, Set<FileReference> fileReferences) {
+        throw new UnsupportedOperationException("Not valid for this Filedistribution implementation");
+    }
 
     @Override
     public void reloadDeployFileDistributor() {