Merge pull request #13519 from vespa-engine/gjoranv/application-package-maintainer

Gjoranv/application package maintainer

6 files changed, 167 insertions, 50 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 9404bc9c279..dd88096c62f 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -74,7 +74,6 @@ import java.nio.file.attribute.BasicFileAttributes;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -89,6 +88,7 @@ import java.util.stream.Collectors;
 import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER;
 import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER;
 import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER;
+import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getFileReferencesOnDisk;
 import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA_TENANT;
 import static com.yahoo.yolean.Exceptions.uncheck;
 import static java.nio.file.Files.readAttributes;
@@ -451,10 +451,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         log.log(Level.FINE, "File references in use : " + fileReferencesInUse);
 
         // Find those on disk that are not in use
-        Set<String> fileReferencesOnDisk = new HashSet<>();
-        File[] filesOnDisk = fileReferencesPath.listFiles();
-        if (filesOnDisk != null)
-            fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet()));
+        Set<String> fileReferencesOnDisk = getFileReferencesOnDisk(fileReferencesPath);
         log.log(Level.FINE, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk);
 
         Instant instant = Instant.now().minus(keepFileReferences);
@@ -512,7 +509,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         }
     }
 
-    Set<ApplicationId> listApplications() {
+    public Set<ApplicationId> listApplications() {
         return tenantRepository.getAllTenants().stream()
                 .flatMap(tenant -> tenant.getApplicationRepo().activeApplications().stream())
                 .collect(Collectors.toSet());
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java
new file mode 100644
index 00000000000..c06e4da2b7b
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java
@@ -0,0 +1,78 @@
+package com.yahoo.vespa.config.server.filedistribution;
+
+import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.config.subscription.ConfigSourceSet;
+import com.yahoo.jrt.Supervisor;
+import com.yahoo.jrt.Transport;
+import com.yahoo.net.HostName;
+import com.yahoo.vespa.config.Connection;
+import com.yahoo.vespa.config.ConnectionPool;
+import com.yahoo.vespa.config.JRTConnectionPool;
+import com.yahoo.vespa.config.server.ConfigServerSpec;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Utilities related to file distribution on config servers.
+ *
+ * @author musum
+ * @author gjoranv
+ */
+public class FileDistributionUtil {
+
+    /**
+     * Returns all files in the given directory, non-recursive.
+     */
+    public static Set<String> getFileReferencesOnDisk(File directory) {
+        Set<String> fileReferencesOnDisk = new HashSet<>();
+        File[] filesOnDisk = directory.listFiles();
+        if (filesOnDisk != null)
+            fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet()));
+        return fileReferencesOnDisk;
+    }
+
+    /**
+     * Returns a connection pool with all config servers except this one, or an empty pool if there
+     * is only one config server.
+     */
+    public static ConnectionPool createConnectionPool(ConfigserverConfig configserverConfig) {
+        List<String> configServers = ConfigServerSpec.fromConfig(configserverConfig)
+                .stream()
+                .filter(spec -> !spec.getHostName().equals(HostName.getLocalhost()))
+                .map(spec -> "tcp/" + spec.getHostName() + ":" + spec.getConfigServerPort())
+                .collect(Collectors.toList());
+
+        return configServers.size() > 0 ? new JRTConnectionPool(new ConfigSourceSet(configServers)) : emptyConnectionPool();
+    }
+
+    static ConnectionPool emptyConnectionPool() {
+        return new EmptyConnectionPool();
+    }
+
+    private static class EmptyConnectionPool implements ConnectionPool {
+
+        @Override
+        public void close() {}
+
+        @Override
+        public void setError(Connection connection, int i) {}
+
+        @Override
+        public Connection getCurrent() { return null; }
+
+        @Override
+        public Connection setNewCurrentConnection() { return null; }
+
+        @Override
+        public int getSize() { return 0; }
+
+        @Override
+        public Supervisor getSupervisor() { return new Supervisor(new Transport()); }
+    }
+
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
index 805ee2bef95..99cdb0a74dc 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
@@ -4,18 +4,10 @@ package com.yahoo.vespa.config.server.filedistribution;
 import com.google.inject.Inject;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.config.FileReference;
-import com.yahoo.config.subscription.ConfigSourceSet;
 import com.yahoo.jrt.Int32Value;
 import com.yahoo.jrt.Request;
 import com.yahoo.jrt.StringValue;
-import com.yahoo.jrt.Supervisor;
-import com.yahoo.jrt.Transport;
-import java.util.logging.Level;
-import com.yahoo.net.HostName;
-import com.yahoo.vespa.config.Connection;
 import com.yahoo.vespa.config.ConnectionPool;
-import com.yahoo.vespa.config.JRTConnectionPool;
-import com.yahoo.vespa.config.server.ConfigServerSpec;
 import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.vespa.filedistribution.CompressedFileReference;
 import com.yahoo.vespa.filedistribution.FileDownloader;
@@ -27,12 +19,14 @@ import com.yahoo.yolean.Exceptions;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.stream.Collectors;
+
+import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.createConnectionPool;
+import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.emptyConnectionPool;
 
 public class FileServer {
     private static final Logger log = Logger.getLogger(FileServer.class.getName());
@@ -79,7 +73,7 @@ public class FileServer {
 
     // For testing only
     public FileServer(File rootDir) {
-        this(new EmptyConnectionPool(), rootDir);
+        this(emptyConnectionPool(), rootDir);
     }
 
     private FileServer(ConnectionPool connectionPool, File rootDir) {
@@ -199,35 +193,4 @@ public class FileServer {
         downloader.close();
     }
 
-    // Connection pool with all config servers except this one (might be an empty pool if there is only one config server)
-    private static ConnectionPool createConnectionPool(ConfigserverConfig configserverConfig) {
-        List<String> configServers = ConfigServerSpec.fromConfig(configserverConfig)
-                .stream()
-                .filter(spec -> !spec.getHostName().equals(HostName.getLocalhost()))
-                .map(spec -> "tcp/" + spec.getHostName() + ":" + spec.getConfigServerPort())
-                .collect(Collectors.toList());
-
-        return configServers.size() > 0 ? new JRTConnectionPool(new ConfigSourceSet(configServers)) : new EmptyConnectionPool();
-    }
-
-    private static class EmptyConnectionPool implements ConnectionPool {
-
-        @Override
-        public void close() {}
-
-        @Override
-        public void setError(Connection connection, int i) {}
-
-        @Override
-        public Connection getCurrent() { return null; }
-
-        @Override
-        public Connection setNewCurrentConnection() { return null; }
-
-        @Override
-        public int getSize() { return 0; }
-
-        @Override
-        public Supervisor getSupervisor() { return new Supervisor(new Transport()); }
-    }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
new file mode 100644
index 00000000000..b7b5c6380ef
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
@@ -0,0 +1,75 @@
+package com.yahoo.vespa.config.server.maintenance;
+
+import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.config.FileReference;
+import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.session.RemoteSession;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.filedistribution.FileDownloader;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
+
+import java.io.File;
+import java.time.Duration;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getFileReferencesOnDisk;
+import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.createConnectionPool;
+
+/**
+ * Verifies that all active sessions has an application package on local disk.
+ * If not, the package is downloaded with file distribution. This can happen e.g.
+ * if a configserver is down when the application is deployed.
+ *
+ * @author gjoranv
+ */
+public class ApplicationPackageMaintainer extends ConfigServerMaintainer {
+    private static final Logger log = Logger.getLogger(ApplicationPackageMaintainer.class.getName());
+
+    private final ApplicationRepository applicationRepository;
+    private final ConfigserverConfig configserverConfig;
+    private final File downloadDirectory;
+    private final BooleanFlag distributeApplicationPackage;
+
+    ApplicationPackageMaintainer(ApplicationRepository applicationRepository,
+                                 Curator curator,
+                                 Duration interval,
+                                 ConfigserverConfig configserverConfig,
+                                 FlagSource flagSource) {
+        super(applicationRepository, curator, interval, interval);
+        this.applicationRepository = applicationRepository;
+        this.configserverConfig = configserverConfig;
+
+        distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource);
+        downloadDirectory = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir()));
+    }
+
+    @Override
+    protected void maintain() {
+        if (! distributeApplicationPackage.value()) return;
+
+        try (var fileDownloader = new FileDownloader(createConnectionPool(configserverConfig), downloadDirectory)){
+            for (var applicationId : applicationRepository.listApplications()) {
+                RemoteSession session = applicationRepository.getActiveSession(applicationId);
+                FileReference applicationPackage = session.getApplicationPackageReference();
+
+                if (applicationPackage != null && missingOnDisk(applicationPackage)) {
+                    log.fine(() -> "Downloading missing application package for application " + applicationId + " - session " + session.getSessionId());
+
+                    if (fileDownloader.getFile(applicationPackage).isEmpty()) {
+                        log.warning("Failed to download application package for application " + applicationId + " - session " + session.getSessionId());
+                    }
+                }
+            }
+        }
+    }
+
+    private boolean missingOnDisk(FileReference applicationPackageReference) {
+        Set<String> fileReferencesOnDisk = getFileReferencesOnDisk(downloadDirectory);
+        return ! fileReferencesOnDisk.contains(applicationPackageReference.value());
+    }
+
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
index afcfe04f4ac..3ea7959c212 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.maintenance;
 
+import com.google.inject.Inject;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.config.provision.SystemName;
@@ -23,8 +24,9 @@ public class ConfigServerMaintenance extends AbstractComponent {
     //private final TenantsMaintainer tenantsMaintainer;
     private final FileDistributionMaintainer fileDistributionMaintainer;
     private final SessionsMaintainer sessionsMaintainer;
+    private final ApplicationPackageMaintainer applicationPackageMaintainer;
 
-    @SuppressWarnings("unused") // instantiated by Dependency Injection
+    @Inject
     public ConfigServerMaintenance(ConfigserverConfig configserverConfig,
                                    ApplicationRepository applicationRepository,
                                    Curator curator,
@@ -35,6 +37,7 @@ public class ConfigServerMaintenance extends AbstractComponent {
         //tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.tenantsMaintainerInterval);
         fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig);
         sessionsMaintainer = new SessionsMaintainer(applicationRepository, curator, defaults.defaultInterval);
+        applicationPackageMaintainer = new ApplicationPackageMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig, flagSource);
     }
 
     @Override
@@ -42,6 +45,7 @@ public class ConfigServerMaintenance extends AbstractComponent {
         //tenantsMaintainer.close();
         fileDistributionMaintainer.close();
         sessionsMaintainer.close();
+        applicationPackageMaintainer.close();
     }
 
     /*
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java
index 8b078f152f3..c553133ba12 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java
@@ -107,7 +107,7 @@ public abstract class Session implements Comparable<Session>  {
 
     public ApplicationId getApplicationId() { return zooKeeperClient.readApplicationId(); }
 
-    FileReference getApplicationPackageReference() {return zooKeeperClient.readApplicationPackageReference(); }
+    public FileReference getApplicationPackageReference() {return zooKeeperClient.readApplicationPackageReference(); }
 
     public Optional<DockerImage> getDockerImageRepository() { return zooKeeperClient.readDockerImageRepository(); }