Merge branch 'master' into hmusum/remove-debug-from-response

6 files changed, 47 insertions, 22 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 5cbe6611a3d..e9d400591e8 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -317,14 +317,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                            .stream()
                                            .map(FileReference::value)
                                            .collect(Collectors.toSet())));
-        log.log(LogLevel.INFO, "File references in use : " + fileReferencesInUse);
+        log.log(LogLevel.DEBUG, "File references in use : " + fileReferencesInUse);
 
         // Find those on disk that are not in use
         Set<String> fileReferencesOnDisk = new HashSet<>();
         File[] filesOnDisk = fileReferencesPath.listFiles();
         if (filesOnDisk != null)
             fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet()));
-        log.log(LogLevel.INFO, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk);
+        log.log(LogLevel.DEBUG, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk);
 
         Instant instant = Instant.now().minus(Duration.ofDays(14));
         Set<String> fileReferencesToDelete = fileReferencesOnDisk
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java
index 22cd2f8d56e..4978f5f274d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java
@@ -20,11 +20,11 @@ import javax.ws.rs.client.WebTarget;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * Checks for convergence of config generation for a given application.
@@ -63,12 +63,7 @@ public class ConfigConvergenceChecker extends AbstractComponent {
                                         .filter(service -> serviceTypesToCheck.contains(service.getServiceType()))
                                         .forEach(service -> getStatePort(service).ifPresent(port -> servicesToCheck.add(service))));
 
-        long currentGeneration = servicesToCheck.stream()
-                                                .map(s -> "http://" + s.getHostName() + ":" + getStatePort(s).get())
-                                                .map(URI::create)
-                                                .map(this::getServiceGeneration)
-                                                .min(Comparator.naturalOrder())
-                                                .orElse(0L);
+        long currentGeneration = getServiceGeneration(servicesToCheck);
         return new ServiceListResponse(200, servicesToCheck, uri, application.getApplicationGeneration(),
                                        currentGeneration);
     }
@@ -112,7 +107,7 @@ public class ConfigConvergenceChecker extends AbstractComponent {
                 .findFirst();
     }
 
-    private long generationFromContainerState(JsonNode state) {
+    private static long generationFromContainerState(JsonNode state) {
         return state.get("config").get("generation").asLong();
     }
 
@@ -121,6 +116,26 @@ public class ConfigConvergenceChecker extends AbstractComponent {
         return WebResourceFactory.newResource(StateApi.class, target);
     }
 
+    /** Get service generation for a list of services. Returns the minimum generation of all services */
+    private long getServiceGeneration(List<ServiceInfo> services) {
+        List<URI> serviceUris = services.stream()
+                                        .map(s -> "http://" + s.getHostName() + ":" + getStatePort(s).get())
+                                        .map(URI::create)
+                                        .collect(Collectors.toList());
+        long generation = -1;
+        for (URI uri : serviceUris) {
+            try {
+                long serviceGeneration = getServiceGeneration(uri);
+                if (generation == -1 || serviceGeneration < generation) {
+                    generation = serviceGeneration;
+                }
+            } catch (ProcessingException e) { // Cannot connect to service to determine service generation
+                return -1;
+            }
+        }
+        return generation;
+    }
+
     private long getServiceGeneration(URI serviceUri) {
         StateApi state = stateApiFactory.createStateApi(client, serviceUri);
         return generationFromContainerState(state.config());
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
index a08b077699c..c6a390caf86 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
@@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.maintenance;
 
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.AbstractComponent;
-import com.yahoo.config.model.api.FileDistribution;
+import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.session.FileDistributionFactory;
@@ -46,10 +46,14 @@ public class ConfigServerMaintenance extends AbstractComponent {
 
         DefaultTimes(ConfigserverConfig configserverConfig) {
             boolean isCd = configserverConfig.system().equals(SystemName.cd.name());
+            boolean isTest = Environment.from(configserverConfig.environment()).isTest();
 
             this.defaultInterval = Duration.ofMinutes(configserverConfig.maintainerIntervalMinutes());
-            // TODO: Want job control or feature flag to control when to run this, for now use a very long interval unless in CD
-            this.tenantsMaintainerInterval = isCd ? defaultInterval : Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes());
+            // TODO: Want job control or feature flag to control when to run this, for now use a very
+            // long interval to avoid running the maintainer
+            this.tenantsMaintainerInterval = isCd || isTest
+                    ? defaultInterval
+                    : Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes());
         }
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
index 58141a3a045..2664a0bde8c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
@@ -2,7 +2,8 @@
 package com.yahoo.vespa.config.server.maintenance;
 
 import com.yahoo.cloud.config.ConfigserverConfig;
-import com.yahoo.config.model.api.FileDistribution;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.defaults.Defaults;
@@ -10,24 +11,29 @@ import com.yahoo.vespa.defaults.Defaults;
 import java.io.File;
 import java.time.Duration;
 
+// Note: Unit test is in ApplicationRepositoryTest
 public class FileDistributionMaintainer extends Maintainer {
 
     private final ApplicationRepository applicationRepository;
     private final File fileReferencesDir;
+    private final ConfigserverConfig configserverConfig;
 
-    public FileDistributionMaintainer(ApplicationRepository applicationRepository,
-                                      Curator curator,
-                                      Duration interval,
-                                      ConfigserverConfig configserverConfig) {
+    FileDistributionMaintainer(ApplicationRepository applicationRepository,
+                               Curator curator,
+                               Duration interval,
+                               ConfigserverConfig configserverConfig) {
         super(applicationRepository, curator, interval);
         this.applicationRepository = applicationRepository;
+        this.configserverConfig = configserverConfig;
         this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir()));;
     }
 
 
     @Override
     protected void maintain() {
-        // TODO: Does not delete, for now just outputs what should be deleted
-        applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, false);
+        // TODO: For now only deletes files in CD system
+        boolean deleteFiles = (SystemName.from(configserverConfig.system()) == SystemName.cd)
+                || Environment.from(configserverConfig.environment()).isTest();
+        applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, deleteFiles);
     }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
index c9f99e3a600..114ad936eda 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
@@ -181,7 +181,7 @@ public class SessionPreparer {
             try {
                 this.applicationPackage = context.getApplicationPackage().preprocess(properties.zone(), logger);
             } catch (IOException | TransformerException | ParserConfigurationException | SAXException e) {
-                throw new RuntimeException("Error preprocessing application package for " + applicationId, e);
+                throw new IllegalArgumentException("Error preprocessing application package for " + applicationId, e);
             }
             checkTimeout("preprocess");
         }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java
index 8f4a90ef85f..779571b737e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java
@@ -265,7 +265,7 @@ public class TenantRepository implements ConnectionStateListener, PathChildrenCa
     public synchronized TenantRepository deleteTenant(TenantName name) {
         if (name.equals(DEFAULT_TENANT))
             throw new IllegalArgumentException("Deleting 'default' tenant is not allowed");
-        log.log(LogLevel.DEBUG, "Deleting tenant '" + name + "'");
+        log.log(LogLevel.INFO, "Deleting tenant '" + name + "'");
         Tenant tenant = tenants.remove(name);
         if (tenant == null) {
             throw new IllegalArgumentException("Deleting '" + name + "' failed, tenant does not exist");