Log when getting possibly unhandled event for application change

* No changes to when we delete applications, but log to hopefully
  get more information if we are in a situation where we could
  have missed an event for deleting an application

6 files changed, 51 insertions, 6 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadHandler.java
index d76342553b2..b493e2c7bc6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ReloadHandler.java
@@ -4,6 +4,8 @@ package com.yahoo.vespa.config.server;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.config.server.application.ApplicationSet;
 
+import java.util.Set;
+
 /**
  * Interface representing a reload handler.
  *
@@ -26,4 +28,11 @@ public interface ReloadHandler {
      */
     void removeApplication(ApplicationId applicationId);
 
+    /**
+     * Remove all applications except those specified in argument.
+     *
+     * @param applicationIds to be kept
+     */
+    void removeApplicationsExcept(Set<ApplicationId> applicationIds);
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java
index 459bee99d75..0d055d3bfc9 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java
@@ -5,8 +5,10 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Version;
 
 import java.time.Instant;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
 import com.yahoo.vespa.config.server.NotFoundException;
@@ -75,4 +77,8 @@ public final class ApplicationMapper {
         return requestHandlers.size();
     }
 
+    public Set<ApplicationId> listApplicationIds() {
+        return Collections.unmodifiableSet(requestHandlers.keySet());
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java
index e651fd6c97c..9e3df5af715 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.application;
 
+import com.google.common.collect.ImmutableSet;
 import com.yahoo.concurrent.ThreadFactoryFactory;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.TenantName;
@@ -22,6 +23,7 @@ import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 /**
@@ -62,6 +64,7 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac
         }
     }
 
+    // TODO: October 2017: Evaluate and remove if possible, seems to be compatibility code that is not needed anymore
     private void rewriteApplicationIds() {
         try {
             List<String> appNodes = curator.framework().getChildren().forPath(applicationsPath.getAbsolute());
@@ -161,6 +164,11 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac
             case CHILD_REMOVED:
                 applicationRemoved(ApplicationId.fromSerializedForm(Path.fromString(event.getData().getPath()).getName()));
                 break;
+            default:
+                // We don't know if applications have been added or removed so possibly need to remove some of them
+                // (new applications are not added here)
+                removeApplications();
+                break;
         }
     }
 
@@ -171,6 +179,13 @@ public class ZKTenantApplications implements TenantApplications, PathChildrenCac
 
     private void applicationAdded(ApplicationId applicationId) {
         log.log(LogLevel.DEBUG, Tenants.logPre(applicationId) + "Application added: " + applicationId);
-    }    
+    }
+
+    private void removeApplications() {
+        ImmutableSet<ApplicationId> allApplications = ImmutableSet.copyOf(listApplications());
+        log.log(Level.INFO, "We probably lost events, need to check if applications have been removed, " +
+                " found these active applications: " + allApplications);
+        reloadHandler.removeApplicationsExcept(allApplications);
+    }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java
index bb3e957e022..ae17d5fc81c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java
@@ -103,6 +103,17 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host
         }
     }
 
+    @Override
+    public void removeApplicationsExcept(Set<ApplicationId> applications) {
+        for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) {
+            if (! applications.contains(activeApplication)) {
+                log.log(LogLevel.INFO, "Will remove deleted application " + activeApplication.toShortString());
+                // TODO: Activate
+                //removeApplication(activeApplication);
+            }
+        }
+    }
+
     private void reloadListenersOnRemove(ApplicationId applicationId) {
         for (ReloadListener listener : reloadListeners) {
             listener.applicationRemoved(applicationId);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockReloadHandler.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockReloadHandler.java
index 651a8424224..ace187489d1 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockReloadHandler.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockReloadHandler.java
@@ -4,6 +4,8 @@ package com.yahoo.vespa.config.server;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.config.server.application.ApplicationSet;
 
+import java.util.Set;
+
 /**
  * @author lulf
  * @since 5.1.24
@@ -23,4 +25,7 @@ public class MockReloadHandler implements ReloadHandler {
         lastRemoved = applicationId;
     }
 
+    @Override
+    public void removeApplicationsExcept(Set<ApplicationId> applicationIds) { }
+
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java
index 51af9c38f90..62ff13093ea 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRequestHandler.java
@@ -20,10 +20,7 @@ import java.util.*;
  */
 public class MockRequestHandler implements RequestHandler, ReloadHandler, TenantHandlerProvider {
 
-    volatile String serverStats = "";
-    volatile boolean reloadResponse = false;
     volatile boolean throwException = false;
-    public long appGeneration = 0;
     private Set<ConfigKey<?>> allConfigs = new HashSet<>();
     public volatile ConfigResponse responseConfig = null; // for some v1 mocking
     public Map<ApplicationId, ConfigResponse> responses = new LinkedHashMap<>(); // for v2 mocking
@@ -52,8 +49,10 @@ public class MockRequestHandler implements RequestHandler, ReloadHandler, Tenant
     }
 
     @Override
-    public void removeApplication(ApplicationId applicationId) {
-    }
+    public void removeApplication(ApplicationId applicationId) { }
+
+    @Override
+    public void removeApplicationsExcept(Set<ApplicationId> applicationIds) { }
 
     @Override
     public void reloadConfig(ApplicationSet application) {