Define completeness of SuperModel and DuperModel

In order for Orchestrator to remove application data from ZooKeeper, it must
know which applications do NOT exist. Since the duper model starts with 0
applications, always, the only way of knowing what applications do not exist is
for the bootstrap code to notify the super model/duper model when bootstrap is
complete. There are 2 sources of applications that must signal completeness:

 - The super model, once all applications have been redeployed in
   ConfigServerBootstrap.
 - The infrastructure application, in the InfrastructureProvisioner the first
   time it runs.

11 files changed, 75 insertions, 11 deletions

diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
index f559e9336c8..1cfb70560b8 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
@@ -4,9 +4,9 @@ package com.yahoo.vespa.service.duper;
 import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
@@ -22,12 +22,16 @@ public class DuperModel {
 
     private final Map<ApplicationId, ApplicationInfo> applications = new TreeMap<>();
     private final List<DuperModelListener> listeners = new ArrayList<>();
+    private boolean isComplete = false;
 
     public void registerListener(DuperModelListener listener) {
         applications.values().forEach(listener::applicationActivated);
         listeners.add(listener);
     }
 
+    public void setCompleteness(boolean isComplete) { this.isComplete = isComplete; }
+    public boolean isComplete() { return isComplete; }
+
     public boolean contains(ApplicationId applicationId) {
         return applications.containsKey(applicationId);
     }
@@ -47,6 +51,6 @@ public class DuperModel {
 
     public List<ApplicationInfo> getApplicationInfos() {
         logger.log(LogLevel.DEBUG, "Applications in duper model: " + applications.values().size());
-        return Collections.unmodifiableList(new ArrayList<>(applications.values()));
+        return List.copyOf(applications.values());
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
index 885368810a8..60e4c30a634 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
@@ -12,6 +12,8 @@ import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.service.monitor.DuperModelInfraApi;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
+import com.yahoo.vespa.service.monitor.DuperModelProvider;
 import com.yahoo.vespa.service.monitor.InfraApplicationApi;
 
 import java.util.ArrayList;
@@ -27,7 +29,7 @@ import java.util.stream.Stream;
 /**
  * @author hakonhall
  */
-public class DuperModelManager implements DuperModelInfraApi {
+public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi {
 
     // Infrastructure applications
     static final ControllerHostApplication controllerHostApplication = new ControllerHostApplication();
@@ -45,6 +47,8 @@ public class DuperModelManager implements DuperModelInfraApi {
     // The set of active infrastructure ApplicationInfo. Not all are necessarily in the DuperModel for historical reasons.
     private final Set<ApplicationId> activeInfraInfos = new HashSet<>(10);
 
+    private boolean superModelIsComplete = false;
+    private boolean infraApplicationsIsComplete = false;
 
     @Inject
     public DuperModelManager(ConfigserverConfig configServerConfig, FlagSource flagSource, SuperModelProvider superModelProvider) {
@@ -53,7 +57,7 @@ public class DuperModelManager implements DuperModelInfraApi {
              superModelProvider, new DuperModel(), flagSource, SystemName.from(configServerConfig.system()));
     }
 
-    /** For testing */
+    /** Non-private for testing */
     DuperModelManager(boolean multitenant, boolean isController, SuperModelProvider superModelProvider, DuperModel duperModel, FlagSource flagSource, SystemName system) {
         this.duperModel = duperModel;
 
@@ -86,6 +90,14 @@ public class DuperModelManager implements DuperModelInfraApi {
                     duperModel.remove(applicationId);
                 }
             }
+
+            @Override
+            public void notifyOfCompleteness(SuperModel superModel) {
+                synchronized (monitor) {
+                    superModelIsComplete = true;
+                    maybeSetDuperModelAsComplete();
+                }
+            }
         });
     }
 
@@ -93,6 +105,7 @@ public class DuperModelManager implements DuperModelInfraApi {
      * Synchronously call {@link DuperModelListener#applicationActivated(ApplicationInfo) listener.applicationActivated()}
      * for each currently active application, and forward future changes.
      */
+    @Override
     public void registerListener(DuperModelListener listener) {
         synchronized (monitor) {
             duperModel.registerListener(listener);
@@ -148,9 +161,23 @@ public class DuperModelManager implements DuperModelInfraApi {
         }
     }
 
+    @Override
+    public void infraApplicationsIsNowComplete() {
+        synchronized (monitor) {
+            this.infraApplicationsIsComplete = true;
+            maybeSetDuperModelAsComplete();
+        }
+    }
+
     public List<ApplicationInfo> getApplicationInfos() {
         synchronized (monitor) {
             return duperModel.getApplicationInfos();
         }
     }
+
+    private void maybeSetDuperModelAsComplete() {
+        if (superModelIsComplete && infraApplicationsIsComplete) {
+            duperModel.setCompleteness(true);
+        }
+    }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
index 3cc7010e209..d6e15f6af4e 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
@@ -92,6 +92,10 @@ public class HealthMonitorManager implements MonitorManager, HealthMonitorApi {
     }
 
     @Override
+    public void bootstrapComplete() {
+    }
+
+    @Override
     public ServiceStatusInfo getStatus(ApplicationId applicationId,
                                        ClusterId clusterId,
                                        ServiceType serviceType,
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java
index dd781a02cef..a7579d3f0da 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java
@@ -1,7 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.manager;
 
-import com.yahoo.vespa.service.duper.DuperModelListener;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
 import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
 
 /**
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
index 3490ad4a5d2..2aacc3eadac 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
@@ -51,4 +51,8 @@ public class UnionMonitorManager implements MonitorManager {
         slobrokMonitorManager.applicationRemoved(id);
         healthMonitorManager.applicationRemoved(id);
     }
+
+    @Override
+    public void bootstrapComplete() {
+    }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
index d08bba2bd3d..f9e47b6b80a 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
@@ -27,4 +27,7 @@ public interface DuperModelInfraApi {
 
     /** Update the DuperModel: A supported infrastructure application has been removed or is not active. */
     void infraApplicationRemoved(ApplicationId applicationId);
+
+    /** All infra applications that are supposed to activate on config server bootstrap has been activated. */
+    void infraApplicationsIsNowComplete();
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelListener.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelListener.java
index a969b6c3f40..f664e5246ca 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelListener.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelListener.java
@@ -1,34 +1,45 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.service.duper;
+package com.yahoo.vespa.service.monitor;
 
 import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.config.model.api.SuperModel;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.service.duper.DuperModel;
 
 /**
  * Interface for listening for changes to the {@link DuperModel}.
  *
- * @author hakon
+ * @author hakonhall
  */
 public interface DuperModelListener {
     /**
      * An application has been activated:
      *
      * <ul>
-     *     <li>A synthetic application like the config server application has been added/"activated"
+     *     <li>A synthetic application like the config server application has been added/activated
      *     <li>A super model application has been activated (see
      *     {@link com.yahoo.config.model.api.SuperModelListener#applicationActivated(SuperModel, ApplicationInfo)
      *     SuperModelListener}
      * </ul>
      *
-     * No other threads will concurrently call any methods on this interface.
+     * <p>No other threads will concurrently call any methods on this interface.</p>
      */
     void applicationActivated(ApplicationInfo application);
 
     /**
      * Application has been removed.
      *
-     * No other threads will concurrently call any methods on this interface.
+     * <p>No other threads will concurrently call any methods on this interface.</p>
      */
     void applicationRemoved(ApplicationId id);
+
+    /**
+     * During bootstrap of the config server, a number of applications are activated before
+     * resuming normal operations: The normal "tenant" application (making the super model) and
+     * the relevant infrastructure applications. Once all of these have been activated, this method
+     * will be invoked.
+     *
+     * <p>No other threads will concurrently call any methods on this interface.</p>
+     */
+    void bootstrapComplete();
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java
new file mode 100644
index 00000000000..a90fa418054
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java
@@ -0,0 +1,6 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor;
+
+public interface DuperModelProvider {
+    void registerListener(DuperModelListener listener);
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
index e3ea48ca9fe..e7a8d33ea14 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
@@ -73,6 +73,10 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager {
     }
 
     @Override
+    public void bootstrapComplete() {
+    }
+
+    @Override
     public List<Mirror.Entry> lookup(ApplicationId id, String pattern) {
         synchronized (monitor) {
             SlobrokMonitor slobrokMonitor = slobrokMonitors.get(id);
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
index 31fd266649a..dc90035be71 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.service.duper;
 
 import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
 import org.junit.Before;
 import org.junit.Test;
 
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java
index 0f7c0dde357..3fb10f1f24e 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java
@@ -47,7 +47,7 @@ public class ExampleModel {
 
         Map<ApplicationId, ApplicationInfo> applicationInfos = new HashMap<>();
         applicationInfos.put(applicationInfo.getApplicationId(), applicationInfo);
-        return new SuperModel(applicationInfos);
+        return new SuperModel(applicationInfos, true);
     }
 
     public static ApplicationBuilder createApplication(String tenant,