Define completeness of SuperModel and DuperModel

In order for Orchestrator to remove application data from ZooKeeper, it must
know which applications do NOT exist. Since the duper model starts with 0
applications, always, the only way of knowing what applications do not exist is
for the bootstrap code to notify the super model/duper model when bootstrap is
complete. There are 2 sources of applications that must signal completeness:

 - The super model, once all applications have been redeployed in
   ConfigServerBootstrap.
 - The infrastructure application, in the InfrastructureProvisioner the first
   time it runs.

3 files changed, 60 insertions, 14 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
index b2a10f4bb21..4e1006213c6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
@@ -62,6 +62,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
     private final StateMonitor stateMonitor;
     private final VipStatus vipStatus;
     private final ConfigserverConfig configserverConfig;
+    private final SuperModelManager superModelManager;
     private final Duration maxDurationOfRedeployment;
     private final Duration sleepTimeWhenRedeployingFails;
     private final RedeployingApplicationsFails exitIfRedeployingApplicationsFails;
@@ -70,29 +71,32 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
     @SuppressWarnings("unused")
     @Inject
     public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server,
-                                 VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus) {
+                                 VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus,
+                                 SuperModelManager superModelManager) {
         this(applicationRepository, server, versionState, stateMonitor, vipStatus, BOOTSTRAP_IN_CONSTRUCTOR, EXIT_JVM,
              applicationRepository.configserverConfig().hostedVespa()
                      ? VipStatusMode.VIP_STATUS_FILE
-                     : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY);
+                     : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY,
+                superModelManager);
     }
 
     // For testing only
     ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState,
                           StateMonitor stateMonitor, VipStatus vipStatus, Mode mode,  VipStatusMode vipStatusMode) {
-        this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode);
+        this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode, null);
     }
 
     private ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server,
                                   VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus,
                                   Mode mode, RedeployingApplicationsFails exitIfRedeployingApplicationsFails,
-                                  VipStatusMode vipStatusMode) {
+                                  VipStatusMode vipStatusMode, SuperModelManager superModelManager) {
         this.applicationRepository = applicationRepository;
         this.server = server;
         this.versionState = versionState;
         this.stateMonitor = stateMonitor;
         this.vipStatus = vipStatus;
         this.configserverConfig = applicationRepository.configserverConfig();
+        this.superModelManager = superModelManager;
         this.maxDurationOfRedeployment = Duration.ofSeconds(configserverConfig.maxDurationOfBootstrap());
         this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails());
         this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails;
@@ -208,6 +212,9 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
     private boolean redeployAllApplications() throws InterruptedException {
         Instant end = Instant.now().plus(maxDurationOfRedeployment);
         Set<ApplicationId> applicationsNotRedeployed = applicationRepository.listApplications();
+        if (superModelManager != null) {
+            superModelManager.setBootstrapApplicationSet(applicationsNotRedeployed);
+        }
         do {
             applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed);
             if ( ! applicationsNotRedeployed.isEmpty()) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
index ea835206b7c..ccd92d4a195 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
@@ -20,6 +20,7 @@ import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 
 /**
  * Provides a SuperModel - a model of all application instances,  and makes it stays
@@ -38,6 +39,9 @@ public class SuperModelManager implements SuperModelProvider {
     private final long masterGeneration; // ConfigserverConfig's generation
     private final GenerationCounter generationCounter;
 
+    // The initial set of applications to be deployed on bootstrap.
+    private Optional<Set<ApplicationId>> bootstrapApplicationSet = Optional.empty();
+
     @Inject
     public SuperModelManager(ConfigserverConfig configserverConfig,
                              NodeFlavors nodeFlavors,
@@ -75,6 +79,10 @@ public class SuperModelManager implements SuperModelProvider {
             listeners.add(listener);
             SuperModel superModel = superModelConfigProvider.getSuperModel();
             superModel.getAllApplicationInfos().forEach(application -> listener.applicationActivated(superModel, application));
+
+            if (superModel.isComplete()) {
+                listener.notifyOfCompleteness(superModel);
+            }
         }
     }
 
@@ -89,13 +97,19 @@ public class SuperModelManager implements SuperModelProvider {
                     .getForVersionOrLatest(Optional.empty(), Instant.now())
                     .toApplicationInfo();
 
-            SuperModel newSuperModel = this.superModelConfigProvider
-                    .getSuperModel()
-                    .cloneAndSetApplication(applicationInfo);
+            SuperModel oldSuperModel = superModelConfigProvider.getSuperModel();
+            SuperModel newSuperModel = oldSuperModel
+                    .cloneAndSetApplication(applicationInfo, isComplete(oldSuperModel));
+
             generationCounter.increment();
             makeNewSuperModelConfigProvider(newSuperModel);
-            listeners.stream().forEach(listener ->
-                    listener.applicationActivated(newSuperModel, applicationInfo));
+            listeners.forEach(listener -> listener.applicationActivated(newSuperModel, applicationInfo));
+
+            if (!oldSuperModel.isComplete() && newSuperModel.isComplete()) {
+                for (var listener : listeners) {
+                    listener.notifyOfCompleteness(newSuperModel);
+                }
+            }
         }
     }
 
@@ -106,11 +120,36 @@ public class SuperModelManager implements SuperModelProvider {
                     .cloneAndRemoveApplication(applicationId);
             generationCounter.increment();
             makeNewSuperModelConfigProvider(newSuperModel);
-            listeners.stream().forEach(listener ->
-                    listener.applicationRemoved(newSuperModel, applicationId));
+            listeners.forEach(listener -> listener.applicationRemoved(newSuperModel, applicationId));
         }
     }
 
+    public void setBootstrapApplicationSet(Set<ApplicationId> bootstrapApplicationSet) {
+        synchronized (monitor) {
+            this.bootstrapApplicationSet = Optional.of(bootstrapApplicationSet);
+
+            SuperModel superModel = superModelConfigProvider.getSuperModel();
+            if (!superModel.isComplete() && isComplete(superModel)) {
+                // We do NOT increment the generation since completeness is not part of the config:
+                // generationCounter.increment()
+
+                SuperModel newSuperModel = superModel.cloneAsComplete();
+                makeNewSuperModelConfigProvider(newSuperModel);
+                listeners.forEach(listener -> listener.notifyOfCompleteness(newSuperModel));
+            }
+        }
+    }
+
+    /** Returns freshly calculated value of isComplete. */
+    private boolean isComplete(SuperModel currentSuperModel) {
+        if (currentSuperModel.isComplete()) return true;
+        if (bootstrapApplicationSet.isEmpty()) return false;
+
+        Set<ApplicationId> currentApplicationIds = superModelConfigProvider.getSuperModel().getApplicationIds();
+        if (currentApplicationIds.size() < bootstrapApplicationSet.get().size()) return false;
+        return currentApplicationIds.containsAll(bootstrapApplicationSet.get());
+    }
+
     private void makeNewSuperModelConfigProvider(SuperModel newSuperModel) {
         generation = masterGeneration + generationCounter.get();
         superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
index b7486dc7951..561422c1cf8 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
@@ -55,7 +55,7 @@ public class SuperModelControllerTest {
         ApplicationId app = ApplicationId.from(TenantName.from("a"),
                                                ApplicationName.from("foo"), InstanceName.defaultName());
         models.put(app, new ApplicationInfo(app, 4L, new VespaModel(FilesApplicationPackage.fromFile(testApp))));
-        SuperModel superModel = new SuperModel(models);
+        SuperModel superModel = new SuperModel(models, true);
         handler = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
     }
     
@@ -98,7 +98,7 @@ public class SuperModelControllerTest {
         models.put(advanced, createApplicationInfo(testApp2, advanced, 4L));
         models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L));
 
-        SuperModel superModel = new SuperModel(models);
+        SuperModel superModel = new SuperModel(models, true);
         SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
         LbServicesConfig.Builder lb = new LbServicesConfig.Builder();
         han.getSuperModel().getConfig(lb);
@@ -126,7 +126,7 @@ public class SuperModelControllerTest {
         models.put(advanced, createApplicationInfo(testApp2, advanced, 4L));
         models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L));
 
-        SuperModel superModel = new SuperModel(models);
+        SuperModel superModel = new SuperModel(models, true);
         SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
         LbServicesConfig.Builder lb = new LbServicesConfig.Builder();
         han.getSuperModel().getConfig(lb);