Merge pull request #12314 from vespa-engine/hakonhall/define-completeness-of-supermodel-and-dupermodel

Define completeness of SuperModel and DuperModel

25 files changed, 208 insertions, 58 deletions

diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java
index 2208b470ed8..fdf2bbfccff 100644
--- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java
+++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java
@@ -201,7 +201,8 @@ public class InstanceValidatorTest {
                                                                   ApplicationInfo::getApplicationId,
                                                                   Function.identity()
                                                                    )
-                                                          ));
+                                                          ),
+                                               true);
 
         SuperModelProvider superModelProvider = mock(SuperModelProvider.class);
         when(superModelProvider.getSuperModel()).thenReturn(superModel);
diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json
index 43527335802..90132d6924a 100644
--- a/config-model-api/abi-spec.json
+++ b/config-model-api/abi-spec.json
@@ -990,13 +990,16 @@
     ],
     "methods": [
       "public void <init>()",
-      "public void <init>(java.util.Map)",
+      "public void <init>(java.util.Map, boolean)",
       "public java.util.Map getModelsPerTenant()",
       "public java.util.Map getModels()",
+      "public boolean isComplete()",
       "public java.util.List getAllApplicationInfos()",
       "public java.util.Optional getApplicationInfo(com.yahoo.config.provision.ApplicationId)",
-      "public com.yahoo.config.model.api.SuperModel cloneAndSetApplication(com.yahoo.config.model.api.ApplicationInfo)",
-      "public com.yahoo.config.model.api.SuperModel cloneAndRemoveApplication(com.yahoo.config.provision.ApplicationId)"
+      "public com.yahoo.config.model.api.SuperModel cloneAndSetApplication(com.yahoo.config.model.api.ApplicationInfo, boolean)",
+      "public com.yahoo.config.model.api.SuperModel cloneAndRemoveApplication(com.yahoo.config.provision.ApplicationId)",
+      "public com.yahoo.config.model.api.SuperModel cloneAsComplete()",
+      "public java.util.Set getApplicationIds()"
     ],
     "fields": []
   },
@@ -1010,7 +1013,8 @@
     ],
     "methods": [
       "public abstract void applicationActivated(com.yahoo.config.model.api.SuperModel, com.yahoo.config.model.api.ApplicationInfo)",
-      "public abstract void applicationRemoved(com.yahoo.config.model.api.SuperModel, com.yahoo.config.provision.ApplicationId)"
+      "public abstract void applicationRemoved(com.yahoo.config.model.api.SuperModel, com.yahoo.config.provision.ApplicationId)",
+      "public abstract void notifyOfCompleteness(com.yahoo.config.model.api.SuperModel)"
     ],
     "fields": []
   },
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java
index 1735e08c930..15502dac1f1 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java
@@ -20,13 +20,15 @@ import java.util.Set;
 public class SuperModel {
 
     private final Map<ApplicationId, ApplicationInfo> models;
+    private final boolean complete;
 
     public SuperModel() {
-        this.models = Collections.emptyMap();
+        this(Collections.emptyMap(), false);
     }
 
-    public SuperModel(Map<ApplicationId, ApplicationInfo> models) {
+    public SuperModel(Map<ApplicationId, ApplicationInfo> models, boolean complete) {
         this.models = models;
+        this.complete = complete;
     }
 
     public Map<TenantName, Set<ApplicationInfo>> getModelsPerTenant() {
@@ -45,6 +47,8 @@ public class SuperModel {
         return ImmutableMap.copyOf(models);
     }
 
+    public boolean isComplete() { return complete; }
+
     public List<ApplicationInfo> getAllApplicationInfos() {
         return new ArrayList<>(models.values());
     }
@@ -54,20 +58,22 @@ public class SuperModel {
         return applicationInfo == null ? Optional.empty() : Optional.of(applicationInfo);
     }
 
-    public SuperModel cloneAndSetApplication(ApplicationInfo application) {
+    public SuperModel cloneAndSetApplication(ApplicationInfo application, boolean complete) {
         Map<ApplicationId, ApplicationInfo> newModels = cloneModels(models);
         newModels.put(application.getApplicationId(), application);
-
-        return new SuperModel(newModels);
+        return new SuperModel(newModels, complete);
     }
 
     public SuperModel cloneAndRemoveApplication(ApplicationId applicationId) {
         Map<ApplicationId, ApplicationInfo> newModels = cloneModels(models);
         newModels.remove(applicationId);
-
-        return new SuperModel(newModels);
+        return new SuperModel(newModels, complete);
     }
 
+    public SuperModel cloneAsComplete() { return new SuperModel(models, true); }
+
+    public Set<ApplicationId> getApplicationIds() { return models.keySet(); }
+
     private static Map<ApplicationId, ApplicationInfo> cloneModels(Map<ApplicationId, ApplicationInfo> models) {
         return new LinkedHashMap<>(models);
     }
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java
index 497c38af908..e66a7e1ef7e 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java
@@ -17,4 +17,12 @@ public interface SuperModelListener {
      * Application has been removed.
      */
     void applicationRemoved(SuperModel superModel, ApplicationId id);
+
+    /**
+     * Invoked once all applications that were supposed to be deployed on bootstrap
+     * have been activated (and the respective {@link #applicationActivated(SuperModel, ApplicationInfo)
+     * applicationActivated} have been invoked). The SuperModel is then said to be "complete".
+     * @param superModel
+     */
+    void notifyOfCompleteness(SuperModel superModel);
 }
diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json
index 45f8171436d..f2ae997a164 100644
--- a/config-provisioning/abi-spec.json
+++ b/config-provisioning/abi-spec.json
@@ -506,7 +506,7 @@
     ],
     "methods": [
       "public abstract java.util.Optional getDeployment(com.yahoo.config.provision.ApplicationId)",
-      "public abstract java.util.Map getSupportedInfraDeployments()"
+      "public abstract void activateAllSupportedInfraApplications(boolean)"
     ],
     "fields": []
   },
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java
index 6fbabfd0c95..363732ee8a7 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java
@@ -1,7 +1,6 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.provision;
 
-import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -17,6 +16,6 @@ public interface InfraDeployer {
      */
     Optional<Deployment> getDeployment(ApplicationId application);
 
-    /** Returns deployments by application id for the supported infrastructure applications in this zone */
-    Map<ApplicationId, Deployment> getSupportedInfraDeployments();
+    /** Deploys all supported infrastructure applications in this zone. */
+    void activateAllSupportedInfraApplications(boolean propagateException);
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
index b2a10f4bb21..4e1006213c6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
@@ -62,6 +62,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
     private final StateMonitor stateMonitor;
     private final VipStatus vipStatus;
     private final ConfigserverConfig configserverConfig;
+    private final SuperModelManager superModelManager;
     private final Duration maxDurationOfRedeployment;
     private final Duration sleepTimeWhenRedeployingFails;
     private final RedeployingApplicationsFails exitIfRedeployingApplicationsFails;
@@ -70,29 +71,32 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
     @SuppressWarnings("unused")
     @Inject
     public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server,
-                                 VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus) {
+                                 VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus,
+                                 SuperModelManager superModelManager) {
         this(applicationRepository, server, versionState, stateMonitor, vipStatus, BOOTSTRAP_IN_CONSTRUCTOR, EXIT_JVM,
              applicationRepository.configserverConfig().hostedVespa()
                      ? VipStatusMode.VIP_STATUS_FILE
-                     : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY);
+                     : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY,
+                superModelManager);
     }
 
     // For testing only
     ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState,
                           StateMonitor stateMonitor, VipStatus vipStatus, Mode mode,  VipStatusMode vipStatusMode) {
-        this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode);
+        this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode, null);
     }
 
     private ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server,
                                   VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus,
                                   Mode mode, RedeployingApplicationsFails exitIfRedeployingApplicationsFails,
-                                  VipStatusMode vipStatusMode) {
+                                  VipStatusMode vipStatusMode, SuperModelManager superModelManager) {
         this.applicationRepository = applicationRepository;
         this.server = server;
         this.versionState = versionState;
         this.stateMonitor = stateMonitor;
         this.vipStatus = vipStatus;
         this.configserverConfig = applicationRepository.configserverConfig();
+        this.superModelManager = superModelManager;
         this.maxDurationOfRedeployment = Duration.ofSeconds(configserverConfig.maxDurationOfBootstrap());
         this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails());
         this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails;
@@ -208,6 +212,9 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
     private boolean redeployAllApplications() throws InterruptedException {
         Instant end = Instant.now().plus(maxDurationOfRedeployment);
         Set<ApplicationId> applicationsNotRedeployed = applicationRepository.listApplications();
+        if (superModelManager != null) {
+            superModelManager.setBootstrapApplicationSet(applicationsNotRedeployed);
+        }
         do {
             applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed);
             if ( ! applicationsNotRedeployed.isEmpty()) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
index ea835206b7c..ccd92d4a195 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
@@ -20,6 +20,7 @@ import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 
 /**
  * Provides a SuperModel - a model of all application instances,  and makes it stays
@@ -38,6 +39,9 @@ public class SuperModelManager implements SuperModelProvider {
     private final long masterGeneration; // ConfigserverConfig's generation
     private final GenerationCounter generationCounter;
 
+    // The initial set of applications to be deployed on bootstrap.
+    private Optional<Set<ApplicationId>> bootstrapApplicationSet = Optional.empty();
+
     @Inject
     public SuperModelManager(ConfigserverConfig configserverConfig,
                              NodeFlavors nodeFlavors,
@@ -75,6 +79,10 @@ public class SuperModelManager implements SuperModelProvider {
             listeners.add(listener);
             SuperModel superModel = superModelConfigProvider.getSuperModel();
             superModel.getAllApplicationInfos().forEach(application -> listener.applicationActivated(superModel, application));
+
+            if (superModel.isComplete()) {
+                listener.notifyOfCompleteness(superModel);
+            }
         }
     }
 
@@ -89,13 +97,19 @@ public class SuperModelManager implements SuperModelProvider {
                     .getForVersionOrLatest(Optional.empty(), Instant.now())
                     .toApplicationInfo();
 
-            SuperModel newSuperModel = this.superModelConfigProvider
-                    .getSuperModel()
-                    .cloneAndSetApplication(applicationInfo);
+            SuperModel oldSuperModel = superModelConfigProvider.getSuperModel();
+            SuperModel newSuperModel = oldSuperModel
+                    .cloneAndSetApplication(applicationInfo, isComplete(oldSuperModel));
+
             generationCounter.increment();
             makeNewSuperModelConfigProvider(newSuperModel);
-            listeners.stream().forEach(listener ->
-                    listener.applicationActivated(newSuperModel, applicationInfo));
+            listeners.forEach(listener -> listener.applicationActivated(newSuperModel, applicationInfo));
+
+            if (!oldSuperModel.isComplete() && newSuperModel.isComplete()) {
+                for (var listener : listeners) {
+                    listener.notifyOfCompleteness(newSuperModel);
+                }
+            }
         }
     }
 
@@ -106,11 +120,36 @@ public class SuperModelManager implements SuperModelProvider {
                     .cloneAndRemoveApplication(applicationId);
             generationCounter.increment();
             makeNewSuperModelConfigProvider(newSuperModel);
-            listeners.stream().forEach(listener ->
-                    listener.applicationRemoved(newSuperModel, applicationId));
+            listeners.forEach(listener -> listener.applicationRemoved(newSuperModel, applicationId));
         }
     }
 
+    public void setBootstrapApplicationSet(Set<ApplicationId> bootstrapApplicationSet) {
+        synchronized (monitor) {
+            this.bootstrapApplicationSet = Optional.of(bootstrapApplicationSet);
+
+            SuperModel superModel = superModelConfigProvider.getSuperModel();
+            if (!superModel.isComplete() && isComplete(superModel)) {
+                // We do NOT increment the generation since completeness is not part of the config:
+                // generationCounter.increment()
+
+                SuperModel newSuperModel = superModel.cloneAsComplete();
+                makeNewSuperModelConfigProvider(newSuperModel);
+                listeners.forEach(listener -> listener.notifyOfCompleteness(newSuperModel));
+            }
+        }
+    }
+
+    /** Returns freshly calculated value of isComplete. */
+    private boolean isComplete(SuperModel currentSuperModel) {
+        if (currentSuperModel.isComplete()) return true;
+        if (bootstrapApplicationSet.isEmpty()) return false;
+
+        Set<ApplicationId> currentApplicationIds = superModelConfigProvider.getSuperModel().getApplicationIds();
+        if (currentApplicationIds.size() < bootstrapApplicationSet.get().size()) return false;
+        return currentApplicationIds.containsAll(bootstrapApplicationSet.get());
+    }
+
     private void makeNewSuperModelConfigProvider(SuperModel newSuperModel) {
         generation = masterGeneration + generationCounter.get();
         superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
index b7486dc7951..561422c1cf8 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
@@ -55,7 +55,7 @@ public class SuperModelControllerTest {
         ApplicationId app = ApplicationId.from(TenantName.from("a"),
                                                ApplicationName.from("foo"), InstanceName.defaultName());
         models.put(app, new ApplicationInfo(app, 4L, new VespaModel(FilesApplicationPackage.fromFile(testApp))));
-        SuperModel superModel = new SuperModel(models);
+        SuperModel superModel = new SuperModel(models, true);
         handler = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
     }
     
@@ -98,7 +98,7 @@ public class SuperModelControllerTest {
         models.put(advanced, createApplicationInfo(testApp2, advanced, 4L));
         models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L));
 
-        SuperModel superModel = new SuperModel(models);
+        SuperModel superModel = new SuperModel(models, true);
         SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
         LbServicesConfig.Builder lb = new LbServicesConfig.Builder();
         han.getSuperModel().getConfig(lb);
@@ -126,7 +126,7 @@ public class SuperModelControllerTest {
         models.put(advanced, createApplicationInfo(testApp2, advanced, 4L));
         models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L));
 
-        SuperModel superModel = new SuperModel(models);
+        SuperModel superModel = new SuperModel(models, true);
         SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
         LbServicesConfig.Builder lb = new LbServicesConfig.Builder();
         han.getSuperModel().getConfig(lb);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
index 3392569d1f2..4d04409aaf0 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
@@ -25,15 +25,20 @@ public class InfrastructureProvisioner extends Maintainer {
         this.infraDeployer = infraDeployer;
     }
 
+    public void maintainButThrowOnException() {
+        try {
+            infraDeployer.activateAllSupportedInfraApplications(true);
+        } catch (RuntimeException e) {
+            logger.log(LogLevel.INFO, "Failed to deploy supported infrastructure applications, " +
+                    "will sleep 30s before propagating failure, to allow inspection of zk",
+                    e.getMessage());
+            try { Thread.sleep(30_000); } catch (InterruptedException ignored) { }
+            throw e;
+        }
+    }
+
     @Override
     protected void maintain() {
-        infraDeployer.getSupportedInfraDeployments().forEach((application, deployment) -> {
-            try {
-                deployment.activate();
-            } catch (RuntimeException e) {
-                logger.log(LogLevel.INFO, "Failed to activate " + application, e);
-                // loop around to activate the next application
-            }
-        });
+        infraDeployer.activateAllSupportedInfraApplications(false);
     }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 063b5ad2c2a..37620e17a95 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -87,7 +87,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval);
 
         // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now
-        infrastructureProvisioner.maintain();
+        infrastructureProvisioner.maintainButThrowOnException();
     }
 
     @Override
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
index a56ace07c82..1086a3a7cd9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
@@ -20,7 +20,6 @@ import com.yahoo.vespa.service.monitor.DuperModelInfraApi;
 import com.yahoo.vespa.service.monitor.InfraApplicationApi;
 
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -51,9 +50,22 @@ public class InfraDeployerImpl implements InfraDeployer {
     }
 
     @Override
-    public Map<ApplicationId, Deployment> getSupportedInfraDeployments() {
-        return duperModel.getSupportedInfraApplications().stream()
-                .collect(Collectors.toMap(InfraApplicationApi::getApplicationId, InfraDeployment::new));
+    public void activateAllSupportedInfraApplications(boolean propagateException) {
+        duperModel.getSupportedInfraApplications().forEach(api -> {
+            var application = api.getApplicationId();
+            var deployment = new InfraDeployment(api);
+            try {
+                deployment.activate();
+            } catch (RuntimeException e) {
+                logger.log(LogLevel.INFO, "Failed to activate " + application, e);
+                if (propagateException) {
+                    throw e;
+                }
+                // loop around to activate the next application
+            }
+        });
+
+        duperModel.infraApplicationsIsNowComplete();
     }
 
     private class InfraDeployment implements Deployment {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
index 62e17ab63ad..915ef0d9125 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
@@ -49,4 +49,8 @@ public class MockDuperModel implements DuperModelInfraApi {
     public void infraApplicationRemoved(ApplicationId applicationId) {
         activeApps.remove(applicationId);
     }
+
+    @Override
+    public void infraApplicationsIsNowComplete() {
+    }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java
index e7bf76986ca..742a863fb38 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java
@@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployment;
 import com.yahoo.config.provision.InfraDeployer;
 
-import java.util.Map;
 import java.util.Optional;
 
 public class MockInfraDeployer implements InfraDeployer {
@@ -15,7 +14,5 @@ public class MockInfraDeployer implements InfraDeployer {
     }
 
     @Override
-    public Map<ApplicationId, Deployment> getSupportedInfraDeployments() {
-        return Map.of();
-    }
+    public void activateAllSupportedInfraApplications(boolean propagateException) { }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
index f559e9336c8..1cfb70560b8 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
@@ -4,9 +4,9 @@ package com.yahoo.vespa.service.duper;
 import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
@@ -22,12 +22,16 @@ public class DuperModel {
 
     private final Map<ApplicationId, ApplicationInfo> applications = new TreeMap<>();
     private final List<DuperModelListener> listeners = new ArrayList<>();
+    private boolean isComplete = false;
 
     public void registerListener(DuperModelListener listener) {
         applications.values().forEach(listener::applicationActivated);
         listeners.add(listener);
     }
 
+    public void setCompleteness(boolean isComplete) { this.isComplete = isComplete; }
+    public boolean isComplete() { return isComplete; }
+
     public boolean contains(ApplicationId applicationId) {
         return applications.containsKey(applicationId);
     }
@@ -47,6 +51,6 @@ public class DuperModel {
 
     public List<ApplicationInfo> getApplicationInfos() {
         logger.log(LogLevel.DEBUG, "Applications in duper model: " + applications.values().size());
-        return Collections.unmodifiableList(new ArrayList<>(applications.values()));
+        return List.copyOf(applications.values());
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
index 885368810a8..15c461c7f59 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
@@ -12,6 +12,8 @@ import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.service.monitor.DuperModelInfraApi;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
+import com.yahoo.vespa.service.monitor.DuperModelProvider;
 import com.yahoo.vespa.service.monitor.InfraApplicationApi;
 
 import java.util.ArrayList;
@@ -27,7 +29,7 @@ import java.util.stream.Stream;
 /**
  * @author hakonhall
  */
-public class DuperModelManager implements DuperModelInfraApi {
+public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi {
 
     // Infrastructure applications
     static final ControllerHostApplication controllerHostApplication = new ControllerHostApplication();
@@ -45,6 +47,8 @@ public class DuperModelManager implements DuperModelInfraApi {
     // The set of active infrastructure ApplicationInfo. Not all are necessarily in the DuperModel for historical reasons.
     private final Set<ApplicationId> activeInfraInfos = new HashSet<>(10);
 
+    private boolean superModelIsComplete = false;
+    private boolean infraApplicationsIsComplete = false;
 
     @Inject
     public DuperModelManager(ConfigserverConfig configServerConfig, FlagSource flagSource, SuperModelProvider superModelProvider) {
@@ -53,7 +57,7 @@ public class DuperModelManager implements DuperModelInfraApi {
              superModelProvider, new DuperModel(), flagSource, SystemName.from(configServerConfig.system()));
     }
 
-    /** For testing */
+    /** Non-private for testing */
     DuperModelManager(boolean multitenant, boolean isController, SuperModelProvider superModelProvider, DuperModel duperModel, FlagSource flagSource, SystemName system) {
         this.duperModel = duperModel;
 
@@ -86,6 +90,16 @@ public class DuperModelManager implements DuperModelInfraApi {
                     duperModel.remove(applicationId);
                 }
             }
+
+            @Override
+            public void notifyOfCompleteness(SuperModel superModel) {
+                synchronized (monitor) {
+                    if (!superModelIsComplete) {
+                        superModelIsComplete = true;
+                        maybeSetDuperModelAsComplete();
+                    }
+                }
+            }
         });
     }
 
@@ -93,6 +107,7 @@ public class DuperModelManager implements DuperModelInfraApi {
      * Synchronously call {@link DuperModelListener#applicationActivated(ApplicationInfo) listener.applicationActivated()}
      * for each currently active application, and forward future changes.
      */
+    @Override
     public void registerListener(DuperModelListener listener) {
         synchronized (monitor) {
             duperModel.registerListener(listener);
@@ -148,9 +163,25 @@ public class DuperModelManager implements DuperModelInfraApi {
         }
     }
 
+    @Override
+    public void infraApplicationsIsNowComplete() {
+        synchronized (monitor) {
+            if (!infraApplicationsIsComplete) {
+                infraApplicationsIsComplete = true;
+                maybeSetDuperModelAsComplete();
+            }
+        }
+    }
+
     public List<ApplicationInfo> getApplicationInfos() {
         synchronized (monitor) {
             return duperModel.getApplicationInfos();
         }
     }
+
+    private void maybeSetDuperModelAsComplete() {
+        if (superModelIsComplete && infraApplicationsIsComplete) {
+            duperModel.setCompleteness(true);
+        }
+    }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
index 3cc7010e209..d6e15f6af4e 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
@@ -92,6 +92,10 @@ public class HealthMonitorManager implements MonitorManager, HealthMonitorApi {
     }
 
     @Override
+    public void bootstrapComplete() {
+    }
+
+    @Override
     public ServiceStatusInfo getStatus(ApplicationId applicationId,
                                        ClusterId clusterId,
                                        ServiceType serviceType,
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java
index dd781a02cef..a7579d3f0da 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java
@@ -1,7 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.manager;
 
-import com.yahoo.vespa.service.duper.DuperModelListener;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
 import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
 
 /**
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
index 3490ad4a5d2..2aacc3eadac 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
@@ -51,4 +51,8 @@ public class UnionMonitorManager implements MonitorManager {
         slobrokMonitorManager.applicationRemoved(id);
         healthMonitorManager.applicationRemoved(id);
     }
+
+    @Override
+    public void bootstrapComplete() {
+    }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
index d08bba2bd3d..f9e47b6b80a 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
@@ -27,4 +27,7 @@ public interface DuperModelInfraApi {
 
     /** Update the DuperModel: A supported infrastructure application has been removed or is not active. */
     void infraApplicationRemoved(ApplicationId applicationId);
+
+    /** All infra applications that are supposed to activate on config server bootstrap has been activated. */
+    void infraApplicationsIsNowComplete();
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelListener.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelListener.java
index a969b6c3f40..f664e5246ca 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelListener.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelListener.java
@@ -1,34 +1,45 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.service.duper;
+package com.yahoo.vespa.service.monitor;
 
 import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.config.model.api.SuperModel;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.service.duper.DuperModel;
 
 /**
  * Interface for listening for changes to the {@link DuperModel}.
  *
- * @author hakon
+ * @author hakonhall
  */
 public interface DuperModelListener {
     /**
      * An application has been activated:
      *
      * <ul>
-     *     <li>A synthetic application like the config server application has been added/"activated"
+     *     <li>A synthetic application like the config server application has been added/activated
      *     <li>A super model application has been activated (see
      *     {@link com.yahoo.config.model.api.SuperModelListener#applicationActivated(SuperModel, ApplicationInfo)
      *     SuperModelListener}
      * </ul>
      *
-     * No other threads will concurrently call any methods on this interface.
+     * <p>No other threads will concurrently call any methods on this interface.</p>
      */
     void applicationActivated(ApplicationInfo application);
 
     /**
      * Application has been removed.
      *
-     * No other threads will concurrently call any methods on this interface.
+     * <p>No other threads will concurrently call any methods on this interface.</p>
      */
     void applicationRemoved(ApplicationId id);
+
+    /**
+     * During bootstrap of the config server, a number of applications are activated before
+     * resuming normal operations: The normal "tenant" application (making the super model) and
+     * the relevant infrastructure applications. Once all of these have been activated, this method
+     * will be invoked.
+     *
+     * <p>No other threads will concurrently call any methods on this interface.</p>
+     */
+    void bootstrapComplete();
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java
new file mode 100644
index 00000000000..a90fa418054
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java
@@ -0,0 +1,6 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor;
+
+public interface DuperModelProvider {
+    void registerListener(DuperModelListener listener);
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
index e3ea48ca9fe..e7a8d33ea14 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
@@ -73,6 +73,10 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager {
     }
 
     @Override
+    public void bootstrapComplete() {
+    }
+
+    @Override
     public List<Mirror.Entry> lookup(ApplicationId id, String pattern) {
         synchronized (monitor) {
             SlobrokMonitor slobrokMonitor = slobrokMonitors.get(id);
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
index 31fd266649a..dc90035be71 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.service.duper;
 
 import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
 import org.junit.Before;
 import org.junit.Test;
 
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java
index 0f7c0dde357..3fb10f1f24e 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java
@@ -47,7 +47,7 @@ public class ExampleModel {
 
         Map<ApplicationId, ApplicationInfo> applicationInfos = new HashMap<>();
         applicationInfos.put(applicationInfo.getApplicationId(), applicationInfo);
-        return new SuperModel(applicationInfos);
+        return new SuperModel(applicationInfos, true);
     }
 
     public static ApplicationBuilder createApplication(String tenant,