Revert "Revert "Use VIP status file for config server (and controller) in hosted""

4 files changed, 166 insertions, 39 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
index 991cd99968d..e8a5f1708a1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
@@ -38,7 +38,9 @@ import static com.yahoo.vespa.config.server.ConfigServerBootstrap.RedeployingApp
  *
  * If config server has been upgraded to a new version since the last time it was running it will redeploy all
  * applications. If that is done successfully the RPC server will start and the health status code will change from
- * 'initializing' to 'up' and the config server will be put into rotation (start serving status.html with 200 OK)
+ * 'initializing' to 'up'. If VIP status mode is VIP_STATUS_PROGRAMMATICALLY the config server
+ * will be put into rotation (start serving status.html with 200 OK), if the mode is VIP_STATUS_FILE a VIP status
+ * file is created or removed ny some external pgrogram based on the health status code.
  *
  * @author Ulf Lilleengen
  * @author hmusum
@@ -50,6 +52,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
     // INITIALIZE_ONLY is for testing only
     enum Mode {BOOTSTRAP_IN_CONSTRUCTOR, BOOTSTRAP_IN_SEPARATE_THREAD, INITIALIZE_ONLY}
     enum RedeployingApplicationsFails {EXIT_JVM, CONTINUE}
+    enum VipStatusMode {VIP_STATUS_FILE, VIP_STATUS_PROGRAMMATICALLY}
 
     private final ApplicationRepository applicationRepository;
     private final RpcServer server;
@@ -72,18 +75,22 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
                 Flags.CONFIG_SERVER_BOOTSTRAP_IN_SEPARATE_THREAD.bindTo(flagSource).value()
                      ? Mode.BOOTSTRAP_IN_SEPARATE_THREAD
                      : Mode.BOOTSTRAP_IN_CONSTRUCTOR,
-             EXIT_JVM);
+             EXIT_JVM,
+             applicationRepository.configserverConfig().hostedVespa()
+                     ? VipStatusMode.VIP_STATUS_FILE
+                     : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY);
     }
 
     // For testing only
     ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState,
-                          StateMonitor stateMonitor, VipStatus vipStatus, Mode mode) {
-        this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE);
+                          StateMonitor stateMonitor, VipStatus vipStatus, Mode mode,  VipStatusMode vipStatusMode) {
+        this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode);
     }
 
     private ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server,
                                   VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus,
-                                  Mode mode, RedeployingApplicationsFails exitIfRedeployingApplicationsFails) {
+                                  Mode mode, RedeployingApplicationsFails exitIfRedeployingApplicationsFails,
+                                  VipStatusMode vipStatusMode) {
         this.applicationRepository = applicationRepository;
         this.server = server;
         this.versionState = versionState;
@@ -94,8 +101,8 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
         this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails());
         this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails;
         rpcServerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server RPC server"));
-        initializing(); // Initially take server out of rotation
-        log.log(LogLevel.INFO, "Mode: " + mode);
+        log.log(LogLevel.INFO, "Bootstrap mode: " + mode + ", VIP status mode: " + vipStatusMode);
+        initializing(vipStatusMode);
         switch (mode) {
             case BOOTSTRAP_IN_SEPARATE_THREAD:
                 this.serverThread = Optional.of(new Thread(this, "config server bootstrap thread"));
@@ -109,7 +116,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
                 this.serverThread = Optional.empty();
                 break;
             default:
-                throw new IllegalArgumentException("Unknown mode " + mode + ", legal values: " + Arrays.toString(Mode.values()));
+                throw new IllegalArgumentException("Unknown bootstrap mode " + mode + ", legal values: " + Arrays.toString(Mode.values()));
         }
     }
 
@@ -145,7 +152,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
 
     public void start() {
         if (versionState.isUpgraded()) {
-            log.log(LogLevel.INFO, "Configserver upgrading from " + versionState.storedVersion() + " to "
+            log.log(LogLevel.INFO, "Config server upgrading from " + versionState.storedVersion() + " to "
                     + versionState.currentVersion() + ". Redeploying all applications");
             try {
                 if ( ! redeployAllApplications()) {
@@ -178,10 +185,10 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
         vipStatus.setInRotation(false);
     }
 
-    private void initializing() {
-        // This is default value (from config), so not strictly necessary
+    private void initializing(VipStatusMode vipStatusMode) {
         stateMonitor.status(StateMonitor.Status.initializing);
-        vipStatus.setInRotation(false);
+        if (vipStatusMode == VipStatusMode.VIP_STATUS_PROGRAMMATICALLY)
+            vipStatus.setInRotation(false);
     }
 
     private void startRpcServer() {
diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml
index 0a8fa6e483e..b2c426e707b 100644
--- a/configserver/src/main/resources/configserver-app/services.xml
+++ b/configserver/src/main/resources/configserver-app/services.xml
@@ -6,10 +6,6 @@
       <initialStatus>initializing</initialStatus>
     </config>
 
-    <config name="container.core.vip-status">
-      <initiallyInRotation>false</initiallyInRotation>
-    </config>
-
     <accesslog type="vespa" fileNamePattern="logs/vespa/configserver/access.log.%Y%m%d%H%M%S" compressOnRotation="true" symlinkName="access.log" />
     <preprocess:include file='access-logging.xml' required='false' />
 
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
index 039146c3cb9..281933e8943 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
@@ -11,6 +11,9 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.container.QrSearchersConfig;
+import com.yahoo.container.core.VipStatusConfig;
+import com.yahoo.container.handler.ClustersStatus;
 import com.yahoo.container.handler.VipStatus;
 import com.yahoo.container.jdisc.config.HealthMonitorConfig;
 import com.yahoo.container.jdisc.state.StateMonitor;
@@ -38,6 +41,10 @@ import java.util.List;
 import java.util.Optional;
 import java.util.function.BooleanSupplier;
 
+import static com.yahoo.vespa.config.server.ConfigServerBootstrap.Mode.BOOTSTRAP_IN_SEPARATE_THREAD;
+import static com.yahoo.vespa.config.server.ConfigServerBootstrap.Mode.INITIALIZE_ONLY;
+import static com.yahoo.vespa.config.server.ConfigServerBootstrap.VipStatusMode.VIP_STATUS_FILE;
+import static com.yahoo.vespa.config.server.ConfigServerBootstrap.VipStatusMode.VIP_STATUS_PROGRAMMATICALLY;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -63,12 +70,12 @@ public class ConfigServerBootstrapTest {
         assertTrue(versionState.isUpgraded());
 
         RpcServer rpcServer = createRpcServer(configserverConfig);
-        VipStatus vipStatus = new VipStatus();
         // Take a host away so that there are too few for the application, to verify we can still bootstrap
         provisioner.allocations().values().iterator().next().remove(0);
+        VipStatus vipStatus = createVipStatus(VIP_STATUS_PROGRAMMATICALLY);
         ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer,
-                                                                    versionState, createStateMonitor(), vipStatus,
-                                                                    ConfigServerBootstrap.Mode.INITIALIZE_ONLY);
+                                                                    versionState, createStateMonitor(),
+                                                                    vipStatus, INITIALIZE_ONLY, VIP_STATUS_PROGRAMMATICALLY);
         assertFalse(vipStatus.isInRotation());
         bootstrap.start();
         waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running");
@@ -81,6 +88,33 @@ public class ConfigServerBootstrapTest {
         assertFalse(vipStatus.isInRotation());
     }
 
+    // Just tests setup, the actual response of accessing /status.html depends on the status
+    // file existing or not, which cannot be tested here
+    @Test
+    public void testBootstrapWithVipStatusFile() throws Exception {
+        ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder);
+        InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3");
+        DeployTester tester = new DeployTester(configserverConfig, provisioner);
+        tester.deployApp("src/test/apps/hosted/");
+
+        File versionFile = temporaryFolder.newFile();
+        VersionState versionState = new VersionState(versionFile);
+        assertTrue(versionState.isUpgraded());
+
+        RpcServer rpcServer = createRpcServer(configserverConfig);
+        VipStatus vipStatus = createVipStatus(VIP_STATUS_FILE);
+        ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer,
+                                                                    versionState, createStateMonitor(),
+                                                                    vipStatus, INITIALIZE_ONLY, VIP_STATUS_FILE);
+        assertTrue(vipStatus.isInRotation()); // default is in rotation when using status file
+
+        bootstrap.start();
+        waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running");
+        waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'");
+        waitUntil(vipStatus::isInRotation, "failed waiting for server to be in rotation");
+        bootstrap.deconstruct();
+    }
+
     @Test
     public void testBootstrapWhenRedeploymentFails() throws Exception {
         ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder);
@@ -98,10 +132,10 @@ public class ConfigServerBootstrapTest {
                                            .resolve("sessions/2/services.xml"));
 
         RpcServer rpcServer = createRpcServer(configserverConfig);
-        VipStatus vipStatus = new VipStatus();
+        VipStatus vipStatus = createVipStatus(VIP_STATUS_PROGRAMMATICALLY);
         ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState,
-                                                                    createStateMonitor(), vipStatus,
-                                                                    ConfigServerBootstrap.Mode.INITIALIZE_ONLY);
+                                                                    createStateMonitor(),
+                                                                    vipStatus, INITIALIZE_ONLY, VIP_STATUS_PROGRAMMATICALLY);
         assertFalse(vipStatus.isInRotation());
         // Call method directly, to be sure that it is finished redeploying all applications and we can check status
         bootstrap.start();
@@ -140,15 +174,13 @@ public class ConfigServerBootstrapTest {
         curator.set(Path.fromString("/config/v2/tenants/" + applicationId.tenant().value() + "/sessions/2/version"), Utf8.toBytes("1.2.2"));
 
         RpcServer rpcServer = createRpcServer(configserverConfig);
-        VipStatus vipStatus = new VipStatus();
+        VipStatus vipStatus = createVipStatus(VIP_STATUS_PROGRAMMATICALLY);
         ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState,
                                                                     createStateMonitor(), vipStatus,
-                                                                    ConfigServerBootstrap.Mode.BOOTSTRAP_IN_SEPARATE_THREAD);
-
+                                                                    BOOTSTRAP_IN_SEPARATE_THREAD, VIP_STATUS_PROGRAMMATICALLY);
         waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running");
         waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'");
         waitUntil(vipStatus::isInRotation, "failed waiting for server to be in rotation");
-        bootstrap.deconstruct();
     }
 
     private void waitUntil(BooleanSupplier booleanSupplier, String messageIfWaitingFails) throws InterruptedException {
@@ -197,6 +229,15 @@ public class ConfigServerBootstrapTest {
         return new Host(hostname, Collections.emptyList(), Optional.empty(), Optional.of(com.yahoo.component.Version.fromString(version)));
     }
 
+    private VipStatus createVipStatus(ConfigServerBootstrap.VipStatusMode vipStatusMode) throws IOException {
+        return new VipStatus(new QrSearchersConfig.Builder().build(),
+                             new VipStatusConfig.Builder()
+                                     .initiallyInRotation(vipStatusMode == VIP_STATUS_FILE)
+                                     .statusfile(temporaryFolder.newFile().getAbsolutePath())
+                                     .accessdisk(vipStatusMode == VIP_STATUS_FILE)
+                                     .build(),
+                             new ClustersStatus());
+    }
 
     public static class MockRpc extends com.yahoo.vespa.config.server.rpc.MockRpc {
 
diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java
index a2e1e36e48f..06e49843eac 100644
--- a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java
+++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java
@@ -15,14 +15,19 @@ import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.config.model.ApplicationConfigProducerRoot;
 import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.model.api.ConfigServerSpec;
+import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.config.model.application.provider.FilesApplicationPackage;
 import com.yahoo.config.model.application.provider.StaticConfigDefinitionRepo;
 import com.yahoo.config.model.builder.xml.ConfigModelId;
 import com.yahoo.config.model.builder.xml.XmlHelper;
 import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Rotation;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.container.di.config.SubscriberFactory;
@@ -43,12 +48,14 @@ import org.w3c.dom.Element;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import static com.yahoo.collections.CollectionUtil.first;
@@ -81,6 +88,7 @@ public class StandaloneContainerApplication implements Application {
     private final Application configuredApplication;
     private final Container container;
 
+    @SuppressWarnings("WeakerAccess")
     @Inject
     public StandaloneContainerApplication(Injector injector) {
         this.injector = injector;
@@ -212,9 +220,9 @@ public class StandaloneContainerApplication implements Application {
         }
     }
 
-    private static ContainerModelBuilder newContainerModelBuilder(Networking networkingOption) {
+    private static ContainerModelBuilder newContainerModelBuilder(Networking networkingOption, CloudConfigInstallVariables cloudConfigInstallVariables) {
         return isConfigServer() ?
-                new ConfigServerContainerModelBuilder(new CloudConfigInstallVariables()) :
+                new ConfigServerContainerModelBuilder(cloudConfigInstallVariables) :
                 new ContainerModelBuilder(true, networkingOption);
     }
 
@@ -238,14 +246,16 @@ public class StandaloneContainerApplication implements Application {
                 .includeSourceFiles(true).preprocessedDir(preprocessedApplicationDir).build();
         ApplicationPackage applicationPackage = rawApplicationPackage.preprocess(getZone(), logger);
         validateApplication(applicationPackage);
-        DeployState deployState = createDeployState(applicationPackage, fileRegistry, logger);
+        CloudConfigInstallVariables cloudConfigInstallVariables = new CloudConfigInstallVariables();
+        DeployState deployState = createDeployState(applicationPackage, fileRegistry, logger, cloudConfigInstallVariables);
 
         VespaModel root = VespaModel.createIncomplete(deployState);
         ApplicationConfigProducerRoot vespaRoot = new ApplicationConfigProducerRoot(root, "vespa", deployState.getDocumentModel(),
                 deployState.getVespaVersion(), deployState.getProperties().applicationId());
 
         Element spec = containerRootElement(applicationPackage);
-        ContainerModel containerModel = newContainerModelBuilder(networkingOption).build(deployState, root, configModelRepo, vespaRoot, spec);
+        ContainerModel containerModel = newContainerModelBuilder(networkingOption, cloudConfigInstallVariables)
+                .build(deployState, root, configModelRepo, vespaRoot, spec);
         containerModel.getCluster().prepare(deployState);
         initializeContainerModel(containerModel, configModelRepo);
         Container container = first(containerModel.getCluster().getContainers());
@@ -274,19 +284,14 @@ public class StandaloneContainerApplication implements Application {
         return new Zone(system, environment, region);
     }
 
-    private static DeployState createDeployState(ApplicationPackage applicationPackage, FileRegistry fileRegistry, DeployLogger logger) {
+    private static DeployState createDeployState(ApplicationPackage applicationPackage, FileRegistry fileRegistry,
+                                                 DeployLogger logger, CloudConfigInstallVariables cloudConfigInstallVariables) {
         DeployState.Builder builder = new DeployState.Builder()
                 .applicationPackage(applicationPackage)
                 .fileRegistry(fileRegistry)
                 .deployLogger(logger)
-                .configDefinitionRepo(configDefinitionRepo);
-
-        /* Temporarily disable until we know how status.html is updated for config servers/controllers
-        if (isConfigServer())
-            builder.properties(new DeployProperties.Builder()
-                                       .hostedVespa(new CloudConfigInstallVariables().hostedVespa().orElse(Boolean.FALSE))
-                                       .build());
-        */
+                .configDefinitionRepo(configDefinitionRepo)
+                .properties(new StandaloneContainerModelContextProperties(cloudConfigInstallVariables));
 
         return builder.build();
     }
@@ -340,4 +345,82 @@ public class StandaloneContainerApplication implements Application {
         return Optional.ofNullable(System.getProperty(name)); // for unit testing
     }
 
+    private static class StandaloneContainerModelContextProperties implements ModelContext.Properties {
+        private final CloudConfigInstallVariables cloudConfigInstallVariables;
+
+        StandaloneContainerModelContextProperties(CloudConfigInstallVariables cloudConfigInstallVariables) {
+            this.cloudConfigInstallVariables = cloudConfigInstallVariables;
+        }
+
+        @Override
+        public boolean multitenant() {
+            return cloudConfigInstallVariables.multiTenant().orElse(Boolean.FALSE);
+        }
+
+        @Override
+        public ApplicationId applicationId() {
+            return ApplicationId.defaultId();
+        }
+
+        @Override
+        public List<ConfigServerSpec> configServerSpecs() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public HostName loadBalancerName() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public URI ztsUrl() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String athenzDnsSuffix() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean hostedVespa() {
+            return cloudConfigInstallVariables.hostedVespa().orElse(Boolean.FALSE);
+        }
+
+        @Override
+        public Zone zone() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Set<Rotation> rotations() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean isBootstrap() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean isFirstTimeDeployment() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean useDedicatedNodeForLogserver() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean useFdispatchByDefault() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean useAdaptiveDispatch() {
+            throw new UnsupportedOperationException();
+        }
+    }
+
 }