Use VIP status file for config server (and controller) in hosted

The code for setting up VIP status is already in place in config model,
but depends on value of hostedVespa() in ModelContext.Properties, so have to
make sure that is set for standalone container as well.
Removing config override for vip status, as setting VIP in or out of rotation when not
using a VIP status file is handled when bootstrapping now.

4 files changed, 167 insertions, 40 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
index 991cd99968d..88745e596e0 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
@@ -37,8 +37,10 @@ import static com.yahoo.vespa.config.server.ConfigServerBootstrap.RedeployingApp
  * Main component that bootstraps and starts config server threads.
  *
  * If config server has been upgraded to a new version since the last time it was running it will redeploy all
- * applications. If that is done successfully the RPC server will start and the health status code will change from
- * 'initializing' to 'up' and the config server will be put into rotation (start serving status.html with 200 OK)
+ * applications. If that is done successfully the RPC server the health status code will change from
+ * 'initializing' to 'up'. If VIP status mode is VIP_STATUS_PROGRAMMATICALLY the config server
+ * will be put into rotation (start serving status.html with 200 OK), if the mode is VIP_STATUS_FILE a VIP status
+ * file is created or removed ny some external pgrogram based on the health status code.
  *
  * @author Ulf Lilleengen
  * @author hmusum
@@ -50,6 +52,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
     // INITIALIZE_ONLY is for testing only
     enum Mode {BOOTSTRAP_IN_CONSTRUCTOR, BOOTSTRAP_IN_SEPARATE_THREAD, INITIALIZE_ONLY}
     enum RedeployingApplicationsFails {EXIT_JVM, CONTINUE}
+    enum VipStatusMode {VIP_STATUS_FILE, VIP_STATUS_PROGRAMMATICALLY}
 
     private final ApplicationRepository applicationRepository;
     private final RpcServer server;
@@ -72,18 +75,22 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
                 Flags.CONFIG_SERVER_BOOTSTRAP_IN_SEPARATE_THREAD.bindTo(flagSource).value()
                      ? Mode.BOOTSTRAP_IN_SEPARATE_THREAD
                      : Mode.BOOTSTRAP_IN_CONSTRUCTOR,
-             EXIT_JVM);
+             EXIT_JVM,
+             applicationRepository.configserverConfig().hostedVespa()
+                     ? VipStatusMode.VIP_STATUS_FILE
+                     : VipStatusMode.VIP_STATUS_PROGRAMMATICALLY);
     }
 
     // For testing only
     ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState,
-                          StateMonitor stateMonitor, VipStatus vipStatus, Mode mode) {
-        this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE);
+                          StateMonitor stateMonitor, VipStatus vipStatus, Mode mode,  VipStatusMode vipStatusMode) {
+        this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode);
     }
 
     private ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server,
                                   VersionState versionState, StateMonitor stateMonitor, VipStatus vipStatus,
-                                  Mode mode, RedeployingApplicationsFails exitIfRedeployingApplicationsFails) {
+                                  Mode mode, RedeployingApplicationsFails exitIfRedeployingApplicationsFails,
+                                  VipStatusMode vipStatusMode) {
         this.applicationRepository = applicationRepository;
         this.server = server;
         this.versionState = versionState;
@@ -94,8 +101,8 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
         this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails());
         this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails;
         rpcServerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server RPC server"));
-        initializing(); // Initially take server out of rotation
-        log.log(LogLevel.INFO, "Mode: " + mode);
+        log.log(LogLevel.INFO, "Bootstrap mode: " + mode + ", VIP status mode: " + vipStatusMode);
+        initializing(vipStatusMode);
         switch (mode) {
             case BOOTSTRAP_IN_SEPARATE_THREAD:
                 this.serverThread = Optional.of(new Thread(this, "config server bootstrap thread"));
@@ -109,7 +116,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
                 this.serverThread = Optional.empty();
                 break;
             default:
-                throw new IllegalArgumentException("Unknown mode " + mode + ", legal values: " + Arrays.toString(Mode.values()));
+                throw new IllegalArgumentException("Unknown bootstrap mode " + mode + ", legal values: " + Arrays.toString(Mode.values()));
         }
     }
 
@@ -145,7 +152,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
 
     public void start() {
         if (versionState.isUpgraded()) {
-            log.log(LogLevel.INFO, "Configserver upgrading from " + versionState.storedVersion() + " to "
+            log.log(LogLevel.INFO, "Config server upgrading from " + versionState.storedVersion() + " to "
                     + versionState.currentVersion() + ". Redeploying all applications");
             try {
                 if ( ! redeployAllApplications()) {
@@ -178,10 +185,10 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
         vipStatus.setInRotation(false);
     }
 
-    private void initializing() {
-        // This is default value (from config), so not strictly necessary
+    private void initializing(VipStatusMode vipStatusMode) {
         stateMonitor.status(StateMonitor.Status.initializing);
-        vipStatus.setInRotation(false);
+        if (vipStatusMode == VipStatusMode.VIP_STATUS_PROGRAMMATICALLY)
+            vipStatus.setInRotation(false);
     }
 
     private void startRpcServer() {
diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml
index 0a8fa6e483e..b2c426e707b 100644
--- a/configserver/src/main/resources/configserver-app/services.xml
+++ b/configserver/src/main/resources/configserver-app/services.xml
@@ -6,10 +6,6 @@
       <initialStatus>initializing</initialStatus>
     </config>
 
-    <config name="container.core.vip-status">
-      <initiallyInRotation>false</initiallyInRotation>
-    </config>
-
     <accesslog type="vespa" fileNamePattern="logs/vespa/configserver/access.log.%Y%m%d%H%M%S" compressOnRotation="true" symlinkName="access.log" />
     <preprocess:include file='access-logging.xml' required='false' />
 
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
index 039146c3cb9..281933e8943 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
@@ -11,6 +11,9 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.container.QrSearchersConfig;
+import com.yahoo.container.core.VipStatusConfig;
+import com.yahoo.container.handler.ClustersStatus;
 import com.yahoo.container.handler.VipStatus;
 import com.yahoo.container.jdisc.config.HealthMonitorConfig;
 import com.yahoo.container.jdisc.state.StateMonitor;
@@ -38,6 +41,10 @@ import java.util.List;
 import java.util.Optional;
 import java.util.function.BooleanSupplier;
 
+import static com.yahoo.vespa.config.server.ConfigServerBootstrap.Mode.BOOTSTRAP_IN_SEPARATE_THREAD;
+import static com.yahoo.vespa.config.server.ConfigServerBootstrap.Mode.INITIALIZE_ONLY;
+import static com.yahoo.vespa.config.server.ConfigServerBootstrap.VipStatusMode.VIP_STATUS_FILE;
+import static com.yahoo.vespa.config.server.ConfigServerBootstrap.VipStatusMode.VIP_STATUS_PROGRAMMATICALLY;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -63,12 +70,12 @@ public class ConfigServerBootstrapTest {
         assertTrue(versionState.isUpgraded());
 
         RpcServer rpcServer = createRpcServer(configserverConfig);
-        VipStatus vipStatus = new VipStatus();
         // Take a host away so that there are too few for the application, to verify we can still bootstrap
         provisioner.allocations().values().iterator().next().remove(0);
+        VipStatus vipStatus = createVipStatus(VIP_STATUS_PROGRAMMATICALLY);
         ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer,
-                                                                    versionState, createStateMonitor(), vipStatus,
-                                                                    ConfigServerBootstrap.Mode.INITIALIZE_ONLY);
+                                                                    versionState, createStateMonitor(),
+                                                                    vipStatus, INITIALIZE_ONLY, VIP_STATUS_PROGRAMMATICALLY);
         assertFalse(vipStatus.isInRotation());
         bootstrap.start();
         waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running");
@@ -81,6 +88,33 @@ public class ConfigServerBootstrapTest {
         assertFalse(vipStatus.isInRotation());
     }
 
+    // Just tests setup, the actual response of accessing /status.html depends on the status
+    // file existing or not, which cannot be tested here
+    @Test
+    public void testBootstrapWithVipStatusFile() throws Exception {
+        ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder);
+        InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3");
+        DeployTester tester = new DeployTester(configserverConfig, provisioner);
+        tester.deployApp("src/test/apps/hosted/");
+
+        File versionFile = temporaryFolder.newFile();
+        VersionState versionState = new VersionState(versionFile);
+        assertTrue(versionState.isUpgraded());
+
+        RpcServer rpcServer = createRpcServer(configserverConfig);
+        VipStatus vipStatus = createVipStatus(VIP_STATUS_FILE);
+        ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer,
+                                                                    versionState, createStateMonitor(),
+                                                                    vipStatus, INITIALIZE_ONLY, VIP_STATUS_FILE);
+        assertTrue(vipStatus.isInRotation()); // default is in rotation when using status file
+
+        bootstrap.start();
+        waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running");
+        waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'");
+        waitUntil(vipStatus::isInRotation, "failed waiting for server to be in rotation");
+        bootstrap.deconstruct();
+    }
+
     @Test
     public void testBootstrapWhenRedeploymentFails() throws Exception {
         ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder);
@@ -98,10 +132,10 @@ public class ConfigServerBootstrapTest {
                                            .resolve("sessions/2/services.xml"));
 
         RpcServer rpcServer = createRpcServer(configserverConfig);
-        VipStatus vipStatus = new VipStatus();
+        VipStatus vipStatus = createVipStatus(VIP_STATUS_PROGRAMMATICALLY);
         ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState,
-                                                                    createStateMonitor(), vipStatus,
-                                                                    ConfigServerBootstrap.Mode.INITIALIZE_ONLY);
+                                                                    createStateMonitor(),
+                                                                    vipStatus, INITIALIZE_ONLY, VIP_STATUS_PROGRAMMATICALLY);
         assertFalse(vipStatus.isInRotation());
         // Call method directly, to be sure that it is finished redeploying all applications and we can check status
         bootstrap.start();
@@ -140,15 +174,13 @@ public class ConfigServerBootstrapTest {
         curator.set(Path.fromString("/config/v2/tenants/" + applicationId.tenant().value() + "/sessions/2/version"), Utf8.toBytes("1.2.2"));
 
         RpcServer rpcServer = createRpcServer(configserverConfig);
-        VipStatus vipStatus = new VipStatus();
+        VipStatus vipStatus = createVipStatus(VIP_STATUS_PROGRAMMATICALLY);
         ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState,
                                                                     createStateMonitor(), vipStatus,
-                                                                    ConfigServerBootstrap.Mode.BOOTSTRAP_IN_SEPARATE_THREAD);
-
+                                                                    BOOTSTRAP_IN_SEPARATE_THREAD, VIP_STATUS_PROGRAMMATICALLY);
         waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running");
         waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'");
         waitUntil(vipStatus::isInRotation, "failed waiting for server to be in rotation");
-        bootstrap.deconstruct();
     }
 
     private void waitUntil(BooleanSupplier booleanSupplier, String messageIfWaitingFails) throws InterruptedException {
@@ -197,6 +229,15 @@ public class ConfigServerBootstrapTest {
         return new Host(hostname, Collections.emptyList(), Optional.empty(), Optional.of(com.yahoo.component.Version.fromString(version)));
     }
 
+    private VipStatus createVipStatus(ConfigServerBootstrap.VipStatusMode vipStatusMode) throws IOException {
+        return new VipStatus(new QrSearchersConfig.Builder().build(),
+                             new VipStatusConfig.Builder()
+                                     .initiallyInRotation(vipStatusMode == VIP_STATUS_FILE)
+                                     .statusfile(temporaryFolder.newFile().getAbsolutePath())
+                                     .accessdisk(vipStatusMode == VIP_STATUS_FILE)
+                                     .build(),
+                             new ClustersStatus());
+    }
 
     public static class MockRpc extends com.yahoo.vespa.config.server.rpc.MockRpc {
 
diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java
index a2e1e36e48f..0c671071ac5 100644
--- a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java
+++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java
@@ -15,14 +15,19 @@ import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.config.model.ApplicationConfigProducerRoot;
 import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.model.api.ConfigServerSpec;
+import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.config.model.application.provider.FilesApplicationPackage;
 import com.yahoo.config.model.application.provider.StaticConfigDefinitionRepo;
 import com.yahoo.config.model.builder.xml.ConfigModelId;
 import com.yahoo.config.model.builder.xml.XmlHelper;
 import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Rotation;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.container.di.config.SubscriberFactory;
@@ -43,12 +48,14 @@ import org.w3c.dom.Element;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import static com.yahoo.collections.CollectionUtil.first;
@@ -81,6 +88,7 @@ public class StandaloneContainerApplication implements Application {
     private final Application configuredApplication;
     private final Container container;
 
+    @SuppressWarnings("WeakerAccess")
     @Inject
     public StandaloneContainerApplication(Injector injector) {
         this.injector = injector;
@@ -212,9 +220,9 @@ public class StandaloneContainerApplication implements Application {
         }
     }
 
-    private static ContainerModelBuilder newContainerModelBuilder(Networking networkingOption) {
+    private static ContainerModelBuilder newContainerModelBuilder(Networking networkingOption, CloudConfigInstallVariables cloudConfigInstallVariables) {
         return isConfigServer() ?
-                new ConfigServerContainerModelBuilder(new CloudConfigInstallVariables()) :
+                new ConfigServerContainerModelBuilder(cloudConfigInstallVariables) :
                 new ContainerModelBuilder(true, networkingOption);
     }
 
@@ -238,14 +246,16 @@ public class StandaloneContainerApplication implements Application {
                 .includeSourceFiles(true).preprocessedDir(preprocessedApplicationDir).build();
         ApplicationPackage applicationPackage = rawApplicationPackage.preprocess(getZone(), logger);
         validateApplication(applicationPackage);
-        DeployState deployState = createDeployState(applicationPackage, fileRegistry, logger);
+        CloudConfigInstallVariables cloudConfigInstallVariables = new CloudConfigInstallVariables();
+        DeployState deployState = createDeployState(applicationPackage, fileRegistry, logger, cloudConfigInstallVariables);
 
         VespaModel root = VespaModel.createIncomplete(deployState);
         ApplicationConfigProducerRoot vespaRoot = new ApplicationConfigProducerRoot(root, "vespa", deployState.getDocumentModel(),
                 deployState.getVespaVersion(), deployState.getProperties().applicationId());
 
         Element spec = containerRootElement(applicationPackage);
-        ContainerModel containerModel = newContainerModelBuilder(networkingOption).build(deployState, root, configModelRepo, vespaRoot, spec);
+        ContainerModel containerModel = newContainerModelBuilder(networkingOption, cloudConfigInstallVariables)
+                .build(deployState, root, configModelRepo, vespaRoot, spec);
         containerModel.getCluster().prepare(deployState);
         initializeContainerModel(containerModel, configModelRepo);
         Container container = first(containerModel.getCluster().getContainers());
@@ -274,19 +284,14 @@ public class StandaloneContainerApplication implements Application {
         return new Zone(system, environment, region);
     }
 
-    private static DeployState createDeployState(ApplicationPackage applicationPackage, FileRegistry fileRegistry, DeployLogger logger) {
+    private static DeployState createDeployState(ApplicationPackage applicationPackage, FileRegistry fileRegistry,
+                                                 DeployLogger logger, CloudConfigInstallVariables cloudConfigInstallVariables) {
         DeployState.Builder builder = new DeployState.Builder()
                 .applicationPackage(applicationPackage)
                 .fileRegistry(fileRegistry)
                 .deployLogger(logger)
-                .configDefinitionRepo(configDefinitionRepo);
-
-        /* Temporarily disable until we know how status.html is updated for config servers/controllers
-        if (isConfigServer())
-            builder.properties(new DeployProperties.Builder()
-                                       .hostedVespa(new CloudConfigInstallVariables().hostedVespa().orElse(Boolean.FALSE))
-                                       .build());
-        */
+                .configDefinitionRepo(configDefinitionRepo)
+                .properties(new StandaloneContainerModelContextProperties(cloudConfigInstallVariables));
 
         return builder.build();
     }
@@ -340,4 +345,82 @@ public class StandaloneContainerApplication implements Application {
         return Optional.ofNullable(System.getProperty(name)); // for unit testing
     }
 
+    private static class StandaloneContainerModelContextProperties implements ModelContext.Properties {
+        private final CloudConfigInstallVariables cloudConfigInstallVariables;
+
+        StandaloneContainerModelContextProperties(CloudConfigInstallVariables cloudConfigInstallVariables) {
+            this.cloudConfigInstallVariables = cloudConfigInstallVariables;
+        }
+
+        @Override
+        public boolean multitenant() {
+            return cloudConfigInstallVariables.multiTenant().orElse(Boolean.FALSE);
+        }
+
+        @Override
+        public ApplicationId applicationId() {
+            return ApplicationId.defaultId();
+        }
+
+        @Override
+        public List<ConfigServerSpec> configServerSpecs() {
+            return null;
+        }
+
+        @Override
+        public HostName loadBalancerName() {
+            return null;
+        }
+
+        @Override
+        public URI ztsUrl() {
+            return null;
+        }
+
+        @Override
+        public String athenzDnsSuffix() {
+            return null;
+        }
+
+        @Override
+        public boolean hostedVespa() {
+            return cloudConfigInstallVariables.hostedVespa().orElse(Boolean.FALSE);
+        }
+
+        @Override
+        public Zone zone() {
+            return null;
+        }
+
+        @Override
+        public Set<Rotation> rotations() {
+            return null;
+        }
+
+        @Override
+        public boolean isBootstrap() {
+            return false;
+        }
+
+        @Override
+        public boolean isFirstTimeDeployment() {
+            return false;
+        }
+
+        @Override
+        public boolean useDedicatedNodeForLogserver() {
+            return false;
+        }
+
+        @Override
+        public boolean useFdispatchByDefault() {
+            return false;
+        }
+
+        @Override
+        public boolean useAdaptiveDispatch() {
+            return false;
+        }
+    }
+
 }