Allow file distribution rpc requests while bootstrapping

We do not want to server config requests while bootstrapping, since we
have not finished deploying all applications yet and don't want to serve
stale or empty config. File distributuion requests should be OK, though,
if we cannot serve the requests (unknown file, host is not allowed to
get file etc.), the client will try another config server.

4 files changed, 45 insertions, 18 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
index 4c2aa33a886..6ad6565e523 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
@@ -41,11 +41,12 @@ import static com.yahoo.vespa.config.server.ConfigServerBootstrap.RedeployingApp
 /**
  * Main component that bootstraps and starts config server threads.
  *
- * If config server has been upgraded to a new version since the last time it was running it will redeploy all
- * applications. If that is done successfully the RPC server will start and the health status code will change from
- * 'initializing' to 'up'. If VIP status mode is VIP_STATUS_PROGRAMMATICALLY the config server
- * will be put into rotation (start serving status.html with 200 OK), if the mode is VIP_STATUS_FILE a VIP status
- * file is created or removed by some external program based on the health status code.
+ * Starts RPC server without allowing config requests. If config server has been upgraded to a new version since the
+ * last time it was running it will redeploy all applications. If that is done successfully the RPC server will start
+ * allowing config requets and the health status code will change from 'initializing' to 'up'. If VIP status mode is
+ * VIP_STATUS_PROGRAMMATICALLY the config server will be put into rotation (start serving status.html with 200 OK),
+ * if the mode is VIP_STATUS_FILE a VIP status file is created or removed by some external program based on the
+ * health status code.
  *
  * @author Ulf Lilleengen
  * @author hmusum
@@ -138,6 +139,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
     }
 
     public void start() {
+        startRpcServerWithFileDistribution(); // No config requests allowed yet, will be allowed after bootstrapping done
         if (versionState.isUpgraded()) {
             log.log(Level.INFO, "Config server upgrading from " + versionState.storedVersion() + " to "
                     + versionState.currentVersion() + ". Redeploying all applications");
@@ -155,7 +157,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
             }
         }
         applicationRepository.bootstrappingDone();
-        startRpcServer();
+        allowConfigRpcRequests(server);
         up();
     }
 
@@ -177,7 +179,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
             vipStatus.setInRotation(false);
     }
 
-    private void startRpcServer() {
+    private void startRpcServerWithFileDistribution() {
         rpcServerExecutor.execute(server);
 
         Instant end = Instant.now().plus(Duration.ofSeconds(10));
@@ -193,6 +195,11 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
             throw new RuntimeException("RPC server not started in 10 seconds");
     }
 
+    private void allowConfigRpcRequests(RpcServer rpcServer) {
+        log.log(Level.INFO, "Allowing RPC config requests");
+        rpcServer.setUpGetConfigHandlers();
+    }
+
     private void redeployingApplicationsFailed() {
         if (exitIfRedeployingApplicationsFails == EXIT_JVM) System.exit(1);
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java
index 5b9b3077008..6d04a6ee8b4 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java
@@ -105,6 +105,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener {
     private final FileDownloader downloader;
     private volatile boolean allTenantsLoaded = false;
     private boolean isRunning = false;
+    boolean isServingConfigRequests = false;
 
     static class ApplicationState {
         private final AtomicLong activeGeneration = new AtomicLong(0);
@@ -143,7 +144,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener {
         this.rpcAuthorizer = rpcAuthorizer;
         downloader = fileServer.downloader();
         handlerProvider.setInstance(this);
-        setUpHandlers();
+        setUpFileDistributionHandlers();
     }
 
     private static int threadsToUse() {
@@ -211,11 +212,9 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener {
     }
 
     /**
-     * Set up RPC method handlers.
+     * Set up RPC method handlers, except handlers for getting config (see #setUpGetConfigHandlers())
      */
-    private void setUpHandlers() {
-        // The getConfig method in this class will handle RPC calls for getting config
-        getSupervisor().addMethod(JRTMethods.createConfigV3GetConfigMethod(this::getConfigV3));
+    private void setUpFileDistributionHandlers() {
         getSupervisor().addMethod(new Method("ping", "", "i", this::ping)
                                   .methodDesc("ping")
                                   .returnDesc(0, "ret code", "return code, 0 is OK"));
@@ -229,6 +228,20 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener {
                                      .returnDesc(0, "ret", "0 if success, 1 otherwise"));
     }
 
+    /**
+     * Set up RPC method handlers for getting config
+     */
+    public void setUpGetConfigHandlers() {
+        // The getConfig method in this class will handle RPC calls for getting config
+        getSupervisor().addMethod(JRTMethods.createConfigV3GetConfigMethod(this::getConfigV3));
+        isServingConfigRequests = true;
+    }
+
+
+    public boolean isServingConfigRequests() {
+        return isServingConfigRequests;
+    }
+
     private ApplicationState getState(ApplicationId id) {
         ApplicationState state = applicationStateMap.get(id);
         if (state == null) {
@@ -328,7 +341,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener {
     }
 
     /**
-     * Returns the tenant for this request, empty if there is no tenant for this request
+     * Returns the tenant for this request, empty if there is none
      * (which on hosted Vespa means that the requesting host is not currently active for any tenant)
      */
     Optional<TenantName> resolveTenant(JRTServerConfigRequest request, Trace trace) {
@@ -337,9 +350,9 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener {
         ApplicationId applicationId = hostRegistry.getKeyForHost(hostname);
         if (applicationId == null) {
             if (GetConfigProcessor.logDebug(trace)) {
-                String message = "Did not find tenant for host '" + hostname + "', using " + TenantName.defaultName();
-                log.log(Level.FINE, message);
-                log.log(Level.FINE, () -> "hosts in host registry: " + hostRegistry.getAllHosts());
+                String message = "Did not find tenant for host '" + hostname + "', using " + TenantName.defaultName() +
+                                 ". Hosts in host registry: " + hostRegistry.getAllHosts();
+                log.log(Level.FINE, () -> message);
                 trace.trace(6, message);
             }
             return Optional.empty();
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
index 71e5adc6fea..57d88891ca3 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
@@ -77,12 +77,14 @@ public class ConfigServerBootstrapTest {
         assertFalse(vipStatus.isInRotation());
         bootstrap.start();
         waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running");
+        assertTrue(rpcServer.isServingConfigRequests());
         waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'");
         waitUntil(vipStatus::isInRotation, "failed waiting for server to be in rotation");
 
         bootstrap.deconstruct();
         assertEquals(StateMonitor.Status.down, bootstrap.status());
         assertFalse(rpcServer.isRunning());
+        assertTrue(rpcServer.isServingConfigRequests());
         assertFalse(vipStatus.isInRotation());
     }
 
@@ -108,6 +110,7 @@ public class ConfigServerBootstrapTest {
 
         bootstrap.start();
         waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running");
+        assertTrue(rpcServer.isServingConfigRequests());
         waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'");
         waitUntil(vipStatus::isInRotation, "failed waiting for server to be in rotation");
         bootstrap.deconstruct();
@@ -140,7 +143,8 @@ public class ConfigServerBootstrapTest {
         // App is invalid, bootstrapping was unsuccessful. Status should be 'initializing',
         // rpc server should not be running and it should be out of rotation
         assertEquals(StateMonitor.Status.initializing, stateMonitor.status());
-        assertFalse(rpcServer.isRunning());
+        assertTrue(rpcServer.isRunning());
+        assertFalse(rpcServer.isServingConfigRequests());
         assertFalse(vipStatus.isInRotation());
 
         bootstrap.deconstruct();
@@ -179,6 +183,7 @@ public class ConfigServerBootstrapTest {
                                                                     stateMonitor, vipStatus, VIP_STATUS_PROGRAMMATICALLY);
         bootstrap.start();
         waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running");
+        assertTrue(rpcServer.isServingConfigRequests());
         waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'");
         waitUntil(vipStatus::isInRotation, "failed waiting for server to be in rotation");
     }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java
index 135603def69..d01fbb06df6 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java
@@ -114,7 +114,7 @@ public class RpcTester implements AutoCloseable {
     }
 
     RpcServer createRpcServer(ConfigserverConfig config) throws IOException {
-        return new RpcServer(config,
+        RpcServer rpcServer = new RpcServer(config,
                              new SuperModelRequestHandler(new TestConfigDefinitionRepo(),
                                                           configserverConfig,
                                                           new SuperModelManager(
@@ -128,6 +128,8 @@ public class RpcTester implements AutoCloseable {
                              new FileServer(temporaryFolder.newFolder()),
                              new NoopRpcAuthorizer(),
                              new RpcRequestHandlerProvider());
+        rpcServer.setUpGetConfigHandlers();
+        return rpcServer;
     }
 
     void startRpcServer()  {