diff options
7 files changed, 69 insertions, 31 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index e8a5f1708a1..d631cc18d75 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -40,7 +40,7 @@ import static com.yahoo.vespa.config.server.ConfigServerBootstrap.RedeployingApp * applications. If that is done successfully the RPC server will start and the health status code will change from * 'initializing' to 'up'. If VIP status mode is VIP_STATUS_PROGRAMMATICALLY the config server * will be put into rotation (start serving status.html with 200 OK), if the mode is VIP_STATUS_FILE a VIP status - * file is created or removed ny some external pgrogram based on the health status code. + * file is created or removed ny some external program based on the health status code. * * @author Ulf Lilleengen * @author hmusum @@ -176,12 +176,10 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable } private void up() { - stateMonitor.status(StateMonitor.Status.up); vipStatus.setInRotation(true); } private void down() { - stateMonitor.status(StateMonitor.Status.down); vipStatus.setInRotation(false); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index df295b2d38b..2e7b5a1f4d9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -72,9 +72,10 @@ public class ConfigServerBootstrapTest { RpcServer rpcServer = createRpcServer(configserverConfig); // Take a host away so that there are too few for the application, to verify we can still bootstrap provisioner.allocations().values().iterator().next().remove(0); - VipStatus vipStatus = createVipStatus(); + StateMonitor stateMonitor = new StateMonitor(); + VipStatus vipStatus = createVipStatus(stateMonitor); ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, - versionState, createStateMonitor(), + versionState, stateMonitor, vipStatus, INITIALIZE_ONLY, VIP_STATUS_PROGRAMMATICALLY); assertFalse(vipStatus.isInRotation()); bootstrap.start(); @@ -102,9 +103,10 @@ public class ConfigServerBootstrapTest { assertTrue(versionState.isUpgraded()); RpcServer rpcServer = createRpcServer(configserverConfig); - VipStatus vipStatus = createVipStatus(); + StateMonitor stateMonitor = new StateMonitor(); + VipStatus vipStatus = createVipStatus(stateMonitor); ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, - versionState, createStateMonitor(), + versionState, stateMonitor, vipStatus, INITIALIZE_ONLY, VIP_STATUS_FILE); assertTrue(vipStatus.isInRotation()); // default is in rotation when using status file @@ -132,16 +134,17 @@ public class ConfigServerBootstrapTest { .resolve("sessions/2/services.xml")); RpcServer rpcServer = createRpcServer(configserverConfig); - VipStatus vipStatus = createVipStatus(); + StateMonitor stateMonitor = new StateMonitor(); + VipStatus vipStatus = createVipStatus(stateMonitor); ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState, - createStateMonitor(), + stateMonitor, vipStatus, INITIALIZE_ONLY, VIP_STATUS_PROGRAMMATICALLY); assertFalse(vipStatus.isInRotation()); // Call method directly, to be sure that it is finished redeploying all applications and we can check status bootstrap.start(); // App is invalid, bootstrapping was unsuccessful. Status should be 'initializing', // rpc server should not be running and it should be out of rotation - assertEquals(StateMonitor.Status.initializing, bootstrap.status()); + assertEquals(StateMonitor.Status.initializing, stateMonitor.status()); assertFalse(rpcServer.isRunning()); assertFalse(vipStatus.isInRotation()); @@ -174,9 +177,10 @@ public class ConfigServerBootstrapTest { curator.set(Path.fromString("/config/v2/tenants/" + applicationId.tenant().value() + "/sessions/2/version"), Utf8.toBytes("1.2.2")); RpcServer rpcServer = createRpcServer(configserverConfig); - VipStatus vipStatus = createVipStatus(); + StateMonitor stateMonitor = createStateMonitor(); + VipStatus vipStatus = createVipStatus(stateMonitor); ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState, - createStateMonitor(), vipStatus, + stateMonitor, vipStatus, BOOTSTRAP_IN_SEPARATE_THREAD, VIP_STATUS_PROGRAMMATICALLY); waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running"); waitUntil(() -> bootstrap.status() == StateMonitor.Status.up, "failed waiting for status 'up'"); @@ -229,9 +233,10 @@ public class ConfigServerBootstrapTest { return new Host(hostname, Collections.emptyList(), Optional.empty(), Optional.of(com.yahoo.component.Version.fromString(version))); } - private VipStatus createVipStatus() { + private VipStatus createVipStatus(StateMonitor stateMonitor) { return new VipStatus(new QrSearchersConfig.Builder().build(), - new ClustersStatus()); + new ClustersStatus(), + stateMonitor); } public static class MockRpc extends com.yahoo.vespa.config.server.rpc.MockRpc { diff --git a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java index 16c2677e1b6..aab13a1cc7b 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java @@ -9,7 +9,9 @@ import java.util.Map; /** * A component which tracks the up/down status of any clusters which should influence - * the up down status of this container itself, as well as the separate fact that such clusters are present. + * the up down status of this container itself, as well as the separate fact (from config) + * that such clusters are present. This is a separate fact because we might know we have clusters configured + * but we don't have positive information that they are up yet, and in this case we should be down. * * This is a separate component which has <b>no dependencies</b> such that the status tracked in this * will survive reconfiguration events and inform other components even immediately after a reconfiguration diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java index 555dc81057d..a0b959a1c26 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java @@ -4,6 +4,7 @@ package com.yahoo.container.handler; import com.google.inject.Inject; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.VipStatusConfig; +import com.yahoo.container.jdisc.state.StateMonitor; /** * API for programmatically removing the container from VIP rotation. @@ -17,6 +18,8 @@ public class VipStatus { private final ClustersStatus clustersStatus; + private final StateMonitor healthState; + /** If this is non-null, its value decides whether this container is in rotation */ private Boolean rotationOverride = null; @@ -25,23 +28,30 @@ public class VipStatus { private final Object mutex = new Object(); + /** For testing */ public VipStatus() { - this(new QrSearchersConfig(new QrSearchersConfig.Builder()), - new VipStatusConfig(new VipStatusConfig.Builder()), - new ClustersStatus()); + this(new ClustersStatus()); } + /** For testing */ public VipStatus(QrSearchersConfig dispatchers) { this(dispatchers, new ClustersStatus()); } + /** For testing */ public VipStatus(ClustersStatus clustersStatus) { - this.clustersStatus = clustersStatus; + this(new QrSearchersConfig.Builder().build(), clustersStatus); } - @Inject public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus) { + this(dispatchers, clustersStatus, new StateMonitor()); + } + + @Inject + public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus, StateMonitor healthState) { this.clustersStatus = clustersStatus; + this.healthState = healthState; + healthState.status(StateMonitor.Status.initializing); clustersStatus.setContainerHasClusters(! dispatchers.searchcluster().isEmpty()); updateCurrentlyInRotation(); } @@ -95,6 +105,12 @@ public class VipStatus { currentlyInRotation = rotationOverride; else currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(); + + // Change to/from 'up' when appropriate but don't change 'initializing' to 'down' + if (currentlyInRotation) + healthState.status(StateMonitor.Status.up); + else if (healthState.status() == StateMonitor.Status.up) + healthState.status(StateMonitor.Status.down); } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index 9d558a9c2a2..f690c240537 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -6,6 +6,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.jdisc.Timer; import com.yahoo.jdisc.application.MetricConsumer; +import com.yahoo.jdisc.core.SystemTimer; import com.yahoo.log.LogLevel; import java.util.Map; @@ -37,6 +38,11 @@ public class StateMonitor extends AbstractComponent { private volatile Status status; private final TreeSet<String> valueNames = new TreeSet<>(); + /** For testing */ + public StateMonitor() { + this(new HealthMonitorConfig.Builder().build(), new SystemTimer()); + } + @Inject public StateMonitor(HealthMonitorConfig config, Timer timer) { this(config, timer, runnable -> { diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java index 4c1c1622140..52679c15957 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.container.handler; import static org.junit.Assert.*; +import com.yahoo.container.QrSearchersConfig; import org.junit.Test; /** @@ -14,13 +15,17 @@ public class VipStatusTestCase { @Test public void testVipStatusWorksWithClusters() { - ClustersStatus clustersStatus = new ClustersStatus(); - clustersStatus.setContainerHasClusters(true); - VipStatus v = new VipStatus(clustersStatus); - - String cluster1 = new String("a"); - String cluster2 = new String("b"); - String cluster3 = new String("c"); + var b = new QrSearchersConfig.Builder(); + var searchClusterB = new QrSearchersConfig.Searchcluster.Builder(); + searchClusterB.name("cluster1"); + searchClusterB.name("cluster2"); + searchClusterB.name("cluster3"); + b.searchcluster(searchClusterB); + VipStatus v = new VipStatus(b.build()); + + String cluster1 = "cluster1"; + String cluster2 = "cluster2"; + String cluster3 = "cluster3"; // initial state assertFalse(v.isInRotation()); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java index 5377eb670c4..0caef371d0c 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java @@ -2,6 +2,7 @@ package com.yahoo.prelude.fastsearch.test; import com.google.common.util.concurrent.MoreExecutors; +import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.ClustersStatus; import com.yahoo.container.handler.VipStatus; import com.yahoo.net.HostName; @@ -45,11 +46,16 @@ class FastSearcherTester { } public FastSearcherTester(int containerClusterSize, List<Node> searchNodes) { - ClustersStatus clustersStatus = new ClustersStatus(); - clustersStatus.setContainerHasClusters(true); - vipStatus = new VipStatus(clustersStatus); + String clusterId = "a"; + + var b = new QrSearchersConfig.Builder(); + var searchClusterB = new QrSearchersConfig.Searchcluster.Builder(); + searchClusterB.name(clusterId); + b.searchcluster(searchClusterB); + vipStatus = new VipStatus(b.build()); + mockFS4ResourcePool = new MockFS4ResourcePool(); - mockDispatcher = new MockDispatcher("a", searchNodes, mockFS4ResourcePool, containerClusterSize, vipStatus); + mockDispatcher = new MockDispatcher(clusterId, searchNodes, mockFS4ResourcePool, containerClusterSize, vipStatus); fastSearcher = new FastSearcher(new MockBackend(selfHostname, 0L, true), mockFS4ResourcePool, mockDispatcher, |