diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-02-03 14:46:33 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2020-02-03 14:46:33 +0000 |
commit | 8bedbd44973bb75d39acde924d86101e1deb8e47 (patch) | |
tree | ca58c32274ebd01a18eae2fef8d297c6f6bd59c2 /container-core/src | |
parent | 6bf6ba447fa72321b6711fabcd7ac1f536969ae5 (diff) |
Require all up when going up, but require all down to go down.
Diffstat (limited to 'container-core/src')
4 files changed, 95 insertions, 33 deletions
diff --git a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java index aab13a1cc7b..0ed0daa2141 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java @@ -27,6 +27,8 @@ public class ClustersStatus extends AbstractComponent { @Inject public ClustersStatus() { } + public enum Require {ONE, ALL} + /** Are there any (in-service influencing) clusters in this container? */ private boolean containerHasClusters; @@ -72,12 +74,25 @@ public class ClustersStatus extends AbstractComponent { setDown((String) clusterIdentifier); } - /** Returns whether this container should receive traffic based on the state of this */ + @Deprecated // TODO: Remove on Vespa 8 public boolean containerShouldReceiveTraffic() { + return containerShouldReceiveTraffic(Require.ONE); + } + /** + * Returns whether this container should receive traffic based on the state of this + * @param require Requirement for being up, ALL or ONE. + */ + public boolean containerShouldReceiveTraffic(Require require) { synchronized (mutex) { if (containerHasClusters) { - // Should receive traffic when at least one cluster is up - return clusterStatus.values().stream().anyMatch(status -> status==true); + switch (require) { + case ONE: + // Should receive traffic when at least one cluster is up + return clusterStatus.values().stream().anyMatch(status -> status == true); + case ALL: + default: + return !clusterStatus.isEmpty() && clusterStatus.values().stream().allMatch(status -> status == true); + } } else { return true; diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java index b9ef1627ce7..f712690efc5 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java @@ -102,10 +102,15 @@ public class VipStatus { private void updateCurrentlyInRotation() { synchronized (mutex) { - if (rotationOverride != null) + if (rotationOverride != null) { currentlyInRotation = rotationOverride; - else - currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(); + } else { + if (healthState.status() == StateMonitor.Status.up) { + currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ONE); + } else { + currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ALL); + } + } // Change to/from 'up' when appropriate but don't change 'initializing' to 'down' if (currentlyInRotation) diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index f690c240537..0be9f47f6cf 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -53,10 +53,16 @@ public class StateMonitor extends AbstractComponent { } StateMonitor(HealthMonitorConfig config, Timer timer, ThreadFactory threadFactory) { + this((long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)), + Status.valueOf(config.initialStatus()), + timer, threadFactory); + } + /* For Testing */ + public StateMonitor(long snapshotIntervalMS, Status status, Timer timer, ThreadFactory threadFactory) { this.timer = timer; - this.snapshotIntervalMs = (long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)); + this.snapshotIntervalMs = snapshotIntervalMS; this.lastSnapshotTimeMs = timer.currentTimeMillis(); - this.status = Status.valueOf(config.initialStatus()); + this.status = status; thread = threadFactory.newThread(this::run); thread.start(); } diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java index 52679c15957..d3479936544 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java @@ -4,6 +4,8 @@ package com.yahoo.container.handler; import static org.junit.Assert.*; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.container.jdisc.state.StateMonitor; +import com.yahoo.jdisc.core.SystemTimer; import org.junit.Test; /** @@ -12,45 +14,79 @@ import org.junit.Test; * @author steinar */ public class VipStatusTestCase { + private static final String [] clusters = {"cluster1", "cluster2", "cluster3"}; - @Test - public void testVipStatusWorksWithClusters() { + private static QrSearchersConfig getSearchersCfg() { var b = new QrSearchersConfig.Builder(); var searchClusterB = new QrSearchersConfig.Searchcluster.Builder(); - searchClusterB.name("cluster1"); - searchClusterB.name("cluster2"); - searchClusterB.name("cluster3"); + for (String cluster : clusters) { + searchClusterB.name(cluster); + } b.searchcluster(searchClusterB); - VipStatus v = new VipStatus(b.build()); + return b.build(); + } + private static VipStatus getVipStatus(StateMonitor.Status startState) { + return new VipStatus(getSearchersCfg(), new ClustersStatus(), new StateMonitor(1000, startState, new SystemTimer(), runnable -> { + Thread thread = new Thread(runnable, "StateMonitor"); + thread.setDaemon(true); + return thread; + })); + } - String cluster1 = "cluster1"; - String cluster2 = "cluster2"; - String cluster3 = "cluster3"; + private static void removeAll(VipStatus v) { + for (String s : clusters) { + v.removeFromRotation(s); + } + } + private static void addAll(VipStatus v) { + for (String s : clusters) { + v.addToRotation(s); + } + } + private static void verifyUpOrDown(StateMonitor.Status status) { + VipStatus v = getVipStatus(status); + removeAll(v); // initial state assertFalse(v.isInRotation()); - - // one cluster becomes up - v.addToRotation(cluster1); + v.addToRotation(clusters[0]); + assertFalse(v.isInRotation()); + v.addToRotation(clusters[1]); + assertFalse(v.isInRotation()); + v.addToRotation(clusters[2]); assertTrue(v.isInRotation()); + } - // all clusters down - v.removeFromRotation(cluster1); - v.removeFromRotation(cluster2); - v.removeFromRotation(cluster3); + @Test + public void testInitializingOrDownRequireAllUp() { + verifyUpOrDown(StateMonitor.Status.initializing); + verifyUpOrDown(StateMonitor.Status.down); + } + + @Test + public void testUpRequireAllDown() { + VipStatus v = getVipStatus(StateMonitor.Status.initializing); assertFalse(v.isInRotation()); - // some clusters down - v.addToRotation(cluster2); + addAll(v); assertTrue(v.isInRotation()); - // all clusters up - v.addToRotation(cluster1); - v.addToRotation(cluster3); + + v.removeFromRotation(clusters[0]); + assertTrue(v.isInRotation()); + v.removeFromRotation(clusters[1]); assertTrue(v.isInRotation()); - // and down again - v.removeFromRotation(cluster1); - v.removeFromRotation(cluster2); - v.removeFromRotation(cluster3); + v.removeFromRotation(clusters[2]); + assertFalse(v.isInRotation()); // All down + v.addToRotation(clusters[1]); assertFalse(v.isInRotation()); + v.addToRotation(clusters[0]); + v.addToRotation(clusters[2]); + assertTrue(v.isInRotation()); // All up + v.removeFromRotation(clusters[0]); + v.removeFromRotation(clusters[2]); + assertTrue(v.isInRotation()); + v.addToRotation(clusters[0]); + v.addToRotation(clusters[2]); + assertTrue(v.isInRotation()); } } |