From 5d6a7d4063aabb369f30944f49b1c2a922e12480 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 8 Dec 2017 15:35:17 +0100 Subject: Make it possible to configure and change status code in /state/v1/health API Make inital value configurable and make it possible to get and set it in StateMonitor --- .../vespa/config/server/ConfigServerBootstrap.java | 8 ++++++-- .../main/resources/configserver-app/services.xml | 4 ++++ .../config/server/ConfigServerBootstrapTest.java | 7 ++++++- .../yahoo/container/jdisc/state/StateHandler.java | 6 +++++- .../yahoo/container/jdisc/state/StateMonitor.java | 24 ++++++++++++++-------- .../resources/configdefinitions/health-monitor.def | 3 +++ 6 files changed, 40 insertions(+), 12 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 58c61134bc4..4cc9bd88dc0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.Deployer; +import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; @@ -23,17 +24,19 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable private final Thread serverThread; private final Deployer deployer; private final VersionState versionState; + private final StateMonitor stateMonitor; // The tenants object is injected so that all initial requests handlers are // added to the rpc server before it starts answering rpc requests. - @SuppressWarnings("UnusedParameters") + @SuppressWarnings("WeakerAccess") @Inject public ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, - Deployer deployer, VersionState versionState) { + Deployer deployer, VersionState versionState, StateMonitor stateMonitor) { this.applicationRepository = applicationRepository; this.server = server; this.deployer = deployer; this.versionState = versionState; + this.stateMonitor = stateMonitor; this.serverThread = new Thread(this, "configserver main"); serverThread.start(); } @@ -62,6 +65,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable log.log(LogLevel.INFO, "All applications redeployed"); } versionState.saveNewVersion(); + stateMonitor.status(StateMonitor.Status.up); log.log(LogLevel.DEBUG, "Starting RPC server"); server.run(); log.log(LogLevel.DEBUG, "RPC server stopped"); diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index eddba1ee768..fd77fedd789 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -6,6 +6,10 @@ 100 + + initializing + + diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index e0d65055f21..384ae0853d8 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -6,7 +6,10 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.config.HealthMonitorConfig; +import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.core.SystemTimer; import com.yahoo.vespa.config.server.deploy.MockDeployer; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.http.SessionHandlerTest; @@ -67,7 +70,9 @@ public class ConfigServerBootstrapTest extends TestWithTenant { assertFalse(myServer.stopped); VersionState versionState = new VersionState(versionFile); assertTrue(versionState.isUpgraded()); - ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(applicationRepository, rpc, (application, timeout) -> Optional.empty(), versionState); + ConfigServerBootstrap bootstrap = + new ConfigServerBootstrap(applicationRepository, rpc, (application, timeout) -> Optional.empty(), versionState, + new StateMonitor(new HealthMonitorConfig(new HealthMonitorConfig.Builder()), new SystemTimer())); waitUntilStarted(rpc, 60000); assertFalse(versionState.isUpgraded()); assertThat(versionState.currentVersion(), is(versionState.storedVersion())); diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java index ce9779b83d9..23c247fa438 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java @@ -190,7 +190,7 @@ public class StateHandler extends AbstractRequestHandler { private JSONObjectWithLegibleException buildJsonForConsumer(String consumer) throws JSONException { JSONObjectWithLegibleException ret = new JSONObjectWithLegibleException(); ret.put("time", timer.currentTimeMillis()); - ret.put("status", new JSONObjectWithLegibleException().put("code", "up")); + ret.put("status", new JSONObjectWithLegibleException().put("code", getStatus().name())); ret.put(METRICS_PATH, buildJsonForSnapshot(consumer, getSnapshot())); return ret; } @@ -203,6 +203,10 @@ public class StateHandler extends AbstractRequestHandler { } } + private StateMonitor.Status getStatus() { + return monitor.status(); + } + private JSONObjectWithLegibleException buildJsonForSnapshot(String consumer, MetricSnapshot metricSnapshot) throws JSONException { if (metricSnapshot == null) { return new JSONObjectWithLegibleException(); diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index 140257cbcef..9af9956df06 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -14,7 +14,7 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Logger; /** - * A statemonitor keeps track of the current metrics state of a container. + * A state monitor keeps track of the current health and metrics state of a container. * It is used by jDisc to hand out metric update API endpoints to workers through {@link #newMetricConsumer}, * and to inspect the current accumulated state of metrics through {@link #snapshot}. * @@ -23,12 +23,16 @@ import java.util.logging.Logger; public class StateMonitor extends AbstractComponent { private final static Logger log = Logger.getLogger(StateMonitor.class.getName()); + + public enum Status {up, down, initializing}; + private final CopyOnWriteArrayList consumers = new CopyOnWriteArrayList<>(); private final Thread thread; private final Timer timer; private final long snapshotIntervalMs; private long lastSnapshotTimeMs; private volatile MetricSnapshot snapshot; + private volatile Status status; private final TreeSet valueNames = new TreeSet<>(); @Inject @@ -36,13 +40,9 @@ public class StateMonitor extends AbstractComponent { this.timer = timer; this.snapshotIntervalMs = (long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)); this.lastSnapshotTimeMs = timer.currentTimeMillis(); - thread = new Thread(new Runnable() { - - @Override - public void run() { - StateMonitor.this.run(); - } - }, "StateMonitor"); + this.status = Status.valueOf(config.initialStatus()); + System.out.println("Initial status: " + status); + thread = new Thread(StateMonitor.this::run, "StateMonitor"); thread.setDaemon(true); thread.start(); } @@ -54,6 +54,14 @@ public class StateMonitor extends AbstractComponent { return consumer; } + public void status(Status status) { + this.status = status; + } + + public Status status() { + return status; + } + /** Returns the last snapshot taken of the metrics in this system */ public MetricSnapshot snapshot() { return snapshot; diff --git a/container-core/src/main/resources/configdefinitions/health-monitor.def b/container-core/src/main/resources/configdefinitions/health-monitor.def index dc5cdbc6ca4..5e70c72ae3f 100644 --- a/container-core/src/main/resources/configdefinitions/health-monitor.def +++ b/container-core/src/main/resources/configdefinitions/health-monitor.def @@ -4,3 +4,6 @@ namespace=container.jdisc.config # How far between snapshots. 5 minutes by default snapshot_interval double default=300 + +# Initial status used in /state/v1/health API (value for 'code' in 'status'). See StateMonitor for valid values +initialStatus string default="up" -- cgit v1.2.3 From 9dbfded041e944d4cec699400374be58e9316411 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 8 Dec 2017 15:48:33 +0100 Subject: Remove debug line --- .../src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index 9af9956df06..f3f2a2e5bcc 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -41,7 +41,6 @@ public class StateMonitor extends AbstractComponent { this.snapshotIntervalMs = (long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)); this.lastSnapshotTimeMs = timer.currentTimeMillis(); this.status = Status.valueOf(config.initialStatus()); - System.out.println("Initial status: " + status); thread = new Thread(StateMonitor.this::run, "StateMonitor"); thread.setDaemon(true); thread.start(); -- cgit v1.2.3 From de3fc73f8e5d1b5f31d12a23d9bb39d634f76f96 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Sat, 9 Dec 2017 08:38:21 +0100 Subject: Log when changing status and set status to 'down' when RPC server is shutdown --- .../java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java | 1 + .../main/java/com/yahoo/container/jdisc/state/StateMonitor.java | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 4cc9bd88dc0..6ab98f5af1c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -69,6 +69,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable log.log(LogLevel.DEBUG, "Starting RPC server"); server.run(); log.log(LogLevel.DEBUG, "RPC server stopped"); + stateMonitor.status(StateMonitor.Status.down); } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index f3f2a2e5bcc..6234a96d7a0 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -6,6 +6,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.jdisc.Timer; import com.yahoo.jdisc.application.MetricConsumer; +import com.yahoo.log.LogLevel; import java.util.Map; import java.util.TreeSet; @@ -54,13 +55,12 @@ public class StateMonitor extends AbstractComponent { } public void status(Status status) { + log.log(LogLevel.INFO, "Changing health status code from '" + this.status + "' to '" + status.name() + "'"); this.status = status; - } - - public Status status() { - return status; } + public Status status() { return status; } + /** Returns the last snapshot taken of the metrics in this system */ public MetricSnapshot snapshot() { return snapshot; -- cgit v1.2.3