diff options
author | Geir Storli <geirst@oath.com> | 2018-03-15 16:58:51 +0100 |
---|---|---|
committer | Geir Storli <geirst@oath.com> | 2018-03-15 16:58:51 +0100 |
commit | a1946bc5279d09e8f41717f2ab5ed590aab5ab67 (patch) | |
tree | e3bc4b46fcda1e79e1546f6664033f9d311c8c1e /clustercontroller-core | |
parent | 42ed2f6d0954acd25a9cd3fb3367710f77144d42 (diff) |
Extend cluster v2 to output the published distribution state.
This is the baseline cluster state + per bucket space states.
Diffstat (limited to 'clustercontroller-core')
13 files changed, 92 insertions, 34 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 2edebd17700..2e2534916a7 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -698,7 +698,8 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd private RemoteClusterControllerTask.Context createRemoteTaskProcessingContext() { final RemoteClusterControllerTask.Context context = new RemoteClusterControllerTask.Context(); context.cluster = cluster; - context.currentState = consolidatedClusterState(); + context.currentConsolidatedState = consolidatedClusterState(); + context.publishedClusterStateBundle = stateVersionTracker.getVersionedClusterStateBundle(); context.masterInfo = masterElectionHandler; context.nodeStateOrHostInfoChangeHandler = this; context.nodeAddedOrRemovedListener = this; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RemoteClusterControllerTask.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RemoteClusterControllerTask.java index d082158edc7..e96209c083a 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RemoteClusterControllerTask.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RemoteClusterControllerTask.java @@ -9,7 +9,8 @@ public abstract class RemoteClusterControllerTask { public static class Context { public ContentCluster cluster; - public ClusterState currentState; + public ClusterState currentConsolidatedState; + public ClusterStateBundle publishedClusterStateBundle; public MasterInterface masterInfo; public NodeStateOrHostInfoChangeHandler nodeStateOrHostInfoChangeHandler; public NodeAddedOrRemovedListener nodeAddedOrRemovedListener; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Response.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Response.java index 26c66fea165..90cbd80f18c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Response.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Response.java @@ -68,12 +68,13 @@ public class Response { } public static abstract class EmptyResponse<T extends UnitResponse> - implements UnitResponse, UnitMetrics, UnitAttributes, CurrentUnitState + implements UnitResponse, UnitMetrics, UnitAttributes, CurrentUnitState, DistributionStates { protected final Map<String, String> attributes = new LinkedHashMap<>(); protected final Map<String, SubUnitList> subUnits = new LinkedHashMap<>(); protected final Map<String, Number> metrics = new LinkedHashMap<>(); protected final Map<String, UnitState> stateMap = new LinkedHashMap<>(); + protected DistributionState publishedState = null; @Override public UnitAttributes getAttributes() { return attributes.isEmpty() ? null : this; } @@ -84,11 +85,20 @@ public class Response { @Override public UnitMetrics getMetrics() { return metrics.isEmpty() ? null : this; } @Override + public DistributionStates getDistributionStates() { + return (publishedState == null) ? null : this; + } + + @Override public Map<String, Number> getMetricMap() { return metrics; } @Override public Map<String, UnitState> getStatePerType() { return stateMap; } @Override public Map<String, String> getAttributeValues() { return attributes; } + @Override + public DistributionState getPublishedState() { + return publishedState; + } public EmptyResponse<T> addLink(String type, String unit, String link) { Link list = (Link) subUnits.get(type); @@ -120,6 +130,10 @@ public class Response { attributes.put(name, value); return this; } + public EmptyResponse<T> setPublishedState(DistributionState publishedState) { + this.publishedState = publishedState; + return this; + } } public static class ClusterListResponse extends EmptyResponse<ClusterResponse> {} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/ClusterStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/ClusterStateRequest.java index 2ecc62c303a..122b9d24d40 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/ClusterStateRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/ClusterStateRequest.java @@ -2,11 +2,15 @@ package com.yahoo.vespa.clustercontroller.core.restapiv2.requests; import com.yahoo.vdslib.state.NodeType; +import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle; import com.yahoo.vespa.clustercontroller.core.RemoteClusterControllerTask; import com.yahoo.vespa.clustercontroller.core.restapiv2.Id; import com.yahoo.vespa.clustercontroller.core.restapiv2.Request; import com.yahoo.vespa.clustercontroller.core.restapiv2.Response; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.StateRestApiException; +import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.DistributionState; + +import java.util.stream.Collectors; public class ClusterStateRequest extends Request<Response.ClusterResponse> { private final Id.Cluster id; @@ -21,7 +25,7 @@ public class ClusterStateRequest extends Request<Response.ClusterResponse> { @Override public Response.ClusterResponse calculateResult(RemoteClusterControllerTask.Context context) throws StateRestApiException { Response.ClusterResponse result = new Response.ClusterResponse(); - result.addState("generated", new Response.UnitStateImpl(context.currentState.getClusterState())); + result.addState("generated", new Response.UnitStateImpl(context.currentConsolidatedState.getClusterState())); for (NodeType type : NodeType.getTypes()) { Id.Service serviceId = new Id.Service(id, type); if (recursive > 0) { @@ -31,6 +35,13 @@ public class ClusterStateRequest extends Request<Response.ClusterResponse> { result.addLink("service", type.toString(), serviceId.toString()); } } + result.setPublishedState(bundleToDistributionState(context.publishedClusterStateBundle)); return result; } + + private static DistributionState bundleToDistributionState(ClusterStateBundle bundle) { + return new DistributionState(bundle.getBaselineClusterState().toString(), + bundle.getDerivedBucketSpaceStates().entrySet().stream() + .collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue().getClusterState().toString()))); + } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/NodeStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/NodeStateRequest.java index 669042c2fd8..7f1bfd20124 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/NodeStateRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/NodeStateRequest.java @@ -35,7 +35,7 @@ public class NodeStateRequest extends Request<Response.NodeResponse> { result.addAttribute("hierarchical-group", info.getGroup().getPath()); } - result.addState("generated", new Response.UnitStateImpl(context.currentState.getNodeState(id.getNode()))); + result.addState("generated", new Response.UnitStateImpl(context.currentConsolidatedState.getNodeState(id.getNode()))); result.addState("unit", new Response.UnitStateImpl(info.getReportedState())); result.addState("user", new Response.UnitStateImpl(info.getWantedState())); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/PartitionStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/PartitionStateRequest.java index f8387984263..b7bcca345e2 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/PartitionStateRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/PartitionStateRequest.java @@ -31,7 +31,7 @@ public class PartitionStateRequest extends Request<Response.PartitionResponse> { if (verboseReports.contains(VerboseReport.STATISTICS)) { fillInMetrics(context.cluster.getNodeInfo(id.getNode()).getHostInfo().getMetrics(), result); } - NodeState nodeState = context.currentState.getNodeState(id.getNode()); + NodeState nodeState = context.currentConsolidatedState.getNodeState(id.getNode()); DiskState diskState = nodeState.getDiskState(id.getPartitionIndex()); result.addState("generated", new Response.UnitStateImpl(diskState)); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java index 5b140ee2d87..ada068808f7 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java @@ -50,7 +50,7 @@ public class SetNodeStateRequest extends Request<SetResponse> { newStates, id.getNode(), context.nodeStateOrHostInfoChangeHandler, - context.currentState); + context.currentConsolidatedState); return setResponse; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStatesForClusterRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStatesForClusterRequest.java index a9667042499..3f9e2b48eb5 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStatesForClusterRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStatesForClusterRequest.java @@ -1,17 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.restapiv2.requests; -import com.yahoo.log.LogLevel; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.state.*; -import com.yahoo.vespa.clustercontroller.core.ContentCluster; -import com.yahoo.vespa.clustercontroller.core.NodeInfo; -import com.yahoo.vespa.clustercontroller.core.NodeStateChangeChecker; import com.yahoo.vespa.clustercontroller.core.RemoteClusterControllerTask; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeAddedOrRemovedListener; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; import com.yahoo.vespa.clustercontroller.core.restapiv2.Id; -import com.yahoo.vespa.clustercontroller.core.restapiv2.MissingIdException; import com.yahoo.vespa.clustercontroller.core.restapiv2.Request; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.InternalFailure; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.InvalidContentException; @@ -70,14 +63,14 @@ public class SetNodeStatesForClusterRequest extends Request<SetResponse> { newStates, node, context.nodeStateOrHostInfoChangeHandler, - context.currentState); + context.currentConsolidatedState); if (!setResponse.getWasModified()) { throw new InternalFailure("We have not yet implemented the meaning of " + "failing to set the wanted state for a subset of nodes: " + "condition = " + condition + ", newStates = " + newStates + - ", currentState = " + context.currentState); + ", currentConsolidatedState = " + context.currentConsolidatedState); } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java index 2aac8a94f52..472301e1dfc 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java @@ -16,11 +16,13 @@ public class ClusterControllerMock implements RemoteClusterControllerTaskSchedul public StringBuilder events = new StringBuilder(); public ClusterControllerMock(ContentCluster cluster, ClusterState state, + ClusterStateBundle publishedClusterStateBundle, int fcIndex, Integer fcMaster) { this.fleetControllerIndex = fcIndex; this.fleetControllerMaster = fcMaster; context.cluster = cluster; - context.currentState = state; + context.currentConsolidatedState = state; + context.publishedClusterStateBundle = publishedClusterStateBundle; context.masterInfo = new MasterInterface() { @Override public boolean isMaster() { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterListTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterListTest.java index 55e8f6bf84b..813b3780091 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterListTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterListTest.java @@ -33,7 +33,20 @@ public class ClusterListTest extends StateRestApiTest { " \"service\": {\n" + " \"storage\": {\"link\": \"\\/cluster\\/v2\\/books\\/storage\"},\n" + " \"distributor\": {\"link\": \"\\/cluster\\/v2\\/books\\/distributor\"}\n" + - " }\n" + + " },\n" + + " \"distribution-states\": {\"published\": {\n" + + " \"baseline\": \"distributor:4 storage:4\",\n" + + " \"bucket-spaces\": [\n" + + " {\n" + + " \"name\": \"default\",\n" + + " \"state\": \"distributor:4 storage:4 .3.s:m\"\n" + + " },\n" + + " {\n" + + " \"name\": \"global\",\n" + + " \"state\": \"distributor:4 storage:4\"\n" + + " }\n" + + " ]\n" + + " }}\n" + " },\n" + " \"music\": {\n" + " \"state\": {\"generated\": {\n" + @@ -43,7 +56,11 @@ public class ClusterListTest extends StateRestApiTest { " \"service\": {\n" + " \"storage\": {\"link\": \"\\/cluster\\/v2\\/music\\/storage\"},\n" + " \"distributor\": {\"link\": \"\\/cluster\\/v2\\/music\\/distributor\"}\n" + - " }\n" + + " },\n" + + " \"distribution-states\": {\"published\": {\n" + + " \"baseline\": \"distributor:8 .0.s:d .2.s:d .4.s:d .6.s:d storage:8 .0.s:d .2.s:d .4.s:d .6.s:d\",\n" + + " \"bucket-spaces\": []\n" + + " }}\n" + " }\n" + "}}"; assertEquals(expected, jsonWriter.createJson(response).toString(2)); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterTest.java index 7f088abce12..635ab1f37b3 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterTest.java @@ -11,7 +11,7 @@ public class ClusterTest extends StateRestApiTest { @Test public void testCluster() throws Exception { setUp(true); - UnitResponse response = restAPI.getState(new StateRequest("music", 0)); + UnitResponse response = restAPI.getState(new StateRequest("books", 0)); String expected = "{\n" + " \"state\": {\"generated\": {\n" + @@ -19,9 +19,22 @@ public class ClusterTest extends StateRestApiTest { " \"reason\": \"\"\n" + " }},\n" + " \"service\": {\n" + - " \"storage\": {\"link\": \"\\/cluster\\/v2\\/music\\/storage\"},\n" + - " \"distributor\": {\"link\": \"\\/cluster\\/v2\\/music\\/distributor\"}\n" + - " }\n" + + " \"storage\": {\"link\": \"\\/cluster\\/v2\\/books\\/storage\"},\n" + + " \"distributor\": {\"link\": \"\\/cluster\\/v2\\/books\\/distributor\"}\n" + + " },\n" + + " \"distribution-states\": {\"published\": {\n" + + " \"baseline\": \"distributor:4 storage:4\",\n" + + " \"bucket-spaces\": [\n" + + " {\n" + + " \"name\": \"default\",\n" + + " \"state\": \"distributor:4 storage:4 .3.s:m\"\n" + + " },\n" + + " {\n" + + " \"name\": \"global\",\n" + + " \"state\": \"distributor:4 storage:4\"\n" + + " }\n" + + " ]\n" + + " }}\n" + "}"; assertEquals(expected, jsonWriter.createJson(response).toString(2)); } @@ -51,7 +64,11 @@ public class ClusterTest extends StateRestApiTest { " \"5\": {\"link\": \"\\/cluster\\/v2\\/music\\/distributor\\/5\"},\n" + " \"7\": {\"link\": \"\\/cluster\\/v2\\/music\\/distributor\\/7\"}\n" + " }}\n" + - " }\n" + + " },\n" + + " \"distribution-states\": {\"published\": {\n" + + " \"baseline\": \"distributor:8 .0.s:d .2.s:d .4.s:d .6.s:d storage:8 .0.s:d .2.s:d .4.s:d .6.s:d\",\n" + + " \"bucket-spaces\": []\n" + + " }}\n" + "}"; assertEquals(expected, jsonWriter.createJson(response).toString(2)); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/NodeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/NodeTest.java index de28867520b..6bd7f086249 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/NodeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/NodeTest.java @@ -124,7 +124,7 @@ public class NodeTest extends StateRestApiTest { music.context.cluster = new ContentCluster(old.getName(), old.getConfiguredNodes().values(), old.getDistribution(), 0, 0.0); NodeState currentState = new NodeState(NodeType.STORAGE, State.DOWN); currentState.setDescription("Not seen"); - music.context.currentState.setNodeState(new Node(NodeType.STORAGE, 1), currentState); + music.context.currentConsolidatedState.setNodeState(new Node(NodeType.STORAGE, 1), currentState); UnitResponse response = restAPI.getState(new StateRequest("music/storage/1", 0)); String expected = "{\n" + diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java index 46a50800c53..5160ecad8b3 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java @@ -4,10 +4,7 @@ package com.yahoo.vespa.clustercontroller.core.restapiv2; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.state.*; -import com.yahoo.vespa.clustercontroller.core.FleetControllerTest; -import com.yahoo.vespa.clustercontroller.core.NodeInfo; -import com.yahoo.vespa.clustercontroller.core.RemoteClusterControllerTaskScheduler; -import com.yahoo.vespa.clustercontroller.core.ContentCluster; +import com.yahoo.vespa.clustercontroller.core.*; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; import com.yahoo.vespa.clustercontroller.utils.staterestapi.StateRestAPI; import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.UnitStateRequest; @@ -47,8 +44,12 @@ public abstract class StateRestApiTest { ContentCluster cluster = new ContentCluster( "books", nodes, distribution, 6 /* minStorageNodesUp*/, 0.9 /* minRatioOfStorageNodesUp */); initializeCluster(cluster, nodes); - ClusterState state = new ClusterState("distributor:4 storage:4"); - books = new ClusterControllerMock(cluster, state, 0, 0); + AnnotatedClusterState baselineState = AnnotatedClusterState.withoutAnnotations(ClusterState.stateFromString("distributor:4 storage:4")); + Map<String, AnnotatedClusterState> bucketSpaceStates = new HashMap<>(); + bucketSpaceStates.put("default", AnnotatedClusterState.withoutAnnotations(ClusterState.stateFromString("distributor:4 storage:4 .3.s:m"))); + bucketSpaceStates.put("global", baselineState); + books = new ClusterControllerMock(cluster, baselineState.getClusterState(), + ClusterStateBundle.of(baselineState, bucketSpaceStates), 0, 0); } { Set<ConfiguredNode> nodes = FleetControllerTest.toNodes(1, 2, 3, 5, 7); @@ -64,9 +65,10 @@ public abstract class StateRestApiTest { else { initializeCluster(cluster, nodes); } - ClusterState state = new ClusterState("distributor:8 .0.s:d .2.s:d .4.s:d .6.s:d " - + "storage:8 .0.s:d .2.s:d .4.s:d .6.s:d"); - music = new ClusterControllerMock(cluster, state, 0, 0); + AnnotatedClusterState baselineState = AnnotatedClusterState.withoutAnnotations(ClusterState.stateFromString("distributor:8 .0.s:d .2.s:d .4.s:d .6.s:d " + + "storage:8 .0.s:d .2.s:d .4.s:d .6.s:d")); + music = new ClusterControllerMock(cluster, baselineState.getClusterState(), + ClusterStateBundle.ofBaselineOnly(baselineState), 0, 0); } ccSockets = new TreeMap<>(); ccSockets.put(0, new ClusterControllerStateRestAPI.Socket("localhost", 80)); |