From 0394a2a4430ee34ea7cafe03c98592199eda492f Mon Sep 17 00:00:00 2001 From: Arnstein Ressem Date: Fri, 12 Mar 2021 13:58:06 +0100 Subject: Revert "GC unused DiskState" --- .../restapiv2/ClusterControllerStateRestAPI.java | 12 ++-- .../vespa/clustercontroller/core/restapiv2/Id.java | 12 ++++ .../clustercontroller/core/restapiv2/Response.java | 5 ++ .../core/restapiv2/UnitPathResolver.java | 14 ++++- .../restapiv2/requests/PartitionStateRequest.java | 69 ++++++++++++++++++++++ .../core/ClusterStateGeneratorTest.java | 1 + .../core/GroupAutoTakedownTest.java | 1 + .../core/restapiv2/NotMasterTest.java | 14 +++++ .../core/restapiv2/PartitionTest.java | 49 +++++++++++++++ .../core/restapiv2/SetNodeStateTest.java | 10 ++++ 10 files changed, 180 insertions(+), 7 deletions(-) create mode 100644 clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/PartitionStateRequest.java create mode 100644 clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/PartitionTest.java (limited to 'clustercontroller-core') diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerStateRestAPI.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerStateRestAPI.java index 6c80973c4af..169684aa742 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerStateRestAPI.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerStateRestAPI.java @@ -2,12 +2,7 @@ package com.yahoo.vespa.clustercontroller.core.restapiv2; import com.yahoo.vespa.clustercontroller.core.RemoteClusterControllerTaskScheduler; -import com.yahoo.vespa.clustercontroller.core.restapiv2.requests.ClusterListRequest; -import com.yahoo.vespa.clustercontroller.core.restapiv2.requests.ClusterStateRequest; -import com.yahoo.vespa.clustercontroller.core.restapiv2.requests.NodeStateRequest; -import com.yahoo.vespa.clustercontroller.core.restapiv2.requests.ServiceStateRequest; -import com.yahoo.vespa.clustercontroller.core.restapiv2.requests.SetNodeStateRequest; -import com.yahoo.vespa.clustercontroller.core.restapiv2.requests.SetNodeStatesForClusterRequest; +import com.yahoo.vespa.clustercontroller.core.restapiv2.requests.*; import com.yahoo.vespa.clustercontroller.utils.staterestapi.StateRestAPI; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.InternalFailure; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.OtherMasterException; @@ -17,6 +12,7 @@ import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.UnitStateRe import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.SetResponse; import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.UnitResponse; +import java.util.EnumSet; import java.util.Map; import java.util.Objects; import java.util.logging.Logger; @@ -90,6 +86,10 @@ public class ClusterControllerStateRestAPI implements StateRestAPI { public Request visitNode(Id.Node id) { return new NodeStateRequest(id); } + @Override + public Request visitPartition(Id.Partition id) { + return new PartitionStateRequest(id, EnumSet.of(VerboseReport.STATISTICS)); + } }); if (req instanceof ClusterListRequest) { log.fine("Got cluster list request"); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Id.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Id.java index 1a0fe6cdf8e..ed5af93d7fb 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Id.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Id.java @@ -52,4 +52,16 @@ public class Id { public String toString() { return super.toString() + "/" + id; } } + public static class Partition extends Node { + private final int id; + + public Partition(Node n, int partition) { + super(n, n.id); + this.id = partition; + } + + public final int getPartitionIndex() { return id; } + public String toString() { return super.toString() + "/" + id; } + } + } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Response.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Response.java index 82512262b4a..17949b82365 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Response.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/Response.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.restapiv2; +import com.yahoo.vdslib.state.DiskState; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.InternalFailure; @@ -31,6 +32,10 @@ public class Response { this.id = parseId(ns.getState()); this.reason = ns.getDescription(); } + public UnitStateImpl(DiskState ds) throws StateRestApiException { + this.id = parseId(ds.getState()); + this.reason = ds.getDescription(); + } public String parseId(State id) throws StateRestApiException { switch (id) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/UnitPathResolver.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/UnitPathResolver.java index 04fd154ff6b..4dc73f4d0dd 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/UnitPathResolver.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/UnitPathResolver.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.clustercontroller.core.RemoteClusterControllerTaskSchedul import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.MissingUnitException; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.OperationNotSupportedForUnitException; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.StateRestApiException; +import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.UnitResponse; import java.util.HashMap; import java.util.Map; @@ -18,6 +19,7 @@ public class UnitPathResolver { Request visitCluster(Id.Cluster id) throws StateRestApiException; Request visitService(Id.Service id) throws StateRestApiException; Request visitNode(Id.Node id) throws StateRestApiException; + Request visitPartition(Id.Partition id) throws StateRestApiException; } @@ -38,6 +40,7 @@ public class UnitPathResolver { public Request visitCluster(Id.Cluster id) throws StateRestApiException { return fail(); } public Request visitService(Id.Service id) throws StateRestApiException { return fail(); } public Request visitNode(Id.Node id) throws StateRestApiException { return fail(); } + public Request visitPartition(Id.Partition id) throws StateRestApiException { return fail(); } } @@ -84,7 +87,16 @@ public class UnitPathResolver { if (path.length == 3) { return visitor.visitNode(node); } - throw new MissingUnitException(path, 3); + Id.Partition partition; + try{ + partition = new Id.Partition(node, Integer.valueOf(path[3])); + } catch (NumberFormatException e) { + throw new MissingUnitException(path, 3); + } + if (path.length == 4) { + return visitor.visitPartition(partition); + } + throw new MissingUnitException(path, 4); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/PartitionStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/PartitionStateRequest.java new file mode 100644 index 00000000000..bb42af45ef3 --- /dev/null +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/PartitionStateRequest.java @@ -0,0 +1,69 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core.restapiv2.requests; + +import com.yahoo.vdslib.state.DiskState; +import com.yahoo.vdslib.state.NodeState; +import com.yahoo.vespa.clustercontroller.core.RemoteClusterControllerTask; +import com.yahoo.vespa.clustercontroller.core.hostinfo.Metrics; +import com.yahoo.vespa.clustercontroller.core.restapiv2.Id; +import com.yahoo.vespa.clustercontroller.core.restapiv2.Request; +import com.yahoo.vespa.clustercontroller.core.restapiv2.Response; +import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.StateRestApiException; + +import java.util.Set; +import java.util.logging.Logger; + +public class PartitionStateRequest extends Request { + private static final Logger log = Logger.getLogger(PartitionStateRequest.class.getName()); + private final Id.Partition id; + private final Set verboseReports; + + public PartitionStateRequest(Id.Partition id, Set verboseReports) { + super(MasterState.MUST_BE_MASTER); + this.id = id; + this.verboseReports = verboseReports; + } + + @Override + public Response.PartitionResponse calculateResult(RemoteClusterControllerTask.Context context) + throws StateRestApiException { + Response.PartitionResponse result = new Response.PartitionResponse(); + if (verboseReports.contains(VerboseReport.STATISTICS)) { + fillInMetrics(context.cluster.getNodeInfo(id.getNode()).getHostInfo().getMetrics(), result); + } + NodeState nodeState = context.currentConsolidatedState.getNodeState(id.getNode()); + DiskState diskState = new DiskState(); + result.addState("generated", new Response.UnitStateImpl(diskState)); + + return result; + } + + private static void fillInMetrics(Metrics metrics, Response.PartitionResponse result) { + for (Metrics.Metric metric: metrics.getMetrics()) { + fillInMetricValue(metric.getName(), metric.getValue(), result); + } + } + + private static void fillInMetricValue( + String name, Metrics.Value value, Response.PartitionResponse result) { + if (name.equals("vds.datastored.alldisks.docs")) { + if (value.getLast() == null) { + log.warning("Proper doc count value did not exist in value set."); + return; + } + result.addMetric("unique-document-count", value.getLast()); + } else if (name.equals("vds.datastored.alldisks.bytes")) { + if (value.getLast() == null) { + log.warning("Proper doc size value did not exist in value set."); + return; + } + result.addMetric("unique-document-total-size", value.getLast()); + } else if (name.equals("vds.datastored.alldisks.buckets")) { + if (value.getLast() == null) { + log.warning("Proper bucket count value did not exist in value set."); + return; + } + result.addMetric("bucket-count", value.getLast()); + } + } +} diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java index 0798e6174f9..bf191768892 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGeneratorTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.distribution.ConfiguredNode; +import com.yahoo.vdslib.state.DiskState; import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java index d33d235bc66..bd77f42ff8a 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.distribution.ConfiguredNode; +import com.yahoo.vdslib.state.DiskState; import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/NotMasterTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/NotMasterTest.java index b0355844341..4ec5893c1e7 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/NotMasterTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/NotMasterTest.java @@ -53,6 +53,12 @@ public class NotMasterTest extends StateRestApiTest { } catch (UnknownMasterException e) { assertTrue(e.getMessage(), e.getMessage().contains("No known master cluster controller")); } + try{ + restAPI.getState(new StateRequest("music/storage/1/0", 0)); + fail(); + } catch (UnknownMasterException e) { + assertTrue(e.getMessage(), e.getMessage().contains("No known master cluster controller")); + } try{ restAPI.setUnitState(new SetNodeStateTest.SetUnitStateRequestImpl("music/storage/1") .setNewState("user", "down", "test")); @@ -111,6 +117,14 @@ public class NotMasterTest extends StateRestApiTest { assertEquals("otherhost", e.getHost()); assertEquals(e.getPort(), getDefaults().vespaWebServicePort()); } + try{ + restAPI.getState(new StateRequest("music/storage/1/0", 0)); + fail(); + } catch (OtherMasterException e) { + assertTrue(e.getMessage(), e.getMessage().contains("Cluster controller not master. Use master at otherhost:" + getDefaults().vespaWebServicePort() + ".")); + assertEquals("otherhost", e.getHost()); + assertEquals(e.getPort(), getDefaults().vespaWebServicePort()); + } try{ restAPI.setUnitState(new SetNodeStateTest.SetUnitStateRequestImpl("music/storage/1") .setNewState("user", "down", "test")); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/PartitionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/PartitionTest.java new file mode 100644 index 00000000000..106803ec3f1 --- /dev/null +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/PartitionTest.java @@ -0,0 +1,49 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core.restapiv2; + +import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.UnitResponse; +import org.codehaus.jettison.json.JSONObject; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class PartitionTest extends StateRestApiTest { + + @Test + public void testPartition() throws Exception { + setUp(true); + UnitResponse response = restAPI.getState(new StateRequest("music/storage/1/0", 0)); + String expected = + "{\n" + + " \"state\": {\"generated\": {\n" + + " \"state\": \"up\",\n" + + " \"reason\": \"\"\n" + + " }},\n" + + " \"metrics\": {\n" + + " \"bucket-count\": 1,\n" + + " \"unique-document-count\": 2,\n" + + " \"unique-document-total-size\": 3\n" + + " }\n" + + "}"; + assertEquals(expected, jsonWriter.createJson(response).toString(2)); + } + + @Test + public void testRecursiveCluster() throws Exception { + setUp(true); + UnitResponse response = restAPI.getState(new StateRequest("music/storage/1/0", 1)); + String expected = + "{\n" + + " \"state\": {\"generated\": {\n" + + " \"state\": \"up\",\n" + + " \"reason\": \"\"\n" + + " }},\n" + + " \"metrics\": {\n" + + " \"bucket-count\": 1,\n" + + " \"unique-document-count\": 2,\n" + + " \"unique-document-total-size\": 3\n" + + " }\n" + + "}"; + assertEquals(expected, jsonWriter.createJson(response).toString(2)); + } +} diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/SetNodeStateTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/SetNodeStateTest.java index 0e9a4808e9b..03fdb15971f 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/SetNodeStateTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/SetNodeStateTest.java @@ -366,6 +366,16 @@ public class SetNodeStateTest extends StateRestApiTest { } catch (OperationNotSupportedForUnitException e) { assertTrue(e.getMessage(), e.getMessage().contains(wrongUnitMessage)); } + + // ... setting at node-level is allowed + + try{ + restAPI.setUnitState(new SetUnitStateRequestImpl( + "music/storage/1/0").setNewState("user", "down", "testing")); + fail(); + } catch (OperationNotSupportedForUnitException e) { + assertTrue(e.getMessage(), e.getMessage().contains(wrongUnitMessage)); + } } @Test -- cgit v1.2.3