diff options
Diffstat (limited to 'clustercontroller-core/src/test/java/com/yahoo')
10 files changed, 235 insertions, 15 deletions
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java index aa47ce2ec82..14276c51416 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStatsAggregatorTest.java @@ -33,6 +33,10 @@ public class ClusterStatsAggregatorTest { assertEquals(expectedStats.build(), aggregator.getAggregatedStatsForDistributor(distributorIndex)); } + public void verifyGlobal(ContentNodeStatsBuilder expectedStats) { + assertEquals(expectedStats.build(), aggregator.getAggregatedStats().getGlobalStats()); + } + boolean hasUpdatesFromAllDistributors() { return aggregator.getAggregatedStats().hasUpdatesFromAllDistributors(); } @@ -64,6 +68,10 @@ public class ClusterStatsAggregatorTest { return Sets.newHashSet(indices); } + private static ContentNodeStatsBuilder globalStatsBuilder() { + return ContentNodeStatsBuilder.forNode(-1); + } + @Test void aggregator_handles_updates_to_single_distributor_and_content_node() { Fixture f = new Fixture(distributorNodes(1), contentNodes(3)); @@ -72,6 +80,9 @@ public class ClusterStatsAggregatorTest { .add(3, "global", 11, 2); f.update(1, stats); f.verify(stats); + f.verifyGlobal(globalStatsBuilder() + .add("default", 10, 1) + .add("global", 11, 2)); } @Test @@ -80,9 +91,13 @@ public class ClusterStatsAggregatorTest { f.verify(new ContentClusterStatsBuilder() .add(3, "default", 10 + 14, 1 + 5) - .add(3, "global", 11 + 15, 2 + 6) + .add(3, "global", 11 + 15, 2 + 6) .add(4, "default", 12 + 16, 3 + 7) - .add(4, "global", 13 + 17, 4 + 8)); + .add(4, "global", 13 + 17, 4 + 8)); + + f.verifyGlobal(globalStatsBuilder() + .add("default", (10 + 14) + (12 + 16), (1 + 5) + (3 + 7)) + .add("global", (11 + 15) + (13 + 17), (2 + 6) + (4 + 8))); } @Test @@ -94,28 +109,34 @@ public class ClusterStatsAggregatorTest { f.update(2, new ContentClusterStatsBuilder().add(3, "default", 10, 1)); f.verify(new ContentClusterStatsBuilder().addInvalid(3, "default", 10, 1)); + f.verifyGlobal(globalStatsBuilder().addInvalid("default", 10, 1)); f.update(1, new ContentClusterStatsBuilder().add(3, "default", 11, 2)); f.verify(new ContentClusterStatsBuilder().add(3, "default", 10 + 11, 1 + 2)); + f.verifyGlobal(globalStatsBuilder().add("default", 10 + 11, 1 + 2)); f.update(2, new ContentClusterStatsBuilder().add(3, "default", 15, 6)); f.verify(new ContentClusterStatsBuilder().add(3, "default", 11 + 15, 2 + 6)); + f.verifyGlobal(globalStatsBuilder().add("default", 11 + 15, 2 + 6)); f.update(1, new ContentClusterStatsBuilder().add(3, "default", 16, 7)); f.verify(new ContentClusterStatsBuilder().add(3, "default", 15 + 16, 6 + 7)); + f.verifyGlobal(globalStatsBuilder().add("default", 15 + 16, 6 + 7)); f.update(2, new ContentClusterStatsBuilder().add(3, "default", 12, 3)); f.verify(new ContentClusterStatsBuilder().add(3, "default", 16 + 12, 7 + 3)); + f.verifyGlobal(globalStatsBuilder().add("default", 16 + 12, 7 + 3)); } @Test - void aggregator_handles_more_content_nodes_that_distributors() { + void aggregator_handles_more_content_nodes_than_distributors() { Fixture f = new Fixture(distributorNodes(1), contentNodes(3, 4)); ContentClusterStatsBuilder stats = new ContentClusterStatsBuilder() .add(3, "default", 10, 1) .add(4, "default", 11, 2); f.update(1, stats); f.verify(stats); + f.verifyGlobal(globalStatsBuilder().add("default", 10 + 11, 1 + 2)); } @Test diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java index 8048e77b05c..af9e2104828 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java @@ -30,8 +30,9 @@ public class ContentClusterHtmlRendererTest { ClusterStateBundle stateBundle = ClusterStateBundle.ofBaselineOnly( AnnotatedClusterState.withoutAnnotations( ClusterState.stateFromString("version:34633 bits:24 distributor:211 storage:211"))); - var metricUpdater = new MetricUpdater(new NoMetricReporter(), 0, clusterName); - EventLog eventLog = new EventLog(new FakeTimer(), metricUpdater); + var timer = new FakeTimer(); + var metricUpdater = new MetricUpdater(new NoMetricReporter(), timer, 0, clusterName); + EventLog eventLog = new EventLog(timer, metricUpdater); VdsClusterHtmlRenderer.Table table = renderer.createNewClusterHtmlTable(clusterName, slobrokGeneration); ContentCluster contentCluster = mock(ContentCluster.class); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStatsBuilder.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStatsBuilder.java index 9d4664a9362..34035793e75 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStatsBuilder.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentNodeStatsBuilder.java @@ -13,7 +13,7 @@ public class ContentNodeStatsBuilder { this.nodeIndex = nodeIndex; } - static ContentNodeStatsBuilder forNode(int nodeIndex) { + public static ContentNodeStatsBuilder forNode(int nodeIndex) { return new ContentNodeStatsBuilder(nodeIndex); } @@ -21,12 +21,16 @@ public class ContentNodeStatsBuilder { return add(bucketSpace, ContentNodeStats.BucketSpaceStats.of(bucketsTotal, bucketsPending)); } + public ContentNodeStatsBuilder addInvalid(String bucketSpace, long bucketsTotal, long bucketsPending) { + return add(bucketSpace, ContentNodeStats.BucketSpaceStats.invalid(bucketsTotal, bucketsPending)); + } + public ContentNodeStatsBuilder add(String bucketSpace, ContentNodeStats.BucketSpaceStats bucketSpaceStats) { stats.put(bucketSpace, bucketSpaceStats); return this; } - ContentNodeStats build() { + public ContentNodeStats build() { return new ContentNodeStats(nodeIndex, stats); } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java index 9146b2812a9..1cefac311d3 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import java.time.Instant; import java.util.logging.Level; import com.yahoo.vespa.clustercontroller.core.testutils.LogFormatter; @@ -22,8 +23,9 @@ public class FakeTimer implements Timer { // Don't start at zero. Clock users may initialize a 'last run' entry with 0, and we want first time to always look like a timeout private long currentTime = (long) 30 * 365 * 24 * 60 * 60 * 1000; - public synchronized long getCurrentTimeInMillis() { - return currentTime; + @Override + public synchronized Instant getCurrentWallClockTime() { + return Instant.ofEpochMilli(currentTime); } public synchronized void advanceTime(long time) { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index 3b26e3b6965..688d82e5ebb 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -120,7 +120,7 @@ public abstract class FleetControllerTest implements Waiter { RpcServer rpcServer, boolean start) { waiter = createWaiter(timer); - var metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex(), options.clusterName()); + var metricUpdater = new MetricUpdater(new NoMetricReporter(), timer, options.fleetControllerIndex(), options.clusterName()); var log = new EventLog(timer, metricUpdater); var cluster = new ContentCluster(options.clusterName(), options.nodes(), options.storageDistribution()); var stateGatherer = new NodeStateGatherer(timer, timer, log); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GlobalBucketSyncStatsCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GlobalBucketSyncStatsCalculatorTest.java new file mode 100644 index 00000000000..d44aaa54a1d --- /dev/null +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GlobalBucketSyncStatsCalculatorTest.java @@ -0,0 +1,59 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class GlobalBucketSyncStatsCalculatorTest { + + private static ContentNodeStatsBuilder globalStatsBuilder() { + return ContentNodeStatsBuilder.forNode(-1); + } + + private static void assertComputedRatio(double expected, ContentNodeStatsBuilder statsBuilder) { + var maybeRatio = GlobalBucketSyncStatsCalculator.clusterBucketsOutOfSyncRatio(statsBuilder.build()); + if (maybeRatio.isEmpty()) { + throw new IllegalArgumentException("Expected calculation to yield a value, but was empty"); + } + assertEquals(expected, maybeRatio.get(), 0.00001); + } + + private static void assertEmptyComputedRatio(ContentNodeStatsBuilder statsBuilder) { + var maybeRatio = GlobalBucketSyncStatsCalculator.clusterBucketsOutOfSyncRatio(statsBuilder.build()); + assertTrue(maybeRatio.isEmpty()); + } + + @Test + void no_buckets_imply_fully_in_sync() { + // Can't have anything out of sync if you don't have anything to be out of sync with *taps side of head* + assertComputedRatio(0.0, globalStatsBuilder().add("default", 0, 0)); + } + + @Test + void no_pending_buckets_implies_fully_in_sync() { + assertComputedRatio(0.0, globalStatsBuilder().add("default", 100, 0)); + assertComputedRatio(0.0, globalStatsBuilder().add("default", 100, 0).add("global", 50, 0)); + } + + @Test + void invalid_stats_returns_empty() { + assertEmptyComputedRatio(globalStatsBuilder().add("default", ContentNodeStats.BucketSpaceStats.invalid())); + assertEmptyComputedRatio(globalStatsBuilder() + .add("default", 100, 0) + .add("global", ContentNodeStats.BucketSpaceStats.invalid())); + } + + @Test + void pending_buckets_return_expected_ratio() { + assertComputedRatio(0.50, globalStatsBuilder().add("default", 10, 5)); + assertComputedRatio(0.80, globalStatsBuilder().add("default", 10, 8)); + assertComputedRatio(0.10, globalStatsBuilder().add("default", 100, 10)); + assertComputedRatio(0.01, globalStatsBuilder().add("default", 100, 1)); + assertComputedRatio(0.05, globalStatsBuilder().add("default", 50, 5).add("global", 50, 0)); + assertComputedRatio(0.05, globalStatsBuilder().add("default", 50, 0).add("global", 50, 5)); + assertComputedRatio(0.10, globalStatsBuilder().add("default", 50, 5).add("global", 50, 5)); + } + +} diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java index 7175aefa97c..5eeb4c55387 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java @@ -2,10 +2,13 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.state.ClusterState; +import com.yahoo.vdslib.state.Node; import com.yahoo.vespa.clustercontroller.core.matchers.HasMetricContext; import com.yahoo.vespa.clustercontroller.utils.util.MetricReporter; import org.junit.jupiter.api.Test; +import java.time.Duration; +import java.time.Instant; import java.util.Map; import static com.yahoo.vespa.clustercontroller.core.matchers.HasMetricContext.hasMetricContext; @@ -25,7 +28,8 @@ public class MetricReporterTest { private static class Fixture { final MetricReporter mockReporter = mock(MetricReporter.class); - final MetricUpdater metricUpdater = new MetricUpdater(mockReporter, 0, CLUSTER_NAME); + final FakeTimer timer = new FakeTimer(); + final MetricUpdater metricUpdater = new MetricUpdater(mockReporter, timer, 0, CLUSTER_NAME); final ClusterFixture clusterFixture; Fixture() { @@ -61,7 +65,7 @@ public class MetricReporterTest { Fixture f = new Fixture(); f.metricUpdater.updateClusterStateMetrics(f.clusterFixture.cluster(), ClusterState.stateFromString("distributor:10 .1.s:d storage:9 .1.s:d .2.s:m .4.s:d"), - new ResourceUsageStats()); + new ResourceUsageStats(), Instant.ofEpochMilli(12345)); verify(f.mockReporter).set(eq("cluster-controller.up.count"), eq(9), argThat(hasMetricContext(withNodeTypeDimension("distributor")))); @@ -78,7 +82,7 @@ public class MetricReporterTest { private void doTestRatiosInState(String clusterState, double distributorRatio, double storageRatio) { Fixture f = new Fixture(); f.metricUpdater.updateClusterStateMetrics(f.clusterFixture.cluster(), ClusterState.stateFromString(clusterState), - new ResourceUsageStats()); + new ResourceUsageStats(), Instant.ofEpochMilli(12345)); verify(f.mockReporter).set(eq("cluster-controller.available-nodes.ratio"), doubleThat(closeTo(distributorRatio, 0.0001)), @@ -115,7 +119,7 @@ public class MetricReporterTest { Fixture f = new Fixture(); f.metricUpdater.updateClusterStateMetrics(f.clusterFixture.cluster(), ClusterState.stateFromString("distributor:10 storage:10"), - new ResourceUsageStats(0.5, 0.6, 5, 0.7, 0.8)); + new ResourceUsageStats(0.5, 0.6, 5, 0.7, 0.8), Instant.ofEpochMilli(12345)); verify(f.mockReporter).set(eq("cluster-controller.resource_usage.max_disk_utilization"), doubleThat(closeTo(0.5, 0.0001)), @@ -138,4 +142,73 @@ public class MetricReporterTest { argThat(hasMetricContext(withClusterDimension()))); } + private static class ConvergenceFixture extends Fixture { + + String stateString; + Instant stateBroadcastTime; + + ConvergenceFixture(String stateString) { + super(5); + this.stateString = stateString; + setUpFixturePendingVersions(); + + metricUpdater.setStateVersionConvergenceGracePeriod(Duration.ofSeconds(10)); + stateBroadcastTime = timer.getCurrentWallClockTime(); + } + + // Sets pending state versions for 5 distributors and storage nodes: + // - distributor: 2 converged, 3 not converged + // - storage: 3 converged, 2 not converged + private void setUpFixturePendingVersions() { + var pendingBundle = ClusterStateBundle.ofBaselineOnly(AnnotatedClusterState.withoutAnnotations( + ClusterState.stateFromString(stateString))); + for (int i = 0; i < 5; ++i) { + clusterFixture.cluster().getNodeInfo(Node.ofDistributor(i)).setClusterStateVersionBundleSent(pendingBundle); + clusterFixture.cluster().getNodeInfo(Node.ofStorage(i)).setClusterStateVersionBundleSent(pendingBundle); + } + clusterFixture.cluster().getNodeInfo(Node.ofDistributor(0)).setClusterStateBundleVersionAcknowledged(100, false); // NACK + clusterFixture.cluster().getNodeInfo(Node.ofDistributor(1)).setClusterStateBundleVersionAcknowledged(100, true); + clusterFixture.cluster().getNodeInfo(Node.ofDistributor(4)).setClusterStateBundleVersionAcknowledged(100, true); + // Heard nothing from distributors {2, 3} yet. + clusterFixture.cluster().getNodeInfo(Node.ofStorage(0)).setClusterStateBundleVersionAcknowledged(100, true); + clusterFixture.cluster().getNodeInfo(Node.ofStorage(1)).setClusterStateBundleVersionAcknowledged(100, true); + clusterFixture.cluster().getNodeInfo(Node.ofStorage(2)).setClusterStateBundleVersionAcknowledged(100, true); + // Heard nothing from storage {3, 4} yet. + } + + void advanceTimeAndVerifyMetrics(Duration delta, int expectedDistr, int expectedStorage) { + timer.advanceTime(delta.toMillis()); + metricUpdater.updateClusterStateMetrics(clusterFixture.cluster(), + ClusterState.stateFromString(stateString), + new ResourceUsageStats(), stateBroadcastTime); + + verify(mockReporter).set(eq("cluster-controller.nodes-not-converged"), eq(expectedDistr), + argThat(hasMetricContext(withNodeTypeDimension("distributor")))); + verify(mockReporter).set(eq("cluster-controller.nodes-not-converged"), eq(expectedStorage), + argThat(hasMetricContext(withNodeTypeDimension("storage")))); + } + + } + + @Test + void nodes_not_converged_metric_not_incremented_when_within_grace_period() { + var f = new ConvergenceFixture("version:100 distributor:5 storage:5"); + // 9 seconds passed since state broadcast; should not tag nodes as not converged + f.advanceTimeAndVerifyMetrics(Duration.ofMillis(9000), 0, 0); + } + + @Test + void nodes_not_converged_metric_incremented_when_outside_grace_period() { + var f = new ConvergenceFixture("version:100 distributor:5 storage:5"); + // 10+ seconds passed since state broadcast; _should_ tag nodes as not converged + f.advanceTimeAndVerifyMetrics(Duration.ofMillis(10001), 3, 2); + } + + @Test + void only_count_nodes_in_available_states_as_non_converging() { + var f = new ConvergenceFixture("version:100 distributor:5 .0.s:d .2.s:d .3.s:d storage:5 .3.s:m .4.s:d"); + // Should not count non-converged nodes, as they are not in an available state + f.advanceTimeAndVerifyMetrics(Duration.ofMillis(10001), 0, 0); + } + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java index d06cc730b3f..902b1bce24a 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java @@ -15,6 +15,8 @@ public class ClusterControllerMock implements RemoteClusterControllerTaskSchedul private final int fleetControllerIndex; Integer fleetControllerMaster; private final StringBuilder events = new StringBuilder(); + ContentNodeStats globalClusterStats = new ContentNodeStats(-1); + boolean enableGlobalStatsReporting = false; ClusterControllerMock(ContentCluster cluster, ClusterState state, ClusterStateBundle publishedClusterStateBundle, @@ -88,6 +90,22 @@ public class ClusterControllerMock implements RemoteClusterControllerTaskSchedul } }; + context.aggregatedClusterStats = new AggregatedClusterStats() { + @Override + public boolean hasUpdatesFromAllDistributors() { + return enableGlobalStatsReporting; + } + + @Override + public ContentClusterStats getStats() { + return null; + } + + @Override + public ContentNodeStats getGlobalStats() { + return globalClusterStats; + } + }; } @Override diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterTest.java index e4b3c0b9f2c..cb1213542ce 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.restapiv2; +import com.yahoo.vespa.clustercontroller.core.ContentNodeStatsBuilder; import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.UnitResponse; import org.junit.jupiter.api.Test; @@ -105,4 +106,45 @@ public class ClusterTest extends StateRestApiTest { }""", jsonWriter.createJson(response).toPrettyString()); } + + @Test + void emit_cluster_stats_if_present() throws Exception { + setUp(true); + books.globalClusterStats.add(ContentNodeStatsBuilder.forNode(-1).add("default", 10, 4).build()); + books.enableGlobalStatsReporting = true; + UnitResponse response = restAPI.getState(new StateRequest("books", 0)); + assertEquals(""" + { + "state" : { + "generated" : { + "state" : "up", + "reason" : "" + } + }, + "metrics" : { + "cluster-buckets-out-of-sync-ratio" : 0.4 + }, + "service" : { + "storage" : { + "link" : "/cluster/v2/books/storage" + }, + "distributor" : { + "link" : "/cluster/v2/books/distributor" + } + }, + "distribution-states" : { + "published" : { + "baseline" : "distributor:4 storage:4", + "bucket-spaces" : [ { + "name" : "default", + "state" : "distributor:4 storage:4 .3.s:m" + }, { + "name" : "global", + "state" : "distributor:4 storage:4" + } ] + } + } + }""", + jsonWriter.createJson(response).toPrettyString()); + } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java index dfd9783ecef..1ad5f6828b7 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java @@ -30,7 +30,7 @@ import java.util.stream.Collectors; public abstract class StateRestApiTest { - private ClusterControllerMock books; + ClusterControllerMock books; ClusterControllerMock music; StateRestAPI restAPI; JsonWriter jsonWriter = new JsonWriter(); |