1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.clustercontroller.core;
import com.yahoo.vdslib.state.ClusterState;
import com.yahoo.vespa.clustercontroller.core.matchers.HasMetricContext;
import com.yahoo.vespa.clustercontroller.utils.util.MetricReporter;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static com.yahoo.vespa.clustercontroller.core.matchers.HasMetricContext.hasMetricContext;
import static com.yahoo.vespa.clustercontroller.core.matchers.HasMetricContext.withDimension;
import static org.hamcrest.Matchers.closeTo;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.hamcrest.MockitoHamcrest.argThat;
import static org.mockito.hamcrest.MockitoHamcrest.doubleThat;
public class MetricReporterTest {
private static final String CLUSTER_NAME = "foo";
private static class Fixture {
final MetricReporter mockReporter = mock(MetricReporter.class);
final MetricUpdater metricUpdater = new MetricUpdater(mockReporter, 0, CLUSTER_NAME);
final ClusterFixture clusterFixture;
Fixture() {
this(10);
}
Fixture(int nodes) {
clusterFixture = ClusterFixture.forFlatCluster(nodes);
when(mockReporter.createContext(any())).then(invocation -> {
@SuppressWarnings("unchecked") Map<String, ?> arg = (Map<String, ?>)invocation.getArguments()[0];
return new HasMetricContext.MockContext(arg);
});
}
}
private static HasMetricContext.Dimension[] withClusterDimension() {
// Dimensions that are always present
HasMetricContext.Dimension controllerDim = withDimension("controller-index", "0");
HasMetricContext.Dimension clusterDim = withDimension("cluster", CLUSTER_NAME);
HasMetricContext.Dimension clusteridDim = withDimension("clusterid", CLUSTER_NAME);
return new HasMetricContext.Dimension[] { controllerDim, clusterDim, clusteridDim };
}
private static HasMetricContext.Dimension[] withNodeTypeDimension(String type) {
// Node type-specific dimension
HasMetricContext.Dimension nodeType = withDimension("node-type", type);
var otherDims = withClusterDimension();
return new HasMetricContext.Dimension[] { otherDims[0], otherDims[1], otherDims[2], nodeType };
}
@Test
void metrics_are_emitted_for_different_node_state_counts() {
Fixture f = new Fixture();
f.metricUpdater.updateClusterStateMetrics(f.clusterFixture.cluster(),
ClusterState.stateFromString("distributor:10 .1.s:d storage:9 .1.s:d .2.s:m .4.s:d"),
new ResourceUsageStats());
verify(f.mockReporter).set(eq("cluster-controller.up.count"), eq(9),
argThat(hasMetricContext(withNodeTypeDimension("distributor"))));
verify(f.mockReporter).set(eq("cluster-controller.up.count"), eq(6),
argThat(hasMetricContext(withNodeTypeDimension("storage"))));
verify(f.mockReporter).set(eq("cluster-controller.down.count"), eq(1),
argThat(hasMetricContext(withNodeTypeDimension("distributor"))));
verify(f.mockReporter).set(eq("cluster-controller.down.count"), eq(3),
argThat(hasMetricContext(withNodeTypeDimension("storage"))));
verify(f.mockReporter).set(eq("cluster-controller.maintenance.count"), eq(1),
argThat(hasMetricContext(withNodeTypeDimension("storage"))));
}
private void doTestRatiosInState(String clusterState, double distributorRatio, double storageRatio) {
Fixture f = new Fixture();
f.metricUpdater.updateClusterStateMetrics(f.clusterFixture.cluster(), ClusterState.stateFromString(clusterState),
new ResourceUsageStats());
verify(f.mockReporter).set(eq("cluster-controller.available-nodes.ratio"),
doubleThat(closeTo(distributorRatio, 0.0001)),
argThat(hasMetricContext(withNodeTypeDimension("distributor"))));
verify(f.mockReporter).set(eq("cluster-controller.available-nodes.ratio"),
doubleThat(closeTo(storageRatio, 0.0001)),
argThat(hasMetricContext(withNodeTypeDimension("storage"))));
}
@Test
void metrics_are_emitted_for_partial_node_availability_ratio() {
// Only Up, Init, Retired and Maintenance are counted as available states
doTestRatiosInState("distributor:10 .1.s:d storage:9 .1.s:d .2.s:m .4.s:r .5.s:i .6.s:s", 0.9, 0.7);
}
@Test
void metrics_are_emitted_for_full_node_availability_ratio() {
doTestRatiosInState("distributor:10 storage:10", 1.0, 1.0);
}
@Test
void metrics_are_emitted_for_zero_node_availability_ratio() {
doTestRatiosInState("cluster:d", 0.0, 0.0);
}
@Test
void maintenance_mode_is_counted_as_available() {
doTestRatiosInState("distributor:10 storage:10 .0.s:m", 1.0, 1.0);
}
@Test
void metrics_are_emitted_for_resource_usage() {
Fixture f = new Fixture();
f.metricUpdater.updateClusterStateMetrics(f.clusterFixture.cluster(),
ClusterState.stateFromString("distributor:10 storage:10"),
new ResourceUsageStats(0.5, 0.6, 5, 0.7, 0.8));
verify(f.mockReporter).set(eq("cluster-controller.resource_usage.max_disk_utilization"),
doubleThat(closeTo(0.5, 0.0001)),
argThat(hasMetricContext(withClusterDimension())));
verify(f.mockReporter).set(eq("cluster-controller.resource_usage.max_memory_utilization"),
doubleThat(closeTo(0.6, 0.0001)),
argThat(hasMetricContext(withClusterDimension())));
verify(f.mockReporter).set(eq("cluster-controller.resource_usage.nodes_above_limit"),
eq(5),
argThat(hasMetricContext(withClusterDimension())));
verify(f.mockReporter).set(eq("cluster-controller.resource_usage.disk_limit"),
doubleThat(closeTo(0.7, 0.0001)),
argThat(hasMetricContext(withClusterDimension())));
verify(f.mockReporter).set(eq("cluster-controller.resource_usage.memory_limit"),
doubleThat(closeTo(0.8, 0.0001)),
argThat(hasMetricContext(withClusterDimension())));
}
}
|
