1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
|
// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.autoscale;
import com.yahoo.collections.Pair;
import com.yahoo.io.IOUtils;
import com.yahoo.test.ManualClock;
import org.junit.Ignore;
import org.junit.Test;
import java.io.File;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
/**
* Tests the Quest metrics db.
*
* @author bratseth
*/
public class QuestMetricsDbTest {
private static final double delta = 0.0000001;
@Test
public void testReadWrite() {
String dataDir = "data/QuestMetricsDbReadWrite";
IOUtils.recursiveDeleteDir(new File(dataDir));
IOUtils.createDirectory(dataDir + "/metrics");
ManualClock clock = new ManualClock("2020-10-01T00:00:00");
QuestMetricsDb db = new QuestMetricsDb(dataDir, clock);
Instant startTime = clock.instant();
clock.advance(Duration.ofSeconds(1));
db.add(timeseries(1000, Duration.ofSeconds(1), clock, "host1", "host2", "host3"));
clock.advance(Duration.ofSeconds(1));
// Read all of one host
List<NodeTimeseries> nodeTimeSeries1 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
Set.of("host1"));
assertEquals(1, nodeTimeSeries1.size());
assertEquals("host1", nodeTimeSeries1.get(0).hostname());
assertEquals(1000, nodeTimeSeries1.get(0).size());
MetricSnapshot snapshot = nodeTimeSeries1.get(0).asList().get(0);
assertEquals(startTime.plus(Duration.ofSeconds(1)), snapshot.at());
assertEquals(0.1, snapshot.cpu(), delta);
assertEquals(0.2, snapshot.memory(), delta);
assertEquals(0.4, snapshot.disk(), delta);
assertEquals(1, snapshot.generation(), delta);
assertEquals(30, snapshot.queryRate(), delta);
// Read all from 2 hosts
List<NodeTimeseries> nodeTimeSeries2 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
Set.of("host2", "host3"));
assertEquals(2, nodeTimeSeries2.size());
assertEquals(Set.of("host2", "host3"), nodeTimeSeries2.stream().map(ts -> ts.hostname()).collect(Collectors.toSet()));
assertEquals(1000, nodeTimeSeries2.get(0).size());
assertEquals(1000, nodeTimeSeries2.get(1).size());
// Read a short interval from 3 hosts
List<NodeTimeseries> nodeTimeSeries3 = db.getNodeTimeseries(Duration.ofSeconds(3),
Set.of("host1", "host2", "host3"));
assertEquals(3, nodeTimeSeries3.size());
assertEquals(Set.of("host1", "host2", "host3"), nodeTimeSeries3.stream().map(ts -> ts.hostname()).collect(Collectors.toSet()));
assertEquals(2, nodeTimeSeries3.get(0).size());
assertEquals(2, nodeTimeSeries3.get(1).size());
assertEquals(2, nodeTimeSeries3.get(2).size());
}
@Test
public void testWriteOldData() {
String dataDir = "data/QuestMetricsDbWriteOldData";
IOUtils.recursiveDeleteDir(new File(dataDir));
IOUtils.createDirectory(dataDir + "/metrics");
ManualClock clock = new ManualClock("2020-10-01T00:00:00");
QuestMetricsDb db = new QuestMetricsDb(dataDir, clock);
Instant startTime = clock.instant();
clock.advance(Duration.ofSeconds(300));
db.add(timeseriesAt(10, clock.instant(), "host1", "host2", "host3"));
clock.advance(Duration.ofSeconds(1));
List<NodeTimeseries> nodeTimeSeries1 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
Set.of("host1"));
assertEquals(10, nodeTimeSeries1.get(0).size());
db.add(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(20)), "host1", "host2", "host3"));
List<NodeTimeseries> nodeTimeSeries2 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
Set.of("host1"));
assertEquals("Recent data is accepted", 20, nodeTimeSeries2.get(0).size());
db.add(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(200)), "host1", "host2", "host3"));
List<NodeTimeseries> nodeTimeSeries3 = db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
Set.of("host1"));
assertEquals("Too old data is rejected", 20, nodeTimeSeries3.get(0).size());
}
@Test
public void testGc() {
String dataDir = "data/QuestMetricsDbGc";
IOUtils.recursiveDeleteDir(new File(dataDir));
IOUtils.createDirectory(dataDir + "/metrics");
ManualClock clock = new ManualClock("2020-10-01T00:00:00");
QuestMetricsDb db = new QuestMetricsDb(dataDir, clock);
Instant startTime = clock.instant();
int dayOffset = 3;
clock.advance(Duration.ofHours(dayOffset));
db.add(timeseries(24 * 10, Duration.ofHours(1), clock, "host1", "host2", "host3"));
assertEquals(24 * 10, db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
Set.of("host1")).get(0).size());
db.gc();
assertEquals(48 * 1 + dayOffset, db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
Set.of("host1")).get(0).size());
db.gc(); // no-op
assertEquals(48 * 1 + dayOffset, db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
Set.of("host1")).get(0).size());
}
/** To manually test that we can read existing data */
@Ignore
@Test
public void testReadingAndAppendingToExistingData() {
String dataDir = "data/QuestMetricsDbExistingData";
if ( ! new File(dataDir).exists()) {
System.out.println("No existing data to check");
return;
}
IOUtils.createDirectory(dataDir + "/metrics");
ManualClock clock = new ManualClock("2020-10-01T00:00:00");
clock.advance(Duration.ofSeconds(9)); // Adjust to last data written
QuestMetricsDb db = new QuestMetricsDb(dataDir, clock);
List<NodeTimeseries> timeseries = db.getNodeTimeseries(Duration.ofSeconds(9), Set.of("host1"));
assertFalse("Could read existing data", timeseries.isEmpty());
assertEquals(10, timeseries.get(0).size());
System.out.println("Existing data read:");
for (var snapshot : timeseries.get(0).asList())
System.out.println(" " + snapshot);
clock.advance(Duration.ofSeconds(1));
db.add(timeseries(2, Duration.ofSeconds(1), clock, "host1"));
System.out.println("New data written and read:");
timeseries = db.getNodeTimeseries(Duration.ofSeconds(2), Set.of("host1"));
for (var snapshot : timeseries.get(0).asList())
System.out.println(" " + snapshot);
}
/** To update data for the manual test above */
@Ignore
@Test
public void updateExistingData() {
String dataDir = "data/QuestMetricsDbExistingData";
IOUtils.recursiveDeleteDir(new File(dataDir));
IOUtils.createDirectory(dataDir + "/metrics");
ManualClock clock = new ManualClock("2020-10-01T00:00:00");
QuestMetricsDb db = new QuestMetricsDb(dataDir, clock);
Instant startTime = clock.instant();
db.add(timeseries(10, Duration.ofSeconds(1), clock, "host1"));
int added = db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
Set.of("host1")).get(0).asList().size();
System.out.println("Added " + added + " rows of data");
db.close();
}
private Collection<Pair<String, MetricSnapshot>> timeseries(int countPerHost, Duration sampleRate, ManualClock clock,
String ... hosts) {
Collection<Pair<String, MetricSnapshot>> timeseries = new ArrayList<>();
for (int i = 1; i <= countPerHost; i++) {
for (String host : hosts)
timeseries.add(new Pair<>(host, new MetricSnapshot(clock.instant(),
i * 0.1,
i * 0.2,
i * 0.4,
i % 100,
true,
true,
30.0)));
clock.advance(sampleRate);
}
return timeseries;
}
private Collection<Pair<String, MetricSnapshot>> timeseriesAt(int countPerHost, Instant at, String ... hosts) {
Collection<Pair<String, MetricSnapshot>> timeseries = new ArrayList<>();
for (int i = 1; i <= countPerHost; i++) {
for (String host : hosts)
timeseries.add(new Pair<>(host, new MetricSnapshot(at,
i * 0.1,
i * 0.2,
i * 0.4,
i % 100,
true,
false,
0.0)));
}
return timeseries;
}
}
|
