aboutsummaryrefslogtreecommitdiffstats
path: root/node-repository
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@gmail.com>2020-10-30 13:21:01 +0100
committerJon Bratseth <bratseth@gmail.com>2020-10-30 13:21:01 +0100
commit209148a9f8b24526871dda8b0b2b639e5ee5cd8a (patch)
treecdfd59a3835390cc5009779106ff0e4af7c48403 /node-repository
parentd10f3a3c184b67cfc77ff8f9dc7456afdb33cbcd (diff)
Accept recent out-of-order metrics
Diffstat (limited to 'node-repository')
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java15
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java36
2 files changed, 49 insertions, 2 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
index 1d6d96c098d..070bf98bf87 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
@@ -80,8 +80,8 @@ public class QuestMetricsDb implements MetricsDb {
public void add(Collection<Pair<String, MetricSnapshot>> snapshots) {
try (TableWriter writer = engine.getWriter(newContext().getCairoSecurityContext(), tableName)) {
for (var snapshot : snapshots) {
- long atMillis = snapshot.getSecond().at().toEpochMilli();
- if (atMillis < highestTimestampAdded) continue; // Ignore old data without causing an exception in QuestDb
+ long atMillis = adjustIfRecent(snapshot.getSecond().at().toEpochMilli(), highestTimestampAdded);
+ if (atMillis < highestTimestampAdded) continue; // Ignore old data
highestTimestampAdded = atMillis;
TableWriter.Row row = writer.newRow(atMillis * 1000); // in microseconds
row.putStr(0, snapshot.getFirst());
@@ -159,6 +159,17 @@ public class QuestMetricsDb implements MetricsDb {
}
}
+ private long adjustIfRecent(long timestamp, long highestTimestampAdded) {
+ if (timestamp >= highestTimestampAdded) return timestamp;
+
+ // We cannot add old data to QuestDb, but we want to use all recent information
+ long oneMinute = 60 * 1000;
+ if (timestamp >= highestTimestampAdded - oneMinute) return highestTimestampAdded;
+
+ // Too old; discard
+ return timestamp;
+ }
+
private ListMap<String, MetricSnapshot> getSnapshots(Instant startTime,
Set<String> hostnames,
SqlCompiler compiler,
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
index a1cc66ffa28..6d52fb29160 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
@@ -69,6 +69,30 @@ public class QuestMetricsDbTest {
}
@Test
+ public void testWriteOldData() {
+ String dataDir = "data/QuestMetricsDbWriteOldData";
+ IOUtils.recursiveDeleteDir(new File(dataDir));
+ IOUtils.createDirectory(dataDir + "/metrics");
+ ManualClock clock = new ManualClock("2020-10-01T00:00:00");
+ QuestMetricsDb db = new QuestMetricsDb(dataDir, clock);
+ Instant startTime = clock.instant();
+ clock.advance(Duration.ofSeconds(300));
+ db.add(timeseriesAt(10, clock.instant(), "host1", "host2", "host3"));
+ clock.advance(Duration.ofSeconds(1));
+
+ List<NodeTimeseries> nodeTimeSeries1 = db.getNodeTimeseries(startTime, Set.of("host1"));
+ assertEquals(10, nodeTimeSeries1.get(0).size());
+
+ db.add(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(20)), "host1", "host2", "host3"));
+ List<NodeTimeseries> nodeTimeSeries2 = db.getNodeTimeseries(startTime, Set.of("host1"));
+ assertEquals("Recent data is accepted", 20, nodeTimeSeries2.get(0).size());
+
+ db.add(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(200)), "host1", "host2", "host3"));
+ List<NodeTimeseries> nodeTimeSeries3 = db.getNodeTimeseries(startTime, Set.of("host1"));
+ assertEquals("Too old data is rejected", 20, nodeTimeSeries3.get(0).size());
+ }
+
+ @Test
public void testGc() {
String dataDir = "data/QuestMetricsDbGc";
IOUtils.recursiveDeleteDir(new File(dataDir));
@@ -102,4 +126,16 @@ public class QuestMetricsDbTest {
return timeseries;
}
+ private Collection<Pair<String, MetricSnapshot>> timeseriesAt(int countPerHost, Instant at, String ... hosts) {
+ Collection<Pair<String, MetricSnapshot>> timeseries = new ArrayList<>();
+ for (int i = 1; i <= countPerHost; i++) {
+ for (String host : hosts)
+ timeseries.add(new Pair<>(host, new MetricSnapshot(at,
+ i * 0.1,
+ i * 0.2,
+ i * 0.4,
+ i % 100)));
+ }
+ return timeseries;
+ }
}