diff options
author | Jon Bratseth <bratseth@gmail.com> | 2020-10-30 13:21:01 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2020-10-30 13:21:01 +0100 |
commit | 209148a9f8b24526871dda8b0b2b639e5ee5cd8a (patch) | |
tree | cdfd59a3835390cc5009779106ff0e4af7c48403 /node-repository | |
parent | d10f3a3c184b67cfc77ff8f9dc7456afdb33cbcd (diff) |
Accept recent out-of-order metrics
Diffstat (limited to 'node-repository')
2 files changed, 49 insertions, 2 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index 1d6d96c098d..070bf98bf87 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -80,8 +80,8 @@ public class QuestMetricsDb implements MetricsDb { public void add(Collection<Pair<String, MetricSnapshot>> snapshots) { try (TableWriter writer = engine.getWriter(newContext().getCairoSecurityContext(), tableName)) { for (var snapshot : snapshots) { - long atMillis = snapshot.getSecond().at().toEpochMilli(); - if (atMillis < highestTimestampAdded) continue; // Ignore old data without causing an exception in QuestDb + long atMillis = adjustIfRecent(snapshot.getSecond().at().toEpochMilli(), highestTimestampAdded); + if (atMillis < highestTimestampAdded) continue; // Ignore old data highestTimestampAdded = atMillis; TableWriter.Row row = writer.newRow(atMillis * 1000); // in microseconds row.putStr(0, snapshot.getFirst()); @@ -159,6 +159,17 @@ public class QuestMetricsDb implements MetricsDb { } } + private long adjustIfRecent(long timestamp, long highestTimestampAdded) { + if (timestamp >= highestTimestampAdded) return timestamp; + + // We cannot add old data to QuestDb, but we want to use all recent information + long oneMinute = 60 * 1000; + if (timestamp >= highestTimestampAdded - oneMinute) return highestTimestampAdded; + + // Too old; discard + return timestamp; + } + private ListMap<String, MetricSnapshot> getSnapshots(Instant startTime, Set<String> hostnames, SqlCompiler compiler, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java index a1cc66ffa28..6d52fb29160 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java @@ -69,6 +69,30 @@ public class QuestMetricsDbTest { } @Test + public void testWriteOldData() { + String dataDir = "data/QuestMetricsDbWriteOldData"; + IOUtils.recursiveDeleteDir(new File(dataDir)); + IOUtils.createDirectory(dataDir + "/metrics"); + ManualClock clock = new ManualClock("2020-10-01T00:00:00"); + QuestMetricsDb db = new QuestMetricsDb(dataDir, clock); + Instant startTime = clock.instant(); + clock.advance(Duration.ofSeconds(300)); + db.add(timeseriesAt(10, clock.instant(), "host1", "host2", "host3")); + clock.advance(Duration.ofSeconds(1)); + + List<NodeTimeseries> nodeTimeSeries1 = db.getNodeTimeseries(startTime, Set.of("host1")); + assertEquals(10, nodeTimeSeries1.get(0).size()); + + db.add(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(20)), "host1", "host2", "host3")); + List<NodeTimeseries> nodeTimeSeries2 = db.getNodeTimeseries(startTime, Set.of("host1")); + assertEquals("Recent data is accepted", 20, nodeTimeSeries2.get(0).size()); + + db.add(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(200)), "host1", "host2", "host3")); + List<NodeTimeseries> nodeTimeSeries3 = db.getNodeTimeseries(startTime, Set.of("host1")); + assertEquals("Too old data is rejected", 20, nodeTimeSeries3.get(0).size()); + } + + @Test public void testGc() { String dataDir = "data/QuestMetricsDbGc"; IOUtils.recursiveDeleteDir(new File(dataDir)); @@ -102,4 +126,16 @@ public class QuestMetricsDbTest { return timeseries; } + private Collection<Pair<String, MetricSnapshot>> timeseriesAt(int countPerHost, Instant at, String ... hosts) { + Collection<Pair<String, MetricSnapshot>> timeseries = new ArrayList<>(); + for (int i = 1; i <= countPerHost; i++) { + for (String host : hosts) + timeseries.add(new Pair<>(host, new MetricSnapshot(at, + i * 0.1, + i * 0.2, + i * 0.4, + i % 100))); + } + return timeseries; + } } |