diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2020-10-30 14:29:51 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-10-30 14:29:51 +0100 |
commit | 9745101f518e293b40442ac4182483060eaabb56 (patch) | |
tree | e73fc37c8ea9edd1c0ad8184ca83ef9850adf867 /node-repository | |
parent | 553fbac3137ffb5fb0559c4052f46e061beb3b0a (diff) | |
parent | 209148a9f8b24526871dda8b0b2b639e5ee5cd8a (diff) |
Merge pull request #15103 from vespa-engine/bratseth/skip-old-metrics
Skip old metrics
Diffstat (limited to 'node-repository')
2 files changed, 53 insertions, 1 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index b1585922f38..070bf98bf87 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -49,6 +49,8 @@ public class QuestMetricsDb implements MetricsDb { private final String dataDir; private final CairoEngine engine; + private long highestTimestampAdded = 0; + @Inject public QuestMetricsDb() { this(Defaults.getDefaults().underVespaHome("var/db/vespa/autoscaling"), Clock.systemUTC()); @@ -67,6 +69,7 @@ public class QuestMetricsDb implements MetricsDb { // silence Questdb's custom logging system IOUtils.writeFile(new File(dataDir, "quest-log.conf"), new byte[0]); System.setProperty("questdbLog", dataDir + "/quest-log.conf"); + System.setProperty("org.jooq.no-logo", "true"); CairoConfiguration configuration = new DefaultCairoConfiguration(dataDir); engine = new CairoEngine(configuration); @@ -77,7 +80,9 @@ public class QuestMetricsDb implements MetricsDb { public void add(Collection<Pair<String, MetricSnapshot>> snapshots) { try (TableWriter writer = engine.getWriter(newContext().getCairoSecurityContext(), tableName)) { for (var snapshot : snapshots) { - long atMillis = snapshot.getSecond().at().toEpochMilli(); + long atMillis = adjustIfRecent(snapshot.getSecond().at().toEpochMilli(), highestTimestampAdded); + if (atMillis < highestTimestampAdded) continue; // Ignore old data + highestTimestampAdded = atMillis; TableWriter.Row row = writer.newRow(atMillis * 1000); // in microseconds row.putStr(0, snapshot.getFirst()); row.putFloat(2, (float)snapshot.getSecond().cpu()); @@ -154,6 +159,17 @@ public class QuestMetricsDb implements MetricsDb { } } + private long adjustIfRecent(long timestamp, long highestTimestampAdded) { + if (timestamp >= highestTimestampAdded) return timestamp; + + // We cannot add old data to QuestDb, but we want to use all recent information + long oneMinute = 60 * 1000; + if (timestamp >= highestTimestampAdded - oneMinute) return highestTimestampAdded; + + // Too old; discard + return timestamp; + } + private ListMap<String, MetricSnapshot> getSnapshots(Instant startTime, Set<String> hostnames, SqlCompiler compiler, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java index a1cc66ffa28..6d52fb29160 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java @@ -69,6 +69,30 @@ public class QuestMetricsDbTest { } @Test + public void testWriteOldData() { + String dataDir = "data/QuestMetricsDbWriteOldData"; + IOUtils.recursiveDeleteDir(new File(dataDir)); + IOUtils.createDirectory(dataDir + "/metrics"); + ManualClock clock = new ManualClock("2020-10-01T00:00:00"); + QuestMetricsDb db = new QuestMetricsDb(dataDir, clock); + Instant startTime = clock.instant(); + clock.advance(Duration.ofSeconds(300)); + db.add(timeseriesAt(10, clock.instant(), "host1", "host2", "host3")); + clock.advance(Duration.ofSeconds(1)); + + List<NodeTimeseries> nodeTimeSeries1 = db.getNodeTimeseries(startTime, Set.of("host1")); + assertEquals(10, nodeTimeSeries1.get(0).size()); + + db.add(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(20)), "host1", "host2", "host3")); + List<NodeTimeseries> nodeTimeSeries2 = db.getNodeTimeseries(startTime, Set.of("host1")); + assertEquals("Recent data is accepted", 20, nodeTimeSeries2.get(0).size()); + + db.add(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(200)), "host1", "host2", "host3")); + List<NodeTimeseries> nodeTimeSeries3 = db.getNodeTimeseries(startTime, Set.of("host1")); + assertEquals("Too old data is rejected", 20, nodeTimeSeries3.get(0).size()); + } + + @Test public void testGc() { String dataDir = "data/QuestMetricsDbGc"; IOUtils.recursiveDeleteDir(new File(dataDir)); @@ -102,4 +126,16 @@ public class QuestMetricsDbTest { return timeseries; } + private Collection<Pair<String, MetricSnapshot>> timeseriesAt(int countPerHost, Instant at, String ... hosts) { + Collection<Pair<String, MetricSnapshot>> timeseries = new ArrayList<>(); + for (int i = 1; i <= countPerHost; i++) { + for (String host : hosts) + timeseries.add(new Pair<>(host, new MetricSnapshot(at, + i * 0.1, + i * 0.2, + i * 0.4, + i % 100))); + } + return timeseries; + } } |