Merge pull request #15103 from vespa-engine/bratseth/skip-old-metrics

Skip old metrics

2 files changed, 53 insertions, 1 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
index b1585922f38..070bf98bf87 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
@@ -49,6 +49,8 @@ public class QuestMetricsDb implements MetricsDb {
     private final String dataDir;
     private final CairoEngine engine;
 
+    private long highestTimestampAdded = 0;
+
     @Inject
     public QuestMetricsDb() {
         this(Defaults.getDefaults().underVespaHome("var/db/vespa/autoscaling"), Clock.systemUTC());
@@ -67,6 +69,7 @@ public class QuestMetricsDb implements MetricsDb {
         // silence Questdb's custom logging system
         IOUtils.writeFile(new File(dataDir, "quest-log.conf"), new byte[0]);
         System.setProperty("questdbLog", dataDir + "/quest-log.conf");
+        System.setProperty("org.jooq.no-logo", "true");
 
         CairoConfiguration configuration = new DefaultCairoConfiguration(dataDir);
         engine = new CairoEngine(configuration);
@@ -77,7 +80,9 @@ public class QuestMetricsDb implements MetricsDb {
     public void add(Collection<Pair<String, MetricSnapshot>> snapshots) {
         try (TableWriter writer = engine.getWriter(newContext().getCairoSecurityContext(), tableName)) {
             for (var snapshot : snapshots) {
-                long atMillis = snapshot.getSecond().at().toEpochMilli();
+                long atMillis = adjustIfRecent(snapshot.getSecond().at().toEpochMilli(), highestTimestampAdded);
+                if (atMillis < highestTimestampAdded) continue; // Ignore old data
+                highestTimestampAdded = atMillis;
                 TableWriter.Row row = writer.newRow(atMillis * 1000); // in microseconds
                 row.putStr(0, snapshot.getFirst());
                 row.putFloat(2, (float)snapshot.getSecond().cpu());
@@ -154,6 +159,17 @@ public class QuestMetricsDb implements MetricsDb {
         }
     }
 
+    private long adjustIfRecent(long timestamp, long highestTimestampAdded) {
+        if (timestamp >= highestTimestampAdded) return timestamp;
+
+        // We cannot add old data to QuestDb, but we want to use all recent information
+        long oneMinute = 60 * 1000;
+        if (timestamp >= highestTimestampAdded - oneMinute) return highestTimestampAdded;
+
+        // Too old; discard
+        return timestamp;
+    }
+
     private ListMap<String, MetricSnapshot> getSnapshots(Instant startTime,
                                                          Set<String> hostnames,
                                                          SqlCompiler compiler,
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
index a1cc66ffa28..6d52fb29160 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
@@ -69,6 +69,30 @@ public class QuestMetricsDbTest {
     }
 
     @Test
+    public void testWriteOldData() {
+        String dataDir = "data/QuestMetricsDbWriteOldData";
+        IOUtils.recursiveDeleteDir(new File(dataDir));
+        IOUtils.createDirectory(dataDir + "/metrics");
+        ManualClock clock = new ManualClock("2020-10-01T00:00:00");
+        QuestMetricsDb db = new QuestMetricsDb(dataDir, clock);
+        Instant startTime = clock.instant();
+        clock.advance(Duration.ofSeconds(300));
+        db.add(timeseriesAt(10, clock.instant(), "host1", "host2", "host3"));
+        clock.advance(Duration.ofSeconds(1));
+
+        List<NodeTimeseries> nodeTimeSeries1 = db.getNodeTimeseries(startTime, Set.of("host1"));
+        assertEquals(10, nodeTimeSeries1.get(0).size());
+
+        db.add(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(20)), "host1", "host2", "host3"));
+        List<NodeTimeseries> nodeTimeSeries2 = db.getNodeTimeseries(startTime, Set.of("host1"));
+        assertEquals("Recent data is accepted", 20, nodeTimeSeries2.get(0).size());
+
+        db.add(timeseriesAt(10, clock.instant().minus(Duration.ofSeconds(200)), "host1", "host2", "host3"));
+        List<NodeTimeseries> nodeTimeSeries3 = db.getNodeTimeseries(startTime, Set.of("host1"));
+        assertEquals("Too old data is rejected", 20, nodeTimeSeries3.get(0).size());
+    }
+
+    @Test
     public void testGc() {
         String dataDir = "data/QuestMetricsDbGc";
         IOUtils.recursiveDeleteDir(new File(dataDir));
@@ -102,4 +126,16 @@ public class QuestMetricsDbTest {
         return timeseries;
     }
 
+    private Collection<Pair<String, MetricSnapshot>> timeseriesAt(int countPerHost, Instant at, String ... hosts) {
+        Collection<Pair<String, MetricSnapshot>> timeseries = new ArrayList<>();
+        for (int i = 1; i <= countPerHost; i++) {
+            for (String host : hosts)
+                timeseries.add(new Pair<>(host, new MetricSnapshot(at,
+                                                                   i * 0.1,
+                                                                   i * 0.2,
+                                                                   i * 0.4,
+                                                                   i % 100)));
+        }
+        return timeseries;
+    }
 }