Use SQL to gc

2 files changed, 19 insertions, 34 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
index 8d97e14fc7c..3e4bf6233cc 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
@@ -358,30 +358,10 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
 
         void gc() {
             synchronized (writeLock) {
-                // We remove full days at once and we want to see at least three days to not every only see weekend data
-                Instant oldestToKeep = clock.instant().minus(Duration.ofDays(4));
-                SqlExecutionContext context = newContext();
-                int partitions = 0;
                 try {
-                    List<String> removeList = new ArrayList<>();
-                    for (String dirEntry : dir.list()) {
-                        File partitionDir = new File(dir, dirEntry);
-                        if (!partitionDir.isDirectory()) continue;
-
-                        partitions++;
-                        DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of("UTC"));
-                        Instant partitionDay = Instant.from(formatter.parse(dirEntry.substring(0, 10) + "T00:00:00"));
-                        if (partitionDay.isBefore(oldestToKeep))
-                            removeList.add(dirEntry);
-
-                    }
-                    // Remove unless all partitions are old: Removing all partitions "will be supported in the future"
-                    if (removeList.size() < partitions && !removeList.isEmpty()) {
-                        issue("alter table " + name + " drop partition list " +
-                              removeList.stream().map(dir -> "'" + dir + "'").collect(Collectors.joining(",")),
-                              context);
-                    }
-                } catch (SqlException e) {
+                    issue("alter table " + name + " drop partition where at < dateadd('d', -4, now());", newContext());
+                }
+                catch (SqlException e) {
                     log.log(Level.WARNING, "Failed to gc old metrics data in " + dir + " table " + name, e);
                 }
             }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
index 34243f4548f..1c458750569 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java
@@ -22,6 +22,7 @@ import java.util.stream.Collectors;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 /**
  * Tests the Quest metrics db.
@@ -163,21 +164,25 @@ public class QuestMetricsDbTest {
         String dataDir = "data/QuestMetricsDbGc";
         IOUtils.recursiveDeleteDir(new File(dataDir));
         IOUtils.createDirectory(dataDir + "/metrics");
-        ManualClock clock = new ManualClock("2020-10-01T00:00:00");
-        QuestMetricsDb db = new QuestMetricsDb(dataDir, clock);
+        ManualClock clock = new ManualClock();
+        int days = 10; // The first metrics are this many days in the past
+        clock.retreat(Duration.ofDays(10));
         Instant startTime = clock.instant();
-        int dayOffset = 3;
-        clock.advance(Duration.ofHours(dayOffset));
-        db.addNodeMetrics(nodeTimeseries(24 * 10, Duration.ofHours(1), clock, "host1", "host2", "host3"));
 
-        assertEquals(24 * 10, db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
-                                                   Set.of("host1")).get(0).size());
-        db.gc();
-        assertEquals(75, db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
+        QuestMetricsDb db = new QuestMetricsDb(dataDir, clock);
+
+        db.addNodeMetrics(nodeTimeseries(24 * days, Duration.ofHours(1), clock, "host1", "host2", "host3"));
+
+        var application1 = ApplicationId.from("t1", "a1", "i1");
+        var cluster1 = new ClusterSpec.Id("cluster1");
+        db.addClusterMetrics(application1, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 30.0, 15.0)));
+
+        assertEquals(24 * days, db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
                                                               Set.of("host1")).get(0).size());
+        db.gc();
+        assertTrue(db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")).get(0).size() < 24 * 4);
         db.gc(); // no-op
-        assertEquals(75, db.getNodeTimeseries(Duration.between(startTime, clock.instant()),
-                                                              Set.of("host1")).get(0).size());
+        assertTrue(db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")).get(0).size() < 24 * 4);
     }
 
     /** To manually test that we can read existing data */