diff options
3 files changed, 25 insertions, 35 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index 8d97e14fc7c..3e4bf6233cc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -358,30 +358,10 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { void gc() { synchronized (writeLock) { - // We remove full days at once and we want to see at least three days to not every only see weekend data - Instant oldestToKeep = clock.instant().minus(Duration.ofDays(4)); - SqlExecutionContext context = newContext(); - int partitions = 0; try { - List<String> removeList = new ArrayList<>(); - for (String dirEntry : dir.list()) { - File partitionDir = new File(dir, dirEntry); - if (!partitionDir.isDirectory()) continue; - - partitions++; - DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of("UTC")); - Instant partitionDay = Instant.from(formatter.parse(dirEntry.substring(0, 10) + "T00:00:00")); - if (partitionDay.isBefore(oldestToKeep)) - removeList.add(dirEntry); - - } - // Remove unless all partitions are old: Removing all partitions "will be supported in the future" - if (removeList.size() < partitions && !removeList.isEmpty()) { - issue("alter table " + name + " drop partition list " + - removeList.stream().map(dir -> "'" + dir + "'").collect(Collectors.joining(",")), - context); - } - } catch (SqlException e) { + issue("alter table " + name + " drop partition where at < dateadd('d', -4, now());", newContext()); + } + catch (SqlException e) { log.log(Level.WARNING, "Failed to gc old metrics data in " + dir + " table " + name, e); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java index 34243f4548f..1c458750569 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java @@ -22,6 +22,7 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Tests the Quest metrics db. @@ -163,21 +164,25 @@ public class QuestMetricsDbTest { String dataDir = "data/QuestMetricsDbGc"; IOUtils.recursiveDeleteDir(new File(dataDir)); IOUtils.createDirectory(dataDir + "/metrics"); - ManualClock clock = new ManualClock("2020-10-01T00:00:00"); - QuestMetricsDb db = new QuestMetricsDb(dataDir, clock); + ManualClock clock = new ManualClock(); + int days = 10; // The first metrics are this many days in the past + clock.retreat(Duration.ofDays(10)); Instant startTime = clock.instant(); - int dayOffset = 3; - clock.advance(Duration.ofHours(dayOffset)); - db.addNodeMetrics(nodeTimeseries(24 * 10, Duration.ofHours(1), clock, "host1", "host2", "host3")); - assertEquals(24 * 10, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), - Set.of("host1")).get(0).size()); - db.gc(); - assertEquals(75, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), + QuestMetricsDb db = new QuestMetricsDb(dataDir, clock); + + db.addNodeMetrics(nodeTimeseries(24 * days, Duration.ofHours(1), clock, "host1", "host2", "host3")); + + var application1 = ApplicationId.from("t1", "a1", "i1"); + var cluster1 = new ClusterSpec.Id("cluster1"); + db.addClusterMetrics(application1, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 30.0, 15.0))); + + assertEquals(24 * days, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")).get(0).size()); + db.gc(); + assertTrue(db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")).get(0).size() < 24 * 4); db.gc(); // no-op - assertEquals(75, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), - Set.of("host1")).get(0).size()); + assertTrue(db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")).get(0).size() < 24 * 4); } /** To manually test that we can read existing data */ diff --git a/testutil/src/main/java/com/yahoo/test/ManualClock.java b/testutil/src/main/java/com/yahoo/test/ManualClock.java index ba7d9698d72..296b4a2b360 100644 --- a/testutil/src/main/java/com/yahoo/test/ManualClock.java +++ b/testutil/src/main/java/com/yahoo/test/ManualClock.java @@ -18,7 +18,7 @@ import java.util.concurrent.atomic.AtomicReference; */ public class ManualClock extends Clock { - private AtomicReference<Instant> currentTime = new AtomicReference<>(Instant.now()); + private final AtomicReference<Instant> currentTime = new AtomicReference<>(Instant.now()); @Inject public ManualClock() {} @@ -35,6 +35,11 @@ public class ManualClock extends Clock { currentTime.updateAndGet(time -> time.plus(temporal)); } + /** Move time backwards by the given amount */ + public void retreat(TemporalAmount temporal) { + currentTime.updateAndGet(time -> time.minus(temporal)); + } + public void setInstant(Instant time) { currentTime.set(time); } |