diff options
3 files changed, 34 insertions, 23 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index a9fde4bc26c..aa2a898d58b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -357,8 +357,29 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { } void gc() { + // We remove full days at once and we want to see at least three days to not every only see weekend data + Instant oldestToKeep = clock.instant().minus(Duration.ofDays(4)); + SqlExecutionContext context = newContext(); + int partitions = 0; try { - issue("alter table " + name + " drop partition where at < dateadd('d', -4, now());", newContext()); + List<String> removeList = new ArrayList<>(); + for (String dirEntry : dir.list()) { + File partitionDir = new File(dir, dirEntry); + if ( ! partitionDir.isDirectory()) continue; + + partitions++; + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of("UTC")); + Instant partitionDay = Instant.from(formatter.parse(dirEntry.substring(0, 10) + "T00:00:00")); + if (partitionDay.isBefore(oldestToKeep)) + removeList.add(dirEntry); + + } + // Remove unless all partitions are old: Removing all partitions "will be supported in the future" + if ( removeList.size() < partitions && ! removeList.isEmpty()) { + issue("alter table " + name + " drop partition list " + + removeList.stream().map(dir -> "'" + dir + "'").collect(Collectors.joining(",")), + context); + } } catch (SqlException e) { log.log(Level.WARNING, "Failed to gc old metrics data in " + dir + " table " + name, e); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java index 1c458750569..34243f4548f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java @@ -22,7 +22,6 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; /** * Tests the Quest metrics db. @@ -164,25 +163,21 @@ public class QuestMetricsDbTest { String dataDir = "data/QuestMetricsDbGc"; IOUtils.recursiveDeleteDir(new File(dataDir)); IOUtils.createDirectory(dataDir + "/metrics"); - ManualClock clock = new ManualClock(); - int days = 10; // The first metrics are this many days in the past - clock.retreat(Duration.ofDays(10)); - Instant startTime = clock.instant(); - + ManualClock clock = new ManualClock("2020-10-01T00:00:00"); QuestMetricsDb db = new QuestMetricsDb(dataDir, clock); + Instant startTime = clock.instant(); + int dayOffset = 3; + clock.advance(Duration.ofHours(dayOffset)); + db.addNodeMetrics(nodeTimeseries(24 * 10, Duration.ofHours(1), clock, "host1", "host2", "host3")); - db.addNodeMetrics(nodeTimeseries(24 * days, Duration.ofHours(1), clock, "host1", "host2", "host3")); - - var application1 = ApplicationId.from("t1", "a1", "i1"); - var cluster1 = new ClusterSpec.Id("cluster1"); - db.addClusterMetrics(application1, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 30.0, 15.0))); - - assertEquals(24 * days, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), - Set.of("host1")).get(0).size()); + assertEquals(24 * 10, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), + Set.of("host1")).get(0).size()); db.gc(); - assertTrue(db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")).get(0).size() < 24 * 4); + assertEquals(75, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), + Set.of("host1")).get(0).size()); db.gc(); // no-op - assertTrue(db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")).get(0).size() < 24 * 4); + assertEquals(75, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), + Set.of("host1")).get(0).size()); } /** To manually test that we can read existing data */ diff --git a/testutil/src/main/java/com/yahoo/test/ManualClock.java b/testutil/src/main/java/com/yahoo/test/ManualClock.java index 296b4a2b360..ba7d9698d72 100644 --- a/testutil/src/main/java/com/yahoo/test/ManualClock.java +++ b/testutil/src/main/java/com/yahoo/test/ManualClock.java @@ -18,7 +18,7 @@ import java.util.concurrent.atomic.AtomicReference; */ public class ManualClock extends Clock { - private final AtomicReference<Instant> currentTime = new AtomicReference<>(Instant.now()); + private AtomicReference<Instant> currentTime = new AtomicReference<>(Instant.now()); @Inject public ManualClock() {} @@ -35,11 +35,6 @@ public class ManualClock extends Clock { currentTime.updateAndGet(time -> time.plus(temporal)); } - /** Move time backwards by the given amount */ - public void retreat(TemporalAmount temporal) { - currentTime.updateAndGet(time -> time.minus(temporal)); - } - public void setInstant(Instant time) { currentTime.set(time); } |