Handle 'wrong order'

1 files changed, 30 insertions, 3 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
index 48e50b935a4..6e8a0ac8309 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java
@@ -84,13 +84,27 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
         System.setProperty("out", logConfig);
 
         this.dataDir = dataDir;
-        engine = new CairoEngine(new DefaultCairoConfiguration(dataDir));
+        engine = createEngine(dataDir);
         sqlCompilerPool = new ConcurrentResourcePool<>(() -> SqlCompilerFactoryImpl.INSTANCE.getInstance(engine()));
         nodeTable = new Table(dataDir, "metrics");
         clusterTable = new Table(dataDir, "clusterMetrics");
         ensureTablesExist();
     }
 
+    private static CairoEngine createEngine(String dataDir) {
+        try {
+            return new CairoEngine(new DefaultCairoConfiguration(dataDir));
+        }
+        catch (CairoException e) {
+            if (e.getMessage().contains("partitions are not ordered")) { // Happens when migrating 6.7 -> 7.3.1
+                repairTables(dataDir, e, "metrics", "clusterMetrics");
+                return new CairoEngine(new DefaultCairoConfiguration(dataDir));
+            }
+            throw new IllegalStateException("Could not create Quest db in " + dataDir, e);
+        }
+
+    }
+
     private CairoEngine engine() {
         if (closed.get())
             throw new IllegalStateException("Attempted to access QuestDb after calling close");
@@ -360,6 +374,19 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
                 .with(AllowAllSecurityContext.INSTANCE, null);
     }
 
+    private static void repairTables(String dataDir, Exception e, String ... tableNames) {
+        log.log(Level.WARNING, "QuestDb seems corrupted, wiping data and starting over", e);
+        for (String name : tableNames)
+            repairTable(dataDir, name);
+    }
+
+    private static void repairTable(String dataDir, String name) {
+        var dir = new File(dataDir, name);
+        IOUtils.createDirectory(dir.getPath());
+        IOUtils.recursiveDeleteDir(dir);
+        IOUtils.createDirectory(dir.getPath());
+    }
+
     /** A questDb table */
     private class Table {
 
@@ -375,6 +402,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
             // https://stackoverflow.com/questions/67785629/what-does-max-txn-txn-inflight-limit-reached-in-questdb-and-how-to-i-avoid-it
             new File(dir + "/_txn_scoreboard").delete();
         }
+
         private TableToken token() { return engine().getTableTokenIfExists(name); }
 
         boolean exists() {
@@ -406,8 +434,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb {
          */
         private void repair(Exception e) {
             log.log(Level.WARNING, "QuestDb seems corrupted, wiping data and starting over", e);
-            IOUtils.recursiveDeleteDir(dir);
-            IOUtils.createDirectory(dir.getPath());
+            repairTable(dataDir, name);
             ensureTablesExist();
         }