Set speed: 1 for config change reindexing, and user-specified in (0, 10] from handler

9 files changed, 67 insertions, 64 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java
index 2403fe9d965..7f099e4cf67 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java
@@ -173,6 +173,7 @@ public class ApplicationCuratorDatabase {
         private static final String NAME = "name";
         private static final String GENERATION = "generation";
         private static final String EPOCH_MILLIS = "epochMillis";
+        private static final String SPEED = "speed";
 
         private static byte[] toBytes(ApplicationReindexing reindexing) {
             Cursor root = new Slime().setObject();
@@ -202,6 +203,7 @@ public class ApplicationCuratorDatabase {
 
         private static void setStatus(Cursor statusObject, Status status) {
             statusObject.setLong(EPOCH_MILLIS, status.ready().toEpochMilli());
+            statusObject.setDouble(SPEED, status.speed());
         }
 
         private static ApplicationReindexing fromBytes(byte[] data) {
@@ -222,7 +224,8 @@ public class ApplicationCuratorDatabase {
         }
 
         private static Status getStatus(Inspector statusObject) {
-            return new Status(Instant.ofEpochMilli(statusObject.field(EPOCH_MILLIS).asLong()));
+            return new Status(Instant.ofEpochMilli(statusObject.field(EPOCH_MILLIS).asLong()),
+                              statusObject.field(SPEED).valid() ? statusObject.field(SPEED).asDouble() : 0.2);
         }
 
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java
index c1ef4127919..19395b10b2b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java
@@ -37,10 +37,10 @@ public class ApplicationReindexing implements Reindexing {
     }
 
     /** Returns a copy of this with reindexing for the given document type in the given cluster ready at the given instant. */
-    public ApplicationReindexing withReady(String cluster, String documentType, Instant readyAt) {
+    public ApplicationReindexing withReady(String cluster, String documentType, Instant readyAt, double speed) {
         Cluster current = clusters.getOrDefault(cluster, Cluster.empty());
         Cluster modified = new Cluster(current.pending,
-                                       with(documentType, new Status(readyAt), current.ready));
+                                       with(documentType, new Status(readyAt, speed), current.ready));
         return new ApplicationReindexing(enabled, with(cluster, modified, clusters));
     }
 
@@ -171,30 +171,40 @@ public class ApplicationReindexing implements Reindexing {
     public static class Status implements Reindexing.Status {
 
         private final Instant ready;
+        private final double speed;
+
+        Status(Instant ready, double speed) {
+            if (speed <= 0 || 10 < speed)
+                throw new IllegalArgumentException("Reindexing speed must be in (0, 10], but was " + speed);
 
-        Status(Instant ready) {
             this.ready = ready.truncatedTo(ChronoUnit.MILLIS);
+            this.speed = speed;
         }
 
         @Override
         public Instant ready() { return ready; }
 
         @Override
+        public double speed() {
+            return speed;
+        }
+
+        @Override
         public boolean equals(Object o) {
             if (this == o) return true;
             if (o == null || getClass() != o.getClass()) return false;
             Status status = (Status) o;
-            return ready.equals(status.ready);
+            return Double.compare(status.speed, speed) == 0 && ready.equals(status.ready);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(ready);
+            return Objects.hash(ready, speed);
         }
 
         @Override
         public String toString() {
-            return "ready at " + ready;
+            return "ready at " + ready + ", with relative speed " + speed;
         }
 
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index e581a1edc21..4dda141491c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -19,6 +19,7 @@ import com.yahoo.restapi.Path;
 import com.yahoo.slime.SlimeUtils;
 import com.yahoo.text.StringUtilities;
 import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.application.ApplicationReindexing;
 import com.yahoo.vespa.config.server.http.ContentHandler;
 import com.yahoo.vespa.config.server.http.ContentRequest;
 import com.yahoo.vespa.config.server.http.HttpHandler;
@@ -35,6 +36,7 @@ import java.io.IOException;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.StringJoiner;
@@ -207,6 +209,7 @@ public class ApplicationHandler extends HttpHandler {
         boolean indexedOnly = request.getBooleanProperty("indexedOnly");
         Set<String> clusters = StringUtilities.split(request.getProperty("clusterId"));
         Set<String> types = StringUtilities.split(request.getProperty("documentType"));
+        double speed = Double.parseDouble(Objects.requireNonNullElse(request.getProperty("speed"), "1"));
 
         Map<String, Set<String>> reindexed = new TreeMap<>();
         Instant now = applicationRepository.clock().instant();
@@ -222,7 +225,7 @@ public class ApplicationHandler extends HttpHandler {
                                                            String.join(", ", documentTypes.get(cluster)));
 
                     if ( ! indexedOnly || indexedDocumentTypes.get(cluster).contains(type)) {
-                        reindexing = reindexing.withReady(cluster, type, now);
+                        reindexing = reindexing.withReady(cluster, type, now, speed);
                         reindexed.computeIfAbsent(cluster, __ -> new TreeSet<>()).add(type);
                     }
                 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ReindexingResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ReindexingResponse.java
index 1a8f3042f13..1844b204a63 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ReindexingResponse.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/response/ReindexingResponse.java
@@ -39,6 +39,7 @@ public class ReindexingResponse extends JSONResponse {
 
     private static void setStatus(Cursor object, ApplicationReindexing.Status readyStatus) {
         object.setLong("readyMillis", readyStatus.ready().toEpochMilli());
+        object.setDouble("speed", readyStatus.speed());
     }
 
     private static void setStatus(Cursor object, ClusterReindexing.Status status) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
index fb4d7ac1fc1..99c3ecae62a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
@@ -39,6 +39,9 @@ public class ReindexingMaintainer extends ConfigServerMaintainer {
     /** Timeout per service when getting config generations. */
     private static final Duration timeout = Duration.ofSeconds(10);
 
+    /** Relative reindexing speed. */
+    static final double SPEED = 1;
+
     private final ConfigConvergenceChecker convergence;
     private final Clock clock;
 
@@ -92,9 +95,10 @@ public class ReindexingMaintainer extends ConfigServerMaintainer {
         // Config convergence means reindexing of detected reindex actions may begin.
         for (var cluster : reindexing.clusters().entrySet())
             for (var pending : cluster.getValue().pending().entrySet())
-                if (pending.getValue() <= oldestGeneration.get())
-                    reindexing = reindexing.withReady(cluster.getKey(), pending.getKey(), now)
+                if (pending.getValue() <= oldestGeneration.get()) {
+                    reindexing = reindexing.withReady(cluster.getKey(), pending.getKey(), now, SPEED)
                                            .withoutPending(cluster.getKey(), pending.getKey());
+                }
 
         return reindexing;
     }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java
index 476ac7172bb..eb66e7abd88 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java
@@ -23,11 +23,8 @@ public class ApplicationCuratorDatabaseTest {
         assertEquals(Optional.empty(), db.readReindexingStatus(id));
 
         ApplicationReindexing reindexing = ApplicationReindexing.empty()
-                                                                .withPending("one", "a", 10)
-                                                                .withReady("two", "b", Instant.ofEpochMilli(2))
-                                                                .withPending("two", "b", 20)
-                                                                .withReady("one", "a", Instant.ofEpochMilli(1))
-                                                                .withReady("two", "c", Instant.ofEpochMilli(3))
+                                                                .withPending("one", "a", 10).withReady("two", "b", Instant.ofEpochMilli(2), 0.2)
+                                                                .withPending("two", "b", 20).withReady("one", "a", Instant.ofEpochMilli(1), 0.2).withReady("two", "c", Instant.ofEpochMilli(3), 0.2)
                                                                 .enabled(false);
 
         db.writeReindexingStatus(id, reindexing);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationReindexingTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationReindexingTest.java
index 182ea108520..3a19cee6675 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationReindexingTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationReindexingTest.java
@@ -22,10 +22,10 @@ public class ApplicationReindexingTest {
     public void test() {
         ApplicationReindexing reindexing = ApplicationReindexing.empty()
                                                                 .withPending("one", "a", 10)
-                                                                .withReady("two", "b", Instant.ofEpochMilli(2))
+                                                                .withReady("two", "b", Instant.ofEpochMilli(2), 3)
                                                                 .withPending("two", "b", 20)
-                                                                .withReady("one", "a", Instant.ofEpochMilli(1))
-                                                                .withReady("two", "a", Instant.ofEpochMilli(3))
+                                                                .withReady("one", "a", Instant.ofEpochMilli(1), 1)
+                                                                .withReady("two", "a", Instant.ofEpochMilli(3), 2)
                                                                 .withoutPending("one", "a");
 
         assertEquals(Instant.ofEpochMilli(1),
@@ -51,14 +51,14 @@ public class ApplicationReindexingTest {
         assertEquals(Set.of("one", "two"),
                      reindexing.clusters().keySet());
 
-        assertEquals(Map.of("a", new Status(Instant.ofEpochMilli(1))),
+        assertEquals(Map.of("a", new Status(Instant.ofEpochMilli(1), 1)),
                      reindexing.clusters().get("one").ready());
 
         assertEquals(Map.of(),
                      reindexing.clusters().get("one").pending());
 
-        assertEquals(Map.of("a", new Status(Instant.ofEpochMilli(3)),
-                            "b", new Status(Instant.ofEpochMilli(2))),
+        assertEquals(Map.of("a", new Status(Instant.ofEpochMilli(3), 2),
+                            "b", new Status(Instant.ofEpochMilli(2), 3)),
                      reindexing.clusters().get("two").ready());
 
         assertEquals(Map.of("b", 20L),
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index 2df7739e552..0b8bf8e84bd 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -228,48 +228,37 @@ public class ApplicationHandlerTest {
 
         clock.advance(Duration.ofSeconds(1));
         reindex(applicationId, "", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar, bax, baz] in 'foo' of application default.default\"}");
-        expected = expected.withReady("boo", "bar", clock.instant())
-                           .withReady("foo", "bar", clock.instant())
-                           .withReady("foo", "baz", clock.instant())
-                           .withReady("foo", "bax", clock.instant());
+        expected = expected.withReady("boo", "bar", clock.instant(), 1).withReady("foo", "bar", clock.instant(), 1).withReady("foo", "baz", clock.instant(), 1).withReady("foo", "bax", clock.instant(), 1);
         assertEquals(expected,
                      database.readReindexingStatus(applicationId).orElseThrow());
 
         clock.advance(Duration.ofSeconds(1));
         reindex(applicationId, "?indexedOnly=true", "{\"message\":\"Reindexing document types [bar] in 'foo' of application default.default\"}");
-        expected = expected.withReady("foo", "bar", clock.instant());
+        expected = expected.withReady("foo", "bar", clock.instant(), 1);
         assertEquals(expected,
                      database.readReindexingStatus(applicationId).orElseThrow());
 
         clock.advance(Duration.ofSeconds(1));
-        expected = expected.withReady("boo", "bar", clock.instant())
-                           .withReady("foo", "bar", clock.instant())
-                           .withReady("foo", "baz", clock.instant())
-                           .withReady("foo", "bax", clock.instant());
+        expected = expected.withReady("boo", "bar", clock.instant(), 1).withReady("foo", "bar", clock.instant(), 1).withReady("foo", "baz", clock.instant(), 1).withReady("foo", "bax", clock.instant(), 1);
         reindex(applicationId, "?clusterId=", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar, bax, baz] in 'foo' of application default.default\"}");
         assertEquals(expected,
                      database.readReindexingStatus(applicationId).orElseThrow());
 
         clock.advance(Duration.ofSeconds(1));
-        expected = expected.withReady("boo", "bar", clock.instant())
-                           .withReady("foo", "bar", clock.instant());
+        expected = expected.withReady("boo", "bar", clock.instant(), 1).withReady("foo", "bar", clock.instant(), 1);
         reindex(applicationId, "?documentType=bar", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar] in 'foo' of application default.default\"}");
         assertEquals(expected,
                      database.readReindexingStatus(applicationId).orElseThrow());
 
         clock.advance(Duration.ofSeconds(1));
         reindex(applicationId, "?clusterId=foo,boo", "{\"message\":\"Reindexing document types [bar] in 'boo', [bar, bax, baz] in 'foo' of application default.default\"}");
-        expected = expected.withReady("boo", "bar", clock.instant())
-                           .withReady("foo", "bar", clock.instant())
-                           .withReady("foo", "baz", clock.instant())
-                           .withReady("foo", "bax", clock.instant());
+        expected = expected.withReady("boo", "bar", clock.instant(), 1).withReady("foo", "bar", clock.instant(), 1).withReady("foo", "baz", clock.instant(), 1).withReady("foo", "bax", clock.instant(), 1);
         assertEquals(expected,
                      database.readReindexingStatus(applicationId).orElseThrow());
 
         clock.advance(Duration.ofSeconds(1));
-        reindex(applicationId, "?clusterId=foo&documentType=bar,baz", "{\"message\":\"Reindexing document types [bar, baz] in 'foo' of application default.default\"}");
-        expected = expected.withReady("foo", "bar", clock.instant())
-                           .withReady("foo", "baz", clock.instant());
+        reindex(applicationId, "?clusterId=foo&documentType=bar,baz&speed=0.1", "{\"message\":\"Reindexing document types [bar, baz] in 'foo' of application default.default\"}");
+        expected = expected.withReady("foo", "bar", clock.instant(), 0.1).withReady("foo", "baz", clock.instant(), 0.1);
         assertEquals(expected,
                      database.readReindexingStatus(applicationId).orElseThrow());
 
@@ -295,7 +284,8 @@ public class ApplicationHandlerTest {
                                        "      }," +
                                        "      \"ready\": {" +
                                        "        \"bar\": {" +
-                                       "          \"readyMillis\": " + (now - 1000) +
+                                       "          \"readyMillis\": " + (now - 1000) + ", " +
+                                       "          \"speed\": 1.0" +
                                        "        }" +
                                        "      }" +
                                        "    }," +
@@ -303,13 +293,16 @@ public class ApplicationHandlerTest {
                                        "      \"pending\": {}," +
                                        "      \"ready\": {" +
                                        "        \"bar\": {" +
-                                       "          \"readyMillis\": " + now +
+                                       "          \"readyMillis\": " + now + ", " +
+                                       "          \"speed\": 0.1" +
                                        "        }," +
                                        "        \"bax\": {" +
-                                       "          \"readyMillis\": " + (now - 1000) +
+                                       "          \"readyMillis\": " + (now - 1000) + ", " +
+                                       "          \"speed\": 1.0" +
                                        "        }," +
                                        "        \"baz\": {" +
-                                       "          \"readyMillis\": " + now +
+                                       "          \"readyMillis\": " + now + ", " +
+                                       "          \"speed\": 0.1" +
                                        "        }" +
                                        "      }" +
                                        "    }" +
@@ -482,8 +475,7 @@ public class ApplicationHandlerTest {
     public void testReindexingSerialization() throws IOException {
         Instant now = Instant.ofEpochMilli(123456);
         ApplicationReindexing applicationReindexing = ApplicationReindexing.empty()
-                                                                           .withPending("foo", "bar", 123L)
-                                                                           .withReady("moo", "baz", now);
+                                                                           .withPending("foo", "bar", 123L).withReady("moo", "baz", now, 1);
         ClusterReindexing clusterReindexing = new ClusterReindexing(Map.of("bax", new Status(now, null, null, null, null),
                                                                            "baz", new Status(now.plusSeconds(1),
                                                                                              now.plusSeconds(2),
@@ -534,6 +526,7 @@ public class ApplicationHandlerTest {
                          "        },\n" +
                          "        \"baz\": {\n" +
                          "          \"readyMillis\": 123456,\n" +
+                         "          \"speed\": 1.0,\n" +
                          "          \"startedMillis\": 124456,\n" +
                          "          \"endedMillis\": 125456,\n" +
                          "          \"state\": \"failed\",\n" +
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java
index 03b872a3d72..26edc5d0f71 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java
@@ -8,6 +8,7 @@ import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 
+import static com.yahoo.vespa.config.server.maintenance.ReindexingMaintainer.SPEED;
 import static com.yahoo.vespa.config.server.maintenance.ReindexingMaintainer.withNewReady;
 import static com.yahoo.vespa.config.server.maintenance.ReindexingMaintainer.withOnlyCurrentData;
 import static org.junit.Assert.assertEquals;
@@ -21,10 +22,7 @@ public class ReindexingMaintainerTest {
     public void testReadyComputation() {
         ApplicationReindexing reindexing = ApplicationReindexing.empty()
                                                                 .withPending("one", "a", 10)
-                                                                .withPending("two", "b", 20)
-                                                                .withReady("one", "a", Instant.ofEpochMilli(3))
-                                                                .withReady("two", "b", Instant.ofEpochMilli(2))
-                                                                .withReady("two", "c", Instant.ofEpochMilli(3));
+                                                                .withPending("two", "b", 20).withReady("one", "a", Instant.ofEpochMilli(3), SPEED).withReady("two", "b", Instant.ofEpochMilli(2), SPEED).withReady("two", "c", Instant.ofEpochMilli(3), SPEED);
 
         // Nothing happens without convergence.
         assertEquals(reindexing,
@@ -32,14 +30,14 @@ public class ReindexingMaintainerTest {
 
         // Status for (one, a) changes, but not (two, b).
         Instant later = Instant.ofEpochMilli(3 << 10);
-        assertEquals(reindexing.withoutPending("one", "a")      // Converged, no longer pending.
-                               .withReady("one", "a", later),   // Converged, now ready.
+        // Converged, no longer pending.
+        assertEquals(reindexing.withoutPending("one", "a").withReady("one", "a", later, SPEED),   // Converged, now ready.
                      withNewReady(reindexing, () -> 19L, later));
 
-        assertEquals(reindexing.withoutPending("one", "a")      // Converged, no longer pending.
-                               .withReady("one", "a", later)
-                               .withoutPending("two", "b")      // Converged, no Longer pending.
-                               .withReady("two", "b", later),
+        // Converged, no longer pending.
+        // Converged, no Longer pending.
+        assertEquals(reindexing.withoutPending("one", "a").withReady("one", "a", later, SPEED)
+                               .withoutPending("two", "b").withReady("two", "b", later, SPEED),
                      withNewReady(reindexing, () -> 20L, later));
 
         // Verify generation supplier isn't called when no pending document types.
@@ -52,10 +50,7 @@ public class ReindexingMaintainerTest {
     public void testGarbageRemoval() {
         ApplicationReindexing reindexing = ApplicationReindexing.empty()
                                                                 .withPending("one", "a", 10)
-                                                                .withPending("two", "b", 20)
-                                                                .withReady("one", "a", Instant.ofEpochMilli(3))
-                                                                .withReady("two", "b", Instant.ofEpochMilli(2))
-                                                                .withReady("two", "c", Instant.ofEpochMilli(3));
+                                                                .withPending("two", "b", 20).withReady("one", "a", Instant.ofEpochMilli(3), SPEED).withReady("two", "b", Instant.ofEpochMilli(2), SPEED).withReady("two", "c", Instant.ofEpochMilli(3), SPEED);
 
         assertEquals(reindexing,
                      withOnlyCurrentData(reindexing, Map.of("one", List.of("a", "b", "c", "d"),
@@ -67,13 +62,10 @@ public class ReindexingMaintainerTest {
                                                             "two", List.of("b", "c"))));
 
         assertEquals(ApplicationReindexing.empty()
-                                          .withPending("two", "b", 20)
-                                          .withReady("two", "b", Instant.ofEpochMilli(2)),
+                                          .withPending("two", "b", 20).withReady("two", "b", Instant.ofEpochMilli(2), SPEED),
                      withOnlyCurrentData(reindexing, Map.of("two", List.of("a", "b"))));
 
-        assertEquals(ApplicationReindexing.empty()
-                                          .withReady("one", "a", Instant.EPOCH).without("one", "a")
-                                          .withReady("two", "c", Instant.ofEpochMilli(3)),
+        assertEquals(ApplicationReindexing.empty().withReady("one", "a", Instant.EPOCH, SPEED).without("one", "a").withReady("two", "c", Instant.ofEpochMilli(3), SPEED),
                      withOnlyCurrentData(reindexing, Map.of("one", List.of("c"),
                                                             "two", List.of("c"))));
     }